From: Hans-Christoph Steiner Date: Tue, 6 Dec 2016 13:03:34 +0000 (+0100) Subject: Merge branch 'feature/dscanner' into master X-Git-Tag: 0.8~140 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=7c8823e94e37b6a6629cf1de0aed73fc92d58ad8;hp=f4392663033508a3b0cdc9b5c18174ad76bf45c8;p=fdroidserver.git Merge branch 'feature/dscanner' into master dscanner - drozer scanner work. closes !187 --- diff --git a/README.md b/README.md index aee5b82d..95cf67f3 100644 --- a/README.md +++ b/README.md @@ -76,3 +76,19 @@ Then here's how to install: source env/bin/activate pip3 install -e . python3 setup.py install + + +### Drozer Scanner + +There is a new feature under development that can scan any APK in a +repo, or any build, using Drozer. Drozer is a dynamic exploit +scanner, it runs an app in the emulator and runs known exploits on it. + +This setup requires specific versions of two Python modules: +_docker-py_ 1.9.0 and _requests_ older than 2.11. Other versions +might cause the docker-py connection to break with the containers. +Newer versions of docker-py might have this fixed already. + +For Debian based distributions: + + apt-get install libffi-dev libssl-dev python-docker diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..9b7b4fb6 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,180 @@ +# This image is intended to be used with fdroidserver for the purpose +# of dynamic scanning of pre-built APKs during the fdroid build process. + +# Start with ubuntu 12.04 (i386). +FROM ubuntu:14.04 +MAINTAINER fdroid.dscanner + +ENV DROZER_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer_2.3.4.deb +ENV DROZER_DEB drozer_2.3.4.deb + +ENV AGENT_URL https://github.com/mwrlabs/drozer/releases/download/2.3.4/drozer-agent-2.3.4.apk +ENV AGENT_APK drozer-agent-2.3.4.apk + +# Specially for SSH access and port redirection +ENV ROOTPASSWORD android + +# Expose ADB, ADB control and VNC ports +EXPOSE 22 +EXPOSE 5037 +EXPOSE 5554 +EXPOSE 5555 +EXPOSE 5900 +EXPOSE 5901 + +ENV DEBIAN_FRONTEND noninteractive +RUN echo "debconf shared/accepted-oracle-license-v1-1 select true" | debconf-set-selections +RUN echo "debconf shared/accepted-oracle-license-v1-1 seen true" | debconf-set-selections + +# Update packages +RUN apt-get -y update + +# Drozer packages +RUN apt-get install wget python2.7 python-dev python2.7-dev python-openssl python-twisted python-protobuf bash-completion -y + +# First, install add-apt-repository, sshd and bzip2 +RUN apt-get -y install python-software-properties bzip2 ssh net-tools + +# ubuntu 14.04 needs this too +RUN apt-get -y install software-properties-common + +# Add oracle-jdk7 to repositories +RUN add-apt-repository ppa:webupd8team/java + +# Make sure the package repository is up to date +RUN echo "deb http://archive.ubuntu.com/ubuntu trusty main universe" > /etc/apt/sources.list + +# Update apt +RUN apt-get update + +# Add drozer +RUN useradd -ms /bin/bash drozer + +# Install oracle-jdk7 +RUN apt-get -y install oracle-java7-installer + +# Install android sdk +RUN wget http://dl.google.com/android/android-sdk_r23-linux.tgz +RUN tar -xvzf android-sdk_r23-linux.tgz +RUN mv -v android-sdk-linux /usr/local/android-sdk + +# Install apache ant +RUN wget http://archive.apache.org/dist/ant/binaries/apache-ant-1.8.4-bin.tar.gz +RUN tar -xvzf apache-ant-1.8.4-bin.tar.gz +RUN mv -v apache-ant-1.8.4 /usr/local/apache-ant + +# Add android tools and platform tools to PATH +ENV ANDROID_HOME /usr/local/android-sdk +ENV PATH $PATH:$ANDROID_HOME/tools +ENV PATH $PATH:$ANDROID_HOME/platform-tools + +# Add ant to PATH +ENV ANT_HOME /usr/local/apache-ant +ENV PATH $PATH:$ANT_HOME/bin + +# Export JAVA_HOME variable +ENV JAVA_HOME /usr/lib/jvm/java-7-oracle + +# Remove compressed files. +RUN cd /; rm android-sdk_r23-linux.tgz && rm apache-ant-1.8.4-bin.tar.gz + +# Some preparation before update +RUN chown -R root:root /usr/local/android-sdk/ + +# Install latest android tools and system images +RUN echo "y" | android update sdk --filter platform-tool --no-ui --force +RUN echo "y" | android update sdk --filter platform --no-ui --force +RUN echo "y" | android update sdk --filter build-tools-22.0.1 --no-ui -a +RUN echo "y" | android update sdk --filter sys-img-x86-android-19 --no-ui -a +#RUN echo "y" | android update sdk --filter sys-img-x86-android-21 --no-ui -a +#RUN echo "y" | android update sdk --filter sys-img-x86-android-22 --no-ui -a +RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-19 --no-ui -a +#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-21 --no-ui -a +#RUN echo "y" | android update sdk --filter sys-img-armeabi-v7a-android-22 --no-ui -a + +# Update ADB +RUN echo "y" | android update adb + +# Create fake keymap file +RUN mkdir /usr/local/android-sdk/tools/keymaps +RUN touch /usr/local/android-sdk/tools/keymaps/en-us + +# Run sshd +RUN apt-get install -y openssh-server +RUN mkdir /var/run/sshd +RUN echo "root:$ROOTPASSWORD" | chpasswd +RUN sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config +RUN sed -i 's/PermitEmptyPasswords no/PermitEmptyPasswords yes/' /etc/ssh/sshd_config + +# SSH login fix. Otherwise user is kicked off after login +RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd + +ENV NOTVISIBLE "in users profile" +RUN echo "export VISIBLE=now" >> /etc/profile + +# Install socat +RUN apt-get install -y socat + +# symlink android bins +RUN ln -sv /usr/local/android-sdk/tools/android /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/emulator /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/ddms /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/scheenshot2 /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/monkeyrunner /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/monitor /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/mksdcard /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/uiautomatorviewer /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/tools/traceview /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/platform-tools/adb /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/platform-tools/fastboot /usr/local/bin/ +RUN ln -sv /usr/local/android-sdk/platform-tools/sqlite3 /usr/local/bin/ + +# Setup DROZER... +# https://labs.mwrinfosecurity.com/tools/drozer/ + +# Run as drozer user +WORKDIR /home/drozer + +# Site lists the shasums, however, I'm not sure the best way to integrate the +# checks here. No real idiomatic way for Dockerfile to do that and most of +# the examples online use chained commands but we want things to *BREAK* when +# the sha doesn't match. So far, I can't seem to reliably make Docker not +# finish the image build process. + +# Download the console +RUN wget -c $DROZER_URL + +# Install the console +RUN dpkg -i $DROZER_DEB + +# Download agent +RUN wget -c $AGENT_URL +# Keep it version agnostic for other scripts such as install_drozer.py +RUN mv -v $AGENT_APK drozer-agent.apk + +# Port forwarding required by drozer +RUN echo 'adb forward tcp:31415 tcp:31415' >> /home/drozer/.bashrc + +# Alias for Drozer +RUN echo "alias drozer='drozer console connect'" >> /home/drozer/.bashrc + +# add extra scripting +COPY install_agent.py /home/drozer/install_agent.py +RUN chmod 755 /home/drozer/install_agent.py +COPY enable_service.py /home/drozer/enable_service.py +RUN chmod 755 /home/drozer/enable_service.py +COPY drozer.py /home/drozer/drozer.py +RUN chmod 755 /home/drozer/drozer.py + +# fix ownerships +RUN chown -R drozer.drozer /home/drozer + +RUN apt-get -y --force-yes install python-pkg-resources=3.3-1ubuntu1 +RUN apt-get -y install python-pip python-setuptools git +RUN pip install "git+https://github.com/dtmilano/AndroidViewClient.git#egg=androidviewclient" +RUN apt-get -y install python-pexpect + +# Add entrypoint +COPY entrypoint.sh /home/drozer/entrypoint.sh +RUN chmod +x /home/drozer/entrypoint.sh +ENTRYPOINT ["/home/drozer/entrypoint.sh"] diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 00000000..eacb3268 --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,48 @@ +SHELL := /bin/bash +ALIAS = "dscanner" +EXISTS := $(shell docker ps -a -q -f name=$(ALIAS)) +RUNNED := $(shell docker ps -q -f name=$(ALIAS)) +ifneq "$(RUNNED)" "" +IP := $(shell docker inspect $(ALIAS) | grep "IPAddress\"" | head -n1 | cut -d '"' -f 4) +endif +STALE_IMAGES := $(shell docker images | grep "" | awk '{print($$3)}') +EMULATOR ?= "android-19" +ARCH ?= "armeabi-v7a" + +COLON := : + +.PHONY = build clean kill info + +all: help + +help: + @echo "usage: make {help|build|clean|kill|info}" + @echo "" + @echo " help this help screen" + @echo " build create docker image" + @echo " clean remove images and containers" + @echo " kill stop running containers" + @echo " info details of running container" + +build: + @docker build -t "dscanner/fdroidserver:latest" . + +clean: kill + @docker ps -a -q | xargs -n 1 -I {} docker rm -f {} +ifneq "$(STALE_IMAGES)" "" + @docker rmi -f $(STALE_IMAGES) +endif + +kill: +ifneq "$(RUNNED)" "" + @docker kill $(ALIAS) +endif + +info: + @docker ps -a -f name=$(ALIAS) +ifneq "$(RUNNED)" "" + $(eval ADBPORT := $(shell docker port $(ALIAS) | grep '5555/tcp' | awk '{split($$3,a,"$(COLON)");print a[2]}')) + @echo -e "Use:\n adb kill-server\n adb connect $(IP):$(ADBPORT)" +else + @echo "Run container" +endif diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 00000000..9f2d657c --- /dev/null +++ b/docker/README.md @@ -0,0 +1,13 @@ +# dscanner docker image # + +Use `make help` for up-to-date instructions. + +``` +usage: make {help|build|clean|kill|info} + + help this help screen + build create docker image + clean remove images and containers + kill stop running containers + info details of running container +``` diff --git a/docker/drozer.py b/docker/drozer.py new file mode 100644 index 00000000..d0546934 --- /dev/null +++ b/docker/drozer.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python2 + +import pexpect +import sys + +prompt = "dz>" +target = sys.argv[1] + +drozer = pexpect.spawn("drozer console connect") +drozer.logfile = open("/tmp/drozer_report.log", "w") + + +# start +drozer.expect(prompt) + + +def send_command(command, target): + cmd = "run {0} -a {1}".format(command, target) + drozer.sendline(cmd) + drozer.expect(prompt) + +scanners = [ + "scanner.misc.native", # Find native components included in packages + #"scanner.misc.readablefiles", # Find world-readable files in the given folder + #"scanner.misc.secretcodes", # Search for secret codes that can be used from the dialer + #"scanner.misc.sflagbinaries", # Find suid/sgid binaries in the given folder (default is /system). + #"scanner.misc.writablefiles", # Find world-writable files in the given folder + "scanner.provider.finduris", # Search for content providers that can be queried. + "scanner.provider.injection", # Test content providers for SQL injection vulnerabilities. + "scanner.provider.sqltables", # Find tables accessible through SQL injection vulnerabilities. + "scanner.provider.traversal" # Test content providers for basic directory traversal +] + +for scanner in scanners: + send_command(scanner, target) diff --git a/docker/enable_service.py b/docker/enable_service.py new file mode 100755 index 00000000..803532c9 --- /dev/null +++ b/docker/enable_service.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python2 + +from com.dtmilano.android.viewclient import ViewClient + +vc = ViewClient(*ViewClient.connectToDeviceOrExit()) + +button = vc.findViewWithText("OFF") + +if button: + (x, y) = button.getXY() + button.touch() +else: + print("Button not found. Is the app currently running?") + exit() + +print("Done!") diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh new file mode 100755 index 00000000..95b5ede1 --- /dev/null +++ b/docker/entrypoint.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +if [[ $EMULATOR == "" ]]; then + EMULATOR="android-19" + echo "Using default emulator $EMULATOR" +fi + +if [[ $ARCH == "" ]]; then + ARCH="x86" + echo "Using default arch $ARCH" +fi +echo EMULATOR = "Requested API: ${EMULATOR} (${ARCH}) emulator." +if [[ -n $1 ]]; then + echo "Last line of file specified as non-opt/last argument:" + tail -1 $1 +fi + +# Run sshd +/usr/sbin/sshd +adb start-server + +# Detect ip and forward ADB ports outside to outside interface +ip=$(ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $1}') +socat tcp-listen:5037,bind=$ip,fork tcp:127.0.0.1:5037 & +socat tcp-listen:5554,bind=$ip,fork tcp:127.0.0.1:5554 & +socat tcp-listen:5555,bind=$ip,fork tcp:127.0.0.1:5555 & + +# Set up and run emulator +if [[ $ARCH == *"x86"* ]] +then + EMU="x86" +else + EMU="arm" +fi + +#FASTDROID_VNC_URL="https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/fastdroid-vnc/fastdroid-vnc" +#wget -c "${FASTDROID_VNC_URL}" + +export PATH="${PATH}:/usr/local/android-sdk/tools/:/usr/local/android-sdk/platform-tools/" + +echo "no" | android create avd -f -n test -t ${EMULATOR} --abi default/${ARCH} +echo "no" | emulator64-${EMU} -avd test -noaudio -no-window -gpu off -verbose -qemu -usbdevice tablet -vnc :0 diff --git a/docker/install_agent.py b/docker/install_agent.py new file mode 100755 index 00000000..1a0f348a --- /dev/null +++ b/docker/install_agent.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python2 + +import os +from subprocess import call, check_output +from time import sleep + +FNULL = open(os.devnull, 'w') + +print("Ensuring device is online") +call("adb wait-for-device", shell=True) + +print("Installing the drozer agent") +print("If the device just came online it is likely the package manager hasn't booted.") +print("Will try multiple attempts to install.") +print("This may need tweaking depending on hardware.") + + +attempts = 0 +time_to_sleep = 30 + +while attempts < 8: + output = check_output('adb shell "pm list packages"', shell=True) + print("Checking whether the package manager is up...") + if "Could not access the Package Manager" in output: + print("Nope. Sleeping for 30 seconds and then trying again.") + sleep(time_to_sleep) + else: + break + +time_to_sleep = 5 +attempts = 0 + +while attempts < 5: + sleep(time_to_sleep) + try: + install_output = check_output("adb install /home/drozer/drozer-agent.apk", shell=True) + except Exception: + print("Failed. Trying again.") + attempts += 1 + else: + attempts += 1 + if "Error: Could not access the Package Manager" not in install_output: + break + +print("Install attempted. Checking everything worked") + +pm_list_output = check_output('adb shell "pm list packages"', shell=True) + +if "com.mwr.dz" not in pm_list_output: + print(install_output) + exit("APK didn't install properly. Exiting.") + +print("Installed ok.") + +print("Starting the drozer agent main activity: com.mwr.dz/.activities.MainActivity") +call('adb shell "am start com.mwr.dz/.activities.MainActivity"', shell=True, stdout=FNULL) + +print("Starting the service") +# start the service +call("python /home/drozer/enable_service.py", shell=True, stdout=FNULL) + +print("Forward dem ports mon.") +call("adb forward tcp:31415 tcp:31415", shell=True, stdout=FNULL) diff --git a/docs/fdroid.texi b/docs/fdroid.texi index 021e8e55..c4c1d909 100644 --- a/docs/fdroid.texi +++ b/docs/fdroid.texi @@ -57,6 +57,7 @@ Free Documentation License". * Update Processing:: * Build Server:: * Signing:: +* Vulnerability Scanning:: * GNU Free Documentation License:: * Index:: @end menu @@ -1697,6 +1698,132 @@ A new key will be generated using these details, for each application that is built. (If a specific key is required for a particular application, this system can be overridden using the @code{keyaliases} config settings. +@node Vulnerability Scanning +@chapter Vulnerability Scanning (dscanner) + +F-Droid now includes a means of running automated vulnerability scanning +using @uref{https://github.com/mwrlabs/drozer, Drozer}. This is achieved +by starting a docker container, with the Android SDK and Emulator +prepared already, installing drozer into the emulator and scripting the +knobs to scan any fully built and signed APKs. + +Note: if your application is not intended to run within an Android +emulator, please do not continue with these instructions. At this time, +the @code{dscanner} feature is fully dependent upon your application +running properly in an emulated environment. + +@section Quick Start + +@enumerate +@item Ensure that your application is a signed release build +@item @code{fdroid dscanner --init-only} from within the repo +@item Go for a coffee, this takes a long time and requires approximately +6 GB of disk space. Once this is complete, you'll be left with a docker +container running and ready to go. +@item @code{fdroid dscanner --latest app.pkg.name} from within the repo +to run drozer on the latest build of @code{app.pkg.name} +@item If all went well, there should be an ``app.pkg.name_CODE.apk.dscanner'' +file in the repo (next to the original APK file) +@item When you're all done scanning packages, you can cleanup the docker +container with: @code{fdroid dscanner --clean-only} +@end enumerate + +You can also run the drozer scan as an optional part of the overall +@code{fdroid build} operation. This option will trigger a drozer scan of +all signed APKs found in the repo. See @code{fdroid build --help} for +more information. + +@section Command Line Help + +@example +usage: fdroid dscanner [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]] + +positional arguments: + app_id app-id with optional versioncode in the form + APPID[:VERCODE] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Spew out even more information than normal + -q, --quiet Restrict output to warnings and errors + -l, --latest Scan only the latest version of each package + --clean-after Clean after all scans have finished. + --clean-before Clean before the scans start and rebuild the + container. + --clean-only Clean up all containers and then exit. + --init-only Prepare drozer to run a scan + --repo-path REPO_PATH + Override repo path for built APK files. +@end example + +@section From Scratch + +Because the docker image used to do the Android Emulator and all of that +takes a considerable amount of time to prepare, one has been uploaded to +dockerhub.com for general use. However, the astute researcher will be +weary of any black boxes and want to build their own black box. This +section elaborates how to build the docker image yourself. + +From within the F-Droid Server source code directory, @code{cd +./docker/} in order to begin. + +Within this directory are the custom scripting used within the docker +image creation. For conveience, there is a simple Makefile that +wraps the process of creating images into convenient pieces. + +@subsection @code{make help} + +@example +usage: make help|build|clean|kill|info + + help this help screen + build create docker image + clean remove images and containers + kill stop running containers + info details of running container +@end example + +@subsection @code{make clean} + +Stops any running containers (@code{make kill}) and then forcully +removes them from docker. After that, all images associated are also +explicitly removed. + +Note: this will destroy docker images! + +@subsection @code{make build} + +Builds the actual docker container, tagged +``dscanner/fdroidserver:latest'' from the local directory. Obviously +this is operating with the @code{Dockerfile} to build and tie everything +together nicely. + +@subsection @code{make kill} + +@code{docker kill} the container tagged ``dscanner''. + +@subsection @code{make info} + +Prints some useful information about the currently running dscanner +container (if it is even running). The output of this command is +confusing and raw but useful none-the-less. See example output below: + +@example +CONTAINER ID IMAGE COMMAND +CREATED STATUS PORTS +NAMES +b90a60afe477 dscanner/fdroidserver "/home/drozer/entrypo" 20 +minutes ago Up 20 minutes 0.0.0.0:32779->22/tcp, +0.0.0.0:32778->5037/tcp, 0.0.0.0:32777->5554/tcp, +0.0.0.0:32776->5555/tcp, 0.0.0.0:32775->5900/tcp, +0.0.0.0:32774->5901/tcp dscanner +Use: + adb kill-server + adb connect 172.17.0.2:32776 +@end example + +Typical usage is for finding the ``adb connect'' line or the ``ssh'' +port (32779 from the @code{0.0.0.0:32779->22/tcp} note). @node GNU Free Documentation License @appendix GNU Free Documentation License diff --git a/fdroid b/fdroid index af670d1b..feea104a 100755 --- a/fdroid +++ b/fdroid @@ -38,6 +38,7 @@ commands = { "rewritemeta": "Rewrite all the metadata files", "lint": "Warn about possible metadata errors", "scanner": "Scan the source code of a package", + "dscanner": "Dynamically scan APKs post build", "stats": "Update the stats of the repo", "server": "Interact with the repo HTTP server", "signindex": "Sign indexes created using update --nosign", diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 281ad523..e34fe15e 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -1006,6 +1006,8 @@ def parse_commandline(): help="Specify that we're running on the build server") parser.add_argument("--skip-scan", dest="skipscan", action="store_true", default=False, help="Skip scanning the source code for binaries and other problems") + parser.add_argument("--dscanner", action="store_true", default=False, + help="Setup an emulator, install the apk on it and perform a drozer scan") parser.add_argument("--no-tarball", dest="notarball", action="store_true", default=False, help="Don't create a source tarball, useful when testing a build") parser.add_argument("--no-refresh", dest="refresh", action="store_false", default=True, @@ -1221,6 +1223,42 @@ def main(): for fa in failed_apps: logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) + # perform a drozer scan of all successful builds + if options.dscanner and build_succeeded: + from .dscanner import DockerDriver + + docker = DockerDriver() + + try: + for app in build_succeeded: + + logging.info("Need to sign the app before we can install it.") + subprocess.call("fdroid publish {0}".format(app.id), shell=True) + + apk_path = None + + for f in os.listdir(repo_dir): + if f.endswith('.apk') and f.startswith(app.id): + apk_path = os.path.join(repo_dir, f) + break + + if not apk_path: + raise Exception("No signed APK found at path: {0}".format(apk_path)) + + if not os.path.isdir(repo_dir): + exit(1) + + logging.info("Performing Drozer scan on {0}.".format(app)) + docker.perform_drozer_scan(apk_path, app.id, repo_dir) + except Exception as e: + logging.error(str(e)) + logging.error("An exception happened. Making sure to clean up") + else: + logging.info("Scan succeeded.") + + logging.info("Cleaning up after ourselves.") + docker.clean() + logging.info("Finished.") if len(build_succeeded) > 0: logging.info(str(len(build_succeeded)) + ' builds succeeded') diff --git a/fdroidserver/dscanner.py b/fdroidserver/dscanner.py new file mode 100644 index 00000000..84915b9a --- /dev/null +++ b/fdroidserver/dscanner.py @@ -0,0 +1,482 @@ +#!/usr/bin/env python3 +# +# dscanner.py - part of the FDroid server tools +# Copyright (C) 2016-2017 Shawn Gustaw +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import logging +import os +import json +import sys +from time import sleep +from argparse import ArgumentParser +from subprocess import CalledProcessError, check_output + +from fdroidserver import common, metadata + +try: + from docker import Client +except ImportError: + logging.error(("Docker client not installed." + "Install it using pip install docker-py")) + +config = None +options = None + + +class DockerConfig: + ALIAS = "dscanner" + CONTAINER = "dscanner/fdroidserver" + EMULATOR = "android-19" + ARCH = "armeabi-v7a" + + +class DockerDriver(object): + """ + Handles all the interactions with the docker container the + Android emulator runs in. + """ + class Commands: + build = ['docker', 'build', '--no-cache=false', '--pull=true', + '--quiet=false', '--rm=true', '-t', + '{0}:latest'.format(DockerConfig.CONTAINER), '.'] + run = [ + 'docker', 'run', + '-e', '"EMULATOR={0}"'.format(DockerConfig.EMULATOR), + '-e', '"ARCH={0}"'.format(DockerConfig.ARCH), + '-d', '-P', '--name', + '{0}'.format(DockerConfig.ALIAS), '--log-driver=json-file', + DockerConfig.CONTAINER] + start = ['docker', 'start', '{0}'.format(DockerConfig.ALIAS)] + inspect = ['docker', 'inspect', '{0}'.format(DockerConfig.ALIAS)] + pm_list = 'adb shell "pm list packages"' + install_drozer = "docker exec {0} python /home/drozer/install_agent.py" + run_drozer = 'python /home/drozer/drozer.py {0}' + copy_to_container = 'docker cp "{0}" {1}:{2}' + copy_from_container = 'docker cp {0}:{1} "{2}"' + + def __init__(self, init_only=False, fresh_start=False, clean_only=False): + self.container_id = None + self.ip_address = None + + self.cli = Client(base_url='unix://var/run/docker.sock') + + if fresh_start or clean_only: + self.clean() + + if clean_only: + logging.info("Cleaned containers and quitting.") + exit(0) + + self.init_docker() + + if init_only: + logging.info("Initialized and quitting.") + exit(0) + + def _copy_to_container(self, src_path, dest_path): + """ + Copies a file (presumed to be an apk) from src_path + to home directory on container. + """ + path = '/home/drozer/{path}.apk'.format(path=dest_path) + command = self.Commands.copy_to_container.format(src_path, + self.container_id, + path) + + try: + check_output(command, shell=True) + except CalledProcessError as e: + logging.error(('Command "{command}" failed with ' + 'error code {code}'.format(command=command, + code=e.returncode))) + raise + + def _copy_from_container(self, src_path, dest_path): + """ + Copies a file from src_path on the container to + dest_path on the host machine. + """ + command = self.Commands.copy_from_container.format(self.container_id, + src_path, + dest_path) + try: + check_output(command, shell=True) + except CalledProcessError as e: + logging.error(('Command "{command}" failed with ' + 'error code {code}'.format(command=command, + code=e.returncode))) + raise + + logging.info("Log stored at {path}".format(path=dest_path)) + + def _adb_install_apk(self, apk_path): + """ + Installs an apk on the device running in the container + using adb. + """ + logging.info("Attempting to install an apk.") + exec_id = self.cli.exec_create( + self.container_id, 'adb install {0}' + .format(apk_path) + )['Id'] + output = self.cli.exec_start(exec_id).decode('utf-8') + + if "INSTALL_PARSE_FAILED_NO_CERTIFICATES" in output: + raise Exception('Install parse failed, no certificates') + elif "INSTALL_FAILED_ALREADY_EXISTS" in output: + logging.info("APK already installed. Skipping.") + elif "Success" not in output: + logging.error("APK didn't install properly") + return False + return True + + def _adb_uninstall_apk(self, app_id): + """ + Uninstalls an application from the device running in the container + via its app_id. + """ + logging.info( + "Uninstalling {app_id} from the emulator." + .format(app_id=app_id) + ) + exec_id = self.cli.exec_create( + self.container_id, + 'adb uninstall {0}'.format(app_id) + )['Id'] + output = self.cli.exec_start(exec_id).decode('utf-8') + + if 'Success' in output: + logging.info("Successfully uninstalled.") + + return True + + def _verify_apk_install(self, app_id): + """ + Checks that the app_id is installed on the device running in the + container. + """ + logging.info( + "Verifying {app} is installed on the device." + .format(app=app_id) + ) + exec_id = self.cli.exec_create( + self.container_id, self.Commands.pm_list + )['Id'] + output = self.cli.exec_start(exec_id).decode('utf-8') + + if ("Could not access the Package Manager" in output or + "device offline" in output): + logging.info("Device or package manager isn't up") + + if app_id.split('_')[0] in output: # TODO: this is a temporary fix + logging.info("{app} is installed.".format(app=app_id)) + return True + + logging.error("APK not found in packages list on emulator.") + + def _delete_file(self, path): + """ + Deletes file off the container to preserve space if scanning many apps + """ + command = "rm {path}".format(path=path) + exec_id = self.cli.exec_create(self.container_id, command)['Id'] + logging.info("Deleting {path} on the container.".format(path=path)) + self.cli.exec_start(exec_id) + + def _install_apk(self, apk_path, app_id): + """ + Installs apk found at apk_path on the emulator. Will then + verify it installed properly by looking up its app_id in + the package manager. + """ + if not all([self.container_id, self.ip_address]): + # TODO: maybe have this fail nicely + raise Exception("Went to install apk and couldn't find container") + + path = "/home/drozer/{app_id}.apk".format(app_id=app_id) + self._copy_to_container(apk_path, app_id) + self._adb_install_apk(path) + self._verify_apk_install(app_id) + self._delete_file(path) + + def _install_drozer(self): + """ + Performs all the initialization of drozer within the emulator. + """ + logging.info("Attempting to install com.mwr.dz on the emulator") + logging.info("This could take a while so be patient...") + logging.info(("We need to wait for the device to boot AND" + " the package manager to come online.")) + command = self.Commands.install_drozer.format(self.container_id) + try: + output = check_output(command, + shell=True).decode('utf-8') + except CalledProcessError as e: + logging.error(('Command "{command}" failed with ' + 'error code {code}'.format(command=command, + code=e.returncode))) + raise + + if 'Installed ok' in output: + return True + + def _run_drozer_scan(self, app): + """ + Runs the drozer agent which connects to the app running + on the emulator. + """ + logging.info("Running the drozer agent") + exec_id = self.cli.exec_create( + self.container_id, + self.Commands.run_drozer.format(app) + )['Id'] + self.cli.exec_start(exec_id) + + def _container_is_running(self): + """ + Checks whether the emulator container is running. + """ + for container in self.cli.containers(): + if DockerConfig.ALIAS in container['Image']: + return True + + def _docker_image_exists(self): + """ + Check whether the docker image exists already. + If this returns false we'll need to build the image + from the DockerFile. + """ + for image in self.cli.images(): + for tag in image['RepoTags']: + if DockerConfig.ALIAS in tag: + return True + + _image_queue = {} + + def _build_docker_image(self): + """ + Builds the docker container so we can run the android emulator + inside it. + """ + logging.info("Pulling the container from docker hub") + logging.info("Image is roughly 5 GB so be patient") + + logging.info("(Progress output is slow and requires a tty.)") + # we pause briefly to narrow race condition windows of opportunity + sleep(1) + + is_a_tty = os.isatty(sys.stdout.fileno()) + + for output in self.cli.pull( + DockerConfig.CONTAINER, + stream=True, + tag="latest"): + if not is_a_tty: + # run silent, run quick + continue + try: + p = json.loads(output.decode('utf-8')) + p_id = p['id'] + self._image_queue[p_id] = p + t, c, j = 1, 1, 0 + for k in sorted(self._image_queue): + j += 1 + v = self._image_queue[k] + vd = v['progressDetail'] + t += vd['total'] + c += vd['current'] + msg = "\rDownloading: {0}/{1} {2}% [{3} jobs]" + msg = msg.format(c, t, int(c / t * 100), j) + sys.stdout.write(msg) + sys.stdout.flush() + except: + pass + print("\nDONE!\n") + + def _verify_apk_exists(self, full_apk_path): + """ + Verifies that the apk path we have is actually a file. + """ + return os.path.isfile(full_apk_path) + + def init_docker(self): + """ + Perform all the initialization required before a drozer scan. + 1. build the image + 2. run the container + 3. install drozer and enable the service within the app + """ + built = self._docker_image_exists() + + if not built: + self._build_docker_image() + + running = self._container_is_running() + + if not running: + logging.info('Trying to run container...') + try: + check_output(self.Commands.run) + except CalledProcessError as e: + logging.error(( + 'Command "{command}" failed with error code {code}' + .format(command=self.Commands.run, code=e.returncode) + )) + running = self._container_is_running() + + if not running: + logging.info('Trying to start container...') + try: + check_output(self.Commands.start) + except CalledProcessError as e: + logging.error(( + 'Command "{command}" failed with error code {code}' + .format(command=self.Commands.run, code=e.returncode) + )) + running = self._container_is_running() + + if not running: + raise Exception("Running container not found, critical error.") + + containers = self.cli.containers() + + for container in containers: + if DockerConfig.ALIAS in container['Image']: + self.container_id = container['Id'] + n = container['NetworkSettings']['Networks'] + self.ip_address = n['bridge']['IPAddress'] + break + + if not self.container_id or not self.ip_address: + logging.error("No ip address or container id found.") + exit(1) + + if self._verify_apk_install('com.mwr.dz'): + return + + self._install_drozer() + + def clean(self): + """ + Clean up all the containers made by this script. + Should be run after the drozer scan completes. + """ + for container in self.cli.containers(): + if DockerConfig.ALIAS in container['Image']: + logging.info("Removing container {0}".format(container['Id'])) + self.cli.remove_container(container['Id'], force=True) + + def perform_drozer_scan(self, apk_path, app_id): + """ + Entrypoint for scanning an android app. Performs the following steps: + 1. installs an apk on the device + 2. runs a drozer scan + 3. copies the report off the container + 4. uninstalls the apk to save space on the device + """ + self._install_apk(apk_path, app_id) + logging.info("Running the drozer scan.") + self._run_drozer_scan(app_id) + logging.info("Scan finished. Moving the report off the container") + dest = apk_path + '.drozer' + self._copy_from_container('/tmp/drozer_report.log', dest) + self._adb_uninstall_apk(app_id) + + +def main(): + global config, options + + # Parse command line... + parser = ArgumentParser( + usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]" + ) + common.setup_global_opts(parser) + + parser.add_argument( + "app_id", nargs='*', + help="app-id with optional versioncode in the form APPID[:VERCODE]") + parser.add_argument( + "-l", "--latest", action="store_true", default=False, + help="Scan only the latest version of each package") + parser.add_argument( + "--clean-after", default=False, action='store_true', + help="Clean after all scans have finished") + parser.add_argument( + "--clean-before", default=False, action='store_true', + help="Clean before the scans start and rebuild the container") + parser.add_argument( + "--clean-only", default=False, action='store_true', + help="Clean up all containers and then exit") + parser.add_argument( + "--init-only", default=False, action='store_true', + help="Prepare drozer to run a scan") + parser.add_argument( + "--repo-path", default="repo", action="store", + help="Override path for repo APKs (default: ./repo)") + + options = parser.parse_args() + config = common.read_config(options) + + if not os.path.isdir(options.repo_path): + sys.stderr.write("repo-path not found: \"" + options.repo_path + "\"") + exit(1) + + # Read all app and srclib metadata + allapps = metadata.read_metadata() + apps = common.read_app_args(options.app_id, allapps, True) + + docker = DockerDriver( + init_only=options.init_only, + fresh_start=options.clean_before, + clean_only=options.clean_only + ) + + if options.clean_before: + docker.clean() + + if options.clean_only: + exit(0) + + for app_id, app in apps.items(): + vercode = 0 + if ':' in app_id: + vercode = app_id.split(':')[1] + for build in reversed(app.builds): + if build.disable: + continue + if options.latest or vercode == 0 or build.vercode == vercode: + app.builds = [build] + break + continue + continue + + for app_id, app in apps.items(): + for build in app.builds: + apks = [] + for f in os.listdir(options.repo_path): + n = "%v_%v.apk".format(app_id, build.vercode) + if f == n: + apks.append(f) + for apk in sorted(apks): + apk_path = os.path.join(options.repo_path, apk) + docker.perform_drozer_scan(apk_path, app.id) + + if options.clean_after: + docker.clean() + + +if __name__ == "__main__": + main() diff --git a/setup.py b/setup.py index 10dfaf0b..a0917810 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,8 @@ setup(name='fdroidserver', 'pyasn1', 'pyasn1-modules', 'PyYAML', - 'requests', + 'requests < 2.11', + 'docker-py == 1.9.0', ], classifiers=[ 'Development Status :: 3 - Alpha',