3 # dscanner.py - part of the FDroid server tools
4 # Copyright (C) 2016-2017 Shawn Gustaw <self@shawngustaw.com>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU Affero General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU Affero General Public License for more details.
16 # You should have received a copy of the GNU Affero General Public License
17 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 from time import sleep
24 from argparse import ArgumentParser
25 from subprocess import CalledProcessError, check_output
29 from . import metadata
32 from docker import Client
34 logging.error(("Docker client not installed."
35 "Install it using pip install docker-py"))
43 CONTAINER = "dscanner/fdroidserver"
44 EMULATOR = "android-19"
48 class DockerDriver(object):
50 Handles all the interactions with the docker container the
51 Android emulator runs in.
54 build = ['docker', 'build', '--no-cache=false', '--pull=true',
55 '--quiet=false', '--rm=true', '-t',
56 '{0}:latest'.format(DockerConfig.CONTAINER), '.']
59 '-e', '"EMULATOR={0}"'.format(DockerConfig.EMULATOR),
60 '-e', '"ARCH={0}"'.format(DockerConfig.ARCH),
62 '{0}'.format(DockerConfig.ALIAS), '--log-driver=json-file',
63 DockerConfig.CONTAINER]
64 start = ['docker', 'start', '{0}'.format(DockerConfig.ALIAS)]
65 inspect = ['docker', 'inspect', '{0}'.format(DockerConfig.ALIAS)]
66 pm_list = 'adb shell "pm list packages"'
67 install_drozer = "docker exec {0} python /home/drozer/install_agent.py"
68 run_drozer = 'python /home/drozer/drozer.py {0}'
69 copy_to_container = 'docker cp "{0}" {1}:{2}'
70 copy_from_container = 'docker cp {0}:{1} "{2}"'
72 def __init__(self, init_only=False, fresh_start=False, clean_only=False):
73 self.container_id = None
74 self.ip_address = None
76 self.cli = Client(base_url='unix://var/run/docker.sock')
78 if fresh_start or clean_only:
82 logging.info("Cleaned containers and quitting.")
88 logging.info("Initialized and quitting.")
91 def _copy_to_container(self, src_path, dest_path):
93 Copies a file (presumed to be an apk) from src_path
94 to home directory on container.
96 path = '/home/drozer/{path}.apk'.format(path=dest_path)
97 command = self.Commands.copy_to_container.format(src_path,
102 check_output(command, shell=True)
103 except CalledProcessError as e:
104 logging.error(('Command "{command}" failed with '
105 'error code {code}'.format(command=command,
109 def _copy_from_container(self, src_path, dest_path):
111 Copies a file from src_path on the container to
112 dest_path on the host machine.
114 command = self.Commands.copy_from_container.format(self.container_id,
118 check_output(command, shell=True)
119 except CalledProcessError as e:
120 logging.error(('Command "{command}" failed with '
121 'error code {code}'.format(command=command,
125 logging.info("Log stored at {path}".format(path=dest_path))
127 def _adb_install_apk(self, apk_path):
129 Installs an apk on the device running in the container
132 logging.info("Attempting to install an apk.")
133 exec_id = self.cli.exec_create(
134 self.container_id, 'adb install {0}'
137 output = self.cli.exec_start(exec_id).decode('utf-8')
139 if "INSTALL_PARSE_FAILED_NO_CERTIFICATES" in output:
140 raise Exception('Install parse failed, no certificates')
141 elif "INSTALL_FAILED_ALREADY_EXISTS" in output:
142 logging.info("APK already installed. Skipping.")
143 elif "Success" not in output:
144 logging.error("APK didn't install properly")
148 def _adb_uninstall_apk(self, app_id):
150 Uninstalls an application from the device running in the container
154 "Uninstalling {app_id} from the emulator."
155 .format(app_id=app_id)
157 exec_id = self.cli.exec_create(
159 'adb uninstall {0}'.format(app_id)
161 output = self.cli.exec_start(exec_id).decode('utf-8')
163 if 'Success' in output:
164 logging.info("Successfully uninstalled.")
168 def _verify_apk_install(self, app_id):
170 Checks that the app_id is installed on the device running in the
174 "Verifying {app} is installed on the device."
177 exec_id = self.cli.exec_create(
178 self.container_id, self.Commands.pm_list
180 output = self.cli.exec_start(exec_id).decode('utf-8')
182 if ("Could not access the Package Manager" in output or
183 "device offline" in output):
184 logging.info("Device or package manager isn't up")
186 if app_id.split('_')[0] in output: # TODO: this is a temporary fix
187 logging.info("{app} is installed.".format(app=app_id))
190 logging.error("APK not found in packages list on emulator.")
192 def _delete_file(self, path):
194 Deletes file off the container to preserve space if scanning many apps
196 command = "rm {path}".format(path=path)
197 exec_id = self.cli.exec_create(self.container_id, command)['Id']
198 logging.info("Deleting {path} on the container.".format(path=path))
199 self.cli.exec_start(exec_id)
201 def _install_apk(self, apk_path, app_id):
203 Installs apk found at apk_path on the emulator. Will then
204 verify it installed properly by looking up its app_id in
207 if not all([self.container_id, self.ip_address]):
208 # TODO: maybe have this fail nicely
209 raise Exception("Went to install apk and couldn't find container")
211 path = "/home/drozer/{app_id}.apk".format(app_id=app_id)
212 self._copy_to_container(apk_path, app_id)
213 self._adb_install_apk(path)
214 self._verify_apk_install(app_id)
215 self._delete_file(path)
217 def _install_drozer(self):
219 Performs all the initialization of drozer within the emulator.
221 logging.info("Attempting to install com.mwr.dz on the emulator")
222 logging.info("This could take a while so be patient...")
223 logging.info(("We need to wait for the device to boot AND"
224 " the package manager to come online."))
225 command = self.Commands.install_drozer.format(self.container_id)
227 output = check_output(command,
228 shell=True).decode('utf-8')
229 except CalledProcessError as e:
230 logging.error(('Command "{command}" failed with '
231 'error code {code}'.format(command=command,
235 if 'Installed ok' in output:
238 def _run_drozer_scan(self, app):
240 Runs the drozer agent which connects to the app running
243 logging.info("Running the drozer agent")
244 exec_id = self.cli.exec_create(
246 self.Commands.run_drozer.format(app)
248 self.cli.exec_start(exec_id)
250 def _container_is_running(self):
252 Checks whether the emulator container is running.
254 for container in self.cli.containers():
255 if DockerConfig.ALIAS in container['Image']:
258 def _docker_image_exists(self):
260 Check whether the docker image exists already.
261 If this returns false we'll need to build the image
264 for image in self.cli.images():
265 for tag in image['RepoTags']:
266 if DockerConfig.ALIAS in tag:
271 def _build_docker_image(self):
273 Builds the docker container so we can run the android emulator
276 logging.info("Pulling the container from docker hub")
277 logging.info("Image is roughly 5 GB so be patient")
279 logging.info("(Progress output is slow and requires a tty.)")
280 # we pause briefly to narrow race condition windows of opportunity
283 is_a_tty = os.isatty(sys.stdout.fileno())
285 for output in self.cli.pull(
286 DockerConfig.CONTAINER,
290 # run silent, run quick
293 p = json.loads(output.decode('utf-8'))
295 self._image_queue[p_id] = p
297 for k in sorted(self._image_queue):
299 v = self._image_queue[k]
300 vd = v['progressDetail']
303 msg = "\rDownloading: {0}/{1} {2}% [{3} jobs]"
304 msg = msg.format(c, t, int(c / t * 100), j)
305 sys.stdout.write(msg)
311 def _verify_apk_exists(self, full_apk_path):
313 Verifies that the apk path we have is actually a file.
315 return os.path.isfile(full_apk_path)
317 def init_docker(self):
319 Perform all the initialization required before a drozer scan.
322 3. install drozer and enable the service within the app
324 built = self._docker_image_exists()
327 self._build_docker_image()
329 running = self._container_is_running()
332 logging.info('Trying to run container...')
334 check_output(self.Commands.run)
335 except CalledProcessError as e:
337 'Command "{command}" failed with error code {code}'
338 .format(command=self.Commands.run, code=e.returncode)
340 running = self._container_is_running()
343 logging.info('Trying to start container...')
345 check_output(self.Commands.start)
346 except CalledProcessError as e:
348 'Command "{command}" failed with error code {code}'
349 .format(command=self.Commands.run, code=e.returncode)
351 running = self._container_is_running()
354 raise Exception("Running container not found, critical error.")
356 containers = self.cli.containers()
358 for container in containers:
359 if DockerConfig.ALIAS in container['Image']:
360 self.container_id = container['Id']
361 n = container['NetworkSettings']['Networks']
362 self.ip_address = n['bridge']['IPAddress']
365 if not self.container_id or not self.ip_address:
366 logging.error("No ip address or container id found.")
369 if self._verify_apk_install('com.mwr.dz'):
372 self._install_drozer()
376 Clean up all the containers made by this script.
377 Should be run after the drozer scan completes.
379 for container in self.cli.containers():
380 if DockerConfig.ALIAS in container['Image']:
381 logging.info("Removing container {0}".format(container['Id']))
382 self.cli.remove_container(container['Id'], force=True)
384 def perform_drozer_scan(self, apk_path, app_id):
386 Entrypoint for scanning an android app. Performs the following steps:
387 1. installs an apk on the device
388 2. runs a drozer scan
389 3. copies the report off the container
390 4. uninstalls the apk to save space on the device
392 self._install_apk(apk_path, app_id)
393 logging.info("Running the drozer scan.")
394 self._run_drozer_scan(app_id)
395 logging.info("Scan finished. Moving the report off the container")
396 dest = apk_path + '.drozer'
397 self._copy_from_container('/tmp/drozer_report.log', dest)
398 self._adb_uninstall_apk(app_id)
402 global config, options
404 # Parse command line...
405 parser = ArgumentParser(
406 usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]"
408 common.setup_global_opts(parser)
412 help=_("applicationId with optional versionCode in the form APPID[:VERCODE]"))
414 "-l", "--latest", action="store_true", default=False,
415 help=_("Scan only the latest version of each package"))
417 "--clean-after", default=False, action='store_true',
418 help=_("Clean after all scans have finished"))
420 "--clean-before", default=False, action='store_true',
421 help=_("Clean before the scans start and rebuild the container"))
423 "--clean-only", default=False, action='store_true',
424 help=_("Clean up all containers and then exit"))
426 "--init-only", default=False, action='store_true',
427 help=_("Prepare Drozer to run a scan"))
429 "--repo-path", default="repo", action="store",
430 help=_("Override path for repo APKs (default: ./repo)"))
432 options = parser.parse_args()
433 config = common.read_config(options)
435 if not os.path.isdir(options.repo_path):
436 sys.stderr.write("repo-path not found: \"" + options.repo_path + "\"")
439 # Read all app and srclib metadata
440 allapps = metadata.read_metadata()
441 apps = common.read_app_args(options.app_id, allapps, True)
443 docker = DockerDriver(
444 init_only=options.init_only,
445 fresh_start=options.clean_before,
446 clean_only=options.clean_only
449 if options.clean_before:
452 if options.clean_only:
455 for app_id, app in apps.items():
458 vercode = app_id.split(':')[1]
459 for build in reversed(app.builds):
462 if options.latest or vercode == 0 or build.versionCode == vercode:
468 for app_id, app in apps.items():
469 for build in app.builds:
471 for f in os.listdir(options.repo_path):
472 n = common.get_release_filename(app, build)
475 for apk in sorted(apks):
476 apk_path = os.path.join(options.repo_path, apk)
477 docker.perform_drozer_scan(apk_path, app.id)
479 if options.clean_after:
483 if __name__ == "__main__":