1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
42 def get_default_config():
44 'sdk_path': os.getenv("ANDROID_HOME") or "",
45 'ndk_path': os.getenv("ANDROID_NDK") or "",
46 'build_tools': "20.0.0",
50 'sync_from_local_copy_dir': False,
51 'update_stats': False,
52 'stats_to_carbon': False,
54 'build_server_always': False,
55 'keystore': os.path.join(os.getenv("HOME"), '.local', 'share', 'fdroidserver', 'keystore.jks'),
56 'smartcardoptions': [],
62 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
63 'repo_name': "My First FDroid Repo Demo",
64 'repo_icon': "fdroid-icon.png",
66 "This is a repository of apps to be used with FDroid. Applications in this "
67 + "repository are either official binaries built by the original application "
68 + "developers, or are binaries built from source by the admin of f-droid.org "
69 + "using the tools on https://gitlab.com/u/fdroid."),
74 def read_config(opts, config_file='config.py'):
75 """Read the repository config
77 The config is read from config_file, which is in the current directory when
78 any of the repo management commands are used.
80 global config, options, env
82 if config is not None:
84 if not os.path.isfile(config_file):
85 logging.critical("Missing config file - is this a repo directory?")
92 logging.debug("Reading %s" % config_file)
93 execfile(config_file, config)
95 # smartcardoptions must be a list since its command line args for Popen
96 if 'smartcardoptions' in config:
97 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
98 elif 'keystore' in config and config['keystore'] == 'NONE':
99 # keystore='NONE' means use smartcard, these are required defaults
100 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
101 'SunPKCS11-OpenSC', '-providerClass',
102 'sun.security.pkcs11.SunPKCS11',
103 '-providerArg', 'opensc-fdroid.cfg']
105 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
106 st = os.stat(config_file)
107 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
108 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
110 defconfig = get_default_config()
111 for k, v in defconfig.items():
115 # Expand environment variables
116 for k, v in config.items():
119 v = os.path.expanduser(v)
120 config[k] = os.path.expandvars(v)
122 if not test_sdk_exists(config):
125 if not test_build_tools_exists(config):
130 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
133 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
134 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
137 os.path.join(config['sdk_path'], 'tools', 'android'),
140 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
144 for b, paths in bin_paths.items():
147 if os.path.isfile(path):
150 if config[b] is None:
151 logging.warn("Could not find %s in any of the following paths:\n%s" % (
152 b, '\n'.join(paths)))
154 # There is no standard, so just set up the most common environment
157 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
158 env[n] = config['sdk_path']
159 for n in ['ANDROID_NDK', 'NDK']:
160 env[n] = config['ndk_path']
162 for k in ["keystorepass", "keypass"]:
164 write_password_file(k)
166 # since this is used with rsync, where trailing slashes have meaning,
167 # ensure there is always a trailing slash
168 if 'serverwebroot' in config:
169 if config['serverwebroot'][-1] != '/':
170 config['serverwebroot'] += '/'
171 config['serverwebroot'] = config['serverwebroot'].replace('//', '/')
176 def test_sdk_exists(c):
177 if c['sdk_path'] is None:
178 # c['sdk_path'] is set to the value of ANDROID_HOME by default
179 logging.error('No Android SDK found! ANDROID_HOME is not set and sdk_path is not in config.py!')
180 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
181 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
183 if not os.path.exists(c['sdk_path']):
184 logging.critical('Android SDK path "' + c['sdk_path'] + '" does not exist!')
186 if not os.path.isdir(c['sdk_path']):
187 logging.critical('Android SDK path "' + c['sdk_path'] + '" is not a directory!')
189 for d in ['build-tools', 'platform-tools', 'tools']:
190 if not os.path.isdir(os.path.join(c['sdk_path'], d)):
191 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
197 def test_build_tools_exists(c):
198 if not test_sdk_exists(c):
200 build_tools = os.path.join(c['sdk_path'], 'build-tools')
201 versioned_build_tools = os.path.join(build_tools, c['build_tools'])
202 if not os.path.isdir(versioned_build_tools):
203 logging.critical('Android Build Tools path "'
204 + versioned_build_tools + '" does not exist!')
209 def write_password_file(pwtype, password=None):
211 writes out passwords to a protected file instead of passing passwords as
212 command line argments
214 filename = '.fdroid.' + pwtype + '.txt'
215 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
217 os.write(fd, config[pwtype])
219 os.write(fd, password)
221 config[pwtype + 'file'] = filename
224 # Given the arguments in the form of multiple appid:[vc] strings, this returns
225 # a dictionary with the set of vercodes specified for each package.
226 def read_pkg_args(args, allow_vercodes=False):
233 if allow_vercodes and ':' in p:
234 package, vercode = p.split(':')
236 package, vercode = p, None
237 if package not in vercodes:
238 vercodes[package] = [vercode] if vercode else []
240 elif vercode and vercode not in vercodes[package]:
241 vercodes[package] += [vercode] if vercode else []
246 # On top of what read_pkg_args does, this returns the whole app metadata, but
247 # limiting the builds list to the builds matching the vercodes specified.
248 def read_app_args(args, allapps, allow_vercodes=False):
250 vercodes = read_pkg_args(args, allow_vercodes)
255 apps = [app for app in allapps if app['id'] in vercodes]
257 if len(apps) != len(vercodes):
258 allids = [app["id"] for app in allapps]
261 logging.critical("No such package: %s" % p)
262 raise Exception("Found invalid app ids in arguments")
264 raise Exception("No packages specified")
268 vc = vercodes[app['id']]
271 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
272 if len(app['builds']) != len(vercodes[app['id']]):
274 allvcs = [b['vercode'] for b in app['builds']]
275 for v in vercodes[app['id']]:
277 logging.critical("No such vercode %s for app %s" % (v, app['id']))
280 raise Exception("Found invalid vercodes for some apps")
285 def has_extension(filename, extension):
286 name, ext = os.path.splitext(filename)
287 ext = ext.lower()[1:]
288 return ext == extension
293 def apknameinfo(filename):
295 filename = os.path.basename(filename)
296 if apk_regex is None:
297 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
298 m = apk_regex.match(filename)
300 result = (m.group(1), m.group(2))
301 except AttributeError:
302 raise Exception("Invalid apk name: %s" % filename)
306 def getapkname(app, build):
307 return "%s_%s.apk" % (app['id'], build['vercode'])
310 def getsrcname(app, build):
311 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
318 return app['Auto Name']
323 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
326 def getvcs(vcstype, remote, local):
328 return vcs_git(remote, local)
330 return vcs_svn(remote, local)
331 if vcstype == 'git-svn':
332 return vcs_gitsvn(remote, local)
334 return vcs_hg(remote, local)
336 return vcs_bzr(remote, local)
337 if vcstype == 'srclib':
338 if local != 'build/srclib/' + remote:
339 raise VCSException("Error: srclib paths are hard-coded!")
340 return getsrclib(remote, 'build/srclib', raw=True)
341 raise VCSException("Invalid vcs type " + vcstype)
344 def getsrclibvcs(name):
345 if name not in metadata.srclibs:
346 raise VCSException("Missing srclib " + name)
347 return metadata.srclibs[name]['Repo Type']
351 def __init__(self, remote, local):
353 # svn, git-svn and bzr may require auth
355 if self.repotype() in ('svn', 'git-svn', 'bzr'):
357 self.username, remote = remote.split('@')
358 if ':' not in self.username:
359 raise VCSException("Password required with username")
360 self.username, self.password = self.username.split(':')
364 self.clone_failed = False
365 self.refreshed = False
371 # Take the local repository to a clean version of the given revision, which
372 # is specificed in the VCS's native format. Beforehand, the repository can
373 # be dirty, or even non-existent. If the repository does already exist
374 # locally, it will be updated from the origin, but only once in the
375 # lifetime of the vcs object.
376 # None is acceptable for 'rev' if you know you are cloning a clean copy of
377 # the repo - otherwise it must specify a valid revision.
378 def gotorevision(self, rev):
380 if self.clone_failed:
381 raise VCSException("Downloading the repository already failed once, not trying again.")
383 # The .fdroidvcs-id file for a repo tells us what VCS type
384 # and remote that directory was created from, allowing us to drop it
385 # automatically if either of those things changes.
386 fdpath = os.path.join(self.local, '..',
387 '.fdroidvcs-' + os.path.basename(self.local))
388 cdata = self.repotype() + ' ' + self.remote
391 if os.path.exists(self.local):
392 if os.path.exists(fdpath):
393 with open(fdpath, 'r') as f:
394 fsdata = f.read().strip()
400 "Repository details for %s changed - deleting" % (
404 logging.info("Repository details for %s missing - deleting" % (
407 shutil.rmtree(self.local)
409 self.gotorevisionx(rev)
411 # If necessary, write the .fdroidvcs file.
413 with open(fdpath, 'w') as f:
416 # Derived classes need to implement this. It's called once basic checking
417 # has been performend.
418 def gotorevisionx(self, rev):
419 raise VCSException("This VCS type doesn't define gotorevisionx")
421 # Initialise and update submodules
422 def initsubmodules(self):
423 raise VCSException('Submodules not supported for this vcs type')
425 # Get a list of all known tags
427 raise VCSException('gettags not supported for this vcs type')
429 # Get a list of latest number tags
430 def latesttags(self, number):
431 raise VCSException('latesttags not supported for this vcs type')
433 # Get current commit reference (hash, revision, etc)
435 raise VCSException('getref not supported for this vcs type')
437 # Returns the srclib (name, path) used in setting up the current
448 # If the local directory exists, but is somehow not a git repository, git
449 # will traverse up the directory tree until it finds one that is (i.e.
450 # fdroidserver) and then we'll proceed to destroy it! This is called as
453 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
454 result = p.output.rstrip()
455 if not result.endswith(self.local):
456 raise VCSException('Repository mismatch')
458 def gotorevisionx(self, rev):
459 if not os.path.exists(self.local):
461 p = FDroidPopen(['git', 'clone', self.remote, self.local])
462 if p.returncode != 0:
463 self.clone_failed = True
464 raise VCSException("Git clone failed", p.output)
468 # Discard any working tree changes
469 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
470 if p.returncode != 0:
471 raise VCSException("Git reset failed", p.output)
472 # Remove untracked files now, in case they're tracked in the target
473 # revision (it happens!)
474 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
475 if p.returncode != 0:
476 raise VCSException("Git clean failed", p.output)
477 if not self.refreshed:
478 # Get latest commits and tags from remote
479 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
480 if p.returncode != 0:
481 raise VCSException("Git fetch failed", p.output)
482 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
483 if p.returncode != 0:
484 raise VCSException("Git fetch failed", p.output)
485 # Recreate origin/HEAD as git clone would do it, in case it disappeared
486 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
487 if p.returncode != 0:
488 lines = p.output.splitlines()
489 if 'Multiple remote HEAD branches' not in lines[0]:
490 raise VCSException("Git remote set-head failed", p.output)
491 branch = lines[1].split(' ')[-1]
492 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
493 if p2.returncode != 0:
494 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
495 self.refreshed = True
496 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
497 # a github repo. Most of the time this is the same as origin/master.
498 rev = rev or 'origin/HEAD'
499 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
500 if p.returncode != 0:
501 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
502 # Get rid of any uncontrolled files left behind
503 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
504 if p.returncode != 0:
505 raise VCSException("Git clean failed", p.output)
507 def initsubmodules(self):
509 submfile = os.path.join(self.local, '.gitmodules')
510 if not os.path.isfile(submfile):
511 raise VCSException("No git submodules available")
513 # fix submodules not accessible without an account and public key auth
514 with open(submfile, 'r') as f:
515 lines = f.readlines()
516 with open(submfile, 'w') as f:
518 if 'git@github.com' in line:
519 line = line.replace('git@github.com:', 'https://github.com/')
523 ['git', 'reset', '--hard'],
524 ['git', 'clean', '-dffx'],
526 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
527 if p.returncode != 0:
528 raise VCSException("Git submodule reset failed", p.output)
529 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local)
530 if p.returncode != 0:
531 raise VCSException("Git submodule sync failed", p.output)
532 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
533 if p.returncode != 0:
534 raise VCSException("Git submodule update failed", p.output)
538 p = SilentPopen(['git', 'tag'], cwd=self.local)
539 return p.output.splitlines()
541 def latesttags(self, alltags, number):
543 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
544 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
545 + 'sort -n | awk \'{print $2}\''],
546 cwd=self.local, shell=True)
547 return p.output.splitlines()[-number:]
550 class vcs_gitsvn(vcs):
555 # Damn git-svn tries to use a graphical password prompt, so we have to
556 # trick it into taking the password from stdin
558 if self.username is None:
560 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
562 # If the local directory exists, but is somehow not a git repository, git
563 # will traverse up the directory tree until it finds one that is (i.e.
564 # fdroidserver) and then we'll proceed to destory it! This is called as
567 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
568 result = p.output.rstrip()
569 if not result.endswith(self.local):
570 raise VCSException('Repository mismatch')
572 def gotorevisionx(self, rev):
573 if not os.path.exists(self.local):
575 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
576 if ';' in self.remote:
577 remote_split = self.remote.split(';')
578 for i in remote_split[1:]:
579 if i.startswith('trunk='):
580 gitsvn_cmd += ' -T %s' % i[6:]
581 elif i.startswith('tags='):
582 gitsvn_cmd += ' -t %s' % i[5:]
583 elif i.startswith('branches='):
584 gitsvn_cmd += ' -b %s' % i[9:]
585 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
586 if p.returncode != 0:
587 self.clone_failed = True
588 raise VCSException("Git clone failed", p.output)
590 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
591 if p.returncode != 0:
592 self.clone_failed = True
593 raise VCSException("Git clone failed", p.output)
597 # Discard any working tree changes
598 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
599 if p.returncode != 0:
600 raise VCSException("Git reset failed", p.output)
601 # Remove untracked files now, in case they're tracked in the target
602 # revision (it happens!)
603 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
604 if p.returncode != 0:
605 raise VCSException("Git clean failed", p.output)
606 if not self.refreshed:
607 # Get new commits, branches and tags from repo
608 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
609 if p.returncode != 0:
610 raise VCSException("Git svn fetch failed")
611 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
612 if p.returncode != 0:
613 raise VCSException("Git svn rebase failed", p.output)
614 self.refreshed = True
616 rev = rev or 'master'
618 nospaces_rev = rev.replace(' ', '%20')
619 # Try finding a svn tag
620 p = SilentPopen(['git', 'checkout', 'tags/' + nospaces_rev], cwd=self.local)
621 if p.returncode != 0:
622 # No tag found, normal svn rev translation
623 # Translate svn rev into git format
624 rev_split = rev.split('/')
627 for treeish in ['origin/', '']:
628 if len(rev_split) > 1:
629 treeish += rev_split[0]
630 svn_rev = rev_split[1]
633 # if no branch is specified, then assume trunk (i.e. 'master' branch):
637 p = SilentPopen(['git', 'svn', 'find-rev', 'r' + svn_rev, treeish], cwd=self.local)
638 git_rev = p.output.rstrip()
640 if p.returncode == 0 and git_rev:
643 if p.returncode != 0 or not git_rev:
644 # Try a plain git checkout as a last resort
645 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
646 if p.returncode != 0:
647 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
649 # Check out the git rev equivalent to the svn rev
650 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
651 if p.returncode != 0:
652 raise VCSException("Git svn checkout of '%s' failed" % rev, p.output)
654 # Get rid of any uncontrolled files left behind
655 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
656 if p.returncode != 0:
657 raise VCSException("Git clean failed", p.output)
661 return os.listdir(os.path.join(self.local, '.git/svn/refs/remotes/tags'))
665 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
666 if p.returncode != 0:
668 return p.output.strip()
677 if self.username is None:
678 return ['--non-interactive']
679 return ['--username', self.username,
680 '--password', self.password,
683 def gotorevisionx(self, rev):
684 if not os.path.exists(self.local):
685 p = SilentPopen(['svn', 'checkout', self.remote, self.local] + self.userargs())
686 if p.returncode != 0:
687 self.clone_failed = True
688 raise VCSException("Svn checkout of '%s' failed" % rev, p.output)
692 r"svn status | awk '/\?/ {print $2}' | xargs rm -rf"):
693 p = SilentPopen([svncommand], cwd=self.local, shell=True)
694 if p.returncode != 0:
695 raise VCSException("Svn reset ({0}) failed in {1}".format(svncommand, self.local), p.output)
696 if not self.refreshed:
697 p = SilentPopen(['svn', 'update'] + self.userargs(), cwd=self.local)
698 if p.returncode != 0:
699 raise VCSException("Svn update failed", p.output)
700 self.refreshed = True
702 revargs = list(['-r', rev] if rev else [])
703 p = SilentPopen(['svn', 'update', '--force'] + revargs + self.userargs(), cwd=self.local)
704 if p.returncode != 0:
705 raise VCSException("Svn update failed", p.output)
708 p = SilentPopen(['svn', 'info'], cwd=self.local)
709 for line in p.output.splitlines():
710 if line and line.startswith('Last Changed Rev: '):
720 def gotorevisionx(self, rev):
721 if not os.path.exists(self.local):
722 p = SilentPopen(['hg', 'clone', self.remote, self.local])
723 if p.returncode != 0:
724 self.clone_failed = True
725 raise VCSException("Hg clone failed", p.output)
727 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
728 if p.returncode != 0:
729 raise VCSException("Hg clean failed", p.output)
730 if not self.refreshed:
731 p = SilentPopen(['hg', 'pull'], cwd=self.local)
732 if p.returncode != 0:
733 raise VCSException("Hg pull failed", p.output)
734 self.refreshed = True
736 rev = rev or 'default'
739 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
740 if p.returncode != 0:
741 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
742 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
743 # Also delete untracked files, we have to enable purge extension for that:
744 if "'purge' is provided by the following extension" in p.output:
745 with open(self.local + "/.hg/hgrc", "a") as myfile:
746 myfile.write("\n[extensions]\nhgext.purge=\n")
747 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
748 if p.returncode != 0:
749 raise VCSException("HG purge failed", p.output)
750 elif p.returncode != 0:
751 raise VCSException("HG purge failed", p.output)
754 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
755 return p.output.splitlines()[1:]
763 def gotorevisionx(self, rev):
764 if not os.path.exists(self.local):
765 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
766 if p.returncode != 0:
767 self.clone_failed = True
768 raise VCSException("Bzr branch failed", p.output)
770 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
771 if p.returncode != 0:
772 raise VCSException("Bzr revert failed", p.output)
773 if not self.refreshed:
774 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
775 if p.returncode != 0:
776 raise VCSException("Bzr update failed", p.output)
777 self.refreshed = True
779 revargs = list(['-r', rev] if rev else [])
780 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
781 if p.returncode != 0:
782 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
785 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
786 return [tag.split(' ')[0].strip() for tag in
787 p.output.splitlines()]
790 def retrieve_string(app_dir, string, xmlfiles=None):
793 os.path.join(app_dir, 'res'),
794 os.path.join(app_dir, 'src/main'),
799 for res_dir in res_dirs:
800 for r, d, f in os.walk(res_dir):
801 if r.endswith('/values'):
802 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
805 if string.startswith('@string/'):
806 string_search = re.compile(r'.*name="' + string[8:] + '".*?>([^<]+?)<.*').search
807 elif string.startswith('&') and string.endswith(';'):
808 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
810 if string_search is not None:
811 for xmlfile in xmlfiles:
812 for line in file(xmlfile):
813 matches = string_search(line)
815 return retrieve_string(app_dir, matches.group(1), xmlfiles)
818 return string.replace("\\'", "'")
821 # Return list of existing files that will be used to find the highest vercode
822 def manifest_paths(app_dir, flavour):
824 possible_manifests = \
825 [os.path.join(app_dir, 'AndroidManifest.xml'),
826 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
827 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
828 os.path.join(app_dir, 'build.gradle')]
831 possible_manifests.append(
832 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
834 return [path for path in possible_manifests if os.path.isfile(path)]
837 # Retrieve the package name. Returns the name, or None if not found.
838 def fetch_real_name(app_dir, flavour):
839 app_search = re.compile(r'.*<application.*').search
840 name_search = re.compile(r'.*android:label="([^"]+)".*').search
842 for f in manifest_paths(app_dir, flavour):
843 if not has_extension(f, 'xml'):
845 logging.debug("fetch_real_name: Checking manifest at " + f)
851 matches = name_search(line)
853 stringname = matches.group(1)
854 logging.debug("fetch_real_name: using string " + stringname)
855 result = retrieve_string(app_dir, stringname)
857 result = result.strip()
862 # Retrieve the version name
863 def version_name(original, app_dir, flavour):
864 for f in manifest_paths(app_dir, flavour):
865 if not has_extension(f, 'xml'):
867 string = retrieve_string(app_dir, original)
873 def get_library_references(root_dir):
875 proppath = os.path.join(root_dir, 'project.properties')
876 if not os.path.isfile(proppath):
878 with open(proppath) as f:
879 for line in f.readlines():
880 if not line.startswith('android.library.reference.'):
882 path = line.split('=')[1].strip()
883 relpath = os.path.join(root_dir, path)
884 if not os.path.isdir(relpath):
886 logging.debug("Found subproject at %s" % path)
887 libraries.append(path)
891 def ant_subprojects(root_dir):
892 subprojects = get_library_references(root_dir)
893 for subpath in subprojects:
894 subrelpath = os.path.join(root_dir, subpath)
895 for p in get_library_references(subrelpath):
896 relp = os.path.normpath(os.path.join(subpath, p))
897 if relp not in subprojects:
898 subprojects.insert(0, relp)
902 def remove_debuggable_flags(root_dir):
903 # Remove forced debuggable flags
904 logging.info("Removing debuggable flags from %s" % root_dir)
905 for root, dirs, files in os.walk(root_dir):
906 if 'AndroidManifest.xml' in files:
907 path = os.path.join(root, 'AndroidManifest.xml')
908 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
909 if p.returncode != 0:
910 raise BuildException("Failed to remove debuggable flags of %s" % path)
913 # Extract some information from the AndroidManifest.xml at the given path.
914 # Returns (version, vercode, package), any or all of which might be None.
915 # All values returned are strings.
916 def parse_androidmanifests(paths, ignoreversions=None):
919 return (None, None, None)
921 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
922 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
923 psearch = re.compile(r'.*package="([^"]+)".*').search
925 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
926 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
927 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
929 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
937 gradle = has_extension(path, 'gradle')
940 # Remember package name, may be defined separately from version+vercode
941 package = max_package
943 for line in file(path):
946 matches = psearch_g(line)
948 matches = psearch(line)
950 package = matches.group(1)
953 matches = vnsearch_g(line)
955 matches = vnsearch(line)
957 version = matches.group(2 if gradle else 1)
960 matches = vcsearch_g(line)
962 matches = vcsearch(line)
964 vercode = matches.group(1)
966 # Always grab the package name and version name in case they are not
967 # together with the highest version code
968 if max_package is None and package is not None:
969 max_package = package
970 if max_version is None and version is not None:
971 max_version = version
973 if max_vercode is None or (vercode is not None and vercode > max_vercode):
974 if not ignoresearch or not ignoresearch(version):
975 if version is not None:
976 max_version = version
977 if vercode is not None:
978 max_vercode = vercode
979 if package is not None:
980 max_package = package
982 max_version = "Ignore"
984 if max_version is None:
985 max_version = "Unknown"
987 return (max_version, max_vercode, max_package)
990 class FDroidException(Exception):
991 def __init__(self, value, detail=None):
995 def get_wikitext(self):
996 ret = repr(self.value) + "\n"
1000 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1008 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1012 class VCSException(FDroidException):
1016 class BuildException(FDroidException):
1020 # Get the specified source library.
1021 # Returns the path to it. Normally this is the path to be used when referencing
1022 # it, which may be a subdirectory of the actual project. If you want the base
1023 # directory of the project, pass 'basepath=True'.
1024 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1025 basepath=False, raw=False, prepare=True, preponly=False):
1033 name, ref = spec.split('@')
1035 number, name = name.split(':', 1)
1037 name, subdir = name.split('/', 1)
1039 if name not in metadata.srclibs:
1040 raise VCSException('srclib ' + name + ' not found.')
1042 srclib = metadata.srclibs[name]
1044 sdir = os.path.join(srclib_dir, name)
1047 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1048 vcs.srclib = (name, number, sdir)
1050 vcs.gotorevision(ref)
1057 libdir = os.path.join(sdir, subdir)
1058 elif srclib["Subdir"]:
1059 for subdir in srclib["Subdir"]:
1060 libdir_candidate = os.path.join(sdir, subdir)
1061 if os.path.exists(libdir_candidate):
1062 libdir = libdir_candidate
1068 if srclib["Srclibs"]:
1070 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1072 for t in srclibpaths:
1077 raise VCSException('Missing recursive srclib %s for %s' % (
1079 place_srclib(libdir, n, s_tuple[2])
1082 remove_signing_keys(sdir)
1083 remove_debuggable_flags(sdir)
1087 if srclib["Prepare"]:
1088 cmd = replace_config_vars(srclib["Prepare"])
1090 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1091 if p.returncode != 0:
1092 raise BuildException("Error running prepare command for srclib %s"
1098 return (name, number, libdir)
1101 # Prepare the source code for a particular build
1102 # 'vcs' - the appropriate vcs object for the application
1103 # 'app' - the application details from the metadata
1104 # 'build' - the build details from the metadata
1105 # 'build_dir' - the path to the build directory, usually
1107 # 'srclib_dir' - the path to the source libraries directory, usually
1109 # 'extlib_dir' - the path to the external libraries directory, usually
1111 # Returns the (root, srclibpaths) where:
1112 # 'root' is the root directory, which may be the same as 'build_dir' or may
1113 # be a subdirectory of it.
1114 # 'srclibpaths' is information on the srclibs being used
1115 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1117 # Optionally, the actual app source can be in a subdirectory
1119 root_dir = os.path.join(build_dir, build['subdir'])
1121 root_dir = build_dir
1123 # Get a working copy of the right revision
1124 logging.info("Getting source for revision " + build['commit'])
1125 vcs.gotorevision(build['commit'])
1127 # Initialise submodules if requred
1128 if build['submodules']:
1129 logging.info("Initialising submodules")
1130 vcs.initsubmodules()
1132 # Check that a subdir (if we're using one) exists. This has to happen
1133 # after the checkout, since it might not exist elsewhere
1134 if not os.path.exists(root_dir):
1135 raise BuildException('Missing subdir ' + root_dir)
1137 # Run an init command if one is required
1139 cmd = replace_config_vars(build['init'])
1140 logging.info("Running 'init' commands in %s" % root_dir)
1142 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1143 if p.returncode != 0:
1144 raise BuildException("Error running init command for %s:%s" %
1145 (app['id'], build['version']), p.output)
1147 # Apply patches if any
1149 logging.info("Applying patches")
1150 for patch in build['patch']:
1151 patch = patch.strip()
1152 logging.info("Applying " + patch)
1153 patch_path = os.path.join('metadata', app['id'], patch)
1154 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1155 if p.returncode != 0:
1156 raise BuildException("Failed to apply patch %s" % patch_path)
1158 # Get required source libraries
1160 if build['srclibs']:
1161 logging.info("Collecting source libraries")
1162 for lib in build['srclibs']:
1163 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1166 for name, number, libpath in srclibpaths:
1167 place_srclib(root_dir, int(number) if number else None, libpath)
1169 basesrclib = vcs.getsrclib()
1170 # If one was used for the main source, add that too.
1172 srclibpaths.append(basesrclib)
1174 # Update the local.properties file
1175 localprops = [os.path.join(build_dir, 'local.properties')]
1177 localprops += [os.path.join(root_dir, 'local.properties')]
1178 for path in localprops:
1179 if not os.path.isfile(path):
1181 logging.info("Updating properties file at %s" % path)
1186 # Fix old-fashioned 'sdk-location' by copying
1187 # from sdk.dir, if necessary
1188 if build['oldsdkloc']:
1189 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1190 re.S | re.M).group(1)
1191 props += "sdk-location=%s\n" % sdkloc
1193 props += "sdk.dir=%s\n" % config['sdk_path']
1194 props += "sdk-location=%s\n" % config['sdk_path']
1195 if 'ndk_path' in config:
1197 props += "ndk.dir=%s\n" % config['ndk_path']
1198 props += "ndk-location=%s\n" % config['ndk_path']
1199 # Add java.encoding if necessary
1200 if build['encoding']:
1201 props += "java.encoding=%s\n" % build['encoding']
1207 if build['type'] == 'gradle':
1208 flavour = build['gradle'].split('@')[0]
1209 if flavour in ['main', 'yes', '']:
1212 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1213 gradlepluginver = None
1215 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1217 # Parent dir build.gradle
1218 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1219 if parent_dir.startswith(build_dir):
1220 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1222 # Gradle execution dir build.gradle
1223 if '@' in build['gradle']:
1224 gradle_file = os.path.join(root_dir, build['gradle'].split('@', 1)[1], 'build.gradle')
1225 gradle_file = os.path.normpath(gradle_file)
1226 if gradle_file not in gradle_files:
1227 gradle_files.append(gradle_file)
1229 for path in gradle_files:
1232 if not os.path.isfile(path):
1234 with open(path) as f:
1236 match = version_regex.match(line)
1238 gradlepluginver = match.group(1)
1242 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1244 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1245 build['gradlepluginver'] = LooseVersion('0.11')
1248 n = build["target"].split('-')[1]
1249 FDroidPopen(['sed', '-i',
1250 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1254 # Remove forced debuggable flags
1255 remove_debuggable_flags(root_dir)
1257 # Insert version code and number into the manifest if necessary
1258 if build['forceversion']:
1259 logging.info("Changing the version name")
1260 for path in manifest_paths(root_dir, flavour):
1261 if not os.path.isfile(path):
1263 if has_extension(path, 'xml'):
1264 p = SilentPopen(['sed', '-i',
1265 's/android:versionName="[^"]*"/android:versionName="'
1266 + build['version'] + '"/g',
1268 if p.returncode != 0:
1269 raise BuildException("Failed to amend manifest")
1270 elif has_extension(path, 'gradle'):
1271 p = SilentPopen(['sed', '-i',
1272 's/versionName *=* *"[^"]*"/versionName = "'
1273 + build['version'] + '"/g',
1275 if p.returncode != 0:
1276 raise BuildException("Failed to amend build.gradle")
1277 if build['forcevercode']:
1278 logging.info("Changing the version code")
1279 for path in manifest_paths(root_dir, flavour):
1280 if not os.path.isfile(path):
1282 if has_extension(path, 'xml'):
1283 p = SilentPopen(['sed', '-i',
1284 's/android:versionCode="[^"]*"/android:versionCode="'
1285 + build['vercode'] + '"/g',
1287 if p.returncode != 0:
1288 raise BuildException("Failed to amend manifest")
1289 elif has_extension(path, 'gradle'):
1290 p = SilentPopen(['sed', '-i',
1291 's/versionCode *=* *[0-9]*/versionCode = '
1292 + build['vercode'] + '/g',
1294 if p.returncode != 0:
1295 raise BuildException("Failed to amend build.gradle")
1297 # Delete unwanted files
1299 logging.info("Removing specified files")
1300 for part in getpaths(build_dir, build, 'rm'):
1301 dest = os.path.join(build_dir, part)
1302 logging.info("Removing {0}".format(part))
1303 if os.path.lexists(dest):
1304 if os.path.islink(dest):
1305 SilentPopen(['unlink ' + dest], shell=True)
1307 SilentPopen(['rm -rf ' + dest], shell=True)
1309 logging.info("...but it didn't exist")
1311 remove_signing_keys(build_dir)
1313 # Add required external libraries
1314 if build['extlibs']:
1315 logging.info("Collecting prebuilt libraries")
1316 libsdir = os.path.join(root_dir, 'libs')
1317 if not os.path.exists(libsdir):
1319 for lib in build['extlibs']:
1321 logging.info("...installing extlib {0}".format(lib))
1322 libf = os.path.basename(lib)
1323 libsrc = os.path.join(extlib_dir, lib)
1324 if not os.path.exists(libsrc):
1325 raise BuildException("Missing extlib file {0}".format(libsrc))
1326 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1328 # Run a pre-build command if one is required
1329 if build['prebuild']:
1330 logging.info("Running 'prebuild' commands in %s" % root_dir)
1332 cmd = replace_config_vars(build['prebuild'])
1334 # Substitute source library paths into prebuild commands
1335 for name, number, libpath in srclibpaths:
1336 libpath = os.path.relpath(libpath, root_dir)
1337 cmd = cmd.replace('$$' + name + '$$', libpath)
1339 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1340 if p.returncode != 0:
1341 raise BuildException("Error running prebuild command for %s:%s" %
1342 (app['id'], build['version']), p.output)
1344 # Generate (or update) the ant build file, build.xml...
1345 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1346 parms = [config['android'], 'update', 'lib-project']
1347 lparms = [config['android'], 'update', 'project']
1350 parms += ['-t', build['target']]
1351 lparms += ['-t', build['target']]
1352 if build['update'] == ['auto']:
1353 update_dirs = ant_subprojects(root_dir) + ['.']
1355 update_dirs = build['update']
1357 for d in update_dirs:
1358 subdir = os.path.join(root_dir, d)
1360 logging.info("Updating main project")
1361 cmd = parms + ['-p', d]
1363 logging.info("Updating subproject %s" % d)
1364 cmd = lparms + ['-p', d]
1365 p = FDroidPopen(cmd, cwd=root_dir)
1366 # Check to see whether an error was returned without a proper exit
1367 # code (this is the case for the 'no target set or target invalid'
1369 if p.returncode != 0 or p.output.startswith("Error: "):
1370 raise BuildException("Failed to update project at %s" % d, p.output)
1371 # Clean update dirs via ant
1373 logging.info("Cleaning subproject %s" % d)
1374 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1376 return (root_dir, srclibpaths)
1379 # Split and extend via globbing the paths from a field
1380 def getpaths(build_dir, build, field):
1382 for p in build[field]:
1384 full_path = os.path.join(build_dir, p)
1385 full_path = os.path.normpath(full_path)
1386 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1390 # Scan the source code in the given directory (and all subdirectories)
1391 # and return the number of fatal problems encountered
1392 def scan_source(build_dir, root_dir, thisbuild):
1396 # Common known non-free blobs (always lower case):
1398 re.compile(r'flurryagent', re.IGNORECASE),
1399 re.compile(r'paypal.*mpl', re.IGNORECASE),
1400 re.compile(r'libgoogleanalytics', re.IGNORECASE),
1401 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1402 re.compile(r'googleadview', re.IGNORECASE),
1403 re.compile(r'googleadmobadssdk', re.IGNORECASE),
1404 re.compile(r'google.*play.*services', re.IGNORECASE),
1405 re.compile(r'crittercism', re.IGNORECASE),
1406 re.compile(r'heyzap', re.IGNORECASE),
1407 re.compile(r'jpct.*ae', re.IGNORECASE),
1408 re.compile(r'youtubeandroidplayerapi', re.IGNORECASE),
1409 re.compile(r'bugsense', re.IGNORECASE),
1410 re.compile(r'crashlytics', re.IGNORECASE),
1411 re.compile(r'ouya.*sdk', re.IGNORECASE),
1412 re.compile(r'libspen23', re.IGNORECASE),
1415 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1416 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1419 ms = magic.open(magic.MIME_TYPE)
1421 except AttributeError:
1425 for i in scanignore:
1426 if fd.startswith(i):
1431 for i in scandelete:
1432 if fd.startswith(i):
1436 def removeproblem(what, fd, fp):
1437 logging.info('Removing %s at %s' % (what, fd))
1440 def warnproblem(what, fd):
1441 logging.warn('Found %s at %s' % (what, fd))
1443 def handleproblem(what, fd, fp):
1445 removeproblem(what, fd, fp)
1447 logging.error('Found %s at %s' % (what, fd))
1451 def insidedir(path, dirname):
1452 return path.endswith('/%s' % dirname) or '/%s/' % dirname in path
1454 # Iterate through all files in the source code
1455 for r, d, f in os.walk(build_dir):
1457 if any(insidedir(r, d) for d in ('.hg', '.git', '.svn', '.bzr')):
1462 # Path (relative) to the file
1463 fp = os.path.join(r, curfile)
1464 fd = fp[len(build_dir) + 1:]
1466 # Check if this file has been explicitly excluded from scanning
1470 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1472 if mime == 'application/x-sharedlib':
1473 count += handleproblem('shared library', fd, fp)
1475 elif mime == 'application/x-archive':
1476 count += handleproblem('static library', fd, fp)
1478 elif mime == 'application/x-executable':
1479 count += handleproblem('binary executable', fd, fp)
1481 elif mime == 'application/x-java-applet':
1482 count += handleproblem('Java compiled class', fd, fp)
1487 'application/java-archive',
1488 'application/octet-stream',
1492 if has_extension(fp, 'apk'):
1493 removeproblem('APK file', fd, fp)
1495 elif has_extension(fp, 'jar'):
1497 if any(suspect.match(curfile) for suspect in usual_suspects):
1498 count += handleproblem('usual supect', fd, fp)
1500 warnproblem('JAR file', fd)
1502 elif has_extension(fp, 'zip'):
1503 warnproblem('ZIP file', fd)
1506 warnproblem('unknown compressed or binary file', fd)
1508 elif has_extension(fp, 'java'):
1509 for line in file(fp):
1510 if 'DexClassLoader' in line:
1511 count += handleproblem('DexClassLoader', fd, fp)
1516 # Presence of a jni directory without buildjni=yes might
1517 # indicate a problem (if it's not a problem, explicitly use
1518 # buildjni=no to bypass this check)
1519 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1520 not thisbuild['buildjni']):
1521 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1530 self.path = os.path.join('stats', 'known_apks.txt')
1532 if os.path.exists(self.path):
1533 for line in file(self.path):
1534 t = line.rstrip().split(' ')
1536 self.apks[t[0]] = (t[1], None)
1538 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1539 self.changed = False
1541 def writeifchanged(self):
1543 if not os.path.exists('stats'):
1545 f = open(self.path, 'w')
1547 for apk, app in self.apks.iteritems():
1549 line = apk + ' ' + appid
1551 line += ' ' + time.strftime('%Y-%m-%d', added)
1553 for line in sorted(lst):
1554 f.write(line + '\n')
1557 # Record an apk (if it's new, otherwise does nothing)
1558 # Returns the date it was added.
1559 def recordapk(self, apk, app):
1560 if apk not in self.apks:
1561 self.apks[apk] = (app, time.gmtime(time.time()))
1563 _, added = self.apks[apk]
1566 # Look up information - given the 'apkname', returns (app id, date added/None).
1567 # Or returns None for an unknown apk.
1568 def getapp(self, apkname):
1569 if apkname in self.apks:
1570 return self.apks[apkname]
1573 # Get the most recent 'num' apps added to the repo, as a list of package ids
1574 # with the most recent first.
1575 def getlatest(self, num):
1577 for apk, app in self.apks.iteritems():
1581 if apps[appid] > added:
1585 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1586 lst = [app for app, _ in sortedapps]
1591 def isApkDebuggable(apkfile, config):
1592 """Returns True if the given apk file is debuggable
1594 :param apkfile: full path to the apk to check"""
1596 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1597 config['build_tools'], 'aapt'),
1598 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1599 if p.returncode != 0:
1600 logging.critical("Failed to get apk manifest information")
1602 for line in p.output.splitlines():
1603 if 'android:debuggable' in line and not line.endswith('0x0'):
1608 class AsynchronousFileReader(threading.Thread):
1610 Helper class to implement asynchronous reading of a file
1611 in a separate thread. Pushes read lines on a queue to
1612 be consumed in another thread.
1615 def __init__(self, fd, queue):
1616 assert isinstance(queue, Queue.Queue)
1617 assert callable(fd.readline)
1618 threading.Thread.__init__(self)
1623 '''The body of the tread: read lines and put them on the queue.'''
1624 for line in iter(self._fd.readline, ''):
1625 self._queue.put(line)
1628 '''Check whether there is no more content to expect.'''
1629 return not self.is_alive() and self._queue.empty()
1637 def SilentPopen(commands, cwd=None, shell=False):
1638 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1641 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1643 Run a command and capture the possibly huge output.
1645 :param commands: command and argument list like in subprocess.Popen
1646 :param cwd: optionally specifies a working directory
1647 :returns: A PopenResult.
1653 cwd = os.path.normpath(cwd)
1654 logging.debug("Directory: %s" % cwd)
1655 logging.debug("> %s" % ' '.join(commands))
1657 result = PopenResult()
1658 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1659 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1661 stdout_queue = Queue.Queue()
1662 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1663 stdout_reader.start()
1665 # Check the queue for output (until there is no more to get)
1666 while not stdout_reader.eof():
1667 while not stdout_queue.empty():
1668 line = stdout_queue.get()
1669 if output and options.verbose:
1670 # Output directly to console
1671 sys.stderr.write(line)
1673 result.output += line
1678 result.returncode = p.returncode
1682 def remove_signing_keys(build_dir):
1683 comment = re.compile(r'[ ]*//')
1684 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1686 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1687 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1688 re.compile(r'.*variant\.outputFile = .*'),
1689 re.compile(r'.*\.readLine\(.*'),
1691 for root, dirs, files in os.walk(build_dir):
1692 if 'build.gradle' in files:
1693 path = os.path.join(root, 'build.gradle')
1695 with open(path, "r") as o:
1696 lines = o.readlines()
1701 with open(path, "w") as o:
1703 if comment.match(line):
1707 opened += line.count('{')
1708 opened -= line.count('}')
1711 if signing_configs.match(line):
1716 if any(s.match(line) for s in line_matches):
1724 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1727 'project.properties',
1729 'default.properties',
1732 if propfile in files:
1733 path = os.path.join(root, propfile)
1735 with open(path, "r") as o:
1736 lines = o.readlines()
1740 with open(path, "w") as o:
1742 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1749 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1752 def replace_config_vars(cmd):
1753 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1754 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1755 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1759 def place_srclib(root_dir, number, libpath):
1762 relpath = os.path.relpath(libpath, root_dir)
1763 proppath = os.path.join(root_dir, 'project.properties')
1766 if os.path.isfile(proppath):
1767 with open(proppath, "r") as o:
1768 lines = o.readlines()
1770 with open(proppath, "w") as o:
1773 if line.startswith('android.library.reference.%d=' % number):
1774 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1779 o.write('android.library.reference.%d=%s\n' % (number, relpath))