1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
43 'sdk_path': "$ANDROID_HOME",
44 'ndk_path': "$ANDROID_NDK",
45 'build_tools': "21.1.2",
49 'sync_from_local_copy_dir': False,
50 'make_current_version_link': True,
51 'current_version_name_source': 'Name',
52 'update_stats': False,
56 'stats_to_carbon': False,
58 'build_server_always': False,
59 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
60 'smartcardoptions': [],
66 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
67 'repo_name': "My First FDroid Repo Demo",
68 'repo_icon': "fdroid-icon.png",
69 'repo_description': '''
70 This is a repository of apps to be used with FDroid. Applications in this
71 repository are either official binaries built by the original application
72 developers, or are binaries built from source by the admin of f-droid.org
73 using the tools on https://gitlab.com/u/fdroid.
79 def fill_config_defaults(thisconfig):
80 for k, v in default_config.items():
81 if k not in thisconfig:
84 # Expand paths (~users and $vars)
85 for k in ['sdk_path', 'ndk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
88 v = os.path.expanduser(v)
89 v = os.path.expandvars(v)
92 thisconfig[k + '_orig'] = orig
95 def read_config(opts, config_file='config.py'):
96 """Read the repository config
98 The config is read from config_file, which is in the current directory when
99 any of the repo management commands are used.
101 global config, options, env
103 if config is not None:
105 if not os.path.isfile(config_file):
106 logging.critical("Missing config file - is this a repo directory?")
113 logging.debug("Reading %s" % config_file)
114 execfile(config_file, config)
116 # smartcardoptions must be a list since its command line args for Popen
117 if 'smartcardoptions' in config:
118 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
119 elif 'keystore' in config and config['keystore'] == 'NONE':
120 # keystore='NONE' means use smartcard, these are required defaults
121 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
122 'SunPKCS11-OpenSC', '-providerClass',
123 'sun.security.pkcs11.SunPKCS11',
124 '-providerArg', 'opensc-fdroid.cfg']
126 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
127 st = os.stat(config_file)
128 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
129 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
131 fill_config_defaults(config)
133 if not test_sdk_exists(config):
136 if not test_build_tools_exists(config):
141 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'aapt'),
144 os.path.join(config['sdk_path'], 'tools', 'zipalign'),
145 os.path.join(config['sdk_path'], 'build-tools', config['build_tools'], 'zipalign'),
148 os.path.join(config['sdk_path'], 'tools', 'android'),
151 os.path.join(config['sdk_path'], 'platform-tools', 'adb'),
155 for b, paths in bin_paths.items():
158 if os.path.isfile(path):
161 if config[b] is None:
162 logging.warn("Could not find %s in any of the following paths:\n%s" % (
163 b, '\n'.join(paths)))
165 # There is no standard, so just set up the most common environment
168 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
169 env[n] = config['sdk_path']
170 for n in ['ANDROID_NDK', 'NDK']:
171 env[n] = config['ndk_path']
173 for k in ["keystorepass", "keypass"]:
175 write_password_file(k)
177 for k in ["repo_description", "archive_description"]:
179 config[k] = clean_description(config[k])
181 if 'serverwebroot' in config:
182 if isinstance(config['serverwebroot'], basestring):
183 roots = [config['serverwebroot']]
184 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
185 roots = config['serverwebroot']
187 raise TypeError('only accepts strings, lists, and tuples')
189 for rootstr in roots:
190 # since this is used with rsync, where trailing slashes have
191 # meaning, ensure there is always a trailing slash
192 if rootstr[-1] != '/':
194 rootlist.append(rootstr.replace('//', '/'))
195 config['serverwebroot'] = rootlist
200 def test_sdk_exists(thisconfig):
201 if thisconfig['sdk_path'] == default_config['sdk_path']:
202 logging.error('No Android SDK found!')
203 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
204 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
206 if not os.path.exists(thisconfig['sdk_path']):
207 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
209 if not os.path.isdir(thisconfig['sdk_path']):
210 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
212 for d in ['build-tools', 'platform-tools', 'tools']:
213 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
214 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
215 thisconfig['sdk_path'], d))
220 def test_build_tools_exists(thisconfig):
221 if not test_sdk_exists(thisconfig):
223 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
224 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
225 if not os.path.isdir(versioned_build_tools):
226 logging.critical('Android Build Tools path "'
227 + versioned_build_tools + '" does not exist!')
232 def write_password_file(pwtype, password=None):
234 writes out passwords to a protected file instead of passing passwords as
235 command line argments
237 filename = '.fdroid.' + pwtype + '.txt'
238 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
240 os.write(fd, config[pwtype])
242 os.write(fd, password)
244 config[pwtype + 'file'] = filename
247 # Given the arguments in the form of multiple appid:[vc] strings, this returns
248 # a dictionary with the set of vercodes specified for each package.
249 def read_pkg_args(args, allow_vercodes=False):
256 if allow_vercodes and ':' in p:
257 package, vercode = p.split(':')
259 package, vercode = p, None
260 if package not in vercodes:
261 vercodes[package] = [vercode] if vercode else []
263 elif vercode and vercode not in vercodes[package]:
264 vercodes[package] += [vercode] if vercode else []
269 # On top of what read_pkg_args does, this returns the whole app metadata, but
270 # limiting the builds list to the builds matching the vercodes specified.
271 def read_app_args(args, allapps, allow_vercodes=False):
273 vercodes = read_pkg_args(args, allow_vercodes)
279 for appid, app in allapps.iteritems():
280 if appid in vercodes:
283 if len(apps) != len(vercodes):
286 logging.critical("No such package: %s" % p)
287 raise FDroidException("Found invalid app ids in arguments")
289 raise FDroidException("No packages specified")
292 for appid, app in apps.iteritems():
296 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
297 if len(app['builds']) != len(vercodes[appid]):
299 allvcs = [b['vercode'] for b in app['builds']]
300 for v in vercodes[appid]:
302 logging.critical("No such vercode %s for app %s" % (v, appid))
305 raise FDroidException("Found invalid vercodes for some apps")
310 def has_extension(filename, extension):
311 name, ext = os.path.splitext(filename)
312 ext = ext.lower()[1:]
313 return ext == extension
318 def clean_description(description):
319 'Remove unneeded newlines and spaces from a block of description text'
321 # this is split up by paragraph to make removing the newlines easier
322 for paragraph in re.split(r'\n\n', description):
323 paragraph = re.sub('\r', '', paragraph)
324 paragraph = re.sub('\n', ' ', paragraph)
325 paragraph = re.sub(' {2,}', ' ', paragraph)
326 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
327 returnstring += paragraph + '\n\n'
328 return returnstring.rstrip('\n')
331 def apknameinfo(filename):
333 filename = os.path.basename(filename)
334 if apk_regex is None:
335 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
336 m = apk_regex.match(filename)
338 result = (m.group(1), m.group(2))
339 except AttributeError:
340 raise FDroidException("Invalid apk name: %s" % filename)
344 def getapkname(app, build):
345 return "%s_%s.apk" % (app['id'], build['vercode'])
348 def getsrcname(app, build):
349 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
356 return app['Auto Name']
361 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
364 def getvcs(vcstype, remote, local):
366 return vcs_git(remote, local)
367 if vcstype == 'git-svn':
368 return vcs_gitsvn(remote, local)
370 return vcs_hg(remote, local)
372 return vcs_bzr(remote, local)
373 if vcstype == 'srclib':
374 if local != os.path.join('build', 'srclib', remote):
375 raise VCSException("Error: srclib paths are hard-coded!")
376 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
378 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
379 raise VCSException("Invalid vcs type " + vcstype)
382 def getsrclibvcs(name):
383 if name not in metadata.srclibs:
384 raise VCSException("Missing srclib " + name)
385 return metadata.srclibs[name]['Repo Type']
389 def __init__(self, remote, local):
391 # svn, git-svn and bzr may require auth
393 if self.repotype() in ('git-svn', 'bzr'):
395 self.username, remote = remote.split('@')
396 if ':' not in self.username:
397 raise VCSException("Password required with username")
398 self.username, self.password = self.username.split(':')
402 self.clone_failed = False
403 self.refreshed = False
409 # Take the local repository to a clean version of the given revision, which
410 # is specificed in the VCS's native format. Beforehand, the repository can
411 # be dirty, or even non-existent. If the repository does already exist
412 # locally, it will be updated from the origin, but only once in the
413 # lifetime of the vcs object.
414 # None is acceptable for 'rev' if you know you are cloning a clean copy of
415 # the repo - otherwise it must specify a valid revision.
416 def gotorevision(self, rev):
418 if self.clone_failed:
419 raise VCSException("Downloading the repository already failed once, not trying again.")
421 # The .fdroidvcs-id file for a repo tells us what VCS type
422 # and remote that directory was created from, allowing us to drop it
423 # automatically if either of those things changes.
424 fdpath = os.path.join(self.local, '..',
425 '.fdroidvcs-' + os.path.basename(self.local))
426 cdata = self.repotype() + ' ' + self.remote
429 if os.path.exists(self.local):
430 if os.path.exists(fdpath):
431 with open(fdpath, 'r') as f:
432 fsdata = f.read().strip()
438 "Repository details for %s changed - deleting" % (
442 logging.info("Repository details for %s missing - deleting" % (
445 shutil.rmtree(self.local)
450 self.gotorevisionx(rev)
451 except FDroidException, e:
454 # If necessary, write the .fdroidvcs file.
455 if writeback and not self.clone_failed:
456 with open(fdpath, 'w') as f:
462 # Derived classes need to implement this. It's called once basic checking
463 # has been performend.
464 def gotorevisionx(self, rev):
465 raise VCSException("This VCS type doesn't define gotorevisionx")
467 # Initialise and update submodules
468 def initsubmodules(self):
469 raise VCSException('Submodules not supported for this vcs type')
471 # Get a list of all known tags
473 raise VCSException('gettags not supported for this vcs type')
475 # Get a list of latest number tags
476 def latesttags(self, number):
477 raise VCSException('latesttags not supported for this vcs type')
479 # Get current commit reference (hash, revision, etc)
481 raise VCSException('getref not supported for this vcs type')
483 # Returns the srclib (name, path) used in setting up the current
494 # If the local directory exists, but is somehow not a git repository, git
495 # will traverse up the directory tree until it finds one that is (i.e.
496 # fdroidserver) and then we'll proceed to destroy it! This is called as
499 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
500 result = p.output.rstrip()
501 if not result.endswith(self.local):
502 raise VCSException('Repository mismatch')
504 def gotorevisionx(self, rev):
505 if not os.path.exists(self.local):
507 p = FDroidPopen(['git', 'clone', self.remote, self.local])
508 if p.returncode != 0:
509 self.clone_failed = True
510 raise VCSException("Git clone failed", p.output)
514 # Discard any working tree changes
515 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
516 if p.returncode != 0:
517 raise VCSException("Git reset failed", p.output)
518 # Remove untracked files now, in case they're tracked in the target
519 # revision (it happens!)
520 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
521 if p.returncode != 0:
522 raise VCSException("Git clean failed", p.output)
523 if not self.refreshed:
524 # Get latest commits and tags from remote
525 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
526 if p.returncode != 0:
527 raise VCSException("Git fetch failed", p.output)
528 p = SilentPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local)
529 if p.returncode != 0:
530 raise VCSException("Git fetch failed", p.output)
531 # Recreate origin/HEAD as git clone would do it, in case it disappeared
532 p = SilentPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local)
533 if p.returncode != 0:
534 lines = p.output.splitlines()
535 if 'Multiple remote HEAD branches' not in lines[0]:
536 raise VCSException("Git remote set-head failed", p.output)
537 branch = lines[1].split(' ')[-1]
538 p2 = SilentPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local)
539 if p2.returncode != 0:
540 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
541 self.refreshed = True
542 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
543 # a github repo. Most of the time this is the same as origin/master.
544 rev = rev or 'origin/HEAD'
545 p = SilentPopen(['git', 'checkout', '-f', rev], cwd=self.local)
546 if p.returncode != 0:
547 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
548 # Get rid of any uncontrolled files left behind
549 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
550 if p.returncode != 0:
551 raise VCSException("Git clean failed", p.output)
553 def initsubmodules(self):
555 submfile = os.path.join(self.local, '.gitmodules')
556 if not os.path.isfile(submfile):
557 raise VCSException("No git submodules available")
559 # fix submodules not accessible without an account and public key auth
560 with open(submfile, 'r') as f:
561 lines = f.readlines()
562 with open(submfile, 'w') as f:
564 if 'git@github.com' in line:
565 line = line.replace('git@github.com:', 'https://github.com/')
569 ['git', 'reset', '--hard'],
570 ['git', 'clean', '-dffx'],
572 p = SilentPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local)
573 if p.returncode != 0:
574 raise VCSException("Git submodule reset failed", p.output)
575 p = SilentPopen(['git', 'submodule', 'sync'], cwd=self.local)
576 if p.returncode != 0:
577 raise VCSException("Git submodule sync failed", p.output)
578 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
579 if p.returncode != 0:
580 raise VCSException("Git submodule update failed", p.output)
584 p = SilentPopen(['git', 'tag'], cwd=self.local)
585 return p.output.splitlines()
587 def latesttags(self, alltags, number):
589 p = SilentPopen(['echo "' + '\n'.join(alltags) + '" | '
590 + 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
591 + 'sort -n | awk \'{print $2}\''],
592 cwd=self.local, shell=True)
593 return p.output.splitlines()[-number:]
596 class vcs_gitsvn(vcs):
601 # Damn git-svn tries to use a graphical password prompt, so we have to
602 # trick it into taking the password from stdin
604 if self.username is None:
606 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
608 # If the local directory exists, but is somehow not a git repository, git
609 # will traverse up the directory tree until it finds one that is (i.e.
610 # fdroidserver) and then we'll proceed to destory it! This is called as
613 p = SilentPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local)
614 result = p.output.rstrip()
615 if not result.endswith(self.local):
616 raise VCSException('Repository mismatch')
618 def gotorevisionx(self, rev):
619 if not os.path.exists(self.local):
621 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
622 if ';' in self.remote:
623 remote_split = self.remote.split(';')
624 for i in remote_split[1:]:
625 if i.startswith('trunk='):
626 gitsvn_cmd += ' -T %s' % i[6:]
627 elif i.startswith('tags='):
628 gitsvn_cmd += ' -t %s' % i[5:]
629 elif i.startswith('branches='):
630 gitsvn_cmd += ' -b %s' % i[9:]
631 p = SilentPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True)
632 if p.returncode != 0:
633 self.clone_failed = True
634 raise VCSException("Git svn clone failed", p.output)
636 p = SilentPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True)
637 if p.returncode != 0:
638 self.clone_failed = True
639 raise VCSException("Git svn clone failed", p.output)
643 # Discard any working tree changes
644 p = SilentPopen(['git', 'reset', '--hard'], cwd=self.local)
645 if p.returncode != 0:
646 raise VCSException("Git reset failed", p.output)
647 # Remove untracked files now, in case they're tracked in the target
648 # revision (it happens!)
649 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
650 if p.returncode != 0:
651 raise VCSException("Git clean failed", p.output)
652 if not self.refreshed:
653 # Get new commits, branches and tags from repo
654 p = SilentPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True)
655 if p.returncode != 0:
656 raise VCSException("Git svn fetch failed")
657 p = SilentPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True)
658 if p.returncode != 0:
659 raise VCSException("Git svn rebase failed", p.output)
660 self.refreshed = True
662 rev = rev or 'master'
664 nospaces_rev = rev.replace(' ', '%20')
665 # Try finding a svn tag
666 for treeish in ['origin/', '']:
667 p = SilentPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev],
669 if p.returncode == 0:
671 if p.returncode != 0:
672 # No tag found, normal svn rev translation
673 # Translate svn rev into git format
674 rev_split = rev.split('/')
677 for treeish in ['origin/', '']:
678 if len(rev_split) > 1:
679 treeish += rev_split[0]
680 svn_rev = rev_split[1]
683 # if no branch is specified, then assume trunk (i.e. 'master' branch):
687 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
689 p = SilentPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish],
691 git_rev = p.output.rstrip()
693 if p.returncode == 0 and git_rev:
696 if p.returncode != 0 or not git_rev:
697 # Try a plain git checkout as a last resort
698 p = SilentPopen(['git', 'checkout', rev], cwd=self.local)
699 if p.returncode != 0:
700 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
702 # Check out the git rev equivalent to the svn rev
703 p = SilentPopen(['git', 'checkout', git_rev], cwd=self.local)
704 if p.returncode != 0:
705 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
707 # Get rid of any uncontrolled files left behind
708 p = SilentPopen(['git', 'clean', '-dffx'], cwd=self.local)
709 if p.returncode != 0:
710 raise VCSException("Git clean failed", p.output)
714 for treeish in ['origin/', '']:
715 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
721 p = SilentPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local)
722 if p.returncode != 0:
724 return p.output.strip()
732 def gotorevisionx(self, rev):
733 if not os.path.exists(self.local):
734 p = SilentPopen(['hg', 'clone', self.remote, self.local])
735 if p.returncode != 0:
736 self.clone_failed = True
737 raise VCSException("Hg clone failed", p.output)
739 p = SilentPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True)
740 if p.returncode != 0:
741 raise VCSException("Hg clean failed", p.output)
742 if not self.refreshed:
743 p = SilentPopen(['hg', 'pull'], cwd=self.local)
744 if p.returncode != 0:
745 raise VCSException("Hg pull failed", p.output)
746 self.refreshed = True
748 rev = rev or 'default'
751 p = SilentPopen(['hg', 'update', '-C', rev], cwd=self.local)
752 if p.returncode != 0:
753 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
754 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
755 # Also delete untracked files, we have to enable purge extension for that:
756 if "'purge' is provided by the following extension" in p.output:
757 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
758 myfile.write("\n[extensions]\nhgext.purge=\n")
759 p = SilentPopen(['hg', 'purge', '--all'], cwd=self.local)
760 if p.returncode != 0:
761 raise VCSException("HG purge failed", p.output)
762 elif p.returncode != 0:
763 raise VCSException("HG purge failed", p.output)
766 p = SilentPopen(['hg', 'tags', '-q'], cwd=self.local)
767 return p.output.splitlines()[1:]
775 def gotorevisionx(self, rev):
776 if not os.path.exists(self.local):
777 p = SilentPopen(['bzr', 'branch', self.remote, self.local])
778 if p.returncode != 0:
779 self.clone_failed = True
780 raise VCSException("Bzr branch failed", p.output)
782 p = SilentPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local)
783 if p.returncode != 0:
784 raise VCSException("Bzr revert failed", p.output)
785 if not self.refreshed:
786 p = SilentPopen(['bzr', 'pull'], cwd=self.local)
787 if p.returncode != 0:
788 raise VCSException("Bzr update failed", p.output)
789 self.refreshed = True
791 revargs = list(['-r', rev] if rev else [])
792 p = SilentPopen(['bzr', 'revert'] + revargs, cwd=self.local)
793 if p.returncode != 0:
794 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
797 p = SilentPopen(['bzr', 'tags'], cwd=self.local)
798 return [tag.split(' ')[0].strip() for tag in
799 p.output.splitlines()]
802 def retrieve_string(app_dir, string, xmlfiles=None):
805 os.path.join(app_dir, 'res'),
806 os.path.join(app_dir, 'src', 'main'),
811 for res_dir in res_dirs:
812 for r, d, f in os.walk(res_dir):
813 if os.path.basename(r) == 'values':
814 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
817 if string.startswith('@string/'):
818 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
819 elif string.startswith('&') and string.endswith(';'):
820 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
822 if string_search is not None:
823 for xmlfile in xmlfiles:
824 for line in file(xmlfile):
825 matches = string_search(line)
827 return retrieve_string(app_dir, matches.group(1), xmlfiles)
830 return string.replace("\\'", "'")
833 # Return list of existing files that will be used to find the highest vercode
834 def manifest_paths(app_dir, flavours):
836 possible_manifests = \
837 [os.path.join(app_dir, 'AndroidManifest.xml'),
838 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
839 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
840 os.path.join(app_dir, 'build.gradle')]
842 for flavour in flavours:
845 possible_manifests.append(
846 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
848 return [path for path in possible_manifests if os.path.isfile(path)]
851 # Retrieve the package name. Returns the name, or None if not found.
852 def fetch_real_name(app_dir, flavours):
853 app_search = re.compile(r'.*<application.*').search
854 name_search = re.compile(r'.*android:label="([^"]+)".*').search
856 for f in manifest_paths(app_dir, flavours):
857 if not has_extension(f, 'xml'):
859 logging.debug("fetch_real_name: Checking manifest at " + f)
865 matches = name_search(line)
867 stringname = matches.group(1)
868 logging.debug("fetch_real_name: using string " + stringname)
869 result = retrieve_string(app_dir, stringname)
871 result = result.strip()
876 # Retrieve the version name
877 def version_name(original, app_dir, flavours):
878 for f in manifest_paths(app_dir, flavours):
879 if not has_extension(f, 'xml'):
881 string = retrieve_string(app_dir, original)
887 def get_library_references(root_dir):
889 proppath = os.path.join(root_dir, 'project.properties')
890 if not os.path.isfile(proppath):
892 with open(proppath) as f:
893 for line in f.readlines():
894 if not line.startswith('android.library.reference.'):
896 path = line.split('=')[1].strip()
897 relpath = os.path.join(root_dir, path)
898 if not os.path.isdir(relpath):
900 logging.debug("Found subproject at %s" % path)
901 libraries.append(path)
905 def ant_subprojects(root_dir):
906 subprojects = get_library_references(root_dir)
907 for subpath in subprojects:
908 subrelpath = os.path.join(root_dir, subpath)
909 for p in get_library_references(subrelpath):
910 relp = os.path.normpath(os.path.join(subpath, p))
911 if relp not in subprojects:
912 subprojects.insert(0, relp)
916 def remove_debuggable_flags(root_dir):
917 # Remove forced debuggable flags
918 logging.debug("Removing debuggable flags from %s" % root_dir)
919 for root, dirs, files in os.walk(root_dir):
920 if 'AndroidManifest.xml' in files:
921 path = os.path.join(root, 'AndroidManifest.xml')
922 p = SilentPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path])
923 if p.returncode != 0:
924 raise BuildException("Failed to remove debuggable flags of %s" % path)
927 # Extract some information from the AndroidManifest.xml at the given path.
928 # Returns (version, vercode, package), any or all of which might be None.
929 # All values returned are strings.
930 def parse_androidmanifests(paths, ignoreversions=None):
933 return (None, None, None)
935 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
936 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
937 psearch = re.compile(r'.*package="([^"]+)".*').search
939 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
940 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
941 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
943 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
951 gradle = has_extension(path, 'gradle')
954 # Remember package name, may be defined separately from version+vercode
955 package = max_package
957 for line in file(path):
960 matches = psearch_g(line)
962 matches = psearch(line)
964 package = matches.group(1)
967 matches = vnsearch_g(line)
969 matches = vnsearch(line)
971 version = matches.group(2 if gradle else 1)
974 matches = vcsearch_g(line)
976 matches = vcsearch(line)
978 vercode = matches.group(1)
980 # Always grab the package name and version name in case they are not
981 # together with the highest version code
982 if max_package is None and package is not None:
983 max_package = package
984 if max_version is None and version is not None:
985 max_version = version
987 if max_vercode is None or (vercode is not None and vercode > max_vercode):
988 if not ignoresearch or not ignoresearch(version):
989 if version is not None:
990 max_version = version
991 if vercode is not None:
992 max_vercode = vercode
993 if package is not None:
994 max_package = package
996 max_version = "Ignore"
998 if max_version is None:
999 max_version = "Unknown"
1001 return (max_version, max_vercode, max_package)
1004 class FDroidException(Exception):
1005 def __init__(self, value, detail=None):
1007 self.detail = detail
1009 def get_wikitext(self):
1010 ret = repr(self.value) + "\n"
1014 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1022 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1026 class VCSException(FDroidException):
1030 class BuildException(FDroidException):
1034 # Get the specified source library.
1035 # Returns the path to it. Normally this is the path to be used when referencing
1036 # it, which may be a subdirectory of the actual project. If you want the base
1037 # directory of the project, pass 'basepath=True'.
1038 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1039 basepath=False, raw=False, prepare=True, preponly=False):
1047 name, ref = spec.split('@')
1049 number, name = name.split(':', 1)
1051 name, subdir = name.split('/', 1)
1053 if name not in metadata.srclibs:
1054 raise VCSException('srclib ' + name + ' not found.')
1056 srclib = metadata.srclibs[name]
1058 sdir = os.path.join(srclib_dir, name)
1061 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1062 vcs.srclib = (name, number, sdir)
1064 vcs.gotorevision(ref)
1071 libdir = os.path.join(sdir, subdir)
1072 elif srclib["Subdir"]:
1073 for subdir in srclib["Subdir"]:
1074 libdir_candidate = os.path.join(sdir, subdir)
1075 if os.path.exists(libdir_candidate):
1076 libdir = libdir_candidate
1082 if srclib["Srclibs"]:
1084 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1086 for t in srclibpaths:
1091 raise VCSException('Missing recursive srclib %s for %s' % (
1093 place_srclib(libdir, n, s_tuple[2])
1096 remove_signing_keys(sdir)
1097 remove_debuggable_flags(sdir)
1101 if srclib["Prepare"]:
1102 cmd = replace_config_vars(srclib["Prepare"])
1104 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1105 if p.returncode != 0:
1106 raise BuildException("Error running prepare command for srclib %s"
1112 return (name, number, libdir)
1115 # Prepare the source code for a particular build
1116 # 'vcs' - the appropriate vcs object for the application
1117 # 'app' - the application details from the metadata
1118 # 'build' - the build details from the metadata
1119 # 'build_dir' - the path to the build directory, usually
1121 # 'srclib_dir' - the path to the source libraries directory, usually
1123 # 'extlib_dir' - the path to the external libraries directory, usually
1125 # Returns the (root, srclibpaths) where:
1126 # 'root' is the root directory, which may be the same as 'build_dir' or may
1127 # be a subdirectory of it.
1128 # 'srclibpaths' is information on the srclibs being used
1129 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1131 # Optionally, the actual app source can be in a subdirectory
1133 root_dir = os.path.join(build_dir, build['subdir'])
1135 root_dir = build_dir
1137 # Get a working copy of the right revision
1138 logging.info("Getting source for revision " + build['commit'])
1139 vcs.gotorevision(build['commit'])
1141 # Initialise submodules if requred
1142 if build['submodules']:
1143 logging.info("Initialising submodules")
1144 vcs.initsubmodules()
1146 # Check that a subdir (if we're using one) exists. This has to happen
1147 # after the checkout, since it might not exist elsewhere
1148 if not os.path.exists(root_dir):
1149 raise BuildException('Missing subdir ' + root_dir)
1151 # Run an init command if one is required
1153 cmd = replace_config_vars(build['init'])
1154 logging.info("Running 'init' commands in %s" % root_dir)
1156 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1157 if p.returncode != 0:
1158 raise BuildException("Error running init command for %s:%s" %
1159 (app['id'], build['version']), p.output)
1161 # Apply patches if any
1163 logging.info("Applying patches")
1164 for patch in build['patch']:
1165 patch = patch.strip()
1166 logging.info("Applying " + patch)
1167 patch_path = os.path.join('metadata', app['id'], patch)
1168 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1169 if p.returncode != 0:
1170 raise BuildException("Failed to apply patch %s" % patch_path)
1172 # Get required source libraries
1174 if build['srclibs']:
1175 logging.info("Collecting source libraries")
1176 for lib in build['srclibs']:
1177 srclibpaths.append(getsrclib(lib, srclib_dir, srclibpaths,
1180 for name, number, libpath in srclibpaths:
1181 place_srclib(root_dir, int(number) if number else None, libpath)
1183 basesrclib = vcs.getsrclib()
1184 # If one was used for the main source, add that too.
1186 srclibpaths.append(basesrclib)
1188 # Update the local.properties file
1189 localprops = [os.path.join(build_dir, 'local.properties')]
1191 localprops += [os.path.join(root_dir, 'local.properties')]
1192 for path in localprops:
1194 if os.path.isfile(path):
1195 logging.info("Updating local.properties file at %s" % path)
1201 logging.info("Creating local.properties file at %s" % path)
1202 # Fix old-fashioned 'sdk-location' by copying
1203 # from sdk.dir, if necessary
1204 if build['oldsdkloc']:
1205 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1206 re.S | re.M).group(1)
1207 props += "sdk-location=%s\n" % sdkloc
1209 props += "sdk.dir=%s\n" % config['sdk_path']
1210 props += "sdk-location=%s\n" % config['sdk_path']
1211 if config['ndk_path']:
1213 props += "ndk.dir=%s\n" % config['ndk_path']
1214 props += "ndk-location=%s\n" % config['ndk_path']
1215 # Add java.encoding if necessary
1216 if build['encoding']:
1217 props += "java.encoding=%s\n" % build['encoding']
1223 if build['type'] == 'gradle':
1224 flavours = build['gradle']
1226 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1227 gradlepluginver = None
1229 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1231 # Parent dir build.gradle
1232 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1233 if parent_dir.startswith(build_dir):
1234 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1236 for path in gradle_files:
1239 if not os.path.isfile(path):
1241 with open(path) as f:
1243 match = version_regex.match(line)
1245 gradlepluginver = match.group(1)
1249 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1251 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1252 build['gradlepluginver'] = LooseVersion('0.11')
1255 n = build["target"].split('-')[1]
1256 SilentPopen(['sed', '-i',
1257 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1261 # Remove forced debuggable flags
1262 remove_debuggable_flags(root_dir)
1264 # Insert version code and number into the manifest if necessary
1265 if build['forceversion']:
1266 logging.info("Changing the version name")
1267 for path in manifest_paths(root_dir, flavours):
1268 if not os.path.isfile(path):
1270 if has_extension(path, 'xml'):
1271 p = SilentPopen(['sed', '-i',
1272 's/android:versionName="[^"]*"/android:versionName="'
1273 + build['version'] + '"/g',
1275 if p.returncode != 0:
1276 raise BuildException("Failed to amend manifest")
1277 elif has_extension(path, 'gradle'):
1278 p = SilentPopen(['sed', '-i',
1279 's/versionName *=* *"[^"]*"/versionName = "'
1280 + build['version'] + '"/g',
1282 if p.returncode != 0:
1283 raise BuildException("Failed to amend build.gradle")
1284 if build['forcevercode']:
1285 logging.info("Changing the version code")
1286 for path in manifest_paths(root_dir, flavours):
1287 if not os.path.isfile(path):
1289 if has_extension(path, 'xml'):
1290 p = SilentPopen(['sed', '-i',
1291 's/android:versionCode="[^"]*"/android:versionCode="'
1292 + build['vercode'] + '"/g',
1294 if p.returncode != 0:
1295 raise BuildException("Failed to amend manifest")
1296 elif has_extension(path, 'gradle'):
1297 p = SilentPopen(['sed', '-i',
1298 's/versionCode *=* *[0-9]*/versionCode = '
1299 + build['vercode'] + '/g',
1301 if p.returncode != 0:
1302 raise BuildException("Failed to amend build.gradle")
1304 # Delete unwanted files
1306 logging.info("Removing specified files")
1307 for part in getpaths(build_dir, build, 'rm'):
1308 dest = os.path.join(build_dir, part)
1309 logging.info("Removing {0}".format(part))
1310 if os.path.lexists(dest):
1311 if os.path.islink(dest):
1312 SilentPopen(['unlink ' + dest], shell=True)
1314 SilentPopen(['rm -rf ' + dest], shell=True)
1316 logging.info("...but it didn't exist")
1318 remove_signing_keys(build_dir)
1320 # Add required external libraries
1321 if build['extlibs']:
1322 logging.info("Collecting prebuilt libraries")
1323 libsdir = os.path.join(root_dir, 'libs')
1324 if not os.path.exists(libsdir):
1326 for lib in build['extlibs']:
1328 logging.info("...installing extlib {0}".format(lib))
1329 libf = os.path.basename(lib)
1330 libsrc = os.path.join(extlib_dir, lib)
1331 if not os.path.exists(libsrc):
1332 raise BuildException("Missing extlib file {0}".format(libsrc))
1333 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1335 # Run a pre-build command if one is required
1336 if build['prebuild']:
1337 logging.info("Running 'prebuild' commands in %s" % root_dir)
1339 cmd = replace_config_vars(build['prebuild'])
1341 # Substitute source library paths into prebuild commands
1342 for name, number, libpath in srclibpaths:
1343 libpath = os.path.relpath(libpath, root_dir)
1344 cmd = cmd.replace('$$' + name + '$$', libpath)
1346 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1347 if p.returncode != 0:
1348 raise BuildException("Error running prebuild command for %s:%s" %
1349 (app['id'], build['version']), p.output)
1351 # Generate (or update) the ant build file, build.xml...
1352 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1353 parms = [config['android'], 'update', 'lib-project']
1354 lparms = [config['android'], 'update', 'project']
1357 parms += ['-t', build['target']]
1358 lparms += ['-t', build['target']]
1359 if build['update'] == ['auto']:
1360 update_dirs = ant_subprojects(root_dir) + ['.']
1362 update_dirs = build['update']
1364 for d in update_dirs:
1365 subdir = os.path.join(root_dir, d)
1367 logging.debug("Updating main project")
1368 cmd = parms + ['-p', d]
1370 logging.debug("Updating subproject %s" % d)
1371 cmd = lparms + ['-p', d]
1372 p = FDroidPopen(cmd, cwd=root_dir)
1373 # Check to see whether an error was returned without a proper exit
1374 # code (this is the case for the 'no target set or target invalid'
1376 if p.returncode != 0 or p.output.startswith("Error: "):
1377 raise BuildException("Failed to update project at %s" % d, p.output)
1378 # Clean update dirs via ant
1380 logging.info("Cleaning subproject %s" % d)
1381 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1383 return (root_dir, srclibpaths)
1386 # Split and extend via globbing the paths from a field
1387 def getpaths(build_dir, build, field):
1389 for p in build[field]:
1391 full_path = os.path.join(build_dir, p)
1392 full_path = os.path.normpath(full_path)
1393 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1397 # Scan the source code in the given directory (and all subdirectories)
1398 # and return the number of fatal problems encountered
1399 def scan_source(build_dir, root_dir, thisbuild):
1403 # Common known non-free blobs (always lower case):
1405 re.compile(r'flurryagent', re.IGNORECASE),
1406 re.compile(r'paypal.*mpl', re.IGNORECASE),
1407 re.compile(r'google.*analytics', re.IGNORECASE),
1408 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1409 re.compile(r'google.*ad.*view', re.IGNORECASE),
1410 re.compile(r'google.*admob', re.IGNORECASE),
1411 re.compile(r'google.*play.*services', re.IGNORECASE),
1412 re.compile(r'crittercism', re.IGNORECASE),
1413 re.compile(r'heyzap', re.IGNORECASE),
1414 re.compile(r'jpct.*ae', re.IGNORECASE),
1415 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1416 re.compile(r'bugsense', re.IGNORECASE),
1417 re.compile(r'crashlytics', re.IGNORECASE),
1418 re.compile(r'ouya.*sdk', re.IGNORECASE),
1419 re.compile(r'libspen23', re.IGNORECASE),
1422 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1423 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1426 ms = magic.open(magic.MIME_TYPE)
1428 except AttributeError:
1432 for i in scanignore:
1433 if fd.startswith(i):
1438 for i in scandelete:
1439 if fd.startswith(i):
1443 def removeproblem(what, fd, fp):
1444 logging.info('Removing %s at %s' % (what, fd))
1447 def warnproblem(what, fd):
1448 logging.warn('Found %s at %s' % (what, fd))
1450 def handleproblem(what, fd, fp):
1452 logging.info('Ignoring %s at %s' % (what, fd))
1454 removeproblem(what, fd, fp)
1456 logging.error('Found %s at %s' % (what, fd))
1460 # Iterate through all files in the source code
1461 for r, d, f in os.walk(build_dir, topdown=True):
1463 # It's topdown, so checking the basename is enough
1464 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1470 # Path (relative) to the file
1471 fp = os.path.join(r, curfile)
1472 fd = fp[len(build_dir) + 1:]
1475 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1476 except UnicodeError:
1477 warnproblem('malformed magic number', fd)
1479 if mime == 'application/x-sharedlib':
1480 count += handleproblem('shared library', fd, fp)
1482 elif mime == 'application/x-archive':
1483 count += handleproblem('static library', fd, fp)
1485 elif mime == 'application/x-executable':
1486 count += handleproblem('binary executable', fd, fp)
1488 elif mime == 'application/x-java-applet':
1489 count += handleproblem('Java compiled class', fd, fp)
1494 'application/java-archive',
1495 'application/octet-stream',
1499 if has_extension(fp, 'apk'):
1500 removeproblem('APK file', fd, fp)
1502 elif has_extension(fp, 'jar'):
1504 if any(suspect.match(curfile) for suspect in usual_suspects):
1505 count += handleproblem('usual supect', fd, fp)
1507 warnproblem('JAR file', fd)
1509 elif has_extension(fp, 'zip'):
1510 warnproblem('ZIP file', fd)
1513 warnproblem('unknown compressed or binary file', fd)
1515 elif has_extension(fp, 'java'):
1516 for line in file(fp):
1517 if 'DexClassLoader' in line:
1518 count += handleproblem('DexClassLoader', fd, fp)
1523 # Presence of a jni directory without buildjni=yes might
1524 # indicate a problem (if it's not a problem, explicitly use
1525 # buildjni=no to bypass this check)
1526 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1527 not thisbuild['buildjni']):
1528 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1537 self.path = os.path.join('stats', 'known_apks.txt')
1539 if os.path.exists(self.path):
1540 for line in file(self.path):
1541 t = line.rstrip().split(' ')
1543 self.apks[t[0]] = (t[1], None)
1545 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1546 self.changed = False
1548 def writeifchanged(self):
1550 if not os.path.exists('stats'):
1552 f = open(self.path, 'w')
1554 for apk, app in self.apks.iteritems():
1556 line = apk + ' ' + appid
1558 line += ' ' + time.strftime('%Y-%m-%d', added)
1560 for line in sorted(lst):
1561 f.write(line + '\n')
1564 # Record an apk (if it's new, otherwise does nothing)
1565 # Returns the date it was added.
1566 def recordapk(self, apk, app):
1567 if apk not in self.apks:
1568 self.apks[apk] = (app, time.gmtime(time.time()))
1570 _, added = self.apks[apk]
1573 # Look up information - given the 'apkname', returns (app id, date added/None).
1574 # Or returns None for an unknown apk.
1575 def getapp(self, apkname):
1576 if apkname in self.apks:
1577 return self.apks[apkname]
1580 # Get the most recent 'num' apps added to the repo, as a list of package ids
1581 # with the most recent first.
1582 def getlatest(self, num):
1584 for apk, app in self.apks.iteritems():
1588 if apps[appid] > added:
1592 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1593 lst = [app for app, _ in sortedapps]
1598 def isApkDebuggable(apkfile, config):
1599 """Returns True if the given apk file is debuggable
1601 :param apkfile: full path to the apk to check"""
1603 p = SilentPopen([os.path.join(config['sdk_path'], 'build-tools',
1604 config['build_tools'], 'aapt'),
1605 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'])
1606 if p.returncode != 0:
1607 logging.critical("Failed to get apk manifest information")
1609 for line in p.output.splitlines():
1610 if 'android:debuggable' in line and not line.endswith('0x0'):
1615 class AsynchronousFileReader(threading.Thread):
1617 Helper class to implement asynchronous reading of a file
1618 in a separate thread. Pushes read lines on a queue to
1619 be consumed in another thread.
1622 def __init__(self, fd, queue):
1623 assert isinstance(queue, Queue.Queue)
1624 assert callable(fd.readline)
1625 threading.Thread.__init__(self)
1630 '''The body of the tread: read lines and put them on the queue.'''
1631 for line in iter(self._fd.readline, ''):
1632 self._queue.put(line)
1635 '''Check whether there is no more content to expect.'''
1636 return not self.is_alive() and self._queue.empty()
1644 def SilentPopen(commands, cwd=None, shell=False):
1645 return FDroidPopen(commands, cwd=cwd, shell=shell, output=False)
1648 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1650 Run a command and capture the possibly huge output.
1652 :param commands: command and argument list like in subprocess.Popen
1653 :param cwd: optionally specifies a working directory
1654 :returns: A PopenResult.
1660 cwd = os.path.normpath(cwd)
1661 logging.debug("Directory: %s" % cwd)
1662 logging.debug("> %s" % ' '.join(commands))
1664 result = PopenResult()
1667 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1668 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1670 raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
1672 stdout_queue = Queue.Queue()
1673 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1674 stdout_reader.start()
1676 # Check the queue for output (until there is no more to get)
1677 while not stdout_reader.eof():
1678 while not stdout_queue.empty():
1679 line = stdout_queue.get()
1680 if output and options.verbose:
1681 # Output directly to console
1682 sys.stderr.write(line)
1684 result.output += line
1688 result.returncode = p.wait()
1692 def remove_signing_keys(build_dir):
1693 comment = re.compile(r'[ ]*//')
1694 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1696 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1697 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1698 re.compile(r'.*variant\.outputFile = .*'),
1699 re.compile(r'.*\.readLine\(.*'),
1701 for root, dirs, files in os.walk(build_dir):
1702 if 'build.gradle' in files:
1703 path = os.path.join(root, 'build.gradle')
1705 with open(path, "r") as o:
1706 lines = o.readlines()
1711 with open(path, "w") as o:
1713 if comment.match(line):
1717 opened += line.count('{')
1718 opened -= line.count('}')
1721 if signing_configs.match(line):
1726 if any(s.match(line) for s in line_matches):
1734 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1737 'project.properties',
1739 'default.properties',
1742 if propfile in files:
1743 path = os.path.join(root, propfile)
1745 with open(path, "r") as o:
1746 lines = o.readlines()
1750 with open(path, "w") as o:
1752 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1759 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1762 def replace_config_vars(cmd):
1763 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1764 cmd = cmd.replace('$$NDK$$', config['ndk_path'])
1765 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1769 def place_srclib(root_dir, number, libpath):
1772 relpath = os.path.relpath(libpath, root_dir)
1773 proppath = os.path.join(root_dir, 'project.properties')
1776 if os.path.isfile(proppath):
1777 with open(proppath, "r") as o:
1778 lines = o.readlines()
1780 with open(proppath, "w") as o:
1783 if line.startswith('android.library.reference.%d=' % number):
1784 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1789 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1792 def compare_apks(apk1, apk2, tmp_dir):
1795 Returns None if the apk content is the same (apart from the signing key),
1796 otherwise a string describing what's different, or what went wrong when
1797 trying to do the comparison.
1800 thisdir = os.path.join(tmp_dir, 'this_apk')
1801 thatdir = os.path.join(tmp_dir, 'that_apk')
1802 for d in [thisdir, thatdir]:
1803 if os.path.exists(d):
1807 if subprocess.call(['jar', 'xf',
1808 os.path.abspath(apk1)],
1810 return("Failed to unpack " + apk1)
1811 if subprocess.call(['jar', 'xf',
1812 os.path.abspath(apk2)],
1814 return("Failed to unpack " + apk2)
1816 p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
1818 lines = p.output.splitlines()
1819 if len(lines) != 1 or 'META-INF' not in lines[0]:
1820 return("Unexpected diff output - " + p.output)
1822 # If we get here, it seems like they're the same!