1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
33 from distutils.version import LooseVersion
43 'sdk_path': "$ANDROID_HOME",
46 'r10d': "$ANDROID_NDK"
48 'build_tools': "21.1.2",
52 'sync_from_local_copy_dir': False,
53 'make_current_version_link': True,
54 'current_version_name_source': 'Name',
55 'update_stats': False,
59 'stats_to_carbon': False,
61 'build_server_always': False,
62 'keystore': os.path.join("$HOME", '.local', 'share', 'fdroidserver', 'keystore.jks'),
63 'smartcardoptions': [],
69 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
70 'repo_name': "My First FDroid Repo Demo",
71 'repo_icon': "fdroid-icon.png",
72 'repo_description': '''
73 This is a repository of apps to be used with FDroid. Applications in this
74 repository are either official binaries built by the original application
75 developers, or are binaries built from source by the admin of f-droid.org
76 using the tools on https://gitlab.com/u/fdroid.
82 def fill_config_defaults(thisconfig):
83 for k, v in default_config.items():
84 if k not in thisconfig:
87 # Expand paths (~users and $vars)
88 def expand_path(path):
92 path = os.path.expanduser(path)
93 path = os.path.expandvars(path)
98 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
103 thisconfig[k + '_orig'] = v
105 for k in ['ndk_paths']:
111 thisconfig[k][k2] = exp
112 thisconfig[k][k2 + '_orig'] = v
115 def read_config(opts, config_file='config.py'):
116 """Read the repository config
118 The config is read from config_file, which is in the current directory when
119 any of the repo management commands are used.
121 global config, options, env
123 if config is not None:
125 if not os.path.isfile(config_file):
126 logging.critical("Missing config file - is this a repo directory?")
133 logging.debug("Reading %s" % config_file)
134 execfile(config_file, config)
136 # smartcardoptions must be a list since its command line args for Popen
137 if 'smartcardoptions' in config:
138 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
139 elif 'keystore' in config and config['keystore'] == 'NONE':
140 # keystore='NONE' means use smartcard, these are required defaults
141 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
142 'SunPKCS11-OpenSC', '-providerClass',
143 'sun.security.pkcs11.SunPKCS11',
144 '-providerArg', 'opensc-fdroid.cfg']
146 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
147 st = os.stat(config_file)
148 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
149 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
151 fill_config_defaults(config)
153 # There is no standard, so just set up the most common environment
156 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
157 env[n] = config['sdk_path']
159 for k in ["keystorepass", "keypass"]:
161 write_password_file(k)
163 for k in ["repo_description", "archive_description"]:
165 config[k] = clean_description(config[k])
167 if 'serverwebroot' in config:
168 if isinstance(config['serverwebroot'], basestring):
169 roots = [config['serverwebroot']]
170 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
171 roots = config['serverwebroot']
173 raise TypeError('only accepts strings, lists, and tuples')
175 for rootstr in roots:
176 # since this is used with rsync, where trailing slashes have
177 # meaning, ensure there is always a trailing slash
178 if rootstr[-1] != '/':
180 rootlist.append(rootstr.replace('//', '/'))
181 config['serverwebroot'] = rootlist
186 def get_ndk_path(version):
188 version = 'r10d' # latest
189 paths = config['ndk_paths']
190 if version not in paths:
192 return paths[version] or ''
195 def find_sdk_tools_cmd(cmd):
196 '''find a working path to a tool from the Android SDK'''
199 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
200 # try to find a working path to this command, in all the recent possible paths
201 if 'build_tools' in config:
202 build_tools = os.path.join(config['sdk_path'], 'build-tools')
203 # if 'build_tools' was manually set and exists, check only that one
204 configed_build_tools = os.path.join(build_tools, config['build_tools'])
205 if os.path.exists(configed_build_tools):
206 tooldirs.append(configed_build_tools)
208 # no configed version, so hunt known paths for it
209 for f in sorted(os.listdir(build_tools), reverse=True):
210 if os.path.isdir(os.path.join(build_tools, f)):
211 tooldirs.append(os.path.join(build_tools, f))
212 tooldirs.append(build_tools)
213 sdk_tools = os.path.join(config['sdk_path'], 'tools')
214 if os.path.exists(sdk_tools):
215 tooldirs.append(sdk_tools)
216 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
217 if os.path.exists(sdk_platform_tools):
218 tooldirs.append(sdk_platform_tools)
219 tooldirs.append('/usr/bin')
221 if os.path.isfile(os.path.join(d, cmd)):
222 return os.path.join(d, cmd)
223 # did not find the command, exit with error message
224 ensure_build_tools_exists(config)
227 def test_sdk_exists(thisconfig):
228 if 'sdk_path' not in thisconfig:
229 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
232 logging.error("'sdk_path' not set in config.py!")
234 if thisconfig['sdk_path'] == default_config['sdk_path']:
235 logging.error('No Android SDK found!')
236 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
237 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
239 if not os.path.exists(thisconfig['sdk_path']):
240 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
242 if not os.path.isdir(thisconfig['sdk_path']):
243 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
245 for d in ['build-tools', 'platform-tools', 'tools']:
246 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
247 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
248 thisconfig['sdk_path'], d))
253 def ensure_build_tools_exists(thisconfig):
254 if not test_sdk_exists(thisconfig):
256 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
257 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
258 if not os.path.isdir(versioned_build_tools):
259 logging.critical('Android Build Tools path "'
260 + versioned_build_tools + '" does not exist!')
264 def write_password_file(pwtype, password=None):
266 writes out passwords to a protected file instead of passing passwords as
267 command line argments
269 filename = '.fdroid.' + pwtype + '.txt'
270 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
272 os.write(fd, config[pwtype])
274 os.write(fd, password)
276 config[pwtype + 'file'] = filename
279 # Given the arguments in the form of multiple appid:[vc] strings, this returns
280 # a dictionary with the set of vercodes specified for each package.
281 def read_pkg_args(args, allow_vercodes=False):
288 if allow_vercodes and ':' in p:
289 package, vercode = p.split(':')
291 package, vercode = p, None
292 if package not in vercodes:
293 vercodes[package] = [vercode] if vercode else []
295 elif vercode and vercode not in vercodes[package]:
296 vercodes[package] += [vercode] if vercode else []
301 # On top of what read_pkg_args does, this returns the whole app metadata, but
302 # limiting the builds list to the builds matching the vercodes specified.
303 def read_app_args(args, allapps, allow_vercodes=False):
305 vercodes = read_pkg_args(args, allow_vercodes)
311 for appid, app in allapps.iteritems():
312 if appid in vercodes:
315 if len(apps) != len(vercodes):
318 logging.critical("No such package: %s" % p)
319 raise FDroidException("Found invalid app ids in arguments")
321 raise FDroidException("No packages specified")
324 for appid, app in apps.iteritems():
328 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
329 if len(app['builds']) != len(vercodes[appid]):
331 allvcs = [b['vercode'] for b in app['builds']]
332 for v in vercodes[appid]:
334 logging.critical("No such vercode %s for app %s" % (v, appid))
337 raise FDroidException("Found invalid vercodes for some apps")
342 def has_extension(filename, extension):
343 name, ext = os.path.splitext(filename)
344 ext = ext.lower()[1:]
345 return ext == extension
350 def clean_description(description):
351 'Remove unneeded newlines and spaces from a block of description text'
353 # this is split up by paragraph to make removing the newlines easier
354 for paragraph in re.split(r'\n\n', description):
355 paragraph = re.sub('\r', '', paragraph)
356 paragraph = re.sub('\n', ' ', paragraph)
357 paragraph = re.sub(' {2,}', ' ', paragraph)
358 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
359 returnstring += paragraph + '\n\n'
360 return returnstring.rstrip('\n')
363 def apknameinfo(filename):
365 filename = os.path.basename(filename)
366 if apk_regex is None:
367 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
368 m = apk_regex.match(filename)
370 result = (m.group(1), m.group(2))
371 except AttributeError:
372 raise FDroidException("Invalid apk name: %s" % filename)
376 def getapkname(app, build):
377 return "%s_%s.apk" % (app['id'], build['vercode'])
380 def getsrcname(app, build):
381 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
388 return app['Auto Name']
393 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
396 def getvcs(vcstype, remote, local):
398 return vcs_git(remote, local)
399 if vcstype == 'git-svn':
400 return vcs_gitsvn(remote, local)
402 return vcs_hg(remote, local)
404 return vcs_bzr(remote, local)
405 if vcstype == 'srclib':
406 if local != os.path.join('build', 'srclib', remote):
407 raise VCSException("Error: srclib paths are hard-coded!")
408 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
410 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
411 raise VCSException("Invalid vcs type " + vcstype)
414 def getsrclibvcs(name):
415 if name not in metadata.srclibs:
416 raise VCSException("Missing srclib " + name)
417 return metadata.srclibs[name]['Repo Type']
422 def __init__(self, remote, local):
424 # svn, git-svn and bzr may require auth
426 if self.repotype() in ('git-svn', 'bzr'):
428 self.username, remote = remote.split('@')
429 if ':' not in self.username:
430 raise VCSException("Password required with username")
431 self.username, self.password = self.username.split(':')
435 self.clone_failed = False
436 self.refreshed = False
442 # Take the local repository to a clean version of the given revision, which
443 # is specificed in the VCS's native format. Beforehand, the repository can
444 # be dirty, or even non-existent. If the repository does already exist
445 # locally, it will be updated from the origin, but only once in the
446 # lifetime of the vcs object.
447 # None is acceptable for 'rev' if you know you are cloning a clean copy of
448 # the repo - otherwise it must specify a valid revision.
449 def gotorevision(self, rev):
451 if self.clone_failed:
452 raise VCSException("Downloading the repository already failed once, not trying again.")
454 # The .fdroidvcs-id file for a repo tells us what VCS type
455 # and remote that directory was created from, allowing us to drop it
456 # automatically if either of those things changes.
457 fdpath = os.path.join(self.local, '..',
458 '.fdroidvcs-' + os.path.basename(self.local))
459 cdata = self.repotype() + ' ' + self.remote
462 if os.path.exists(self.local):
463 if os.path.exists(fdpath):
464 with open(fdpath, 'r') as f:
465 fsdata = f.read().strip()
470 logging.info("Repository details for %s changed - deleting" % (
474 logging.info("Repository details for %s missing - deleting" % (
477 shutil.rmtree(self.local)
482 self.gotorevisionx(rev)
483 except FDroidException, e:
486 # If necessary, write the .fdroidvcs file.
487 if writeback and not self.clone_failed:
488 with open(fdpath, 'w') as f:
494 # Derived classes need to implement this. It's called once basic checking
495 # has been performend.
496 def gotorevisionx(self, rev):
497 raise VCSException("This VCS type doesn't define gotorevisionx")
499 # Initialise and update submodules
500 def initsubmodules(self):
501 raise VCSException('Submodules not supported for this vcs type')
503 # Get a list of all known tags
505 raise VCSException('gettags not supported for this vcs type')
507 # Get a list of latest number tags
508 def latesttags(self, number):
509 raise VCSException('latesttags not supported for this vcs type')
511 # Get current commit reference (hash, revision, etc)
513 raise VCSException('getref not supported for this vcs type')
515 # Returns the srclib (name, path) used in setting up the current
526 # If the local directory exists, but is somehow not a git repository, git
527 # will traverse up the directory tree until it finds one that is (i.e.
528 # fdroidserver) and then we'll proceed to destroy it! This is called as
531 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
532 result = p.output.rstrip()
533 if not result.endswith(self.local):
534 raise VCSException('Repository mismatch')
536 def gotorevisionx(self, rev):
537 if not os.path.exists(self.local):
539 p = FDroidPopen(['git', 'clone', self.remote, self.local])
540 if p.returncode != 0:
541 self.clone_failed = True
542 raise VCSException("Git clone failed", p.output)
546 # Discard any working tree changes
547 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
548 if p.returncode != 0:
549 raise VCSException("Git reset failed", p.output)
550 # Remove untracked files now, in case they're tracked in the target
551 # revision (it happens!)
552 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
553 if p.returncode != 0:
554 raise VCSException("Git clean failed", p.output)
555 if not self.refreshed:
556 # Get latest commits and tags from remote
557 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
558 if p.returncode != 0:
559 raise VCSException("Git fetch failed", p.output)
560 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
561 if p.returncode != 0:
562 raise VCSException("Git fetch failed", p.output)
563 # Recreate origin/HEAD as git clone would do it, in case it disappeared
564 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
565 if p.returncode != 0:
566 lines = p.output.splitlines()
567 if 'Multiple remote HEAD branches' not in lines[0]:
568 raise VCSException("Git remote set-head failed", p.output)
569 branch = lines[1].split(' ')[-1]
570 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
571 if p2.returncode != 0:
572 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
573 self.refreshed = True
574 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
575 # a github repo. Most of the time this is the same as origin/master.
576 rev = rev or 'origin/HEAD'
577 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
578 if p.returncode != 0:
579 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
580 # Get rid of any uncontrolled files left behind
581 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
582 if p.returncode != 0:
583 raise VCSException("Git clean failed", p.output)
585 def initsubmodules(self):
587 submfile = os.path.join(self.local, '.gitmodules')
588 if not os.path.isfile(submfile):
589 raise VCSException("No git submodules available")
591 # fix submodules not accessible without an account and public key auth
592 with open(submfile, 'r') as f:
593 lines = f.readlines()
594 with open(submfile, 'w') as f:
596 if 'git@github.com' in line:
597 line = line.replace('git@github.com:', 'https://github.com/')
601 ['git', 'reset', '--hard'],
602 ['git', 'clean', '-dffx'],
604 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
605 if p.returncode != 0:
606 raise VCSException("Git submodule reset failed", p.output)
607 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
608 if p.returncode != 0:
609 raise VCSException("Git submodule sync failed", p.output)
610 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
611 if p.returncode != 0:
612 raise VCSException("Git submodule update failed", p.output)
616 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
617 return p.output.splitlines()
619 def latesttags(self, alltags, number):
621 p = FDroidPopen(['echo "' + '\n'.join(alltags) + '" | '
623 'xargs -I@ git log --format=format:"%at @%n" -1 @ | '
624 + 'sort -n | awk \'{print $2}\''],
625 cwd=self.local, shell=True, output=False)
626 return p.output.splitlines()[-number:]
629 class vcs_gitsvn(vcs):
634 # Damn git-svn tries to use a graphical password prompt, so we have to
635 # trick it into taking the password from stdin
637 if self.username is None:
639 return ('echo "%s" | DISPLAY="" ' % self.password, ' --username "%s"' % self.username)
641 # If the local directory exists, but is somehow not a git repository, git
642 # will traverse up the directory tree until it finds one that is (i.e.
643 # fdroidserver) and then we'll proceed to destory it! This is called as
646 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
647 result = p.output.rstrip()
648 if not result.endswith(self.local):
649 raise VCSException('Repository mismatch')
651 def gotorevisionx(self, rev):
652 if not os.path.exists(self.local):
654 gitsvn_cmd = '%sgit svn clone%s' % self.userargs()
655 if ';' in self.remote:
656 remote_split = self.remote.split(';')
657 for i in remote_split[1:]:
658 if i.startswith('trunk='):
659 gitsvn_cmd += ' -T %s' % i[6:]
660 elif i.startswith('tags='):
661 gitsvn_cmd += ' -t %s' % i[5:]
662 elif i.startswith('branches='):
663 gitsvn_cmd += ' -b %s' % i[9:]
664 p = FDroidPopen([gitsvn_cmd + " %s %s" % (remote_split[0], self.local)], shell=True, output=False)
665 if p.returncode != 0:
666 self.clone_failed = True
667 raise VCSException("Git svn clone failed", p.output)
669 p = FDroidPopen([gitsvn_cmd + " %s %s" % (self.remote, self.local)], shell=True, output=False)
670 if p.returncode != 0:
671 self.clone_failed = True
672 raise VCSException("Git svn clone failed", p.output)
676 # Discard any working tree changes
677 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
678 if p.returncode != 0:
679 raise VCSException("Git reset failed", p.output)
680 # Remove untracked files now, in case they're tracked in the target
681 # revision (it happens!)
682 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
683 if p.returncode != 0:
684 raise VCSException("Git clean failed", p.output)
685 if not self.refreshed:
686 # Get new commits, branches and tags from repo
687 p = FDroidPopen(['%sgit svn fetch %s' % self.userargs()], cwd=self.local, shell=True, output=False)
688 if p.returncode != 0:
689 raise VCSException("Git svn fetch failed")
690 p = FDroidPopen(['%sgit svn rebase %s' % self.userargs()], cwd=self.local, shell=True, output=False)
691 if p.returncode != 0:
692 raise VCSException("Git svn rebase failed", p.output)
693 self.refreshed = True
695 rev = rev or 'master'
697 nospaces_rev = rev.replace(' ', '%20')
698 # Try finding a svn tag
699 for treeish in ['origin/', '']:
700 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
701 if p.returncode == 0:
703 if p.returncode != 0:
704 # No tag found, normal svn rev translation
705 # Translate svn rev into git format
706 rev_split = rev.split('/')
709 for treeish in ['origin/', '']:
710 if len(rev_split) > 1:
711 treeish += rev_split[0]
712 svn_rev = rev_split[1]
715 # if no branch is specified, then assume trunk (i.e. 'master' branch):
719 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
721 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
722 git_rev = p.output.rstrip()
724 if p.returncode == 0 and git_rev:
727 if p.returncode != 0 or not git_rev:
728 # Try a plain git checkout as a last resort
729 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
730 if p.returncode != 0:
731 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
733 # Check out the git rev equivalent to the svn rev
734 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
735 if p.returncode != 0:
736 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
738 # Get rid of any uncontrolled files left behind
739 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
740 if p.returncode != 0:
741 raise VCSException("Git clean failed", p.output)
745 for treeish in ['origin/', '']:
746 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
752 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
753 if p.returncode != 0:
755 return p.output.strip()
763 def gotorevisionx(self, rev):
764 if not os.path.exists(self.local):
765 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
766 if p.returncode != 0:
767 self.clone_failed = True
768 raise VCSException("Hg clone failed", p.output)
770 p = FDroidPopen(['hg status -uS | xargs rm -rf'], cwd=self.local, shell=True, output=False)
771 if p.returncode != 0:
772 raise VCSException("Hg clean failed", p.output)
773 if not self.refreshed:
774 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
775 if p.returncode != 0:
776 raise VCSException("Hg pull failed", p.output)
777 self.refreshed = True
779 rev = rev or 'default'
782 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
785 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
786 # Also delete untracked files, we have to enable purge extension for that:
787 if "'purge' is provided by the following extension" in p.output:
788 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
789 myfile.write("\n[extensions]\nhgext.purge=\n")
790 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
791 if p.returncode != 0:
792 raise VCSException("HG purge failed", p.output)
793 elif p.returncode != 0:
794 raise VCSException("HG purge failed", p.output)
797 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
798 return p.output.splitlines()[1:]
806 def gotorevisionx(self, rev):
807 if not os.path.exists(self.local):
808 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
809 if p.returncode != 0:
810 self.clone_failed = True
811 raise VCSException("Bzr branch failed", p.output)
813 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
814 if p.returncode != 0:
815 raise VCSException("Bzr revert failed", p.output)
816 if not self.refreshed:
817 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
818 if p.returncode != 0:
819 raise VCSException("Bzr update failed", p.output)
820 self.refreshed = True
822 revargs = list(['-r', rev] if rev else [])
823 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
824 if p.returncode != 0:
825 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
828 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
829 return [tag.split(' ')[0].strip() for tag in
830 p.output.splitlines()]
833 def retrieve_string(app_dir, string, xmlfiles=None):
836 os.path.join(app_dir, 'res'),
837 os.path.join(app_dir, 'src', 'main'),
842 for res_dir in res_dirs:
843 for r, d, f in os.walk(res_dir):
844 if os.path.basename(r) == 'values':
845 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
848 if string.startswith('@string/'):
849 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
850 elif string.startswith('&') and string.endswith(';'):
851 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
853 if string_search is not None:
854 for xmlfile in xmlfiles:
855 for line in file(xmlfile):
856 matches = string_search(line)
858 return retrieve_string(app_dir, matches.group(1), xmlfiles)
861 return string.replace("\\'", "'")
864 # Return list of existing files that will be used to find the highest vercode
865 def manifest_paths(app_dir, flavours):
867 possible_manifests = \
868 [os.path.join(app_dir, 'AndroidManifest.xml'),
869 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
870 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
871 os.path.join(app_dir, 'build.gradle')]
873 for flavour in flavours:
876 possible_manifests.append(
877 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
879 return [path for path in possible_manifests if os.path.isfile(path)]
882 # Retrieve the package name. Returns the name, or None if not found.
883 def fetch_real_name(app_dir, flavours):
884 app_search = re.compile(r'.*<application.*').search
885 name_search = re.compile(r'.*android:label="([^"]+)".*').search
887 for f in manifest_paths(app_dir, flavours):
888 if not has_extension(f, 'xml'):
890 logging.debug("fetch_real_name: Checking manifest at " + f)
896 matches = name_search(line)
898 stringname = matches.group(1)
899 logging.debug("fetch_real_name: using string " + stringname)
900 result = retrieve_string(app_dir, stringname)
902 result = result.strip()
907 # Retrieve the version name
908 def version_name(original, app_dir, flavours):
909 for f in manifest_paths(app_dir, flavours):
910 if not has_extension(f, 'xml'):
912 string = retrieve_string(app_dir, original)
918 def get_library_references(root_dir):
920 proppath = os.path.join(root_dir, 'project.properties')
921 if not os.path.isfile(proppath):
923 with open(proppath) as f:
924 for line in f.readlines():
925 if not line.startswith('android.library.reference.'):
927 path = line.split('=')[1].strip()
928 relpath = os.path.join(root_dir, path)
929 if not os.path.isdir(relpath):
931 logging.debug("Found subproject at %s" % path)
932 libraries.append(path)
936 def ant_subprojects(root_dir):
937 subprojects = get_library_references(root_dir)
938 for subpath in subprojects:
939 subrelpath = os.path.join(root_dir, subpath)
940 for p in get_library_references(subrelpath):
941 relp = os.path.normpath(os.path.join(subpath, p))
942 if relp not in subprojects:
943 subprojects.insert(0, relp)
947 def remove_debuggable_flags(root_dir):
948 # Remove forced debuggable flags
949 logging.debug("Removing debuggable flags from %s" % root_dir)
950 for root, dirs, files in os.walk(root_dir):
951 if 'AndroidManifest.xml' in files:
952 path = os.path.join(root, 'AndroidManifest.xml')
953 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
954 if p.returncode != 0:
955 raise BuildException("Failed to remove debuggable flags of %s" % path)
958 # Extract some information from the AndroidManifest.xml at the given path.
959 # Returns (version, vercode, package), any or all of which might be None.
960 # All values returned are strings.
961 def parse_androidmanifests(paths, ignoreversions=None):
964 return (None, None, None)
966 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
967 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
968 psearch = re.compile(r'.*package="([^"]+)".*').search
970 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
971 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
972 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
974 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
982 gradle = has_extension(path, 'gradle')
985 # Remember package name, may be defined separately from version+vercode
986 package = max_package
988 for line in file(path):
991 matches = psearch_g(line)
993 matches = psearch(line)
995 package = matches.group(1)
998 matches = vnsearch_g(line)
1000 matches = vnsearch(line)
1002 version = matches.group(2 if gradle else 1)
1005 matches = vcsearch_g(line)
1007 matches = vcsearch(line)
1009 vercode = matches.group(1)
1011 # Always grab the package name and version name in case they are not
1012 # together with the highest version code
1013 if max_package is None and package is not None:
1014 max_package = package
1015 if max_version is None and version is not None:
1016 max_version = version
1018 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1019 if not ignoresearch or not ignoresearch(version):
1020 if version is not None:
1021 max_version = version
1022 if vercode is not None:
1023 max_vercode = vercode
1024 if package is not None:
1025 max_package = package
1027 max_version = "Ignore"
1029 if max_version is None:
1030 max_version = "Unknown"
1032 return (max_version, max_vercode, max_package)
1035 class FDroidException(Exception):
1037 def __init__(self, value, detail=None):
1039 self.detail = detail
1041 def get_wikitext(self):
1042 ret = repr(self.value) + "\n"
1046 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1054 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1058 class VCSException(FDroidException):
1062 class BuildException(FDroidException):
1066 # Get the specified source library.
1067 # Returns the path to it. Normally this is the path to be used when referencing
1068 # it, which may be a subdirectory of the actual project. If you want the base
1069 # directory of the project, pass 'basepath=True'.
1070 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1071 basepath=False, raw=False, prepare=True, preponly=False):
1079 name, ref = spec.split('@')
1081 number, name = name.split(':', 1)
1083 name, subdir = name.split('/', 1)
1085 if name not in metadata.srclibs:
1086 raise VCSException('srclib ' + name + ' not found.')
1088 srclib = metadata.srclibs[name]
1090 sdir = os.path.join(srclib_dir, name)
1093 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1094 vcs.srclib = (name, number, sdir)
1096 vcs.gotorevision(ref)
1103 libdir = os.path.join(sdir, subdir)
1104 elif srclib["Subdir"]:
1105 for subdir in srclib["Subdir"]:
1106 libdir_candidate = os.path.join(sdir, subdir)
1107 if os.path.exists(libdir_candidate):
1108 libdir = libdir_candidate
1114 if srclib["Srclibs"]:
1116 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1118 for t in srclibpaths:
1123 raise VCSException('Missing recursive srclib %s for %s' % (
1125 place_srclib(libdir, n, s_tuple[2])
1128 remove_signing_keys(sdir)
1129 remove_debuggable_flags(sdir)
1133 if srclib["Prepare"]:
1134 cmd = replace_config_vars(srclib["Prepare"])
1136 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1137 if p.returncode != 0:
1138 raise BuildException("Error running prepare command for srclib %s"
1144 return (name, number, libdir)
1147 # Prepare the source code for a particular build
1148 # 'vcs' - the appropriate vcs object for the application
1149 # 'app' - the application details from the metadata
1150 # 'build' - the build details from the metadata
1151 # 'build_dir' - the path to the build directory, usually
1153 # 'srclib_dir' - the path to the source libraries directory, usually
1155 # 'extlib_dir' - the path to the external libraries directory, usually
1157 # Returns the (root, srclibpaths) where:
1158 # 'root' is the root directory, which may be the same as 'build_dir' or may
1159 # be a subdirectory of it.
1160 # 'srclibpaths' is information on the srclibs being used
1161 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1163 # Optionally, the actual app source can be in a subdirectory
1165 root_dir = os.path.join(build_dir, build['subdir'])
1167 root_dir = build_dir
1169 # Get a working copy of the right revision
1170 logging.info("Getting source for revision " + build['commit'])
1171 vcs.gotorevision(build['commit'])
1173 # Initialise submodules if requred
1174 if build['submodules']:
1175 logging.info("Initialising submodules")
1176 vcs.initsubmodules()
1178 # Check that a subdir (if we're using one) exists. This has to happen
1179 # after the checkout, since it might not exist elsewhere
1180 if not os.path.exists(root_dir):
1181 raise BuildException('Missing subdir ' + root_dir)
1183 # Run an init command if one is required
1185 cmd = replace_config_vars(build['init'])
1186 logging.info("Running 'init' commands in %s" % root_dir)
1188 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1189 if p.returncode != 0:
1190 raise BuildException("Error running init command for %s:%s" %
1191 (app['id'], build['version']), p.output)
1193 # Apply patches if any
1195 logging.info("Applying patches")
1196 for patch in build['patch']:
1197 patch = patch.strip()
1198 logging.info("Applying " + patch)
1199 patch_path = os.path.join('metadata', app['id'], patch)
1200 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1201 if p.returncode != 0:
1202 raise BuildException("Failed to apply patch %s" % patch_path)
1204 # Get required source libraries
1206 if build['srclibs']:
1207 logging.info("Collecting source libraries")
1208 for lib in build['srclibs']:
1209 srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
1212 for name, number, libpath in srclibpaths:
1213 place_srclib(root_dir, int(number) if number else None, libpath)
1215 basesrclib = vcs.getsrclib()
1216 # If one was used for the main source, add that too.
1218 srclibpaths.append(basesrclib)
1220 # Update the local.properties file
1221 localprops = [os.path.join(build_dir, 'local.properties')]
1223 localprops += [os.path.join(root_dir, 'local.properties')]
1224 for path in localprops:
1226 if os.path.isfile(path):
1227 logging.info("Updating local.properties file at %s" % path)
1233 logging.info("Creating local.properties file at %s" % path)
1234 # Fix old-fashioned 'sdk-location' by copying
1235 # from sdk.dir, if necessary
1236 if build['oldsdkloc']:
1237 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1238 re.S | re.M).group(1)
1239 props += "sdk-location=%s\n" % sdkloc
1241 props += "sdk.dir=%s\n" % config['sdk_path']
1242 props += "sdk-location=%s\n" % config['sdk_path']
1243 if build['ndk_path']:
1245 props += "ndk.dir=%s\n" % build['ndk_path']
1246 props += "ndk-location=%s\n" % build['ndk_path']
1247 # Add java.encoding if necessary
1248 if build['encoding']:
1249 props += "java.encoding=%s\n" % build['encoding']
1255 if build['type'] == 'gradle':
1256 flavours = build['gradle']
1258 version_regex = re.compile(r".*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1259 gradlepluginver = None
1261 gradle_files = [os.path.join(root_dir, 'build.gradle')]
1263 # Parent dir build.gradle
1264 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1265 if parent_dir.startswith(build_dir):
1266 gradle_files.append(os.path.join(parent_dir, 'build.gradle'))
1268 for path in gradle_files:
1271 if not os.path.isfile(path):
1273 with open(path) as f:
1275 match = version_regex.match(line)
1277 gradlepluginver = match.group(1)
1281 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1283 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1284 build['gradlepluginver'] = LooseVersion('0.11')
1287 n = build["target"].split('-')[1]
1288 FDroidPopen(['sed', '-i',
1289 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1290 'build.gradle'], cwd=root_dir, output=False)
1292 # Remove forced debuggable flags
1293 remove_debuggable_flags(root_dir)
1295 # Insert version code and number into the manifest if necessary
1296 if build['forceversion']:
1297 logging.info("Changing the version name")
1298 for path in manifest_paths(root_dir, flavours):
1299 if not os.path.isfile(path):
1301 if has_extension(path, 'xml'):
1302 p = FDroidPopen(['sed', '-i',
1303 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1304 path], output=False)
1305 if p.returncode != 0:
1306 raise BuildException("Failed to amend manifest")
1307 elif has_extension(path, 'gradle'):
1308 p = FDroidPopen(['sed', '-i',
1309 's/versionName *=* *"[^"]*"/versionName = "' + build['version'] + '"/g',
1310 path], output=False)
1311 if p.returncode != 0:
1312 raise BuildException("Failed to amend build.gradle")
1313 if build['forcevercode']:
1314 logging.info("Changing the version code")
1315 for path in manifest_paths(root_dir, flavours):
1316 if not os.path.isfile(path):
1318 if has_extension(path, 'xml'):
1319 p = FDroidPopen(['sed', '-i',
1320 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1321 path], output=False)
1322 if p.returncode != 0:
1323 raise BuildException("Failed to amend manifest")
1324 elif has_extension(path, 'gradle'):
1325 p = FDroidPopen(['sed', '-i',
1326 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1327 path], output=False)
1328 if p.returncode != 0:
1329 raise BuildException("Failed to amend build.gradle")
1331 # Delete unwanted files
1333 logging.info("Removing specified files")
1334 for part in getpaths(build_dir, build, 'rm'):
1335 dest = os.path.join(build_dir, part)
1336 logging.info("Removing {0}".format(part))
1337 if os.path.lexists(dest):
1338 if os.path.islink(dest):
1339 FDroidPopen(['unlink ' + dest], shell=True, output=False)
1341 FDroidPopen(['rm -rf ' + dest], shell=True, output=False)
1343 logging.info("...but it didn't exist")
1345 remove_signing_keys(build_dir)
1347 # Add required external libraries
1348 if build['extlibs']:
1349 logging.info("Collecting prebuilt libraries")
1350 libsdir = os.path.join(root_dir, 'libs')
1351 if not os.path.exists(libsdir):
1353 for lib in build['extlibs']:
1355 logging.info("...installing extlib {0}".format(lib))
1356 libf = os.path.basename(lib)
1357 libsrc = os.path.join(extlib_dir, lib)
1358 if not os.path.exists(libsrc):
1359 raise BuildException("Missing extlib file {0}".format(libsrc))
1360 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1362 # Run a pre-build command if one is required
1363 if build['prebuild']:
1364 logging.info("Running 'prebuild' commands in %s" % root_dir)
1366 cmd = replace_config_vars(build['prebuild'])
1368 # Substitute source library paths into prebuild commands
1369 for name, number, libpath in srclibpaths:
1370 libpath = os.path.relpath(libpath, root_dir)
1371 cmd = cmd.replace('$$' + name + '$$', libpath)
1373 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1374 if p.returncode != 0:
1375 raise BuildException("Error running prebuild command for %s:%s" %
1376 (app['id'], build['version']), p.output)
1378 # Generate (or update) the ant build file, build.xml...
1379 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1380 parms = ['android', 'update', 'lib-project']
1381 lparms = ['android', 'update', 'project']
1384 parms += ['-t', build['target']]
1385 lparms += ['-t', build['target']]
1386 if build['update'] == ['auto']:
1387 update_dirs = ant_subprojects(root_dir) + ['.']
1389 update_dirs = build['update']
1391 for d in update_dirs:
1392 subdir = os.path.join(root_dir, d)
1394 logging.debug("Updating main project")
1395 cmd = parms + ['-p', d]
1397 logging.debug("Updating subproject %s" % d)
1398 cmd = lparms + ['-p', d]
1399 p = SdkToolsPopen(cmd, cwd=root_dir)
1400 # Check to see whether an error was returned without a proper exit
1401 # code (this is the case for the 'no target set or target invalid'
1403 if p.returncode != 0 or p.output.startswith("Error: "):
1404 raise BuildException("Failed to update project at %s" % d, p.output)
1405 # Clean update dirs via ant
1407 logging.info("Cleaning subproject %s" % d)
1408 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1410 return (root_dir, srclibpaths)
1413 # Split and extend via globbing the paths from a field
1414 def getpaths(build_dir, build, field):
1416 for p in build[field]:
1418 full_path = os.path.join(build_dir, p)
1419 full_path = os.path.normpath(full_path)
1420 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1424 # Scan the source code in the given directory (and all subdirectories)
1425 # and return the number of fatal problems encountered
1426 def scan_source(build_dir, root_dir, thisbuild):
1430 # Common known non-free blobs (always lower case):
1432 re.compile(r'flurryagent', re.IGNORECASE),
1433 re.compile(r'paypal.*mpl', re.IGNORECASE),
1434 re.compile(r'google.*analytics', re.IGNORECASE),
1435 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1436 re.compile(r'google.*ad.*view', re.IGNORECASE),
1437 re.compile(r'google.*admob', re.IGNORECASE),
1438 re.compile(r'google.*play.*services', re.IGNORECASE),
1439 re.compile(r'crittercism', re.IGNORECASE),
1440 re.compile(r'heyzap', re.IGNORECASE),
1441 re.compile(r'jpct.*ae', re.IGNORECASE),
1442 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1443 re.compile(r'bugsense', re.IGNORECASE),
1444 re.compile(r'crashlytics', re.IGNORECASE),
1445 re.compile(r'ouya.*sdk', re.IGNORECASE),
1446 re.compile(r'libspen23', re.IGNORECASE),
1449 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1450 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1453 ms = magic.open(magic.MIME_TYPE)
1455 except AttributeError:
1459 for i in scanignore:
1460 if fd.startswith(i):
1465 for i in scandelete:
1466 if fd.startswith(i):
1470 def removeproblem(what, fd, fp):
1471 logging.info('Removing %s at %s' % (what, fd))
1474 def warnproblem(what, fd):
1475 logging.warn('Found %s at %s' % (what, fd))
1477 def handleproblem(what, fd, fp):
1479 logging.info('Ignoring %s at %s' % (what, fd))
1481 removeproblem(what, fd, fp)
1483 logging.error('Found %s at %s' % (what, fd))
1487 # Iterate through all files in the source code
1488 for r, d, f in os.walk(build_dir, topdown=True):
1490 # It's topdown, so checking the basename is enough
1491 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1497 # Path (relative) to the file
1498 fp = os.path.join(r, curfile)
1499 fd = fp[len(build_dir) + 1:]
1502 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1503 except UnicodeError:
1504 warnproblem('malformed magic number', fd)
1506 if mime == 'application/x-sharedlib':
1507 count += handleproblem('shared library', fd, fp)
1509 elif mime == 'application/x-archive':
1510 count += handleproblem('static library', fd, fp)
1512 elif mime == 'application/x-executable':
1513 count += handleproblem('binary executable', fd, fp)
1515 elif mime == 'application/x-java-applet':
1516 count += handleproblem('Java compiled class', fd, fp)
1521 'application/java-archive',
1522 'application/octet-stream',
1526 if has_extension(fp, 'apk'):
1527 removeproblem('APK file', fd, fp)
1529 elif has_extension(fp, 'jar'):
1531 if any(suspect.match(curfile) for suspect in usual_suspects):
1532 count += handleproblem('usual supect', fd, fp)
1534 warnproblem('JAR file', fd)
1536 elif has_extension(fp, 'zip'):
1537 warnproblem('ZIP file', fd)
1540 warnproblem('unknown compressed or binary file', fd)
1542 elif has_extension(fp, 'java'):
1543 for line in file(fp):
1544 if 'DexClassLoader' in line:
1545 count += handleproblem('DexClassLoader', fd, fp)
1550 # Presence of a jni directory without buildjni=yes might
1551 # indicate a problem (if it's not a problem, explicitly use
1552 # buildjni=no to bypass this check)
1553 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1554 not thisbuild['buildjni']):
1555 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1564 self.path = os.path.join('stats', 'known_apks.txt')
1566 if os.path.exists(self.path):
1567 for line in file(self.path):
1568 t = line.rstrip().split(' ')
1570 self.apks[t[0]] = (t[1], None)
1572 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1573 self.changed = False
1575 def writeifchanged(self):
1577 if not os.path.exists('stats'):
1579 f = open(self.path, 'w')
1581 for apk, app in self.apks.iteritems():
1583 line = apk + ' ' + appid
1585 line += ' ' + time.strftime('%Y-%m-%d', added)
1587 for line in sorted(lst):
1588 f.write(line + '\n')
1591 # Record an apk (if it's new, otherwise does nothing)
1592 # Returns the date it was added.
1593 def recordapk(self, apk, app):
1594 if apk not in self.apks:
1595 self.apks[apk] = (app, time.gmtime(time.time()))
1597 _, added = self.apks[apk]
1600 # Look up information - given the 'apkname', returns (app id, date added/None).
1601 # Or returns None for an unknown apk.
1602 def getapp(self, apkname):
1603 if apkname in self.apks:
1604 return self.apks[apkname]
1607 # Get the most recent 'num' apps added to the repo, as a list of package ids
1608 # with the most recent first.
1609 def getlatest(self, num):
1611 for apk, app in self.apks.iteritems():
1615 if apps[appid] > added:
1619 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1620 lst = [app for app, _ in sortedapps]
1625 def isApkDebuggable(apkfile, config):
1626 """Returns True if the given apk file is debuggable
1628 :param apkfile: full path to the apk to check"""
1630 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1632 if p.returncode != 0:
1633 logging.critical("Failed to get apk manifest information")
1635 for line in p.output.splitlines():
1636 if 'android:debuggable' in line and not line.endswith('0x0'):
1641 class AsynchronousFileReader(threading.Thread):
1644 Helper class to implement asynchronous reading of a file
1645 in a separate thread. Pushes read lines on a queue to
1646 be consumed in another thread.
1649 def __init__(self, fd, queue):
1650 assert isinstance(queue, Queue.Queue)
1651 assert callable(fd.readline)
1652 threading.Thread.__init__(self)
1657 '''The body of the tread: read lines and put them on the queue.'''
1658 for line in iter(self._fd.readline, ''):
1659 self._queue.put(line)
1662 '''Check whether there is no more content to expect.'''
1663 return not self.is_alive() and self._queue.empty()
1671 def SdkToolsPopen(commands, cwd=None, shell=False, output=True):
1673 if cmd not in config:
1674 config[cmd] = find_sdk_tools_cmd(commands[0])
1675 return FDroidPopen([config[cmd]] + commands[1:],
1676 cwd=cwd, shell=shell, output=output)
1679 def FDroidPopen(commands, cwd=None, shell=False, output=True):
1681 Run a command and capture the possibly huge output.
1683 :param commands: command and argument list like in subprocess.Popen
1684 :param cwd: optionally specifies a working directory
1685 :returns: A PopenResult.
1691 cwd = os.path.normpath(cwd)
1692 logging.debug("Directory: %s" % cwd)
1693 logging.debug("> %s" % ' '.join(commands))
1695 result = PopenResult()
1698 p = subprocess.Popen(commands, cwd=cwd, shell=shell, env=env,
1699 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1701 raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e))
1703 stdout_queue = Queue.Queue()
1704 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1705 stdout_reader.start()
1707 # Check the queue for output (until there is no more to get)
1708 while not stdout_reader.eof():
1709 while not stdout_queue.empty():
1710 line = stdout_queue.get()
1711 if output and options.verbose:
1712 # Output directly to console
1713 sys.stderr.write(line)
1715 result.output += line
1719 result.returncode = p.wait()
1723 def remove_signing_keys(build_dir):
1724 comment = re.compile(r'[ ]*//')
1725 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1727 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1728 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1729 re.compile(r'.*variant\.outputFile = .*'),
1730 re.compile(r'.*output\.outputFile = .*'),
1731 re.compile(r'.*\.readLine\(.*'),
1733 for root, dirs, files in os.walk(build_dir):
1734 if 'build.gradle' in files:
1735 path = os.path.join(root, 'build.gradle')
1737 with open(path, "r") as o:
1738 lines = o.readlines()
1743 with open(path, "w") as o:
1745 if comment.match(line):
1749 opened += line.count('{')
1750 opened -= line.count('}')
1753 if signing_configs.match(line):
1758 if any(s.match(line) for s in line_matches):
1766 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1769 'project.properties',
1771 'default.properties',
1774 if propfile in files:
1775 path = os.path.join(root, propfile)
1777 with open(path, "r") as o:
1778 lines = o.readlines()
1782 with open(path, "w") as o:
1784 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1791 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1794 def replace_config_vars(cmd):
1796 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1797 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1798 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1799 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1803 def place_srclib(root_dir, number, libpath):
1806 relpath = os.path.relpath(libpath, root_dir)
1807 proppath = os.path.join(root_dir, 'project.properties')
1810 if os.path.isfile(proppath):
1811 with open(proppath, "r") as o:
1812 lines = o.readlines()
1814 with open(proppath, "w") as o:
1817 if line.startswith('android.library.reference.%d=' % number):
1818 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1823 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1826 def compare_apks(apk1, apk2, tmp_dir):
1829 Returns None if the apk content is the same (apart from the signing key),
1830 otherwise a string describing what's different, or what went wrong when
1831 trying to do the comparison.
1834 thisdir = os.path.join(tmp_dir, 'this_apk')
1835 thatdir = os.path.join(tmp_dir, 'that_apk')
1836 for d in [thisdir, thatdir]:
1837 if os.path.exists(d):
1841 if subprocess.call(['jar', 'xf',
1842 os.path.abspath(apk1)],
1844 return("Failed to unpack " + apk1)
1845 if subprocess.call(['jar', 'xf',
1846 os.path.abspath(apk2)],
1848 return("Failed to unpack " + apk2)
1850 p = FDroidPopen(['diff', '-r', 'this_apk', 'that_apk'], cwd=tmp_dir,
1852 lines = p.output.splitlines()
1853 if len(lines) != 1 or 'META-INF' not in lines[0]:
1854 return("Unexpected diff output - " + p.output)
1856 # If we get here, it seems like they're the same!