1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
36 from distutils.version import LooseVersion
37 from zipfile import ZipFile
48 'sdk_path': "$ANDROID_HOME",
51 'r10d': "$ANDROID_NDK"
53 'build_tools': "22.0.0",
57 'sync_from_local_copy_dir': False,
58 'make_current_version_link': True,
59 'current_version_name_source': 'Name',
60 'update_stats': False,
64 'stats_to_carbon': False,
66 'build_server_always': False,
67 'keystore': 'keystore.jks',
68 'smartcardoptions': [],
74 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
75 'repo_name': "My First FDroid Repo Demo",
76 'repo_icon': "fdroid-icon.png",
77 'repo_description': '''
78 This is a repository of apps to be used with FDroid. Applications in this
79 repository are either official binaries built by the original application
80 developers, or are binaries built from source by the admin of f-droid.org
81 using the tools on https://gitlab.com/u/fdroid.
87 def fill_config_defaults(thisconfig):
88 for k, v in default_config.items():
89 if k not in thisconfig:
92 # Expand paths (~users and $vars)
93 def expand_path(path):
97 path = os.path.expanduser(path)
98 path = os.path.expandvars(path)
103 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
108 thisconfig[k + '_orig'] = v
110 for k in ['ndk_paths']:
116 thisconfig[k][k2] = exp
117 thisconfig[k][k2 + '_orig'] = v
120 def read_config(opts, config_file='config.py'):
121 """Read the repository config
123 The config is read from config_file, which is in the current directory when
124 any of the repo management commands are used.
126 global config, options, env, orig_path
128 if config is not None:
130 if not os.path.isfile(config_file):
131 logging.critical("Missing config file - is this a repo directory?")
138 logging.debug("Reading %s" % config_file)
139 execfile(config_file, config)
141 # smartcardoptions must be a list since its command line args for Popen
142 if 'smartcardoptions' in config:
143 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
144 elif 'keystore' in config and config['keystore'] == 'NONE':
145 # keystore='NONE' means use smartcard, these are required defaults
146 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
147 'SunPKCS11-OpenSC', '-providerClass',
148 'sun.security.pkcs11.SunPKCS11',
149 '-providerArg', 'opensc-fdroid.cfg']
151 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
152 st = os.stat(config_file)
153 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
154 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
156 fill_config_defaults(config)
158 # There is no standard, so just set up the most common environment
161 orig_path = env['PATH']
162 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
163 env[n] = config['sdk_path']
165 for k in ["keystorepass", "keypass"]:
167 write_password_file(k)
169 for k in ["repo_description", "archive_description"]:
171 config[k] = clean_description(config[k])
173 if 'serverwebroot' in config:
174 if isinstance(config['serverwebroot'], basestring):
175 roots = [config['serverwebroot']]
176 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
177 roots = config['serverwebroot']
179 raise TypeError('only accepts strings, lists, and tuples')
181 for rootstr in roots:
182 # since this is used with rsync, where trailing slashes have
183 # meaning, ensure there is always a trailing slash
184 if rootstr[-1] != '/':
186 rootlist.append(rootstr.replace('//', '/'))
187 config['serverwebroot'] = rootlist
192 def get_ndk_path(version):
194 version = 'r10d' # latest
195 paths = config['ndk_paths']
196 if version not in paths:
198 return paths[version] or ''
201 def find_sdk_tools_cmd(cmd):
202 '''find a working path to a tool from the Android SDK'''
205 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
206 # try to find a working path to this command, in all the recent possible paths
207 if 'build_tools' in config:
208 build_tools = os.path.join(config['sdk_path'], 'build-tools')
209 # if 'build_tools' was manually set and exists, check only that one
210 configed_build_tools = os.path.join(build_tools, config['build_tools'])
211 if os.path.exists(configed_build_tools):
212 tooldirs.append(configed_build_tools)
214 # no configed version, so hunt known paths for it
215 for f in sorted(os.listdir(build_tools), reverse=True):
216 if os.path.isdir(os.path.join(build_tools, f)):
217 tooldirs.append(os.path.join(build_tools, f))
218 tooldirs.append(build_tools)
219 sdk_tools = os.path.join(config['sdk_path'], 'tools')
220 if os.path.exists(sdk_tools):
221 tooldirs.append(sdk_tools)
222 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
223 if os.path.exists(sdk_platform_tools):
224 tooldirs.append(sdk_platform_tools)
225 tooldirs.append('/usr/bin')
227 if os.path.isfile(os.path.join(d, cmd)):
228 return os.path.join(d, cmd)
229 # did not find the command, exit with error message
230 ensure_build_tools_exists(config)
233 def test_sdk_exists(thisconfig):
234 if 'sdk_path' not in thisconfig:
235 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
238 logging.error("'sdk_path' not set in config.py!")
240 if thisconfig['sdk_path'] == default_config['sdk_path']:
241 logging.error('No Android SDK found!')
242 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
243 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
245 if not os.path.exists(thisconfig['sdk_path']):
246 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
248 if not os.path.isdir(thisconfig['sdk_path']):
249 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
251 for d in ['build-tools', 'platform-tools', 'tools']:
252 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
253 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
254 thisconfig['sdk_path'], d))
259 def ensure_build_tools_exists(thisconfig):
260 if not test_sdk_exists(thisconfig):
262 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
263 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
264 if not os.path.isdir(versioned_build_tools):
265 logging.critical('Android Build Tools path "'
266 + versioned_build_tools + '" does not exist!')
270 def write_password_file(pwtype, password=None):
272 writes out passwords to a protected file instead of passing passwords as
273 command line argments
275 filename = '.fdroid.' + pwtype + '.txt'
276 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
278 os.write(fd, config[pwtype])
280 os.write(fd, password)
282 config[pwtype + 'file'] = filename
285 # Given the arguments in the form of multiple appid:[vc] strings, this returns
286 # a dictionary with the set of vercodes specified for each package.
287 def read_pkg_args(args, allow_vercodes=False):
294 if allow_vercodes and ':' in p:
295 package, vercode = p.split(':')
297 package, vercode = p, None
298 if package not in vercodes:
299 vercodes[package] = [vercode] if vercode else []
301 elif vercode and vercode not in vercodes[package]:
302 vercodes[package] += [vercode] if vercode else []
307 # On top of what read_pkg_args does, this returns the whole app metadata, but
308 # limiting the builds list to the builds matching the vercodes specified.
309 def read_app_args(args, allapps, allow_vercodes=False):
311 vercodes = read_pkg_args(args, allow_vercodes)
317 for appid, app in allapps.iteritems():
318 if appid in vercodes:
321 if len(apps) != len(vercodes):
324 logging.critical("No such package: %s" % p)
325 raise FDroidException("Found invalid app ids in arguments")
327 raise FDroidException("No packages specified")
330 for appid, app in apps.iteritems():
334 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
335 if len(app['builds']) != len(vercodes[appid]):
337 allvcs = [b['vercode'] for b in app['builds']]
338 for v in vercodes[appid]:
340 logging.critical("No such vercode %s for app %s" % (v, appid))
343 raise FDroidException("Found invalid vercodes for some apps")
348 def has_extension(filename, extension):
349 name, ext = os.path.splitext(filename)
350 ext = ext.lower()[1:]
351 return ext == extension
356 def clean_description(description):
357 'Remove unneeded newlines and spaces from a block of description text'
359 # this is split up by paragraph to make removing the newlines easier
360 for paragraph in re.split(r'\n\n', description):
361 paragraph = re.sub('\r', '', paragraph)
362 paragraph = re.sub('\n', ' ', paragraph)
363 paragraph = re.sub(' {2,}', ' ', paragraph)
364 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
365 returnstring += paragraph + '\n\n'
366 return returnstring.rstrip('\n')
369 def apknameinfo(filename):
371 filename = os.path.basename(filename)
372 if apk_regex is None:
373 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
374 m = apk_regex.match(filename)
376 result = (m.group(1), m.group(2))
377 except AttributeError:
378 raise FDroidException("Invalid apk name: %s" % filename)
382 def getapkname(app, build):
383 return "%s_%s.apk" % (app['id'], build['vercode'])
386 def getsrcname(app, build):
387 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
394 return app['Auto Name']
399 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
402 def getvcs(vcstype, remote, local):
404 return vcs_git(remote, local)
405 if vcstype == 'git-svn':
406 return vcs_gitsvn(remote, local)
408 return vcs_hg(remote, local)
410 return vcs_bzr(remote, local)
411 if vcstype == 'srclib':
412 if local != os.path.join('build', 'srclib', remote):
413 raise VCSException("Error: srclib paths are hard-coded!")
414 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
416 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
417 raise VCSException("Invalid vcs type " + vcstype)
420 def getsrclibvcs(name):
421 if name not in metadata.srclibs:
422 raise VCSException("Missing srclib " + name)
423 return metadata.srclibs[name]['Repo Type']
428 def __init__(self, remote, local):
430 # svn, git-svn and bzr may require auth
432 if self.repotype() in ('git-svn', 'bzr'):
434 if self.repotype == 'git-svn':
435 raise VCSException("Authentication is not supported for git-svn")
436 self.username, remote = remote.split('@')
437 if ':' not in self.username:
438 raise VCSException("Password required with username")
439 self.username, self.password = self.username.split(':')
443 self.clone_failed = False
444 self.refreshed = False
450 # Take the local repository to a clean version of the given revision, which
451 # is specificed in the VCS's native format. Beforehand, the repository can
452 # be dirty, or even non-existent. If the repository does already exist
453 # locally, it will be updated from the origin, but only once in the
454 # lifetime of the vcs object.
455 # None is acceptable for 'rev' if you know you are cloning a clean copy of
456 # the repo - otherwise it must specify a valid revision.
457 def gotorevision(self, rev):
459 if self.clone_failed:
460 raise VCSException("Downloading the repository already failed once, not trying again.")
462 # The .fdroidvcs-id file for a repo tells us what VCS type
463 # and remote that directory was created from, allowing us to drop it
464 # automatically if either of those things changes.
465 fdpath = os.path.join(self.local, '..',
466 '.fdroidvcs-' + os.path.basename(self.local))
467 cdata = self.repotype() + ' ' + self.remote
470 if os.path.exists(self.local):
471 if os.path.exists(fdpath):
472 with open(fdpath, 'r') as f:
473 fsdata = f.read().strip()
478 logging.info("Repository details for %s changed - deleting" % (
482 logging.info("Repository details for %s missing - deleting" % (
485 shutil.rmtree(self.local)
490 self.gotorevisionx(rev)
491 except FDroidException, e:
494 # If necessary, write the .fdroidvcs file.
495 if writeback and not self.clone_failed:
496 with open(fdpath, 'w') as f:
502 # Derived classes need to implement this. It's called once basic checking
503 # has been performend.
504 def gotorevisionx(self, rev):
505 raise VCSException("This VCS type doesn't define gotorevisionx")
507 # Initialise and update submodules
508 def initsubmodules(self):
509 raise VCSException('Submodules not supported for this vcs type')
511 # Get a list of all known tags
513 if not self._gettags:
514 raise VCSException('gettags not supported for this vcs type')
516 for tag in self._gettags():
517 if re.match('[-A-Za-z0-9_. ]+$', tag):
521 def latesttags(self, tags, number):
522 """Get the most recent tags in a given list.
524 :param tags: a list of tags
525 :param number: the number to return
526 :returns: A list containing the most recent tags in the provided
527 list, up to the maximum number given.
529 raise VCSException('latesttags not supported for this vcs type')
531 # Get current commit reference (hash, revision, etc)
533 raise VCSException('getref not supported for this vcs type')
535 # Returns the srclib (name, path) used in setting up the current
546 # If the local directory exists, but is somehow not a git repository, git
547 # will traverse up the directory tree until it finds one that is (i.e.
548 # fdroidserver) and then we'll proceed to destroy it! This is called as
551 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
552 result = p.output.rstrip()
553 if not result.endswith(self.local):
554 raise VCSException('Repository mismatch')
556 def gotorevisionx(self, rev):
557 if not os.path.exists(self.local):
559 p = FDroidPopen(['git', 'clone', self.remote, self.local])
560 if p.returncode != 0:
561 self.clone_failed = True
562 raise VCSException("Git clone failed", p.output)
566 # Discard any working tree changes
567 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
568 if p.returncode != 0:
569 raise VCSException("Git reset failed", p.output)
570 # Remove untracked files now, in case they're tracked in the target
571 # revision (it happens!)
572 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
573 if p.returncode != 0:
574 raise VCSException("Git clean failed", p.output)
575 if not self.refreshed:
576 # Get latest commits and tags from remote
577 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
578 if p.returncode != 0:
579 raise VCSException("Git fetch failed", p.output)
580 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
581 if p.returncode != 0:
582 raise VCSException("Git fetch failed", p.output)
583 # Recreate origin/HEAD as git clone would do it, in case it disappeared
584 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
585 if p.returncode != 0:
586 lines = p.output.splitlines()
587 if 'Multiple remote HEAD branches' not in lines[0]:
588 raise VCSException("Git remote set-head failed", p.output)
589 branch = lines[1].split(' ')[-1]
590 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
591 if p2.returncode != 0:
592 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
593 self.refreshed = True
594 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
595 # a github repo. Most of the time this is the same as origin/master.
596 rev = rev or 'origin/HEAD'
597 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
598 if p.returncode != 0:
599 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
600 # Get rid of any uncontrolled files left behind
601 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
602 if p.returncode != 0:
603 raise VCSException("Git clean failed", p.output)
605 def initsubmodules(self):
607 submfile = os.path.join(self.local, '.gitmodules')
608 if not os.path.isfile(submfile):
609 raise VCSException("No git submodules available")
611 # fix submodules not accessible without an account and public key auth
612 with open(submfile, 'r') as f:
613 lines = f.readlines()
614 with open(submfile, 'w') as f:
616 if 'git@github.com' in line:
617 line = line.replace('git@github.com:', 'https://github.com/')
621 ['git', 'reset', '--hard'],
622 ['git', 'clean', '-dffx'],
624 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive'] + cmd, cwd=self.local, output=False)
625 if p.returncode != 0:
626 raise VCSException("Git submodule reset failed", p.output)
627 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
628 if p.returncode != 0:
629 raise VCSException("Git submodule sync failed", p.output)
630 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
631 if p.returncode != 0:
632 raise VCSException("Git submodule update failed", p.output)
636 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
637 return p.output.splitlines()
639 def latesttags(self, tags, number):
644 ['git', 'show', '--format=format:%ct', '-s', tag],
645 cwd=self.local, output=False)
646 # Timestamp is on the last line. For a normal tag, it's the only
647 # line, but for annotated tags, the rest of the info precedes it.
648 ts = int(p.output.splitlines()[-1])
651 for _, t in sorted(tl)[-number:]:
656 class vcs_gitsvn(vcs):
661 # If the local directory exists, but is somehow not a git repository, git
662 # will traverse up the directory tree until it finds one that is (i.e.
663 # fdroidserver) and then we'll proceed to destory it! This is called as
666 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
667 result = p.output.rstrip()
668 if not result.endswith(self.local):
669 raise VCSException('Repository mismatch')
671 def gotorevisionx(self, rev):
672 if not os.path.exists(self.local):
674 gitsvn_args = ['git', 'svn', 'clone']
675 if ';' in self.remote:
676 remote_split = self.remote.split(';')
677 for i in remote_split[1:]:
678 if i.startswith('trunk='):
679 gitsvn_args.extend(['-T', i[6:]])
680 elif i.startswith('tags='):
681 gitsvn_args.extend(['-t', i[5:]])
682 elif i.startswith('branches='):
683 gitsvn_args.extend(['-b', i[9:]])
684 gitsvn_args.extend([remote_split[0], self.local])
685 p = FDroidPopen(gitsvn_args, output=False)
686 if p.returncode != 0:
687 self.clone_failed = True
688 raise VCSException("Git svn clone failed", p.output)
690 gitsvn_args.extend([self.remote, self.local])
691 p = FDroidPopen(gitsvn_args, output=False)
692 if p.returncode != 0:
693 self.clone_failed = True
694 raise VCSException("Git svn clone failed", p.output)
698 # Discard any working tree changes
699 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
700 if p.returncode != 0:
701 raise VCSException("Git reset failed", p.output)
702 # Remove untracked files now, in case they're tracked in the target
703 # revision (it happens!)
704 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
705 if p.returncode != 0:
706 raise VCSException("Git clean failed", p.output)
707 if not self.refreshed:
708 # Get new commits, branches and tags from repo
709 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
710 if p.returncode != 0:
711 raise VCSException("Git svn fetch failed")
712 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
713 if p.returncode != 0:
714 raise VCSException("Git svn rebase failed", p.output)
715 self.refreshed = True
717 rev = rev or 'master'
719 nospaces_rev = rev.replace(' ', '%20')
720 # Try finding a svn tag
721 for treeish in ['origin/', '']:
722 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
723 if p.returncode == 0:
725 if p.returncode != 0:
726 # No tag found, normal svn rev translation
727 # Translate svn rev into git format
728 rev_split = rev.split('/')
731 for treeish in ['origin/', '']:
732 if len(rev_split) > 1:
733 treeish += rev_split[0]
734 svn_rev = rev_split[1]
737 # if no branch is specified, then assume trunk (i.e. 'master' branch):
741 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
743 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
744 git_rev = p.output.rstrip()
746 if p.returncode == 0 and git_rev:
749 if p.returncode != 0 or not git_rev:
750 # Try a plain git checkout as a last resort
751 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
752 if p.returncode != 0:
753 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
755 # Check out the git rev equivalent to the svn rev
756 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
757 if p.returncode != 0:
758 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
760 # Get rid of any uncontrolled files left behind
761 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
762 if p.returncode != 0:
763 raise VCSException("Git clean failed", p.output)
767 for treeish in ['origin/', '']:
768 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
774 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
775 if p.returncode != 0:
777 return p.output.strip()
785 def gotorevisionx(self, rev):
786 if not os.path.exists(self.local):
787 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
788 if p.returncode != 0:
789 self.clone_failed = True
790 raise VCSException("Hg clone failed", p.output)
792 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
793 if p.returncode != 0:
794 raise VCSException("Hg status failed", p.output)
795 for line in p.output.splitlines():
796 if not line.startswith('? '):
797 raise VCSException("Unexpected output from hg status -uS: " + line)
798 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
799 if not self.refreshed:
800 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
801 if p.returncode != 0:
802 raise VCSException("Hg pull failed", p.output)
803 self.refreshed = True
805 rev = rev or 'default'
808 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
809 if p.returncode != 0:
810 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
811 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
812 # Also delete untracked files, we have to enable purge extension for that:
813 if "'purge' is provided by the following extension" in p.output:
814 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
815 myfile.write("\n[extensions]\nhgext.purge=\n")
816 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
817 if p.returncode != 0:
818 raise VCSException("HG purge failed", p.output)
819 elif p.returncode != 0:
820 raise VCSException("HG purge failed", p.output)
823 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
824 return p.output.splitlines()[1:]
832 def gotorevisionx(self, rev):
833 if not os.path.exists(self.local):
834 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
835 if p.returncode != 0:
836 self.clone_failed = True
837 raise VCSException("Bzr branch failed", p.output)
839 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
840 if p.returncode != 0:
841 raise VCSException("Bzr revert failed", p.output)
842 if not self.refreshed:
843 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
844 if p.returncode != 0:
845 raise VCSException("Bzr update failed", p.output)
846 self.refreshed = True
848 revargs = list(['-r', rev] if rev else [])
849 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
850 if p.returncode != 0:
851 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
854 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
855 return [tag.split(' ')[0].strip() for tag in
856 p.output.splitlines()]
859 def retrieve_string(app_dir, string, xmlfiles=None):
862 os.path.join(app_dir, 'res'),
863 os.path.join(app_dir, 'src', 'main'),
868 for res_dir in res_dirs:
869 for r, d, f in os.walk(res_dir):
870 if os.path.basename(r) == 'values':
871 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
874 if string.startswith('@string/'):
875 string_search = re.compile(r'.*name="' + string[8:] + '".*?>"?([^<]+?)"?<.*').search
876 elif string.startswith('&') and string.endswith(';'):
877 string_search = re.compile(r'.*<!ENTITY.*' + string[1:-1] + '.*?"([^"]+?)".*>').search
879 if string_search is not None:
880 for xmlfile in xmlfiles:
881 if not os.path.isfile(xmlfile):
883 for line in file(xmlfile):
884 matches = string_search(line)
886 return retrieve_string(app_dir, matches.group(1), xmlfiles)
889 return string.replace("\\'", "'")
892 # Return list of existing files that will be used to find the highest vercode
893 def manifest_paths(app_dir, flavours):
895 possible_manifests = \
896 [os.path.join(app_dir, 'AndroidManifest.xml'),
897 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
898 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
899 os.path.join(app_dir, 'build.gradle')]
901 for flavour in flavours:
904 possible_manifests.append(
905 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
907 return [path for path in possible_manifests if os.path.isfile(path)]
910 # Retrieve the package name. Returns the name, or None if not found.
911 def fetch_real_name(app_dir, flavours):
912 app_search = re.compile(r'.*<application.*').search
913 name_search = re.compile(r'.*android:label="([^"]+)".*').search
915 for f in manifest_paths(app_dir, flavours):
916 if not has_extension(f, 'xml') or not os.path.isfile(f):
918 logging.debug("fetch_real_name: Checking manifest at " + f)
924 matches = name_search(line)
926 stringname = matches.group(1)
927 logging.debug("fetch_real_name: using string " + stringname)
928 result = retrieve_string(app_dir, stringname)
930 result = result.strip()
935 # Retrieve the version name
936 def version_name(original, app_dir, flavours):
937 for f in manifest_paths(app_dir, flavours):
938 if not has_extension(f, 'xml'):
940 string = retrieve_string(app_dir, original)
946 def get_library_references(root_dir):
948 proppath = os.path.join(root_dir, 'project.properties')
949 if not os.path.isfile(proppath):
951 for line in file(proppath):
952 if not line.startswith('android.library.reference.'):
954 path = line.split('=')[1].strip()
955 relpath = os.path.join(root_dir, path)
956 if not os.path.isdir(relpath):
958 logging.debug("Found subproject at %s" % path)
959 libraries.append(path)
963 def ant_subprojects(root_dir):
964 subprojects = get_library_references(root_dir)
965 for subpath in subprojects:
966 subrelpath = os.path.join(root_dir, subpath)
967 for p in get_library_references(subrelpath):
968 relp = os.path.normpath(os.path.join(subpath, p))
969 if relp not in subprojects:
970 subprojects.insert(0, relp)
974 def remove_debuggable_flags(root_dir):
975 # Remove forced debuggable flags
976 logging.debug("Removing debuggable flags from %s" % root_dir)
977 for root, dirs, files in os.walk(root_dir):
978 if 'AndroidManifest.xml' in files:
979 path = os.path.join(root, 'AndroidManifest.xml')
980 p = FDroidPopen(['sed', '-i', 's/android:debuggable="[^"]*"//g', path], output=False)
981 if p.returncode != 0:
982 raise BuildException("Failed to remove debuggable flags of %s" % path)
985 # Extract some information from the AndroidManifest.xml at the given path.
986 # Returns (version, vercode, package), any or all of which might be None.
987 # All values returned are strings.
988 def parse_androidmanifests(paths, ignoreversions=None):
991 return (None, None, None)
993 vcsearch = re.compile(r'.*:versionCode="([0-9]+?)".*').search
994 vnsearch = re.compile(r'.*:versionName="([^"]+?)".*').search
995 psearch = re.compile(r'.*package="([^"]+)".*').search
997 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
998 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
999 psearch_g = re.compile(r'.*packageName *=* *["\']([^"]+)["\'].*').search
1001 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1009 if not os.path.isfile(path):
1012 logging.debug("Parsing manifest at {0}".format(path))
1013 gradle = has_extension(path, 'gradle')
1016 # Remember package name, may be defined separately from version+vercode
1017 package = max_package
1019 for line in file(path):
1022 matches = psearch_g(line)
1024 matches = psearch(line)
1026 package = matches.group(1)
1029 matches = vnsearch_g(line)
1031 matches = vnsearch(line)
1033 version = matches.group(2 if gradle else 1)
1036 matches = vcsearch_g(line)
1038 matches = vcsearch(line)
1040 vercode = matches.group(1)
1042 logging.debug("..got package={0}, version={1}, vercode={2}"
1043 .format(package, version, vercode))
1045 # Always grab the package name and version name in case they are not
1046 # together with the highest version code
1047 if max_package is None and package is not None:
1048 max_package = package
1049 if max_version is None and version is not None:
1050 max_version = version
1052 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1053 if not ignoresearch or not ignoresearch(version):
1054 if version is not None:
1055 max_version = version
1056 if vercode is not None:
1057 max_vercode = vercode
1058 if package is not None:
1059 max_package = package
1061 max_version = "Ignore"
1063 if max_version is None:
1064 max_version = "Unknown"
1066 if max_package and not is_valid_package_name(max_package):
1067 raise FDroidException("Invalid package name {0}".format(max_package))
1069 return (max_version, max_vercode, max_package)
1072 def is_valid_package_name(name):
1073 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1076 class FDroidException(Exception):
1078 def __init__(self, value, detail=None):
1080 self.detail = detail
1082 def get_wikitext(self):
1083 ret = repr(self.value) + "\n"
1087 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1095 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1099 class VCSException(FDroidException):
1103 class BuildException(FDroidException):
1107 # Get the specified source library.
1108 # Returns the path to it. Normally this is the path to be used when referencing
1109 # it, which may be a subdirectory of the actual project. If you want the base
1110 # directory of the project, pass 'basepath=True'.
1111 def getsrclib(spec, srclib_dir, srclibpaths=[], subdir=None,
1112 basepath=False, raw=False, prepare=True, preponly=False):
1120 name, ref = spec.split('@')
1122 number, name = name.split(':', 1)
1124 name, subdir = name.split('/', 1)
1126 if name not in metadata.srclibs:
1127 raise VCSException('srclib ' + name + ' not found.')
1129 srclib = metadata.srclibs[name]
1131 sdir = os.path.join(srclib_dir, name)
1134 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1135 vcs.srclib = (name, number, sdir)
1137 vcs.gotorevision(ref)
1144 libdir = os.path.join(sdir, subdir)
1145 elif srclib["Subdir"]:
1146 for subdir in srclib["Subdir"]:
1147 libdir_candidate = os.path.join(sdir, subdir)
1148 if os.path.exists(libdir_candidate):
1149 libdir = libdir_candidate
1155 if srclib["Srclibs"]:
1157 for lib in srclib["Srclibs"].replace(';', ',').split(','):
1159 for t in srclibpaths:
1164 raise VCSException('Missing recursive srclib %s for %s' % (
1166 place_srclib(libdir, n, s_tuple[2])
1169 remove_signing_keys(sdir)
1170 remove_debuggable_flags(sdir)
1174 if srclib["Prepare"]:
1175 cmd = replace_config_vars(srclib["Prepare"], None)
1177 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1178 if p.returncode != 0:
1179 raise BuildException("Error running prepare command for srclib %s"
1185 return (name, number, libdir)
1188 # Prepare the source code for a particular build
1189 # 'vcs' - the appropriate vcs object for the application
1190 # 'app' - the application details from the metadata
1191 # 'build' - the build details from the metadata
1192 # 'build_dir' - the path to the build directory, usually
1194 # 'srclib_dir' - the path to the source libraries directory, usually
1196 # 'extlib_dir' - the path to the external libraries directory, usually
1198 # Returns the (root, srclibpaths) where:
1199 # 'root' is the root directory, which may be the same as 'build_dir' or may
1200 # be a subdirectory of it.
1201 # 'srclibpaths' is information on the srclibs being used
1202 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False):
1204 # Optionally, the actual app source can be in a subdirectory
1206 root_dir = os.path.join(build_dir, build['subdir'])
1208 root_dir = build_dir
1210 # Get a working copy of the right revision
1211 logging.info("Getting source for revision " + build['commit'])
1212 vcs.gotorevision(build['commit'])
1214 # Initialise submodules if requred
1215 if build['submodules']:
1216 logging.info("Initialising submodules")
1217 vcs.initsubmodules()
1219 # Check that a subdir (if we're using one) exists. This has to happen
1220 # after the checkout, since it might not exist elsewhere
1221 if not os.path.exists(root_dir):
1222 raise BuildException('Missing subdir ' + root_dir)
1224 # Run an init command if one is required
1226 cmd = replace_config_vars(build['init'], build)
1227 logging.info("Running 'init' commands in %s" % root_dir)
1229 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1230 if p.returncode != 0:
1231 raise BuildException("Error running init command for %s:%s" %
1232 (app['id'], build['version']), p.output)
1234 # Apply patches if any
1236 logging.info("Applying patches")
1237 for patch in build['patch']:
1238 patch = patch.strip()
1239 logging.info("Applying " + patch)
1240 patch_path = os.path.join('metadata', app['id'], patch)
1241 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1242 if p.returncode != 0:
1243 raise BuildException("Failed to apply patch %s" % patch_path)
1245 # Get required source libraries
1247 if build['srclibs']:
1248 logging.info("Collecting source libraries")
1249 for lib in build['srclibs']:
1250 srclibpaths.append(getsrclib(lib, srclib_dir, build, srclibpaths,
1253 for name, number, libpath in srclibpaths:
1254 place_srclib(root_dir, int(number) if number else None, libpath)
1256 basesrclib = vcs.getsrclib()
1257 # If one was used for the main source, add that too.
1259 srclibpaths.append(basesrclib)
1261 # Update the local.properties file
1262 localprops = [os.path.join(build_dir, 'local.properties')]
1264 localprops += [os.path.join(root_dir, 'local.properties')]
1265 for path in localprops:
1267 if os.path.isfile(path):
1268 logging.info("Updating local.properties file at %s" % path)
1274 logging.info("Creating local.properties file at %s" % path)
1275 # Fix old-fashioned 'sdk-location' by copying
1276 # from sdk.dir, if necessary
1277 if build['oldsdkloc']:
1278 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1279 re.S | re.M).group(1)
1280 props += "sdk-location=%s\n" % sdkloc
1282 props += "sdk.dir=%s\n" % config['sdk_path']
1283 props += "sdk-location=%s\n" % config['sdk_path']
1284 if build['ndk_path']:
1286 props += "ndk.dir=%s\n" % build['ndk_path']
1287 props += "ndk-location=%s\n" % build['ndk_path']
1288 # Add java.encoding if necessary
1289 if build['encoding']:
1290 props += "java.encoding=%s\n" % build['encoding']
1296 if build['type'] == 'gradle':
1297 flavours = build['gradle']
1299 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1300 gradlepluginver = None
1302 gradle_dirs = [root_dir]
1304 # Parent dir build.gradle
1305 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1306 if parent_dir.startswith(build_dir):
1307 gradle_dirs.append(parent_dir)
1309 for dir_path in gradle_dirs:
1312 if not os.path.isdir(dir_path):
1314 for filename in os.listdir(dir_path):
1315 if not filename.endswith('.gradle'):
1317 path = os.path.join(dir_path, filename)
1318 if not os.path.isfile(path):
1320 for line in file(path):
1321 match = version_regex.match(line)
1323 gradlepluginver = match.group(1)
1327 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1329 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1330 build['gradlepluginver'] = LooseVersion('0.11')
1333 n = build["target"].split('-')[1]
1334 FDroidPopen(['sed', '-i',
1335 's@compileSdkVersion *[0-9]*@compileSdkVersion ' + n + '@g',
1336 'build.gradle'], cwd=root_dir, output=False)
1338 # Remove forced debuggable flags
1339 remove_debuggable_flags(root_dir)
1341 # Insert version code and number into the manifest if necessary
1342 if build['forceversion']:
1343 logging.info("Changing the version name")
1344 for path in manifest_paths(root_dir, flavours):
1345 if not os.path.isfile(path):
1347 if has_extension(path, 'xml'):
1348 p = FDroidPopen(['sed', '-i',
1349 's/android:versionName="[^"]*"/android:versionName="' + build['version'] + '"/g',
1350 path], output=False)
1351 if p.returncode != 0:
1352 raise BuildException("Failed to amend manifest")
1353 elif has_extension(path, 'gradle'):
1354 p = FDroidPopen(['sed', '-i',
1355 's/versionName *=* *.*/versionName = "' + build['version'] + '"/g',
1356 path], output=False)
1357 if p.returncode != 0:
1358 raise BuildException("Failed to amend build.gradle")
1359 if build['forcevercode']:
1360 logging.info("Changing the version code")
1361 for path in manifest_paths(root_dir, flavours):
1362 if not os.path.isfile(path):
1364 if has_extension(path, 'xml'):
1365 p = FDroidPopen(['sed', '-i',
1366 's/android:versionCode="[^"]*"/android:versionCode="' + build['vercode'] + '"/g',
1367 path], output=False)
1368 if p.returncode != 0:
1369 raise BuildException("Failed to amend manifest")
1370 elif has_extension(path, 'gradle'):
1371 p = FDroidPopen(['sed', '-i',
1372 's/versionCode *=* *[0-9]*/versionCode = ' + build['vercode'] + '/g',
1373 path], output=False)
1374 if p.returncode != 0:
1375 raise BuildException("Failed to amend build.gradle")
1377 # Delete unwanted files
1379 logging.info("Removing specified files")
1380 for part in getpaths(build_dir, build, 'rm'):
1381 dest = os.path.join(build_dir, part)
1382 logging.info("Removing {0}".format(part))
1383 if os.path.lexists(dest):
1384 if os.path.islink(dest):
1385 FDroidPopen(['unlink', dest], output=False)
1387 FDroidPopen(['rm', '-rf', dest], output=False)
1389 logging.info("...but it didn't exist")
1391 remove_signing_keys(build_dir)
1393 # Add required external libraries
1394 if build['extlibs']:
1395 logging.info("Collecting prebuilt libraries")
1396 libsdir = os.path.join(root_dir, 'libs')
1397 if not os.path.exists(libsdir):
1399 for lib in build['extlibs']:
1401 logging.info("...installing extlib {0}".format(lib))
1402 libf = os.path.basename(lib)
1403 libsrc = os.path.join(extlib_dir, lib)
1404 if not os.path.exists(libsrc):
1405 raise BuildException("Missing extlib file {0}".format(libsrc))
1406 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1408 # Run a pre-build command if one is required
1409 if build['prebuild']:
1410 logging.info("Running 'prebuild' commands in %s" % root_dir)
1412 cmd = replace_config_vars(build['prebuild'], build)
1414 # Substitute source library paths into prebuild commands
1415 for name, number, libpath in srclibpaths:
1416 libpath = os.path.relpath(libpath, root_dir)
1417 cmd = cmd.replace('$$' + name + '$$', libpath)
1419 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1420 if p.returncode != 0:
1421 raise BuildException("Error running prebuild command for %s:%s" %
1422 (app['id'], build['version']), p.output)
1424 # Generate (or update) the ant build file, build.xml...
1425 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1426 parms = ['android', 'update', 'lib-project']
1427 lparms = ['android', 'update', 'project']
1430 parms += ['-t', build['target']]
1431 lparms += ['-t', build['target']]
1432 if build['update'] == ['auto']:
1433 update_dirs = ant_subprojects(root_dir) + ['.']
1435 update_dirs = build['update']
1437 for d in update_dirs:
1438 subdir = os.path.join(root_dir, d)
1440 logging.debug("Updating main project")
1441 cmd = parms + ['-p', d]
1443 logging.debug("Updating subproject %s" % d)
1444 cmd = lparms + ['-p', d]
1445 p = SdkToolsPopen(cmd, cwd=root_dir)
1446 # Check to see whether an error was returned without a proper exit
1447 # code (this is the case for the 'no target set or target invalid'
1449 if p.returncode != 0 or p.output.startswith("Error: "):
1450 raise BuildException("Failed to update project at %s" % d, p.output)
1451 # Clean update dirs via ant
1453 logging.info("Cleaning subproject %s" % d)
1454 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1456 return (root_dir, srclibpaths)
1459 # Split and extend via globbing the paths from a field
1460 def getpaths(build_dir, build, field):
1462 for p in build[field]:
1464 full_path = os.path.join(build_dir, p)
1465 full_path = os.path.normpath(full_path)
1466 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1470 # Scan the source code in the given directory (and all subdirectories)
1471 # and return the number of fatal problems encountered
1472 def scan_source(build_dir, root_dir, thisbuild):
1476 # Common known non-free blobs (always lower case):
1478 re.compile(r'flurryagent', re.IGNORECASE),
1479 re.compile(r'paypal.*mpl', re.IGNORECASE),
1480 re.compile(r'google.*analytics', re.IGNORECASE),
1481 re.compile(r'admob.*sdk.*android', re.IGNORECASE),
1482 re.compile(r'google.*ad.*view', re.IGNORECASE),
1483 re.compile(r'google.*admob', re.IGNORECASE),
1484 re.compile(r'google.*play.*services', re.IGNORECASE),
1485 re.compile(r'crittercism', re.IGNORECASE),
1486 re.compile(r'heyzap', re.IGNORECASE),
1487 re.compile(r'jpct.*ae', re.IGNORECASE),
1488 re.compile(r'youtube.*android.*player.*api', re.IGNORECASE),
1489 re.compile(r'bugsense', re.IGNORECASE),
1490 re.compile(r'crashlytics', re.IGNORECASE),
1491 re.compile(r'ouya.*sdk', re.IGNORECASE),
1492 re.compile(r'libspen23', re.IGNORECASE),
1495 scanignore = getpaths(build_dir, thisbuild, 'scanignore')
1496 scandelete = getpaths(build_dir, thisbuild, 'scandelete')
1498 scanignore_worked = set()
1499 scandelete_worked = set()
1502 ms = magic.open(magic.MIME_TYPE)
1504 except AttributeError:
1508 for p in scanignore:
1509 if fd.startswith(p):
1510 scanignore_worked.add(p)
1515 for p in scandelete:
1516 if fd.startswith(p):
1517 scandelete_worked.add(p)
1521 def ignoreproblem(what, fd, fp):
1522 logging.info('Ignoring %s at %s' % (what, fd))
1525 def removeproblem(what, fd, fp):
1526 logging.info('Removing %s at %s' % (what, fd))
1530 def warnproblem(what, fd):
1531 logging.warn('Found %s at %s' % (what, fd))
1533 def handleproblem(what, fd, fp):
1535 return ignoreproblem(what, fd, fp)
1537 return removeproblem(what, fd, fp)
1538 logging.error('Found %s at %s' % (what, fd))
1541 # Iterate through all files in the source code
1542 for r, d, f in os.walk(build_dir, topdown=True):
1544 # It's topdown, so checking the basename is enough
1545 for ignoredir in ('.hg', '.git', '.svn', '.bzr'):
1551 # Path (relative) to the file
1552 fp = os.path.join(r, curfile)
1553 fd = fp[len(build_dir) + 1:]
1556 mime = magic.from_file(fp, mime=True) if ms is None else ms.file(fp)
1557 except UnicodeError:
1558 warnproblem('malformed magic number', fd)
1560 if mime == 'application/x-sharedlib':
1561 count += handleproblem('shared library', fd, fp)
1563 elif mime == 'application/x-archive':
1564 count += handleproblem('static library', fd, fp)
1566 elif mime == 'application/x-executable':
1567 count += handleproblem('binary executable', fd, fp)
1569 elif mime == 'application/x-java-applet':
1570 count += handleproblem('Java compiled class', fd, fp)
1575 'application/java-archive',
1576 'application/octet-stream',
1579 if has_extension(fp, 'apk'):
1580 removeproblem('APK file', fd, fp)
1582 elif has_extension(fp, 'jar'):
1584 if any(suspect.match(curfile) for suspect in usual_suspects):
1585 count += handleproblem('usual supect', fd, fp)
1587 warnproblem('JAR file', fd)
1589 elif has_extension(fp, 'zip'):
1590 warnproblem('ZIP file', fd)
1593 warnproblem('unknown compressed or binary file', fd)
1595 elif has_extension(fp, 'java') and os.path.isfile(fp):
1596 if not os.path.isfile(fp):
1598 for line in file(fp):
1599 if 'DexClassLoader' in line:
1600 count += handleproblem('DexClassLoader', fd, fp)
1605 for p in scanignore:
1606 if p not in scanignore_worked:
1607 logging.error('Unused scanignore path: %s' % p)
1610 for p in scandelete:
1611 if p not in scandelete_worked:
1612 logging.error('Unused scandelete path: %s' % p)
1615 # Presence of a jni directory without buildjni=yes might
1616 # indicate a problem (if it's not a problem, explicitly use
1617 # buildjni=no to bypass this check)
1618 if (os.path.exists(os.path.join(root_dir, 'jni')) and
1619 not thisbuild['buildjni']):
1620 logging.error('Found jni directory, but buildjni is not enabled. Set it to \'no\' to ignore.')
1629 self.path = os.path.join('stats', 'known_apks.txt')
1631 if os.path.isfile(self.path):
1632 for line in file(self.path):
1633 t = line.rstrip().split(' ')
1635 self.apks[t[0]] = (t[1], None)
1637 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1638 self.changed = False
1640 def writeifchanged(self):
1642 if not os.path.exists('stats'):
1644 f = open(self.path, 'w')
1646 for apk, app in self.apks.iteritems():
1648 line = apk + ' ' + appid
1650 line += ' ' + time.strftime('%Y-%m-%d', added)
1652 for line in sorted(lst):
1653 f.write(line + '\n')
1656 # Record an apk (if it's new, otherwise does nothing)
1657 # Returns the date it was added.
1658 def recordapk(self, apk, app):
1659 if apk not in self.apks:
1660 self.apks[apk] = (app, time.gmtime(time.time()))
1662 _, added = self.apks[apk]
1665 # Look up information - given the 'apkname', returns (app id, date added/None).
1666 # Or returns None for an unknown apk.
1667 def getapp(self, apkname):
1668 if apkname in self.apks:
1669 return self.apks[apkname]
1672 # Get the most recent 'num' apps added to the repo, as a list of package ids
1673 # with the most recent first.
1674 def getlatest(self, num):
1676 for apk, app in self.apks.iteritems():
1680 if apps[appid] > added:
1684 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1685 lst = [app for app, _ in sortedapps]
1690 def isApkDebuggable(apkfile, config):
1691 """Returns True if the given apk file is debuggable
1693 :param apkfile: full path to the apk to check"""
1695 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1697 if p.returncode != 0:
1698 logging.critical("Failed to get apk manifest information")
1700 for line in p.output.splitlines():
1701 if 'android:debuggable' in line and not line.endswith('0x0'):
1706 class AsynchronousFileReader(threading.Thread):
1709 Helper class to implement asynchronous reading of a file
1710 in a separate thread. Pushes read lines on a queue to
1711 be consumed in another thread.
1714 def __init__(self, fd, queue):
1715 assert isinstance(queue, Queue.Queue)
1716 assert callable(fd.readline)
1717 threading.Thread.__init__(self)
1722 '''The body of the tread: read lines and put them on the queue.'''
1723 for line in iter(self._fd.readline, ''):
1724 self._queue.put(line)
1727 '''Check whether there is no more content to expect.'''
1728 return not self.is_alive() and self._queue.empty()
1736 def SdkToolsPopen(commands, cwd=None, output=True):
1738 if cmd not in config:
1739 config[cmd] = find_sdk_tools_cmd(commands[0])
1740 return FDroidPopen([config[cmd]] + commands[1:],
1741 cwd=cwd, output=output)
1744 def FDroidPopen(commands, cwd=None, output=True):
1746 Run a command and capture the possibly huge output.
1748 :param commands: command and argument list like in subprocess.Popen
1749 :param cwd: optionally specifies a working directory
1750 :returns: A PopenResult.
1756 cwd = os.path.normpath(cwd)
1757 logging.debug("Directory: %s" % cwd)
1758 logging.debug("> %s" % ' '.join(commands))
1760 result = PopenResult()
1763 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1764 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1766 raise BuildException("OSError while trying to execute " +
1767 ' '.join(commands) + ': ' + str(e))
1769 stdout_queue = Queue.Queue()
1770 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1771 stdout_reader.start()
1773 # Check the queue for output (until there is no more to get)
1774 while not stdout_reader.eof():
1775 while not stdout_queue.empty():
1776 line = stdout_queue.get()
1777 if output and options.verbose:
1778 # Output directly to console
1779 sys.stderr.write(line)
1781 result.output += line
1785 result.returncode = p.wait()
1789 def remove_signing_keys(build_dir):
1790 comment = re.compile(r'[ ]*//')
1791 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1793 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1794 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1795 re.compile(r'.*variant\.outputFile = .*'),
1796 re.compile(r'.*output\.outputFile = .*'),
1797 re.compile(r'.*\.readLine\(.*'),
1799 for root, dirs, files in os.walk(build_dir):
1800 if 'build.gradle' in files:
1801 path = os.path.join(root, 'build.gradle')
1803 with open(path, "r") as o:
1804 lines = o.readlines()
1810 with open(path, "w") as o:
1811 while i < len(lines):
1814 while line.endswith('\\\n'):
1815 line = line.rstrip('\\\n') + lines[i]
1818 if comment.match(line):
1822 opened += line.count('{')
1823 opened -= line.count('}')
1826 if signing_configs.match(line):
1831 if any(s.match(line) for s in line_matches):
1839 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1842 'project.properties',
1844 'default.properties',
1845 'ant.properties', ]:
1846 if propfile in files:
1847 path = os.path.join(root, propfile)
1849 with open(path, "r") as o:
1850 lines = o.readlines()
1854 with open(path, "w") as o:
1856 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1863 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1866 def reset_env_path():
1867 global env, orig_path
1868 env['PATH'] = orig_path
1871 def add_to_env_path(path):
1873 paths = env['PATH'].split(os.pathsep)
1877 env['PATH'] = os.pathsep.join(paths)
1880 def replace_config_vars(cmd, build):
1882 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1883 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1884 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1885 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1886 if build is not None:
1887 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1888 cmd = cmd.replace('$$VERSION$$', build['version'])
1889 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1893 def place_srclib(root_dir, number, libpath):
1896 relpath = os.path.relpath(libpath, root_dir)
1897 proppath = os.path.join(root_dir, 'project.properties')
1900 if os.path.isfile(proppath):
1901 with open(proppath, "r") as o:
1902 lines = o.readlines()
1904 with open(proppath, "w") as o:
1907 if line.startswith('android.library.reference.%d=' % number):
1908 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1913 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1916 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1917 """Verify that two apks are the same
1919 One of the inputs is signed, the other is unsigned. The signature metadata
1920 is transferred from the signed to the unsigned apk, and then jarsigner is
1921 used to verify that the signature from the signed apk is also varlid for
1923 :param signed_apk: Path to a signed apk file
1924 :param unsigned_apk: Path to an unsigned apk file expected to match it
1925 :param tmp_dir: Path to directory for temporary files
1926 :returns: None if the verification is successful, otherwise a string
1927 describing what went wrong.
1929 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1930 with ZipFile(signed_apk) as signed_apk_as_zip:
1931 meta_inf_files = ['META-INF/MANIFEST.MF']
1932 for f in signed_apk_as_zip.namelist():
1933 if sigfile.match(f):
1934 meta_inf_files.append(f)
1935 if len(meta_inf_files) < 3:
1936 return "Signature files missing from {0}".format(signed_apk)
1937 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1938 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1939 for meta_inf_file in meta_inf_files:
1940 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1942 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1943 logging.info("...NOT verified - {0}".format(signed_apk))
1944 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1945 logging.info("...successfully verified")
1949 def compare_apks(apk1, apk2, tmp_dir):
1952 Returns None if the apk content is the same (apart from the signing key),
1953 otherwise a string describing what's different, or what went wrong when
1954 trying to do the comparison.
1957 badchars = re.compile('''[/ :;'"]''')
1958 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1959 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1960 for d in [apk1dir, apk2dir]:
1961 if os.path.exists(d):
1964 os.mkdir(os.path.join(d, 'jar-xf'))
1966 if subprocess.call(['jar', 'xf',
1967 os.path.abspath(apk1)],
1968 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1969 return("Failed to unpack " + apk1)
1970 if subprocess.call(['jar', 'xf',
1971 os.path.abspath(apk2)],
1972 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1973 return("Failed to unpack " + apk2)
1975 # try to find apktool in the path, if it hasn't been manually configed
1976 if 'apktool' not in config:
1977 tmp = find_command('apktool')
1979 config['apktool'] = tmp
1980 if 'apktool' in config:
1981 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1983 return("Failed to unpack " + apk1)
1984 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1986 return("Failed to unpack " + apk2)
1988 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1989 lines = p.output.splitlines()
1990 if len(lines) != 1 or 'META-INF' not in lines[0]:
1991 meld = find_command('meld')
1992 if meld is not None:
1993 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1994 return("Unexpected diff output - " + p.output)
1996 # since everything verifies, delete the comparison to keep cruft down
1997 shutil.rmtree(apk1dir)
1998 shutil.rmtree(apk2dir)
2000 # If we get here, it seems like they're the same!
2004 def find_command(command):
2005 '''find the full path of a command, or None if it can't be found in the PATH'''
2008 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2010 fpath, fname = os.path.split(command)
2015 for path in os.environ["PATH"].split(os.pathsep):
2016 path = path.strip('"')
2017 exe_file = os.path.join(path, command)
2018 if is_exe(exe_file):
2025 '''generate a random password for when generating keys'''
2026 h = hashlib.sha256()
2027 h.update(os.urandom(16)) # salt
2028 h.update(bytes(socket.getfqdn()))
2029 return h.digest().encode('base64').strip()
2032 def genkeystore(localconfig):
2033 '''Generate a new key with random passwords and add it to new keystore'''
2034 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2035 keystoredir = os.path.dirname(localconfig['keystore'])
2036 if keystoredir is None or keystoredir == '':
2037 keystoredir = os.path.join(os.getcwd(), keystoredir)
2038 if not os.path.exists(keystoredir):
2039 os.makedirs(keystoredir, mode=0o700)
2041 write_password_file("keystorepass", localconfig['keystorepass'])
2042 write_password_file("keypass", localconfig['keypass'])
2043 p = FDroidPopen(['keytool', '-genkey',
2044 '-keystore', localconfig['keystore'],
2045 '-alias', localconfig['repo_keyalias'],
2046 '-keyalg', 'RSA', '-keysize', '4096',
2047 '-sigalg', 'SHA256withRSA',
2048 '-validity', '10000',
2049 '-storepass:file', config['keystorepassfile'],
2050 '-keypass:file', config['keypassfile'],
2051 '-dname', localconfig['keydname']])
2052 # TODO keypass should be sent via stdin
2053 os.chmod(localconfig['keystore'], 0o0600)
2054 if p.returncode != 0:
2055 raise BuildException("Failed to generate key", p.output)
2056 # now show the lovely key that was just generated
2057 p = FDroidPopen(['keytool', '-list', '-v',
2058 '-keystore', localconfig['keystore'],
2059 '-alias', localconfig['repo_keyalias'],
2060 '-storepass:file', config['keystorepassfile']])
2061 logging.info(p.output.strip() + '\n\n')
2064 def write_to_config(thisconfig, key, value=None):
2065 '''write a key/value to the local config.py'''
2067 origkey = key + '_orig'
2068 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2069 with open('config.py', 'r') as f:
2071 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
2072 repl = '\n' + key + ' = "' + value + '"'
2073 data = re.sub(pattern, repl, data)
2074 # if this key is not in the file, append it
2075 if not re.match('\s*' + key + '\s*=\s*"', data):
2077 # make sure the file ends with a carraige return
2078 if not re.match('\n$', data):
2080 with open('config.py', 'w') as f: