1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.2",
61 '1.7': "/usr/lib/jvm/java-7-openjdk",
67 'accepted_formats': ['txt', 'yaml'],
68 'sync_from_local_copy_dir': False,
69 'per_app_repos': False,
70 'make_current_version_link': True,
71 'current_version_name_source': 'Name',
72 'update_stats': False,
76 'stats_to_carbon': False,
78 'build_server_always': False,
79 'keystore': 'keystore.jks',
80 'smartcardoptions': [],
86 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
87 'repo_name': "My First FDroid Repo Demo",
88 'repo_icon': "fdroid-icon.png",
89 'repo_description': '''
90 This is a repository of apps to be used with FDroid. Applications in this
91 repository are either official binaries built by the original application
92 developers, or are binaries built from source by the admin of f-droid.org
93 using the tools on https://gitlab.com/u/fdroid.
99 def setup_global_opts(parser):
100 parser.add_argument("-v", "--verbose", action="store_true", default=False,
101 help="Spew out even more information than normal")
102 parser.add_argument("-q", "--quiet", action="store_true", default=False,
103 help="Restrict output to warnings and errors")
106 def fill_config_defaults(thisconfig):
107 for k, v in default_config.items():
108 if k not in thisconfig:
111 # Expand paths (~users and $vars)
112 def expand_path(path):
116 path = os.path.expanduser(path)
117 path = os.path.expandvars(path)
122 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
127 thisconfig[k + '_orig'] = v
129 for k in ['ndk_paths', 'java_paths']:
135 thisconfig[k][k2] = exp
136 thisconfig[k][k2 + '_orig'] = v
139 def regsub_file(pattern, repl, path):
140 with open(path, 'r') as f:
142 text = re.sub(pattern, repl, text)
143 with open(path, 'w') as f:
147 def read_config(opts, config_file='config.py'):
148 """Read the repository config
150 The config is read from config_file, which is in the current directory when
151 any of the repo management commands are used.
153 global config, options, env, orig_path
155 if config is not None:
157 if not os.path.isfile(config_file):
158 logging.critical("Missing config file - is this a repo directory?")
165 logging.debug("Reading %s" % config_file)
166 execfile(config_file, config)
168 # smartcardoptions must be a list since its command line args for Popen
169 if 'smartcardoptions' in config:
170 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
171 elif 'keystore' in config and config['keystore'] == 'NONE':
172 # keystore='NONE' means use smartcard, these are required defaults
173 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
174 'SunPKCS11-OpenSC', '-providerClass',
175 'sun.security.pkcs11.SunPKCS11',
176 '-providerArg', 'opensc-fdroid.cfg']
178 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
179 st = os.stat(config_file)
180 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
181 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
183 fill_config_defaults(config)
185 # There is no standard, so just set up the most common environment
188 orig_path = env['PATH']
189 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
190 env[n] = config['sdk_path']
193 cpath = config['java_paths']['1.%s' % v]
195 env['JAVA%s_HOME' % v] = cpath
197 for k in ["keystorepass", "keypass"]:
199 write_password_file(k)
201 for k in ["repo_description", "archive_description"]:
203 config[k] = clean_description(config[k])
205 if 'serverwebroot' in config:
206 if isinstance(config['serverwebroot'], basestring):
207 roots = [config['serverwebroot']]
208 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
209 roots = config['serverwebroot']
211 raise TypeError('only accepts strings, lists, and tuples')
213 for rootstr in roots:
214 # since this is used with rsync, where trailing slashes have
215 # meaning, ensure there is always a trailing slash
216 if rootstr[-1] != '/':
218 rootlist.append(rootstr.replace('//', '/'))
219 config['serverwebroot'] = rootlist
224 def get_ndk_path(version):
226 version = 'r10e' # falls back to latest
227 paths = config['ndk_paths']
228 if version not in paths:
230 return paths[version] or ''
233 def find_sdk_tools_cmd(cmd):
234 '''find a working path to a tool from the Android SDK'''
237 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
238 # try to find a working path to this command, in all the recent possible paths
239 if 'build_tools' in config:
240 build_tools = os.path.join(config['sdk_path'], 'build-tools')
241 # if 'build_tools' was manually set and exists, check only that one
242 configed_build_tools = os.path.join(build_tools, config['build_tools'])
243 if os.path.exists(configed_build_tools):
244 tooldirs.append(configed_build_tools)
246 # no configed version, so hunt known paths for it
247 for f in sorted(os.listdir(build_tools), reverse=True):
248 if os.path.isdir(os.path.join(build_tools, f)):
249 tooldirs.append(os.path.join(build_tools, f))
250 tooldirs.append(build_tools)
251 sdk_tools = os.path.join(config['sdk_path'], 'tools')
252 if os.path.exists(sdk_tools):
253 tooldirs.append(sdk_tools)
254 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
255 if os.path.exists(sdk_platform_tools):
256 tooldirs.append(sdk_platform_tools)
257 tooldirs.append('/usr/bin')
259 if os.path.isfile(os.path.join(d, cmd)):
260 return os.path.join(d, cmd)
261 # did not find the command, exit with error message
262 ensure_build_tools_exists(config)
265 def test_sdk_exists(thisconfig):
266 if 'sdk_path' not in thisconfig:
267 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
270 logging.error("'sdk_path' not set in config.py!")
272 if thisconfig['sdk_path'] == default_config['sdk_path']:
273 logging.error('No Android SDK found!')
274 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
275 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
277 if not os.path.exists(thisconfig['sdk_path']):
278 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
280 if not os.path.isdir(thisconfig['sdk_path']):
281 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
283 for d in ['build-tools', 'platform-tools', 'tools']:
284 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
285 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
286 thisconfig['sdk_path'], d))
291 def ensure_build_tools_exists(thisconfig):
292 if not test_sdk_exists(thisconfig):
294 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
295 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
296 if not os.path.isdir(versioned_build_tools):
297 logging.critical('Android Build Tools path "'
298 + versioned_build_tools + '" does not exist!')
302 def write_password_file(pwtype, password=None):
304 writes out passwords to a protected file instead of passing passwords as
305 command line argments
307 filename = '.fdroid.' + pwtype + '.txt'
308 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
310 os.write(fd, config[pwtype])
312 os.write(fd, password)
314 config[pwtype + 'file'] = filename
317 # Given the arguments in the form of multiple appid:[vc] strings, this returns
318 # a dictionary with the set of vercodes specified for each package.
319 def read_pkg_args(args, allow_vercodes=False):
326 if allow_vercodes and ':' in p:
327 package, vercode = p.split(':')
329 package, vercode = p, None
330 if package not in vercodes:
331 vercodes[package] = [vercode] if vercode else []
333 elif vercode and vercode not in vercodes[package]:
334 vercodes[package] += [vercode] if vercode else []
339 # On top of what read_pkg_args does, this returns the whole app metadata, but
340 # limiting the builds list to the builds matching the vercodes specified.
341 def read_app_args(args, allapps, allow_vercodes=False):
343 vercodes = read_pkg_args(args, allow_vercodes)
349 for appid, app in allapps.iteritems():
350 if appid in vercodes:
353 if len(apps) != len(vercodes):
356 logging.critical("No such package: %s" % p)
357 raise FDroidException("Found invalid app ids in arguments")
359 raise FDroidException("No packages specified")
362 for appid, app in apps.iteritems():
366 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
367 if len(app['builds']) != len(vercodes[appid]):
369 allvcs = [b['vercode'] for b in app['builds']]
370 for v in vercodes[appid]:
372 logging.critical("No such vercode %s for app %s" % (v, appid))
375 raise FDroidException("Found invalid vercodes for some apps")
380 def get_extension(filename):
381 base, ext = os.path.splitext(filename)
384 return base, ext.lower()[1:]
387 def has_extension(filename, ext):
388 _, f_ext = get_extension(filename)
395 def clean_description(description):
396 'Remove unneeded newlines and spaces from a block of description text'
398 # this is split up by paragraph to make removing the newlines easier
399 for paragraph in re.split(r'\n\n', description):
400 paragraph = re.sub('\r', '', paragraph)
401 paragraph = re.sub('\n', ' ', paragraph)
402 paragraph = re.sub(' {2,}', ' ', paragraph)
403 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
404 returnstring += paragraph + '\n\n'
405 return returnstring.rstrip('\n')
408 def apknameinfo(filename):
410 filename = os.path.basename(filename)
411 if apk_regex is None:
412 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
413 m = apk_regex.match(filename)
415 result = (m.group(1), m.group(2))
416 except AttributeError:
417 raise FDroidException("Invalid apk name: %s" % filename)
421 def getapkname(app, build):
422 return "%s_%s.apk" % (app['id'], build['vercode'])
425 def getsrcname(app, build):
426 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
433 return app['Auto Name']
438 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
441 def getvcs(vcstype, remote, local):
443 return vcs_git(remote, local)
444 if vcstype == 'git-svn':
445 return vcs_gitsvn(remote, local)
447 return vcs_hg(remote, local)
449 return vcs_bzr(remote, local)
450 if vcstype == 'srclib':
451 if local != os.path.join('build', 'srclib', remote):
452 raise VCSException("Error: srclib paths are hard-coded!")
453 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
455 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
456 raise VCSException("Invalid vcs type " + vcstype)
459 def getsrclibvcs(name):
460 if name not in metadata.srclibs:
461 raise VCSException("Missing srclib " + name)
462 return metadata.srclibs[name]['Repo Type']
467 def __init__(self, remote, local):
469 # svn, git-svn and bzr may require auth
471 if self.repotype() in ('git-svn', 'bzr'):
473 if self.repotype == 'git-svn':
474 raise VCSException("Authentication is not supported for git-svn")
475 self.username, remote = remote.split('@')
476 if ':' not in self.username:
477 raise VCSException("Password required with username")
478 self.username, self.password = self.username.split(':')
482 self.clone_failed = False
483 self.refreshed = False
489 # Take the local repository to a clean version of the given revision, which
490 # is specificed in the VCS's native format. Beforehand, the repository can
491 # be dirty, or even non-existent. If the repository does already exist
492 # locally, it will be updated from the origin, but only once in the
493 # lifetime of the vcs object.
494 # None is acceptable for 'rev' if you know you are cloning a clean copy of
495 # the repo - otherwise it must specify a valid revision.
496 def gotorevision(self, rev, refresh=True):
498 if self.clone_failed:
499 raise VCSException("Downloading the repository already failed once, not trying again.")
501 # The .fdroidvcs-id file for a repo tells us what VCS type
502 # and remote that directory was created from, allowing us to drop it
503 # automatically if either of those things changes.
504 fdpath = os.path.join(self.local, '..',
505 '.fdroidvcs-' + os.path.basename(self.local))
506 cdata = self.repotype() + ' ' + self.remote
509 if os.path.exists(self.local):
510 if os.path.exists(fdpath):
511 with open(fdpath, 'r') as f:
512 fsdata = f.read().strip()
517 logging.info("Repository details for %s changed - deleting" % (
521 logging.info("Repository details for %s missing - deleting" % (
524 shutil.rmtree(self.local)
528 self.refreshed = True
531 self.gotorevisionx(rev)
532 except FDroidException, e:
535 # If necessary, write the .fdroidvcs file.
536 if writeback and not self.clone_failed:
537 with open(fdpath, 'w') as f:
543 # Derived classes need to implement this. It's called once basic checking
544 # has been performend.
545 def gotorevisionx(self, rev):
546 raise VCSException("This VCS type doesn't define gotorevisionx")
548 # Initialise and update submodules
549 def initsubmodules(self):
550 raise VCSException('Submodules not supported for this vcs type')
552 # Get a list of all known tags
554 if not self._gettags:
555 raise VCSException('gettags not supported for this vcs type')
557 for tag in self._gettags():
558 if re.match('[-A-Za-z0-9_. /]+$', tag):
562 def latesttags(self, tags, number):
563 """Get the most recent tags in a given list.
565 :param tags: a list of tags
566 :param number: the number to return
567 :returns: A list containing the most recent tags in the provided
568 list, up to the maximum number given.
570 raise VCSException('latesttags not supported for this vcs type')
572 # Get current commit reference (hash, revision, etc)
574 raise VCSException('getref not supported for this vcs type')
576 # Returns the srclib (name, path) used in setting up the current
587 # If the local directory exists, but is somehow not a git repository, git
588 # will traverse up the directory tree until it finds one that is (i.e.
589 # fdroidserver) and then we'll proceed to destroy it! This is called as
592 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
593 result = p.output.rstrip()
594 if not result.endswith(self.local):
595 raise VCSException('Repository mismatch')
597 def gotorevisionx(self, rev):
598 if not os.path.exists(self.local):
600 p = FDroidPopen(['git', 'clone', self.remote, self.local])
601 if p.returncode != 0:
602 self.clone_failed = True
603 raise VCSException("Git clone failed", p.output)
607 # Discard any working tree changes
608 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
609 'git', 'reset', '--hard'], cwd=self.local, output=False)
610 if p.returncode != 0:
611 raise VCSException("Git reset failed", p.output)
612 # Remove untracked files now, in case they're tracked in the target
613 # revision (it happens!)
614 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
615 'git', 'clean', '-dffx'], cwd=self.local, output=False)
616 if p.returncode != 0:
617 raise VCSException("Git clean failed", p.output)
618 if not self.refreshed:
619 # Get latest commits and tags from remote
620 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
621 if p.returncode != 0:
622 raise VCSException("Git fetch failed", p.output)
623 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
624 if p.returncode != 0:
625 raise VCSException("Git fetch failed", p.output)
626 # Recreate origin/HEAD as git clone would do it, in case it disappeared
627 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
628 if p.returncode != 0:
629 lines = p.output.splitlines()
630 if 'Multiple remote HEAD branches' not in lines[0]:
631 raise VCSException("Git remote set-head failed", p.output)
632 branch = lines[1].split(' ')[-1]
633 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
634 if p2.returncode != 0:
635 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
636 self.refreshed = True
637 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
638 # a github repo. Most of the time this is the same as origin/master.
639 rev = rev or 'origin/HEAD'
640 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
641 if p.returncode != 0:
642 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
643 # Get rid of any uncontrolled files left behind
644 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
645 if p.returncode != 0:
646 raise VCSException("Git clean failed", p.output)
648 def initsubmodules(self):
650 submfile = os.path.join(self.local, '.gitmodules')
651 if not os.path.isfile(submfile):
652 raise VCSException("No git submodules available")
654 # fix submodules not accessible without an account and public key auth
655 with open(submfile, 'r') as f:
656 lines = f.readlines()
657 with open(submfile, 'w') as f:
659 if 'git@github.com' in line:
660 line = line.replace('git@github.com:', 'https://github.com/')
661 if 'git@gitlab.com' in line:
662 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
665 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
666 if p.returncode != 0:
667 raise VCSException("Git submodule sync failed", p.output)
668 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
669 if p.returncode != 0:
670 raise VCSException("Git submodule update failed", p.output)
674 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
675 return p.output.splitlines()
677 def latesttags(self, tags, number):
682 ['git', 'show', '--format=format:%ct', '-s', tag],
683 cwd=self.local, output=False)
684 # Timestamp is on the last line. For a normal tag, it's the only
685 # line, but for annotated tags, the rest of the info precedes it.
686 ts = int(p.output.splitlines()[-1])
689 for _, t in sorted(tl)[-number:]:
694 class vcs_gitsvn(vcs):
699 # If the local directory exists, but is somehow not a git repository, git
700 # will traverse up the directory tree until it finds one that is (i.e.
701 # fdroidserver) and then we'll proceed to destory it! This is called as
704 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
705 result = p.output.rstrip()
706 if not result.endswith(self.local):
707 raise VCSException('Repository mismatch')
709 def gotorevisionx(self, rev):
710 if not os.path.exists(self.local):
712 gitsvn_args = ['git', 'svn', 'clone']
713 if ';' in self.remote:
714 remote_split = self.remote.split(';')
715 for i in remote_split[1:]:
716 if i.startswith('trunk='):
717 gitsvn_args.extend(['-T', i[6:]])
718 elif i.startswith('tags='):
719 gitsvn_args.extend(['-t', i[5:]])
720 elif i.startswith('branches='):
721 gitsvn_args.extend(['-b', i[9:]])
722 gitsvn_args.extend([remote_split[0], self.local])
723 p = FDroidPopen(gitsvn_args, output=False)
724 if p.returncode != 0:
725 self.clone_failed = True
726 raise VCSException("Git svn clone failed", p.output)
728 gitsvn_args.extend([self.remote, self.local])
729 p = FDroidPopen(gitsvn_args, output=False)
730 if p.returncode != 0:
731 self.clone_failed = True
732 raise VCSException("Git svn clone failed", p.output)
736 # Discard any working tree changes
737 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
738 if p.returncode != 0:
739 raise VCSException("Git reset failed", p.output)
740 # Remove untracked files now, in case they're tracked in the target
741 # revision (it happens!)
742 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
743 if p.returncode != 0:
744 raise VCSException("Git clean failed", p.output)
745 if not self.refreshed:
746 # Get new commits, branches and tags from repo
747 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
748 if p.returncode != 0:
749 raise VCSException("Git svn fetch failed")
750 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
751 if p.returncode != 0:
752 raise VCSException("Git svn rebase failed", p.output)
753 self.refreshed = True
755 rev = rev or 'master'
757 nospaces_rev = rev.replace(' ', '%20')
758 # Try finding a svn tag
759 for treeish in ['origin/', '']:
760 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
761 if p.returncode == 0:
763 if p.returncode != 0:
764 # No tag found, normal svn rev translation
765 # Translate svn rev into git format
766 rev_split = rev.split('/')
769 for treeish in ['origin/', '']:
770 if len(rev_split) > 1:
771 treeish += rev_split[0]
772 svn_rev = rev_split[1]
775 # if no branch is specified, then assume trunk (i.e. 'master' branch):
779 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
781 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
782 git_rev = p.output.rstrip()
784 if p.returncode == 0 and git_rev:
787 if p.returncode != 0 or not git_rev:
788 # Try a plain git checkout as a last resort
789 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
790 if p.returncode != 0:
791 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
793 # Check out the git rev equivalent to the svn rev
794 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
795 if p.returncode != 0:
796 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
798 # Get rid of any uncontrolled files left behind
799 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
800 if p.returncode != 0:
801 raise VCSException("Git clean failed", p.output)
805 for treeish in ['origin/', '']:
806 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
812 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
813 if p.returncode != 0:
815 return p.output.strip()
823 def gotorevisionx(self, rev):
824 if not os.path.exists(self.local):
825 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
826 if p.returncode != 0:
827 self.clone_failed = True
828 raise VCSException("Hg clone failed", p.output)
830 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
831 if p.returncode != 0:
832 raise VCSException("Hg status failed", p.output)
833 for line in p.output.splitlines():
834 if not line.startswith('? '):
835 raise VCSException("Unexpected output from hg status -uS: " + line)
836 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
837 if not self.refreshed:
838 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
839 if p.returncode != 0:
840 raise VCSException("Hg pull failed", p.output)
841 self.refreshed = True
843 rev = rev or 'default'
846 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
847 if p.returncode != 0:
848 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
849 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
850 # Also delete untracked files, we have to enable purge extension for that:
851 if "'purge' is provided by the following extension" in p.output:
852 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
853 myfile.write("\n[extensions]\nhgext.purge=\n")
854 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
855 if p.returncode != 0:
856 raise VCSException("HG purge failed", p.output)
857 elif p.returncode != 0:
858 raise VCSException("HG purge failed", p.output)
861 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
862 return p.output.splitlines()[1:]
870 def gotorevisionx(self, rev):
871 if not os.path.exists(self.local):
872 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
873 if p.returncode != 0:
874 self.clone_failed = True
875 raise VCSException("Bzr branch failed", p.output)
877 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException("Bzr revert failed", p.output)
880 if not self.refreshed:
881 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 raise VCSException("Bzr update failed", p.output)
884 self.refreshed = True
886 revargs = list(['-r', rev] if rev else [])
887 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
888 if p.returncode != 0:
889 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
892 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
893 return [tag.split(' ')[0].strip() for tag in
894 p.output.splitlines()]
897 def unescape_string(string):
900 if string[0] == '"' and string[-1] == '"':
903 return string.replace("\\'", "'")
906 def retrieve_string(app_dir, string, xmlfiles=None):
908 if not string.startswith('@string/'):
909 return unescape_string(string)
914 os.path.join(app_dir, 'res'),
915 os.path.join(app_dir, 'src', 'main', 'res'),
917 for r, d, f in os.walk(res_dir):
918 if os.path.basename(r) == 'values':
919 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
921 name = string[len('@string/'):]
923 def element_content(element):
924 if element.text is None:
926 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
929 for path in xmlfiles:
930 if not os.path.isfile(path):
932 xml = parse_xml(path)
933 element = xml.find('string[@name="' + name + '"]')
934 if element is not None:
935 content = element_content(element)
936 return retrieve_string(app_dir, content, xmlfiles)
941 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
942 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
945 # Return list of existing files that will be used to find the highest vercode
946 def manifest_paths(app_dir, flavours):
948 possible_manifests = \
949 [os.path.join(app_dir, 'AndroidManifest.xml'),
950 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
951 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
952 os.path.join(app_dir, 'build.gradle')]
954 for flavour in flavours:
957 possible_manifests.append(
958 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
960 return [path for path in possible_manifests if os.path.isfile(path)]
963 # Retrieve the package name. Returns the name, or None if not found.
964 def fetch_real_name(app_dir, flavours):
965 for path in manifest_paths(app_dir, flavours):
966 if not has_extension(path, 'xml') or not os.path.isfile(path):
968 logging.debug("fetch_real_name: Checking manifest at " + path)
969 xml = parse_xml(path)
970 app = xml.find('application')
973 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
975 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
976 result = retrieve_string_singleline(app_dir, label)
978 result = result.strip()
983 def get_library_references(root_dir):
985 proppath = os.path.join(root_dir, 'project.properties')
986 if not os.path.isfile(proppath):
988 for line in file(proppath):
989 if not line.startswith('android.library.reference.'):
991 path = line.split('=')[1].strip()
992 relpath = os.path.join(root_dir, path)
993 if not os.path.isdir(relpath):
995 logging.debug("Found subproject at %s" % path)
996 libraries.append(path)
1000 def ant_subprojects(root_dir):
1001 subprojects = get_library_references(root_dir)
1002 for subpath in subprojects:
1003 subrelpath = os.path.join(root_dir, subpath)
1004 for p in get_library_references(subrelpath):
1005 relp = os.path.normpath(os.path.join(subpath, p))
1006 if relp not in subprojects:
1007 subprojects.insert(0, relp)
1011 def remove_debuggable_flags(root_dir):
1012 # Remove forced debuggable flags
1013 logging.debug("Removing debuggable flags from %s" % root_dir)
1014 for root, dirs, files in os.walk(root_dir):
1015 if 'AndroidManifest.xml' in files:
1016 regsub_file(r'android:debuggable="[^"]*"',
1018 os.path.join(root, 'AndroidManifest.xml'))
1021 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1022 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1023 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1026 def app_matches_packagename(app, package):
1029 appid = app['Update Check Name'] or app['id']
1030 if appid == "Ignore":
1032 return appid == package
1035 # Extract some information from the AndroidManifest.xml at the given path.
1036 # Returns (version, vercode, package), any or all of which might be None.
1037 # All values returned are strings.
1038 def parse_androidmanifests(paths, app):
1040 ignoreversions = app['Update Check Ignore']
1041 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1044 return (None, None, None)
1052 if not os.path.isfile(path):
1055 logging.debug("Parsing manifest at {0}".format(path))
1056 gradle = has_extension(path, 'gradle')
1062 for line in file(path):
1063 if gradle_comment.match(line):
1065 # Grab first occurence of each to avoid running into
1066 # alternative flavours and builds.
1068 matches = psearch_g(line)
1070 s = matches.group(2)
1071 if app_matches_packagename(app, s):
1074 matches = vnsearch_g(line)
1076 version = matches.group(2)
1078 matches = vcsearch_g(line)
1080 vercode = matches.group(1)
1082 xml = parse_xml(path)
1083 if "package" in xml.attrib:
1084 s = xml.attrib["package"].encode('utf-8')
1085 if app_matches_packagename(app, s):
1087 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1088 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1089 base_dir = os.path.dirname(path)
1090 version = retrieve_string_singleline(base_dir, version)
1091 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1092 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1093 if string_is_integer(a):
1096 # Remember package name, may be defined separately from version+vercode
1098 package = max_package
1100 logging.debug("..got package={0}, version={1}, vercode={2}"
1101 .format(package, version, vercode))
1103 # Always grab the package name and version name in case they are not
1104 # together with the highest version code
1105 if max_package is None and package is not None:
1106 max_package = package
1107 if max_version is None and version is not None:
1108 max_version = version
1110 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1111 if not ignoresearch or not ignoresearch(version):
1112 if version is not None:
1113 max_version = version
1114 if vercode is not None:
1115 max_vercode = vercode
1116 if package is not None:
1117 max_package = package
1119 max_version = "Ignore"
1121 if max_version is None:
1122 max_version = "Unknown"
1124 if max_package and not is_valid_package_name(max_package):
1125 raise FDroidException("Invalid package name {0}".format(max_package))
1127 return (max_version, max_vercode, max_package)
1130 def is_valid_package_name(name):
1131 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1134 class FDroidException(Exception):
1136 def __init__(self, value, detail=None):
1138 self.detail = detail
1140 def shortened_detail(self):
1141 if len(self.detail) < 16000:
1143 return '[...]\n' + self.detail[-16000:]
1145 def get_wikitext(self):
1146 ret = repr(self.value) + "\n"
1149 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1155 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1159 class VCSException(FDroidException):
1163 class BuildException(FDroidException):
1167 # Get the specified source library.
1168 # Returns the path to it. Normally this is the path to be used when referencing
1169 # it, which may be a subdirectory of the actual project. If you want the base
1170 # directory of the project, pass 'basepath=True'.
1171 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1172 raw=False, prepare=True, preponly=False, refresh=True):
1180 name, ref = spec.split('@')
1182 number, name = name.split(':', 1)
1184 name, subdir = name.split('/', 1)
1186 if name not in metadata.srclibs:
1187 raise VCSException('srclib ' + name + ' not found.')
1189 srclib = metadata.srclibs[name]
1191 sdir = os.path.join(srclib_dir, name)
1194 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1195 vcs.srclib = (name, number, sdir)
1197 vcs.gotorevision(ref, refresh)
1204 libdir = os.path.join(sdir, subdir)
1205 elif srclib["Subdir"]:
1206 for subdir in srclib["Subdir"]:
1207 libdir_candidate = os.path.join(sdir, subdir)
1208 if os.path.exists(libdir_candidate):
1209 libdir = libdir_candidate
1215 remove_signing_keys(sdir)
1216 remove_debuggable_flags(sdir)
1220 if srclib["Prepare"]:
1221 cmd = replace_config_vars(srclib["Prepare"], None)
1223 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1224 if p.returncode != 0:
1225 raise BuildException("Error running prepare command for srclib %s"
1231 return (name, number, libdir)
1233 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1236 # Prepare the source code for a particular build
1237 # 'vcs' - the appropriate vcs object for the application
1238 # 'app' - the application details from the metadata
1239 # 'build' - the build details from the metadata
1240 # 'build_dir' - the path to the build directory, usually
1242 # 'srclib_dir' - the path to the source libraries directory, usually
1244 # 'extlib_dir' - the path to the external libraries directory, usually
1246 # Returns the (root, srclibpaths) where:
1247 # 'root' is the root directory, which may be the same as 'build_dir' or may
1248 # be a subdirectory of it.
1249 # 'srclibpaths' is information on the srclibs being used
1250 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1252 # Optionally, the actual app source can be in a subdirectory
1254 root_dir = os.path.join(build_dir, build['subdir'])
1256 root_dir = build_dir
1258 # Get a working copy of the right revision
1259 logging.info("Getting source for revision " + build['commit'])
1260 vcs.gotorevision(build['commit'], refresh)
1262 # Initialise submodules if required
1263 if build['submodules']:
1264 logging.info("Initialising submodules")
1265 vcs.initsubmodules()
1267 # Check that a subdir (if we're using one) exists. This has to happen
1268 # after the checkout, since it might not exist elsewhere
1269 if not os.path.exists(root_dir):
1270 raise BuildException('Missing subdir ' + root_dir)
1272 # Run an init command if one is required
1274 cmd = replace_config_vars(build['init'], build)
1275 logging.info("Running 'init' commands in %s" % root_dir)
1277 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1278 if p.returncode != 0:
1279 raise BuildException("Error running init command for %s:%s" %
1280 (app['id'], build['version']), p.output)
1282 # Apply patches if any
1284 logging.info("Applying patches")
1285 for patch in build['patch']:
1286 patch = patch.strip()
1287 logging.info("Applying " + patch)
1288 patch_path = os.path.join('metadata', app['id'], patch)
1289 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1290 if p.returncode != 0:
1291 raise BuildException("Failed to apply patch %s" % patch_path)
1293 # Get required source libraries
1295 if build['srclibs']:
1296 logging.info("Collecting source libraries")
1297 for lib in build['srclibs']:
1298 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1300 for name, number, libpath in srclibpaths:
1301 place_srclib(root_dir, int(number) if number else None, libpath)
1303 basesrclib = vcs.getsrclib()
1304 # If one was used for the main source, add that too.
1306 srclibpaths.append(basesrclib)
1308 # Update the local.properties file
1309 localprops = [os.path.join(build_dir, 'local.properties')]
1311 parts = build['subdir'].split(os.sep)
1314 cur = os.path.join(cur, d)
1315 localprops += [os.path.join(cur, 'local.properties')]
1316 for path in localprops:
1318 if os.path.isfile(path):
1319 logging.info("Updating local.properties file at %s" % path)
1320 with open(path, 'r') as f:
1324 logging.info("Creating local.properties file at %s" % path)
1325 # Fix old-fashioned 'sdk-location' by copying
1326 # from sdk.dir, if necessary
1327 if build['oldsdkloc']:
1328 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1329 re.S | re.M).group(1)
1330 props += "sdk-location=%s\n" % sdkloc
1332 props += "sdk.dir=%s\n" % config['sdk_path']
1333 props += "sdk-location=%s\n" % config['sdk_path']
1334 if build['ndk_path']:
1336 props += "ndk.dir=%s\n" % build['ndk_path']
1337 props += "ndk-location=%s\n" % build['ndk_path']
1338 # Add java.encoding if necessary
1339 if build['encoding']:
1340 props += "java.encoding=%s\n" % build['encoding']
1341 with open(path, 'w') as f:
1345 if build['type'] == 'gradle':
1346 flavours = build['gradle']
1348 gradlepluginver = None
1350 gradle_dirs = [root_dir]
1352 # Parent dir build.gradle
1353 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1354 if parent_dir.startswith(build_dir):
1355 gradle_dirs.append(parent_dir)
1357 for dir_path in gradle_dirs:
1360 if not os.path.isdir(dir_path):
1362 for filename in os.listdir(dir_path):
1363 if not filename.endswith('.gradle'):
1365 path = os.path.join(dir_path, filename)
1366 if not os.path.isfile(path):
1368 for line in file(path):
1369 match = gradle_version_regex.match(line)
1371 gradlepluginver = match.group(1)
1375 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1377 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1378 build['gradlepluginver'] = LooseVersion('0.11')
1381 n = build["target"].split('-')[1]
1382 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1383 r'compileSdkVersion %s' % n,
1384 os.path.join(root_dir, 'build.gradle'))
1386 # Remove forced debuggable flags
1387 remove_debuggable_flags(root_dir)
1389 # Insert version code and number into the manifest if necessary
1390 if build['forceversion']:
1391 logging.info("Changing the version name")
1392 for path in manifest_paths(root_dir, flavours):
1393 if not os.path.isfile(path):
1395 if has_extension(path, 'xml'):
1396 regsub_file(r'android:versionName="[^"]*"',
1397 r'android:versionName="%s"' % build['version'],
1399 elif has_extension(path, 'gradle'):
1400 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1401 r"""\1versionName '%s'""" % build['version'],
1404 if build['forcevercode']:
1405 logging.info("Changing the version code")
1406 for path in manifest_paths(root_dir, flavours):
1407 if not os.path.isfile(path):
1409 if has_extension(path, 'xml'):
1410 regsub_file(r'android:versionCode="[^"]*"',
1411 r'android:versionCode="%s"' % build['vercode'],
1413 elif has_extension(path, 'gradle'):
1414 regsub_file(r'versionCode[ =]+[0-9]+',
1415 r'versionCode %s' % build['vercode'],
1418 # Delete unwanted files
1420 logging.info("Removing specified files")
1421 for part in getpaths(build_dir, build['rm']):
1422 dest = os.path.join(build_dir, part)
1423 logging.info("Removing {0}".format(part))
1424 if os.path.lexists(dest):
1425 if os.path.islink(dest):
1426 FDroidPopen(['unlink', dest], output=False)
1428 FDroidPopen(['rm', '-rf', dest], output=False)
1430 logging.info("...but it didn't exist")
1432 remove_signing_keys(build_dir)
1434 # Add required external libraries
1435 if build['extlibs']:
1436 logging.info("Collecting prebuilt libraries")
1437 libsdir = os.path.join(root_dir, 'libs')
1438 if not os.path.exists(libsdir):
1440 for lib in build['extlibs']:
1442 logging.info("...installing extlib {0}".format(lib))
1443 libf = os.path.basename(lib)
1444 libsrc = os.path.join(extlib_dir, lib)
1445 if not os.path.exists(libsrc):
1446 raise BuildException("Missing extlib file {0}".format(libsrc))
1447 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1449 # Run a pre-build command if one is required
1450 if build['prebuild']:
1451 logging.info("Running 'prebuild' commands in %s" % root_dir)
1453 cmd = replace_config_vars(build['prebuild'], build)
1455 # Substitute source library paths into prebuild commands
1456 for name, number, libpath in srclibpaths:
1457 libpath = os.path.relpath(libpath, root_dir)
1458 cmd = cmd.replace('$$' + name + '$$', libpath)
1460 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1461 if p.returncode != 0:
1462 raise BuildException("Error running prebuild command for %s:%s" %
1463 (app['id'], build['version']), p.output)
1465 # Generate (or update) the ant build file, build.xml...
1466 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1467 parms = ['android', 'update', 'lib-project']
1468 lparms = ['android', 'update', 'project']
1471 parms += ['-t', build['target']]
1472 lparms += ['-t', build['target']]
1473 if build['update'] == ['auto']:
1474 update_dirs = ant_subprojects(root_dir) + ['.']
1476 update_dirs = build['update']
1478 for d in update_dirs:
1479 subdir = os.path.join(root_dir, d)
1481 logging.debug("Updating main project")
1482 cmd = parms + ['-p', d]
1484 logging.debug("Updating subproject %s" % d)
1485 cmd = lparms + ['-p', d]
1486 p = SdkToolsPopen(cmd, cwd=root_dir)
1487 # Check to see whether an error was returned without a proper exit
1488 # code (this is the case for the 'no target set or target invalid'
1490 if p.returncode != 0 or p.output.startswith("Error: "):
1491 raise BuildException("Failed to update project at %s" % d, p.output)
1492 # Clean update dirs via ant
1494 logging.info("Cleaning subproject %s" % d)
1495 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1497 return (root_dir, srclibpaths)
1500 # Extend via globbing the paths from a field and return them as a map from
1501 # original path to resulting paths
1502 def getpaths_map(build_dir, globpaths):
1506 full_path = os.path.join(build_dir, p)
1507 full_path = os.path.normpath(full_path)
1508 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1510 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1514 # Extend via globbing the paths from a field and return them as a set
1515 def getpaths(build_dir, globpaths):
1516 paths_map = getpaths_map(build_dir, globpaths)
1518 for k, v in paths_map.iteritems():
1525 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1531 self.path = os.path.join('stats', 'known_apks.txt')
1533 if os.path.isfile(self.path):
1534 for line in file(self.path):
1535 t = line.rstrip().split(' ')
1537 self.apks[t[0]] = (t[1], None)
1539 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1540 self.changed = False
1542 def writeifchanged(self):
1543 if not self.changed:
1546 if not os.path.exists('stats'):
1550 for apk, app in self.apks.iteritems():
1552 line = apk + ' ' + appid
1554 line += ' ' + time.strftime('%Y-%m-%d', added)
1557 with open(self.path, 'w') as f:
1558 for line in sorted(lst, key=natural_key):
1559 f.write(line + '\n')
1561 # Record an apk (if it's new, otherwise does nothing)
1562 # Returns the date it was added.
1563 def recordapk(self, apk, app):
1564 if apk not in self.apks:
1565 self.apks[apk] = (app, time.gmtime(time.time()))
1567 _, added = self.apks[apk]
1570 # Look up information - given the 'apkname', returns (app id, date added/None).
1571 # Or returns None for an unknown apk.
1572 def getapp(self, apkname):
1573 if apkname in self.apks:
1574 return self.apks[apkname]
1577 # Get the most recent 'num' apps added to the repo, as a list of package ids
1578 # with the most recent first.
1579 def getlatest(self, num):
1581 for apk, app in self.apks.iteritems():
1585 if apps[appid] > added:
1589 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1590 lst = [app for app, _ in sortedapps]
1595 def isApkDebuggable(apkfile, config):
1596 """Returns True if the given apk file is debuggable
1598 :param apkfile: full path to the apk to check"""
1600 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1602 if p.returncode != 0:
1603 logging.critical("Failed to get apk manifest information")
1605 for line in p.output.splitlines():
1606 if 'android:debuggable' in line and not line.endswith('0x0'):
1616 def SdkToolsPopen(commands, cwd=None, output=True):
1618 if cmd not in config:
1619 config[cmd] = find_sdk_tools_cmd(commands[0])
1620 abscmd = config[cmd]
1622 logging.critical("Could not find '%s' on your system" % cmd)
1624 return FDroidPopen([abscmd] + commands[1:],
1625 cwd=cwd, output=output)
1628 def FDroidPopen(commands, cwd=None, output=True):
1630 Run a command and capture the possibly huge output.
1632 :param commands: command and argument list like in subprocess.Popen
1633 :param cwd: optionally specifies a working directory
1634 :returns: A PopenResult.
1640 cwd = os.path.normpath(cwd)
1641 logging.debug("Directory: %s" % cwd)
1642 logging.debug("> %s" % ' '.join(commands))
1644 result = PopenResult()
1647 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1648 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1650 raise BuildException("OSError while trying to execute " +
1651 ' '.join(commands) + ': ' + str(e))
1653 stdout_queue = Queue.Queue()
1654 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1656 # Check the queue for output (until there is no more to get)
1657 while not stdout_reader.eof():
1658 while not stdout_queue.empty():
1659 line = stdout_queue.get()
1660 if output and options.verbose:
1661 # Output directly to console
1662 sys.stderr.write(line)
1664 result.output += line
1668 result.returncode = p.wait()
1672 gradle_comment = re.compile(r'[ ]*//')
1673 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1674 gradle_line_matches = [
1675 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1676 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1677 re.compile(r'.*variant\.outputFile = .*'),
1678 re.compile(r'.*output\.outputFile = .*'),
1679 re.compile(r'.*\.readLine\(.*'),
1683 def remove_signing_keys(build_dir):
1684 for root, dirs, files in os.walk(build_dir):
1685 if 'build.gradle' in files:
1686 path = os.path.join(root, 'build.gradle')
1688 with open(path, "r") as o:
1689 lines = o.readlines()
1695 with open(path, "w") as o:
1696 while i < len(lines):
1699 while line.endswith('\\\n'):
1700 line = line.rstrip('\\\n') + lines[i]
1703 if gradle_comment.match(line):
1708 opened += line.count('{')
1709 opened -= line.count('}')
1712 if gradle_signing_configs.match(line):
1717 if any(s.match(line) for s in gradle_line_matches):
1725 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1728 'project.properties',
1730 'default.properties',
1731 'ant.properties', ]:
1732 if propfile in files:
1733 path = os.path.join(root, propfile)
1735 with open(path, "r") as o:
1736 lines = o.readlines()
1740 with open(path, "w") as o:
1742 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1749 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1752 def reset_env_path():
1753 global env, orig_path
1754 env['PATH'] = orig_path
1757 def add_to_env_path(path):
1759 paths = env['PATH'].split(os.pathsep)
1763 env['PATH'] = os.pathsep.join(paths)
1766 def replace_config_vars(cmd, build):
1768 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1769 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1770 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1771 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1772 if build is not None:
1773 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1774 cmd = cmd.replace('$$VERSION$$', build['version'])
1775 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1779 def place_srclib(root_dir, number, libpath):
1782 relpath = os.path.relpath(libpath, root_dir)
1783 proppath = os.path.join(root_dir, 'project.properties')
1786 if os.path.isfile(proppath):
1787 with open(proppath, "r") as o:
1788 lines = o.readlines()
1790 with open(proppath, "w") as o:
1793 if line.startswith('android.library.reference.%d=' % number):
1794 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1799 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1801 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1804 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1805 """Verify that two apks are the same
1807 One of the inputs is signed, the other is unsigned. The signature metadata
1808 is transferred from the signed to the unsigned apk, and then jarsigner is
1809 used to verify that the signature from the signed apk is also varlid for
1811 :param signed_apk: Path to a signed apk file
1812 :param unsigned_apk: Path to an unsigned apk file expected to match it
1813 :param tmp_dir: Path to directory for temporary files
1814 :returns: None if the verification is successful, otherwise a string
1815 describing what went wrong.
1817 with ZipFile(signed_apk) as signed_apk_as_zip:
1818 meta_inf_files = ['META-INF/MANIFEST.MF']
1819 for f in signed_apk_as_zip.namelist():
1820 if apk_sigfile.match(f):
1821 meta_inf_files.append(f)
1822 if len(meta_inf_files) < 3:
1823 return "Signature files missing from {0}".format(signed_apk)
1824 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1825 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1826 for meta_inf_file in meta_inf_files:
1827 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1829 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1830 logging.info("...NOT verified - {0}".format(signed_apk))
1831 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1832 logging.info("...successfully verified")
1835 apk_badchars = re.compile('''[/ :;'"]''')
1838 def compare_apks(apk1, apk2, tmp_dir):
1841 Returns None if the apk content is the same (apart from the signing key),
1842 otherwise a string describing what's different, or what went wrong when
1843 trying to do the comparison.
1846 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1847 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1848 for d in [apk1dir, apk2dir]:
1849 if os.path.exists(d):
1852 os.mkdir(os.path.join(d, 'jar-xf'))
1854 if subprocess.call(['jar', 'xf',
1855 os.path.abspath(apk1)],
1856 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1857 return("Failed to unpack " + apk1)
1858 if subprocess.call(['jar', 'xf',
1859 os.path.abspath(apk2)],
1860 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1861 return("Failed to unpack " + apk2)
1863 # try to find apktool in the path, if it hasn't been manually configed
1864 if 'apktool' not in config:
1865 tmp = find_command('apktool')
1867 config['apktool'] = tmp
1868 if 'apktool' in config:
1869 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1871 return("Failed to unpack " + apk1)
1872 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1874 return("Failed to unpack " + apk2)
1876 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1877 lines = p.output.splitlines()
1878 if len(lines) != 1 or 'META-INF' not in lines[0]:
1879 meld = find_command('meld')
1880 if meld is not None:
1881 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1882 return("Unexpected diff output - " + p.output)
1884 # since everything verifies, delete the comparison to keep cruft down
1885 shutil.rmtree(apk1dir)
1886 shutil.rmtree(apk2dir)
1888 # If we get here, it seems like they're the same!
1892 def find_command(command):
1893 '''find the full path of a command, or None if it can't be found in the PATH'''
1896 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1898 fpath, fname = os.path.split(command)
1903 for path in os.environ["PATH"].split(os.pathsep):
1904 path = path.strip('"')
1905 exe_file = os.path.join(path, command)
1906 if is_exe(exe_file):
1913 '''generate a random password for when generating keys'''
1914 h = hashlib.sha256()
1915 h.update(os.urandom(16)) # salt
1916 h.update(bytes(socket.getfqdn()))
1917 return h.digest().encode('base64').strip()
1920 def genkeystore(localconfig):
1921 '''Generate a new key with random passwords and add it to new keystore'''
1922 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1923 keystoredir = os.path.dirname(localconfig['keystore'])
1924 if keystoredir is None or keystoredir == '':
1925 keystoredir = os.path.join(os.getcwd(), keystoredir)
1926 if not os.path.exists(keystoredir):
1927 os.makedirs(keystoredir, mode=0o700)
1929 write_password_file("keystorepass", localconfig['keystorepass'])
1930 write_password_file("keypass", localconfig['keypass'])
1931 p = FDroidPopen(['keytool', '-genkey',
1932 '-keystore', localconfig['keystore'],
1933 '-alias', localconfig['repo_keyalias'],
1934 '-keyalg', 'RSA', '-keysize', '4096',
1935 '-sigalg', 'SHA256withRSA',
1936 '-validity', '10000',
1937 '-storepass:file', config['keystorepassfile'],
1938 '-keypass:file', config['keypassfile'],
1939 '-dname', localconfig['keydname']])
1940 # TODO keypass should be sent via stdin
1941 if p.returncode != 0:
1942 raise BuildException("Failed to generate key", p.output)
1943 os.chmod(localconfig['keystore'], 0o0600)
1944 # now show the lovely key that was just generated
1945 p = FDroidPopen(['keytool', '-list', '-v',
1946 '-keystore', localconfig['keystore'],
1947 '-alias', localconfig['repo_keyalias'],
1948 '-storepass:file', config['keystorepassfile']])
1949 logging.info(p.output.strip() + '\n\n')
1952 def write_to_config(thisconfig, key, value=None):
1953 '''write a key/value to the local config.py'''
1955 origkey = key + '_orig'
1956 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1957 with open('config.py', 'r') as f:
1959 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1960 repl = '\n' + key + ' = "' + value + '"'
1961 data = re.sub(pattern, repl, data)
1962 # if this key is not in the file, append it
1963 if not re.match('\s*' + key + '\s*=\s*"', data):
1965 # make sure the file ends with a carraige return
1966 if not re.match('\n$', data):
1968 with open('config.py', 'w') as f:
1972 def parse_xml(path):
1973 return XMLElementTree.parse(path).getroot()
1976 def string_is_integer(string):
1984 def get_per_app_repos():
1985 '''per-app repos are dirs named with the packageName of a single app'''
1987 # Android packageNames are Java packages, they may contain uppercase or
1988 # lowercase letters ('A' through 'Z'), numbers, and underscores
1989 # ('_'). However, individual package name parts may only start with
1990 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1991 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1994 for root, dirs, files in os.walk(os.getcwd()):
1996 print 'checking', root, 'for', d
1997 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1998 # standard parts of an fdroid repo, so never packageNames
2001 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):