1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
36 import xml.etree.ElementTree as XMLElementTree
38 from distutils.version import LooseVersion
39 from zipfile import ZipFile
42 from fdroidserver.asynchronousfilereader import AsynchronousFileReader
45 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
54 'sdk_path': "$ANDROID_HOME",
57 'r10e': "$ANDROID_NDK",
59 'build_tools': "23.0.2",
61 '1.7': "/usr/lib/jvm/java-7-openjdk",
67 'accepted_formats': ['txt', 'yaml'],
68 'sync_from_local_copy_dir': False,
69 'per_app_repos': False,
70 'make_current_version_link': True,
71 'current_version_name_source': 'Name',
72 'update_stats': False,
76 'stats_to_carbon': False,
78 'build_server_always': False,
79 'keystore': 'keystore.jks',
80 'smartcardoptions': [],
86 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
87 'repo_name': "My First FDroid Repo Demo",
88 'repo_icon': "fdroid-icon.png",
89 'repo_description': '''
90 This is a repository of apps to be used with FDroid. Applications in this
91 repository are either official binaries built by the original application
92 developers, or are binaries built from source by the admin of f-droid.org
93 using the tools on https://gitlab.com/u/fdroid.
99 def setup_global_opts(parser):
100 parser.add_argument("-v", "--verbose", action="store_true", default=False,
101 help="Spew out even more information than normal")
102 parser.add_argument("-q", "--quiet", action="store_true", default=False,
103 help="Restrict output to warnings and errors")
106 def fill_config_defaults(thisconfig):
107 for k, v in default_config.items():
108 if k not in thisconfig:
111 # Expand paths (~users and $vars)
112 def expand_path(path):
116 path = os.path.expanduser(path)
117 path = os.path.expandvars(path)
122 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
127 thisconfig[k + '_orig'] = v
129 for k in ['ndk_paths', 'java_paths']:
135 thisconfig[k][k2] = exp
136 thisconfig[k][k2 + '_orig'] = v
139 def regsub_file(pattern, repl, path):
140 with open(path, 'r') as f:
142 text = re.sub(pattern, repl, text)
143 with open(path, 'w') as f:
147 def read_config(opts, config_file='config.py'):
148 """Read the repository config
150 The config is read from config_file, which is in the current directory when
151 any of the repo management commands are used.
153 global config, options, env, orig_path
155 if config is not None:
157 if not os.path.isfile(config_file):
158 logging.critical("Missing config file - is this a repo directory?")
165 logging.debug("Reading %s" % config_file)
166 execfile(config_file, config)
168 # smartcardoptions must be a list since its command line args for Popen
169 if 'smartcardoptions' in config:
170 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
171 elif 'keystore' in config and config['keystore'] == 'NONE':
172 # keystore='NONE' means use smartcard, these are required defaults
173 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
174 'SunPKCS11-OpenSC', '-providerClass',
175 'sun.security.pkcs11.SunPKCS11',
176 '-providerArg', 'opensc-fdroid.cfg']
178 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
179 st = os.stat(config_file)
180 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
181 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
183 fill_config_defaults(config)
185 # There is no standard, so just set up the most common environment
188 orig_path = env['PATH']
189 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
190 env[n] = config['sdk_path']
193 cpath = config['java_paths']['1.%s' % v]
195 env['JAVA%s_HOME' % v] = cpath
197 for k in ["keystorepass", "keypass"]:
199 write_password_file(k)
201 for k in ["repo_description", "archive_description"]:
203 config[k] = clean_description(config[k])
205 if 'serverwebroot' in config:
206 if isinstance(config['serverwebroot'], basestring):
207 roots = [config['serverwebroot']]
208 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
209 roots = config['serverwebroot']
211 raise TypeError('only accepts strings, lists, and tuples')
213 for rootstr in roots:
214 # since this is used with rsync, where trailing slashes have
215 # meaning, ensure there is always a trailing slash
216 if rootstr[-1] != '/':
218 rootlist.append(rootstr.replace('//', '/'))
219 config['serverwebroot'] = rootlist
224 def find_sdk_tools_cmd(cmd):
225 '''find a working path to a tool from the Android SDK'''
228 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
229 # try to find a working path to this command, in all the recent possible paths
230 if 'build_tools' in config:
231 build_tools = os.path.join(config['sdk_path'], 'build-tools')
232 # if 'build_tools' was manually set and exists, check only that one
233 configed_build_tools = os.path.join(build_tools, config['build_tools'])
234 if os.path.exists(configed_build_tools):
235 tooldirs.append(configed_build_tools)
237 # no configed version, so hunt known paths for it
238 for f in sorted(os.listdir(build_tools), reverse=True):
239 if os.path.isdir(os.path.join(build_tools, f)):
240 tooldirs.append(os.path.join(build_tools, f))
241 tooldirs.append(build_tools)
242 sdk_tools = os.path.join(config['sdk_path'], 'tools')
243 if os.path.exists(sdk_tools):
244 tooldirs.append(sdk_tools)
245 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
246 if os.path.exists(sdk_platform_tools):
247 tooldirs.append(sdk_platform_tools)
248 tooldirs.append('/usr/bin')
250 if os.path.isfile(os.path.join(d, cmd)):
251 return os.path.join(d, cmd)
252 # did not find the command, exit with error message
253 ensure_build_tools_exists(config)
256 def test_sdk_exists(thisconfig):
257 if 'sdk_path' not in thisconfig:
258 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
261 logging.error("'sdk_path' not set in config.py!")
263 if thisconfig['sdk_path'] == default_config['sdk_path']:
264 logging.error('No Android SDK found!')
265 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
266 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
268 if not os.path.exists(thisconfig['sdk_path']):
269 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
271 if not os.path.isdir(thisconfig['sdk_path']):
272 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
274 for d in ['build-tools', 'platform-tools', 'tools']:
275 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
276 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
277 thisconfig['sdk_path'], d))
282 def ensure_build_tools_exists(thisconfig):
283 if not test_sdk_exists(thisconfig):
285 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
286 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
287 if not os.path.isdir(versioned_build_tools):
288 logging.critical('Android Build Tools path "'
289 + versioned_build_tools + '" does not exist!')
293 def write_password_file(pwtype, password=None):
295 writes out passwords to a protected file instead of passing passwords as
296 command line argments
298 filename = '.fdroid.' + pwtype + '.txt'
299 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
301 os.write(fd, config[pwtype])
303 os.write(fd, password)
305 config[pwtype + 'file'] = filename
308 # Given the arguments in the form of multiple appid:[vc] strings, this returns
309 # a dictionary with the set of vercodes specified for each package.
310 def read_pkg_args(args, allow_vercodes=False):
317 if allow_vercodes and ':' in p:
318 package, vercode = p.split(':')
320 package, vercode = p, None
321 if package not in vercodes:
322 vercodes[package] = [vercode] if vercode else []
324 elif vercode and vercode not in vercodes[package]:
325 vercodes[package] += [vercode] if vercode else []
330 # On top of what read_pkg_args does, this returns the whole app metadata, but
331 # limiting the builds list to the builds matching the vercodes specified.
332 def read_app_args(args, allapps, allow_vercodes=False):
334 vercodes = read_pkg_args(args, allow_vercodes)
340 for appid, app in allapps.iteritems():
341 if appid in vercodes:
344 if len(apps) != len(vercodes):
347 logging.critical("No such package: %s" % p)
348 raise FDroidException("Found invalid app ids in arguments")
350 raise FDroidException("No packages specified")
353 for appid, app in apps.iteritems():
357 app.builds = [b for b in app.builds if b.vercode in vc]
358 if len(app.builds) != len(vercodes[appid]):
360 allvcs = [b.vercode for b in app.builds]
361 for v in vercodes[appid]:
363 logging.critical("No such vercode %s for app %s" % (v, appid))
366 raise FDroidException("Found invalid vercodes for some apps")
371 def get_extension(filename):
372 base, ext = os.path.splitext(filename)
375 return base, ext.lower()[1:]
378 def has_extension(filename, ext):
379 _, f_ext = get_extension(filename)
383 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
386 def clean_description(description):
387 'Remove unneeded newlines and spaces from a block of description text'
389 # this is split up by paragraph to make removing the newlines easier
390 for paragraph in re.split(r'\n\n', description):
391 paragraph = re.sub('\r', '', paragraph)
392 paragraph = re.sub('\n', ' ', paragraph)
393 paragraph = re.sub(' {2,}', ' ', paragraph)
394 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
395 returnstring += paragraph + '\n\n'
396 return returnstring.rstrip('\n')
399 def apknameinfo(filename):
400 filename = os.path.basename(filename)
401 m = apk_regex.match(filename)
403 result = (m.group(1), m.group(2))
404 except AttributeError:
405 raise FDroidException("Invalid apk name: %s" % filename)
409 def getapkname(app, build):
410 return "%s_%s.apk" % (app.id, build.vercode)
413 def getsrcname(app, build):
414 return "%s_%s_src.tar.gz" % (app.id, build.vercode)
426 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
429 def getvcs(vcstype, remote, local):
431 return vcs_git(remote, local)
432 if vcstype == 'git-svn':
433 return vcs_gitsvn(remote, local)
435 return vcs_hg(remote, local)
437 return vcs_bzr(remote, local)
438 if vcstype == 'srclib':
439 if local != os.path.join('build', 'srclib', remote):
440 raise VCSException("Error: srclib paths are hard-coded!")
441 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
443 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
444 raise VCSException("Invalid vcs type " + vcstype)
447 def getsrclibvcs(name):
448 if name not in metadata.srclibs:
449 raise VCSException("Missing srclib " + name)
450 return metadata.srclibs[name]['Repo Type']
455 def __init__(self, remote, local):
457 # svn, git-svn and bzr may require auth
459 if self.repotype() in ('git-svn', 'bzr'):
461 if self.repotype == 'git-svn':
462 raise VCSException("Authentication is not supported for git-svn")
463 self.username, remote = remote.split('@')
464 if ':' not in self.username:
465 raise VCSException("Password required with username")
466 self.username, self.password = self.username.split(':')
470 self.clone_failed = False
471 self.refreshed = False
477 # Take the local repository to a clean version of the given revision, which
478 # is specificed in the VCS's native format. Beforehand, the repository can
479 # be dirty, or even non-existent. If the repository does already exist
480 # locally, it will be updated from the origin, but only once in the
481 # lifetime of the vcs object.
482 # None is acceptable for 'rev' if you know you are cloning a clean copy of
483 # the repo - otherwise it must specify a valid revision.
484 def gotorevision(self, rev, refresh=True):
486 if self.clone_failed:
487 raise VCSException("Downloading the repository already failed once, not trying again.")
489 # The .fdroidvcs-id file for a repo tells us what VCS type
490 # and remote that directory was created from, allowing us to drop it
491 # automatically if either of those things changes.
492 fdpath = os.path.join(self.local, '..',
493 '.fdroidvcs-' + os.path.basename(self.local))
494 cdata = self.repotype() + ' ' + self.remote
497 if os.path.exists(self.local):
498 if os.path.exists(fdpath):
499 with open(fdpath, 'r') as f:
500 fsdata = f.read().strip()
505 logging.info("Repository details for %s changed - deleting" % (
509 logging.info("Repository details for %s missing - deleting" % (
512 shutil.rmtree(self.local)
516 self.refreshed = True
519 self.gotorevisionx(rev)
520 except FDroidException, e:
523 # If necessary, write the .fdroidvcs file.
524 if writeback and not self.clone_failed:
525 with open(fdpath, 'w') as f:
531 # Derived classes need to implement this. It's called once basic checking
532 # has been performend.
533 def gotorevisionx(self, rev):
534 raise VCSException("This VCS type doesn't define gotorevisionx")
536 # Initialise and update submodules
537 def initsubmodules(self):
538 raise VCSException('Submodules not supported for this vcs type')
540 # Get a list of all known tags
542 if not self._gettags:
543 raise VCSException('gettags not supported for this vcs type')
545 for tag in self._gettags():
546 if re.match('[-A-Za-z0-9_. /]+$', tag):
550 def latesttags(self, tags, number):
551 """Get the most recent tags in a given list.
553 :param tags: a list of tags
554 :param number: the number to return
555 :returns: A list containing the most recent tags in the provided
556 list, up to the maximum number given.
558 raise VCSException('latesttags not supported for this vcs type')
560 # Get current commit reference (hash, revision, etc)
562 raise VCSException('getref not supported for this vcs type')
564 # Returns the srclib (name, path) used in setting up the current
575 # If the local directory exists, but is somehow not a git repository, git
576 # will traverse up the directory tree until it finds one that is (i.e.
577 # fdroidserver) and then we'll proceed to destroy it! This is called as
580 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
581 result = p.output.rstrip()
582 if not result.endswith(self.local):
583 raise VCSException('Repository mismatch')
585 def gotorevisionx(self, rev):
586 if not os.path.exists(self.local):
588 p = FDroidPopen(['git', 'clone', self.remote, self.local])
589 if p.returncode != 0:
590 self.clone_failed = True
591 raise VCSException("Git clone failed", p.output)
595 # Discard any working tree changes
596 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
597 'git', 'reset', '--hard'], cwd=self.local, output=False)
598 if p.returncode != 0:
599 raise VCSException("Git reset failed", p.output)
600 # Remove untracked files now, in case they're tracked in the target
601 # revision (it happens!)
602 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
603 'git', 'clean', '-dffx'], cwd=self.local, output=False)
604 if p.returncode != 0:
605 raise VCSException("Git clean failed", p.output)
606 if not self.refreshed:
607 # Get latest commits and tags from remote
608 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
609 if p.returncode != 0:
610 raise VCSException("Git fetch failed", p.output)
611 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
612 if p.returncode != 0:
613 raise VCSException("Git fetch failed", p.output)
614 # Recreate origin/HEAD as git clone would do it, in case it disappeared
615 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
616 if p.returncode != 0:
617 lines = p.output.splitlines()
618 if 'Multiple remote HEAD branches' not in lines[0]:
619 raise VCSException("Git remote set-head failed", p.output)
620 branch = lines[1].split(' ')[-1]
621 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
622 if p2.returncode != 0:
623 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
624 self.refreshed = True
625 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
626 # a github repo. Most of the time this is the same as origin/master.
627 rev = rev or 'origin/HEAD'
628 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
629 if p.returncode != 0:
630 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
631 # Get rid of any uncontrolled files left behind
632 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
633 if p.returncode != 0:
634 raise VCSException("Git clean failed", p.output)
636 def initsubmodules(self):
638 submfile = os.path.join(self.local, '.gitmodules')
639 if not os.path.isfile(submfile):
640 raise VCSException("No git submodules available")
642 # fix submodules not accessible without an account and public key auth
643 with open(submfile, 'r') as f:
644 lines = f.readlines()
645 with open(submfile, 'w') as f:
647 if 'git@github.com' in line:
648 line = line.replace('git@github.com:', 'https://github.com/')
649 if 'git@gitlab.com' in line:
650 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
653 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
654 if p.returncode != 0:
655 raise VCSException("Git submodule sync failed", p.output)
656 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
657 if p.returncode != 0:
658 raise VCSException("Git submodule update failed", p.output)
662 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
663 return p.output.splitlines()
665 def latesttags(self, tags, number):
670 ['git', 'show', '--format=format:%ct', '-s', tag],
671 cwd=self.local, output=False)
672 # Timestamp is on the last line. For a normal tag, it's the only
673 # line, but for annotated tags, the rest of the info precedes it.
674 ts = int(p.output.splitlines()[-1])
677 for _, t in sorted(tl)[-number:]:
682 class vcs_gitsvn(vcs):
687 # If the local directory exists, but is somehow not a git repository, git
688 # will traverse up the directory tree until it finds one that is (i.e.
689 # fdroidserver) and then we'll proceed to destory it! This is called as
692 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
693 result = p.output.rstrip()
694 if not result.endswith(self.local):
695 raise VCSException('Repository mismatch')
697 def gotorevisionx(self, rev):
698 if not os.path.exists(self.local):
700 gitsvn_args = ['git', 'svn', 'clone']
701 if ';' in self.remote:
702 remote_split = self.remote.split(';')
703 for i in remote_split[1:]:
704 if i.startswith('trunk='):
705 gitsvn_args.extend(['-T', i[6:]])
706 elif i.startswith('tags='):
707 gitsvn_args.extend(['-t', i[5:]])
708 elif i.startswith('branches='):
709 gitsvn_args.extend(['-b', i[9:]])
710 gitsvn_args.extend([remote_split[0], self.local])
711 p = FDroidPopen(gitsvn_args, output=False)
712 if p.returncode != 0:
713 self.clone_failed = True
714 raise VCSException("Git svn clone failed", p.output)
716 gitsvn_args.extend([self.remote, self.local])
717 p = FDroidPopen(gitsvn_args, output=False)
718 if p.returncode != 0:
719 self.clone_failed = True
720 raise VCSException("Git svn clone failed", p.output)
724 # Discard any working tree changes
725 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
726 if p.returncode != 0:
727 raise VCSException("Git reset failed", p.output)
728 # Remove untracked files now, in case they're tracked in the target
729 # revision (it happens!)
730 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
731 if p.returncode != 0:
732 raise VCSException("Git clean failed", p.output)
733 if not self.refreshed:
734 # Get new commits, branches and tags from repo
735 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
736 if p.returncode != 0:
737 raise VCSException("Git svn fetch failed")
738 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
739 if p.returncode != 0:
740 raise VCSException("Git svn rebase failed", p.output)
741 self.refreshed = True
743 rev = rev or 'master'
745 nospaces_rev = rev.replace(' ', '%20')
746 # Try finding a svn tag
747 for treeish in ['origin/', '']:
748 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
749 if p.returncode == 0:
751 if p.returncode != 0:
752 # No tag found, normal svn rev translation
753 # Translate svn rev into git format
754 rev_split = rev.split('/')
757 for treeish in ['origin/', '']:
758 if len(rev_split) > 1:
759 treeish += rev_split[0]
760 svn_rev = rev_split[1]
763 # if no branch is specified, then assume trunk (i.e. 'master' branch):
767 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
769 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
770 git_rev = p.output.rstrip()
772 if p.returncode == 0 and git_rev:
775 if p.returncode != 0 or not git_rev:
776 # Try a plain git checkout as a last resort
777 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
778 if p.returncode != 0:
779 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
781 # Check out the git rev equivalent to the svn rev
782 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
783 if p.returncode != 0:
784 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
786 # Get rid of any uncontrolled files left behind
787 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
788 if p.returncode != 0:
789 raise VCSException("Git clean failed", p.output)
793 for treeish in ['origin/', '']:
794 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
800 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
801 if p.returncode != 0:
803 return p.output.strip()
811 def gotorevisionx(self, rev):
812 if not os.path.exists(self.local):
813 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
814 if p.returncode != 0:
815 self.clone_failed = True
816 raise VCSException("Hg clone failed", p.output)
818 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
819 if p.returncode != 0:
820 raise VCSException("Hg status failed", p.output)
821 for line in p.output.splitlines():
822 if not line.startswith('? '):
823 raise VCSException("Unexpected output from hg status -uS: " + line)
824 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
825 if not self.refreshed:
826 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
827 if p.returncode != 0:
828 raise VCSException("Hg pull failed", p.output)
829 self.refreshed = True
831 rev = rev or 'default'
834 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
835 if p.returncode != 0:
836 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
837 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
838 # Also delete untracked files, we have to enable purge extension for that:
839 if "'purge' is provided by the following extension" in p.output:
840 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
841 myfile.write("\n[extensions]\nhgext.purge=\n")
842 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException("HG purge failed", p.output)
845 elif p.returncode != 0:
846 raise VCSException("HG purge failed", p.output)
849 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
850 return p.output.splitlines()[1:]
858 def gotorevisionx(self, rev):
859 if not os.path.exists(self.local):
860 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
861 if p.returncode != 0:
862 self.clone_failed = True
863 raise VCSException("Bzr branch failed", p.output)
865 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
866 if p.returncode != 0:
867 raise VCSException("Bzr revert failed", p.output)
868 if not self.refreshed:
869 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
870 if p.returncode != 0:
871 raise VCSException("Bzr update failed", p.output)
872 self.refreshed = True
874 revargs = list(['-r', rev] if rev else [])
875 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
880 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
881 return [tag.split(' ')[0].strip() for tag in
882 p.output.splitlines()]
885 def unescape_string(string):
888 if string[0] == '"' and string[-1] == '"':
891 return string.replace("\\'", "'")
894 def retrieve_string(app_dir, string, xmlfiles=None):
896 if not string.startswith('@string/'):
897 return unescape_string(string)
902 os.path.join(app_dir, 'res'),
903 os.path.join(app_dir, 'src', 'main', 'res'),
905 for r, d, f in os.walk(res_dir):
906 if os.path.basename(r) == 'values':
907 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
909 name = string[len('@string/'):]
911 def element_content(element):
912 if element.text is None:
914 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
917 for path in xmlfiles:
918 if not os.path.isfile(path):
920 xml = parse_xml(path)
921 element = xml.find('string[@name="' + name + '"]')
922 if element is not None:
923 content = element_content(element)
924 return retrieve_string(app_dir, content, xmlfiles)
929 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
930 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
933 # Return list of existing files that will be used to find the highest vercode
934 def manifest_paths(app_dir, flavours):
936 possible_manifests = \
937 [os.path.join(app_dir, 'AndroidManifest.xml'),
938 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
939 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
940 os.path.join(app_dir, 'build.gradle')]
942 for flavour in flavours:
945 possible_manifests.append(
946 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
948 return [path for path in possible_manifests if os.path.isfile(path)]
951 # Retrieve the package name. Returns the name, or None if not found.
952 def fetch_real_name(app_dir, flavours):
953 for path in manifest_paths(app_dir, flavours):
954 if not has_extension(path, 'xml') or not os.path.isfile(path):
956 logging.debug("fetch_real_name: Checking manifest at " + path)
957 xml = parse_xml(path)
958 app = xml.find('application')
961 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
963 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
964 result = retrieve_string_singleline(app_dir, label)
966 result = result.strip()
971 def get_library_references(root_dir):
973 proppath = os.path.join(root_dir, 'project.properties')
974 if not os.path.isfile(proppath):
976 for line in file(proppath):
977 if not line.startswith('android.library.reference.'):
979 path = line.split('=')[1].strip()
980 relpath = os.path.join(root_dir, path)
981 if not os.path.isdir(relpath):
983 logging.debug("Found subproject at %s" % path)
984 libraries.append(path)
988 def ant_subprojects(root_dir):
989 subprojects = get_library_references(root_dir)
990 for subpath in subprojects:
991 subrelpath = os.path.join(root_dir, subpath)
992 for p in get_library_references(subrelpath):
993 relp = os.path.normpath(os.path.join(subpath, p))
994 if relp not in subprojects:
995 subprojects.insert(0, relp)
999 def remove_debuggable_flags(root_dir):
1000 # Remove forced debuggable flags
1001 logging.debug("Removing debuggable flags from %s" % root_dir)
1002 for root, dirs, files in os.walk(root_dir):
1003 if 'AndroidManifest.xml' in files:
1004 regsub_file(r'android:debuggable="[^"]*"',
1006 os.path.join(root, 'AndroidManifest.xml'))
1009 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1010 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1011 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1014 def app_matches_packagename(app, package):
1017 appid = app.UpdateCheckName or app.id
1018 if appid is None or appid == "Ignore":
1020 return appid == package
1023 # Extract some information from the AndroidManifest.xml at the given path.
1024 # Returns (version, vercode, package), any or all of which might be None.
1025 # All values returned are strings.
1026 def parse_androidmanifests(paths, app):
1028 ignoreversions = app.UpdateCheckIgnore
1029 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1032 return (None, None, None)
1040 if not os.path.isfile(path):
1043 logging.debug("Parsing manifest at {0}".format(path))
1044 gradle = has_extension(path, 'gradle')
1050 for line in file(path):
1051 if gradle_comment.match(line):
1053 # Grab first occurence of each to avoid running into
1054 # alternative flavours and builds.
1056 matches = psearch_g(line)
1058 s = matches.group(2)
1059 if app_matches_packagename(app, s):
1062 matches = vnsearch_g(line)
1064 version = matches.group(2)
1066 matches = vcsearch_g(line)
1068 vercode = matches.group(1)
1070 xml = parse_xml(path)
1071 if "package" in xml.attrib:
1072 s = xml.attrib["package"].encode('utf-8')
1073 if app_matches_packagename(app, s):
1075 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1076 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1077 base_dir = os.path.dirname(path)
1078 version = retrieve_string_singleline(base_dir, version)
1079 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1080 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1081 if string_is_integer(a):
1084 # Remember package name, may be defined separately from version+vercode
1086 package = max_package
1088 logging.debug("..got package={0}, version={1}, vercode={2}"
1089 .format(package, version, vercode))
1091 # Always grab the package name and version name in case they are not
1092 # together with the highest version code
1093 if max_package is None and package is not None:
1094 max_package = package
1095 if max_version is None and version is not None:
1096 max_version = version
1098 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1099 if not ignoresearch or not ignoresearch(version):
1100 if version is not None:
1101 max_version = version
1102 if vercode is not None:
1103 max_vercode = vercode
1104 if package is not None:
1105 max_package = package
1107 max_version = "Ignore"
1109 if max_version is None:
1110 max_version = "Unknown"
1112 if max_package and not is_valid_package_name(max_package):
1113 raise FDroidException("Invalid package name {0}".format(max_package))
1115 return (max_version, max_vercode, max_package)
1118 def is_valid_package_name(name):
1119 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1122 class FDroidException(Exception):
1124 def __init__(self, value, detail=None):
1126 self.detail = detail
1128 def shortened_detail(self):
1129 if len(self.detail) < 16000:
1131 return '[...]\n' + self.detail[-16000:]
1133 def get_wikitext(self):
1134 ret = repr(self.value) + "\n"
1137 ret += "<pre>\n" + self.shortened_detail() + "</pre>\n"
1143 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1147 class VCSException(FDroidException):
1151 class BuildException(FDroidException):
1155 # Get the specified source library.
1156 # Returns the path to it. Normally this is the path to be used when referencing
1157 # it, which may be a subdirectory of the actual project. If you want the base
1158 # directory of the project, pass 'basepath=True'.
1159 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1160 raw=False, prepare=True, preponly=False, refresh=True):
1168 name, ref = spec.split('@')
1170 number, name = name.split(':', 1)
1172 name, subdir = name.split('/', 1)
1174 if name not in metadata.srclibs:
1175 raise VCSException('srclib ' + name + ' not found.')
1177 srclib = metadata.srclibs[name]
1179 sdir = os.path.join(srclib_dir, name)
1182 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1183 vcs.srclib = (name, number, sdir)
1185 vcs.gotorevision(ref, refresh)
1192 libdir = os.path.join(sdir, subdir)
1193 elif srclib["Subdir"]:
1194 for subdir in srclib["Subdir"]:
1195 libdir_candidate = os.path.join(sdir, subdir)
1196 if os.path.exists(libdir_candidate):
1197 libdir = libdir_candidate
1203 remove_signing_keys(sdir)
1204 remove_debuggable_flags(sdir)
1208 if srclib["Prepare"]:
1209 cmd = replace_config_vars(srclib["Prepare"], None)
1211 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1212 if p.returncode != 0:
1213 raise BuildException("Error running prepare command for srclib %s"
1219 return (name, number, libdir)
1221 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1224 # Prepare the source code for a particular build
1225 # 'vcs' - the appropriate vcs object for the application
1226 # 'app' - the application details from the metadata
1227 # 'build' - the build details from the metadata
1228 # 'build_dir' - the path to the build directory, usually
1230 # 'srclib_dir' - the path to the source libraries directory, usually
1232 # 'extlib_dir' - the path to the external libraries directory, usually
1234 # Returns the (root, srclibpaths) where:
1235 # 'root' is the root directory, which may be the same as 'build_dir' or may
1236 # be a subdirectory of it.
1237 # 'srclibpaths' is information on the srclibs being used
1238 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1240 # Optionally, the actual app source can be in a subdirectory
1242 root_dir = os.path.join(build_dir, build.subdir)
1244 root_dir = build_dir
1246 # Get a working copy of the right revision
1247 logging.info("Getting source for revision " + build.commit)
1248 vcs.gotorevision(build.commit, refresh)
1250 # Initialise submodules if required
1251 if build.submodules:
1252 logging.info("Initialising submodules")
1253 vcs.initsubmodules()
1255 # Check that a subdir (if we're using one) exists. This has to happen
1256 # after the checkout, since it might not exist elsewhere
1257 if not os.path.exists(root_dir):
1258 raise BuildException('Missing subdir ' + root_dir)
1260 # Run an init command if one is required
1262 cmd = replace_config_vars(build.init, build)
1263 logging.info("Running 'init' commands in %s" % root_dir)
1265 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1266 if p.returncode != 0:
1267 raise BuildException("Error running init command for %s:%s" %
1268 (app.id, build.version), p.output)
1270 # Apply patches if any
1272 logging.info("Applying patches")
1273 for patch in build.patch:
1274 patch = patch.strip()
1275 logging.info("Applying " + patch)
1276 patch_path = os.path.join('metadata', app.id, patch)
1277 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1278 if p.returncode != 0:
1279 raise BuildException("Failed to apply patch %s" % patch_path)
1281 # Get required source libraries
1284 logging.info("Collecting source libraries")
1285 for lib in build.srclibs:
1286 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1288 for name, number, libpath in srclibpaths:
1289 place_srclib(root_dir, int(number) if number else None, libpath)
1291 basesrclib = vcs.getsrclib()
1292 # If one was used for the main source, add that too.
1294 srclibpaths.append(basesrclib)
1296 # Update the local.properties file
1297 localprops = [os.path.join(build_dir, 'local.properties')]
1299 parts = build.subdir.split(os.sep)
1302 cur = os.path.join(cur, d)
1303 localprops += [os.path.join(cur, 'local.properties')]
1304 for path in localprops:
1306 if os.path.isfile(path):
1307 logging.info("Updating local.properties file at %s" % path)
1308 with open(path, 'r') as f:
1312 logging.info("Creating local.properties file at %s" % path)
1313 # Fix old-fashioned 'sdk-location' by copying
1314 # from sdk.dir, if necessary
1316 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1317 re.S | re.M).group(1)
1318 props += "sdk-location=%s\n" % sdkloc
1320 props += "sdk.dir=%s\n" % config['sdk_path']
1321 props += "sdk-location=%s\n" % config['sdk_path']
1322 ndk_path = build.ndk_path()
1325 props += "ndk.dir=%s\n" % ndk_path
1326 props += "ndk-location=%s\n" % ndk_path
1327 # Add java.encoding if necessary
1329 props += "java.encoding=%s\n" % build.encoding
1330 with open(path, 'w') as f:
1334 if build.method() == 'gradle':
1335 flavours = build.gradle
1337 gradlepluginver = None
1339 gradle_dirs = [root_dir]
1341 # Parent dir build.gradle
1342 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1343 if parent_dir.startswith(build_dir):
1344 gradle_dirs.append(parent_dir)
1346 for dir_path in gradle_dirs:
1349 if not os.path.isdir(dir_path):
1351 for filename in os.listdir(dir_path):
1352 if not filename.endswith('.gradle'):
1354 path = os.path.join(dir_path, filename)
1355 if not os.path.isfile(path):
1357 for line in file(path):
1358 match = gradle_version_regex.match(line)
1360 gradlepluginver = match.group(1)
1364 build.gradlepluginver = LooseVersion(gradlepluginver)
1366 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1367 build.gradlepluginver = LooseVersion('0.11')
1370 n = build.target.split('-')[1]
1371 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1372 r'compileSdkVersion %s' % n,
1373 os.path.join(root_dir, 'build.gradle'))
1375 # Remove forced debuggable flags
1376 remove_debuggable_flags(root_dir)
1378 # Insert version code and number into the manifest if necessary
1379 if build.forceversion:
1380 logging.info("Changing the version name")
1381 for path in manifest_paths(root_dir, flavours):
1382 if not os.path.isfile(path):
1384 if has_extension(path, 'xml'):
1385 regsub_file(r'android:versionName="[^"]*"',
1386 r'android:versionName="%s"' % build.version,
1388 elif has_extension(path, 'gradle'):
1389 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1390 r"""\1versionName '%s'""" % build.version,
1393 if build.forcevercode:
1394 logging.info("Changing the version code")
1395 for path in manifest_paths(root_dir, flavours):
1396 if not os.path.isfile(path):
1398 if has_extension(path, 'xml'):
1399 regsub_file(r'android:versionCode="[^"]*"',
1400 r'android:versionCode="%s"' % build.vercode,
1402 elif has_extension(path, 'gradle'):
1403 regsub_file(r'versionCode[ =]+[0-9]+',
1404 r'versionCode %s' % build.vercode,
1407 # Delete unwanted files
1409 logging.info("Removing specified files")
1410 for part in getpaths(build_dir, build.rm):
1411 dest = os.path.join(build_dir, part)
1412 logging.info("Removing {0}".format(part))
1413 if os.path.lexists(dest):
1414 if os.path.islink(dest):
1415 FDroidPopen(['unlink', dest], output=False)
1417 FDroidPopen(['rm', '-rf', dest], output=False)
1419 logging.info("...but it didn't exist")
1421 remove_signing_keys(build_dir)
1423 # Add required external libraries
1425 logging.info("Collecting prebuilt libraries")
1426 libsdir = os.path.join(root_dir, 'libs')
1427 if not os.path.exists(libsdir):
1429 for lib in build.extlibs:
1431 logging.info("...installing extlib {0}".format(lib))
1432 libf = os.path.basename(lib)
1433 libsrc = os.path.join(extlib_dir, lib)
1434 if not os.path.exists(libsrc):
1435 raise BuildException("Missing extlib file {0}".format(libsrc))
1436 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1438 # Run a pre-build command if one is required
1440 logging.info("Running 'prebuild' commands in %s" % root_dir)
1442 cmd = replace_config_vars(build.prebuild, build)
1444 # Substitute source library paths into prebuild commands
1445 for name, number, libpath in srclibpaths:
1446 libpath = os.path.relpath(libpath, root_dir)
1447 cmd = cmd.replace('$$' + name + '$$', libpath)
1449 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1450 if p.returncode != 0:
1451 raise BuildException("Error running prebuild command for %s:%s" %
1452 (app.id, build.version), p.output)
1454 # Generate (or update) the ant build file, build.xml...
1455 if build.method() == 'ant' and build.update != ['no']:
1456 parms = ['android', 'update', 'lib-project']
1457 lparms = ['android', 'update', 'project']
1460 parms += ['-t', build.target]
1461 lparms += ['-t', build.target]
1463 update_dirs = build.update
1465 update_dirs = ant_subprojects(root_dir) + ['.']
1467 for d in update_dirs:
1468 subdir = os.path.join(root_dir, d)
1470 logging.debug("Updating main project")
1471 cmd = parms + ['-p', d]
1473 logging.debug("Updating subproject %s" % d)
1474 cmd = lparms + ['-p', d]
1475 p = SdkToolsPopen(cmd, cwd=root_dir)
1476 # Check to see whether an error was returned without a proper exit
1477 # code (this is the case for the 'no target set or target invalid'
1479 if p.returncode != 0 or p.output.startswith("Error: "):
1480 raise BuildException("Failed to update project at %s" % d, p.output)
1481 # Clean update dirs via ant
1483 logging.info("Cleaning subproject %s" % d)
1484 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1486 return (root_dir, srclibpaths)
1489 # Extend via globbing the paths from a field and return them as a map from
1490 # original path to resulting paths
1491 def getpaths_map(build_dir, globpaths):
1495 full_path = os.path.join(build_dir, p)
1496 full_path = os.path.normpath(full_path)
1497 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1499 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1503 # Extend via globbing the paths from a field and return them as a set
1504 def getpaths(build_dir, globpaths):
1505 paths_map = getpaths_map(build_dir, globpaths)
1507 for k, v in paths_map.iteritems():
1514 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1520 self.path = os.path.join('stats', 'known_apks.txt')
1522 if os.path.isfile(self.path):
1523 for line in file(self.path):
1524 t = line.rstrip().split(' ')
1526 self.apks[t[0]] = (t[1], None)
1528 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1529 self.changed = False
1531 def writeifchanged(self):
1532 if not self.changed:
1535 if not os.path.exists('stats'):
1539 for apk, app in self.apks.iteritems():
1541 line = apk + ' ' + appid
1543 line += ' ' + time.strftime('%Y-%m-%d', added)
1546 with open(self.path, 'w') as f:
1547 for line in sorted(lst, key=natural_key):
1548 f.write(line + '\n')
1550 # Record an apk (if it's new, otherwise does nothing)
1551 # Returns the date it was added.
1552 def recordapk(self, apk, app):
1553 if apk not in self.apks:
1554 self.apks[apk] = (app, time.gmtime(time.time()))
1556 _, added = self.apks[apk]
1559 # Look up information - given the 'apkname', returns (app id, date added/None).
1560 # Or returns None for an unknown apk.
1561 def getapp(self, apkname):
1562 if apkname in self.apks:
1563 return self.apks[apkname]
1566 # Get the most recent 'num' apps added to the repo, as a list of package ids
1567 # with the most recent first.
1568 def getlatest(self, num):
1570 for apk, app in self.apks.iteritems():
1574 if apps[appid] > added:
1578 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1579 lst = [app for app, _ in sortedapps]
1584 def isApkDebuggable(apkfile, config):
1585 """Returns True if the given apk file is debuggable
1587 :param apkfile: full path to the apk to check"""
1589 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1591 if p.returncode != 0:
1592 logging.critical("Failed to get apk manifest information")
1594 for line in p.output.splitlines():
1595 if 'android:debuggable' in line and not line.endswith('0x0'):
1605 def SdkToolsPopen(commands, cwd=None, output=True):
1607 if cmd not in config:
1608 config[cmd] = find_sdk_tools_cmd(commands[0])
1609 abscmd = config[cmd]
1611 logging.critical("Could not find '%s' on your system" % cmd)
1613 return FDroidPopen([abscmd] + commands[1:],
1614 cwd=cwd, output=output)
1617 def FDroidPopen(commands, cwd=None, output=True):
1619 Run a command and capture the possibly huge output.
1621 :param commands: command and argument list like in subprocess.Popen
1622 :param cwd: optionally specifies a working directory
1623 :returns: A PopenResult.
1629 cwd = os.path.normpath(cwd)
1630 logging.debug("Directory: %s" % cwd)
1631 logging.debug("> %s" % ' '.join(commands))
1633 result = PopenResult()
1636 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1637 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1639 raise BuildException("OSError while trying to execute " +
1640 ' '.join(commands) + ': ' + str(e))
1642 stdout_queue = Queue.Queue()
1643 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1645 # Check the queue for output (until there is no more to get)
1646 while not stdout_reader.eof():
1647 while not stdout_queue.empty():
1648 line = stdout_queue.get()
1649 if output and options.verbose:
1650 # Output directly to console
1651 sys.stderr.write(line)
1653 result.output += line
1657 result.returncode = p.wait()
1661 gradle_comment = re.compile(r'[ ]*//')
1662 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1663 gradle_line_matches = [
1664 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1665 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1666 re.compile(r'.*variant\.outputFile = .*'),
1667 re.compile(r'.*output\.outputFile = .*'),
1668 re.compile(r'.*\.readLine\(.*'),
1672 def remove_signing_keys(build_dir):
1673 for root, dirs, files in os.walk(build_dir):
1674 if 'build.gradle' in files:
1675 path = os.path.join(root, 'build.gradle')
1677 with open(path, "r") as o:
1678 lines = o.readlines()
1684 with open(path, "w") as o:
1685 while i < len(lines):
1688 while line.endswith('\\\n'):
1689 line = line.rstrip('\\\n') + lines[i]
1692 if gradle_comment.match(line):
1697 opened += line.count('{')
1698 opened -= line.count('}')
1701 if gradle_signing_configs.match(line):
1706 if any(s.match(line) for s in gradle_line_matches):
1714 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1717 'project.properties',
1719 'default.properties',
1720 'ant.properties', ]:
1721 if propfile in files:
1722 path = os.path.join(root, propfile)
1724 with open(path, "r") as o:
1725 lines = o.readlines()
1729 with open(path, "w") as o:
1731 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1738 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1741 def reset_env_path():
1742 global env, orig_path
1743 env['PATH'] = orig_path
1746 def add_to_env_path(path):
1748 paths = env['PATH'].split(os.pathsep)
1752 env['PATH'] = os.pathsep.join(paths)
1755 def replace_config_vars(cmd, build):
1757 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1758 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1759 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1760 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1761 if build is not None:
1762 cmd = cmd.replace('$$COMMIT$$', build.commit)
1763 cmd = cmd.replace('$$VERSION$$', build.version)
1764 cmd = cmd.replace('$$VERCODE$$', build.vercode)
1768 def place_srclib(root_dir, number, libpath):
1771 relpath = os.path.relpath(libpath, root_dir)
1772 proppath = os.path.join(root_dir, 'project.properties')
1775 if os.path.isfile(proppath):
1776 with open(proppath, "r") as o:
1777 lines = o.readlines()
1779 with open(proppath, "w") as o:
1782 if line.startswith('android.library.reference.%d=' % number):
1783 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1788 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1790 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1793 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1794 """Verify that two apks are the same
1796 One of the inputs is signed, the other is unsigned. The signature metadata
1797 is transferred from the signed to the unsigned apk, and then jarsigner is
1798 used to verify that the signature from the signed apk is also varlid for
1800 :param signed_apk: Path to a signed apk file
1801 :param unsigned_apk: Path to an unsigned apk file expected to match it
1802 :param tmp_dir: Path to directory for temporary files
1803 :returns: None if the verification is successful, otherwise a string
1804 describing what went wrong.
1806 with ZipFile(signed_apk) as signed_apk_as_zip:
1807 meta_inf_files = ['META-INF/MANIFEST.MF']
1808 for f in signed_apk_as_zip.namelist():
1809 if apk_sigfile.match(f):
1810 meta_inf_files.append(f)
1811 if len(meta_inf_files) < 3:
1812 return "Signature files missing from {0}".format(signed_apk)
1813 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1814 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1815 for meta_inf_file in meta_inf_files:
1816 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1818 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1819 logging.info("...NOT verified - {0}".format(signed_apk))
1820 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1821 logging.info("...successfully verified")
1824 apk_badchars = re.compile('''[/ :;'"]''')
1827 def compare_apks(apk1, apk2, tmp_dir):
1830 Returns None if the apk content is the same (apart from the signing key),
1831 otherwise a string describing what's different, or what went wrong when
1832 trying to do the comparison.
1835 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
1836 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
1837 for d in [apk1dir, apk2dir]:
1838 if os.path.exists(d):
1841 os.mkdir(os.path.join(d, 'jar-xf'))
1843 if subprocess.call(['jar', 'xf',
1844 os.path.abspath(apk1)],
1845 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1846 return("Failed to unpack " + apk1)
1847 if subprocess.call(['jar', 'xf',
1848 os.path.abspath(apk2)],
1849 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1850 return("Failed to unpack " + apk2)
1852 # try to find apktool in the path, if it hasn't been manually configed
1853 if 'apktool' not in config:
1854 tmp = find_command('apktool')
1856 config['apktool'] = tmp
1857 if 'apktool' in config:
1858 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1860 return("Failed to unpack " + apk1)
1861 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1863 return("Failed to unpack " + apk2)
1865 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1866 lines = p.output.splitlines()
1867 if len(lines) != 1 or 'META-INF' not in lines[0]:
1868 meld = find_command('meld')
1869 if meld is not None:
1870 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1871 return("Unexpected diff output - " + p.output)
1873 # since everything verifies, delete the comparison to keep cruft down
1874 shutil.rmtree(apk1dir)
1875 shutil.rmtree(apk2dir)
1877 # If we get here, it seems like they're the same!
1881 def find_command(command):
1882 '''find the full path of a command, or None if it can't be found in the PATH'''
1885 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1887 fpath, fname = os.path.split(command)
1892 for path in os.environ["PATH"].split(os.pathsep):
1893 path = path.strip('"')
1894 exe_file = os.path.join(path, command)
1895 if is_exe(exe_file):
1902 '''generate a random password for when generating keys'''
1903 h = hashlib.sha256()
1904 h.update(os.urandom(16)) # salt
1905 h.update(bytes(socket.getfqdn()))
1906 return h.digest().encode('base64').strip()
1909 def genkeystore(localconfig):
1910 '''Generate a new key with random passwords and add it to new keystore'''
1911 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1912 keystoredir = os.path.dirname(localconfig['keystore'])
1913 if keystoredir is None or keystoredir == '':
1914 keystoredir = os.path.join(os.getcwd(), keystoredir)
1915 if not os.path.exists(keystoredir):
1916 os.makedirs(keystoredir, mode=0o700)
1918 write_password_file("keystorepass", localconfig['keystorepass'])
1919 write_password_file("keypass", localconfig['keypass'])
1920 p = FDroidPopen(['keytool', '-genkey',
1921 '-keystore', localconfig['keystore'],
1922 '-alias', localconfig['repo_keyalias'],
1923 '-keyalg', 'RSA', '-keysize', '4096',
1924 '-sigalg', 'SHA256withRSA',
1925 '-validity', '10000',
1926 '-storepass:file', config['keystorepassfile'],
1927 '-keypass:file', config['keypassfile'],
1928 '-dname', localconfig['keydname']])
1929 # TODO keypass should be sent via stdin
1930 if p.returncode != 0:
1931 raise BuildException("Failed to generate key", p.output)
1932 os.chmod(localconfig['keystore'], 0o0600)
1933 # now show the lovely key that was just generated
1934 p = FDroidPopen(['keytool', '-list', '-v',
1935 '-keystore', localconfig['keystore'],
1936 '-alias', localconfig['repo_keyalias'],
1937 '-storepass:file', config['keystorepassfile']])
1938 logging.info(p.output.strip() + '\n\n')
1941 def write_to_config(thisconfig, key, value=None):
1942 '''write a key/value to the local config.py'''
1944 origkey = key + '_orig'
1945 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1946 with open('config.py', 'r') as f:
1948 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1949 repl = '\n' + key + ' = "' + value + '"'
1950 data = re.sub(pattern, repl, data)
1951 # if this key is not in the file, append it
1952 if not re.match('\s*' + key + '\s*=\s*"', data):
1954 # make sure the file ends with a carraige return
1955 if not re.match('\n$', data):
1957 with open('config.py', 'w') as f:
1961 def parse_xml(path):
1962 return XMLElementTree.parse(path).getroot()
1965 def string_is_integer(string):
1973 def get_per_app_repos():
1974 '''per-app repos are dirs named with the packageName of a single app'''
1976 # Android packageNames are Java packages, they may contain uppercase or
1977 # lowercase letters ('A' through 'Z'), numbers, and underscores
1978 # ('_'). However, individual package name parts may only start with
1979 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1980 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1983 for root, dirs, files in os.walk(os.getcwd()):
1985 print 'checking', root, 'for', d
1986 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1987 # standard parts of an fdroid repo, so never packageNames
1990 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):