1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
37 import xml.etree.ElementTree as XMLElementTree
39 from distutils.version import LooseVersion
40 from zipfile import ZipFile
44 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
53 'sdk_path': "$ANDROID_HOME",
56 'r10e': "$ANDROID_NDK",
58 'build_tools': "23.0.1",
62 'accepted_formats': ['txt', 'yaml'],
63 'sync_from_local_copy_dir': False,
64 'per_app_repos': False,
65 'make_current_version_link': True,
66 'current_version_name_source': 'Name',
67 'update_stats': False,
71 'stats_to_carbon': False,
73 'build_server_always': False,
74 'keystore': 'keystore.jks',
75 'smartcardoptions': [],
81 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
82 'repo_name': "My First FDroid Repo Demo",
83 'repo_icon': "fdroid-icon.png",
84 'repo_description': '''
85 This is a repository of apps to be used with FDroid. Applications in this
86 repository are either official binaries built by the original application
87 developers, or are binaries built from source by the admin of f-droid.org
88 using the tools on https://gitlab.com/u/fdroid.
94 def setup_global_opts(parser):
95 parser.add_argument("-v", "--verbose", action="store_true", default=False,
96 help="Spew out even more information than normal")
97 parser.add_argument("-q", "--quiet", action="store_true", default=False,
98 help="Restrict output to warnings and errors")
101 def fill_config_defaults(thisconfig):
102 for k, v in default_config.items():
103 if k not in thisconfig:
106 # Expand paths (~users and $vars)
107 def expand_path(path):
111 path = os.path.expanduser(path)
112 path = os.path.expandvars(path)
117 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
122 thisconfig[k + '_orig'] = v
124 for k in ['ndk_paths']:
130 thisconfig[k][k2] = exp
131 thisconfig[k][k2 + '_orig'] = v
134 def regsub_file(pattern, repl, path):
135 with open(path, 'r') as f:
137 text = re.sub(pattern, repl, text)
138 with open(path, 'w') as f:
142 def read_config(opts, config_file='config.py'):
143 """Read the repository config
145 The config is read from config_file, which is in the current directory when
146 any of the repo management commands are used.
148 global config, options, env, orig_path
150 if config is not None:
152 if not os.path.isfile(config_file):
153 logging.critical("Missing config file - is this a repo directory?")
160 logging.debug("Reading %s" % config_file)
161 execfile(config_file, config)
163 # smartcardoptions must be a list since its command line args for Popen
164 if 'smartcardoptions' in config:
165 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
166 elif 'keystore' in config and config['keystore'] == 'NONE':
167 # keystore='NONE' means use smartcard, these are required defaults
168 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
169 'SunPKCS11-OpenSC', '-providerClass',
170 'sun.security.pkcs11.SunPKCS11',
171 '-providerArg', 'opensc-fdroid.cfg']
173 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
174 st = os.stat(config_file)
175 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
176 logging.warn("unsafe permissions on {0} (should be 0600)!".format(config_file))
178 fill_config_defaults(config)
180 # There is no standard, so just set up the most common environment
183 orig_path = env['PATH']
184 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
185 env[n] = config['sdk_path']
187 for k in ["keystorepass", "keypass"]:
189 write_password_file(k)
191 for k in ["repo_description", "archive_description"]:
193 config[k] = clean_description(config[k])
195 if 'serverwebroot' in config:
196 if isinstance(config['serverwebroot'], basestring):
197 roots = [config['serverwebroot']]
198 elif all(isinstance(item, basestring) for item in config['serverwebroot']):
199 roots = config['serverwebroot']
201 raise TypeError('only accepts strings, lists, and tuples')
203 for rootstr in roots:
204 # since this is used with rsync, where trailing slashes have
205 # meaning, ensure there is always a trailing slash
206 if rootstr[-1] != '/':
208 rootlist.append(rootstr.replace('//', '/'))
209 config['serverwebroot'] = rootlist
214 def get_ndk_path(version):
216 version = 'r10e' # falls back to latest
217 paths = config['ndk_paths']
218 if version not in paths:
220 return paths[version] or ''
223 def find_sdk_tools_cmd(cmd):
224 '''find a working path to a tool from the Android SDK'''
227 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
228 # try to find a working path to this command, in all the recent possible paths
229 if 'build_tools' in config:
230 build_tools = os.path.join(config['sdk_path'], 'build-tools')
231 # if 'build_tools' was manually set and exists, check only that one
232 configed_build_tools = os.path.join(build_tools, config['build_tools'])
233 if os.path.exists(configed_build_tools):
234 tooldirs.append(configed_build_tools)
236 # no configed version, so hunt known paths for it
237 for f in sorted(os.listdir(build_tools), reverse=True):
238 if os.path.isdir(os.path.join(build_tools, f)):
239 tooldirs.append(os.path.join(build_tools, f))
240 tooldirs.append(build_tools)
241 sdk_tools = os.path.join(config['sdk_path'], 'tools')
242 if os.path.exists(sdk_tools):
243 tooldirs.append(sdk_tools)
244 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
245 if os.path.exists(sdk_platform_tools):
246 tooldirs.append(sdk_platform_tools)
247 tooldirs.append('/usr/bin')
249 if os.path.isfile(os.path.join(d, cmd)):
250 return os.path.join(d, cmd)
251 # did not find the command, exit with error message
252 ensure_build_tools_exists(config)
255 def test_sdk_exists(thisconfig):
256 if 'sdk_path' not in thisconfig:
257 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
260 logging.error("'sdk_path' not set in config.py!")
262 if thisconfig['sdk_path'] == default_config['sdk_path']:
263 logging.error('No Android SDK found!')
264 logging.error('You can use ANDROID_HOME to set the path to your SDK, i.e.:')
265 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
267 if not os.path.exists(thisconfig['sdk_path']):
268 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" does not exist!')
270 if not os.path.isdir(thisconfig['sdk_path']):
271 logging.critical('Android SDK path "' + thisconfig['sdk_path'] + '" is not a directory!')
273 for d in ['build-tools', 'platform-tools', 'tools']:
274 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
275 logging.critical('Android SDK path "%s" does not contain "%s/"!' % (
276 thisconfig['sdk_path'], d))
281 def ensure_build_tools_exists(thisconfig):
282 if not test_sdk_exists(thisconfig):
284 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
285 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
286 if not os.path.isdir(versioned_build_tools):
287 logging.critical('Android Build Tools path "'
288 + versioned_build_tools + '" does not exist!')
292 def write_password_file(pwtype, password=None):
294 writes out passwords to a protected file instead of passing passwords as
295 command line argments
297 filename = '.fdroid.' + pwtype + '.txt'
298 fd = os.open(filename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0600)
300 os.write(fd, config[pwtype])
302 os.write(fd, password)
304 config[pwtype + 'file'] = filename
307 # Given the arguments in the form of multiple appid:[vc] strings, this returns
308 # a dictionary with the set of vercodes specified for each package.
309 def read_pkg_args(args, allow_vercodes=False):
316 if allow_vercodes and ':' in p:
317 package, vercode = p.split(':')
319 package, vercode = p, None
320 if package not in vercodes:
321 vercodes[package] = [vercode] if vercode else []
323 elif vercode and vercode not in vercodes[package]:
324 vercodes[package] += [vercode] if vercode else []
329 # On top of what read_pkg_args does, this returns the whole app metadata, but
330 # limiting the builds list to the builds matching the vercodes specified.
331 def read_app_args(args, allapps, allow_vercodes=False):
333 vercodes = read_pkg_args(args, allow_vercodes)
339 for appid, app in allapps.iteritems():
340 if appid in vercodes:
343 if len(apps) != len(vercodes):
346 logging.critical("No such package: %s" % p)
347 raise FDroidException("Found invalid app ids in arguments")
349 raise FDroidException("No packages specified")
352 for appid, app in apps.iteritems():
356 app['builds'] = [b for b in app['builds'] if b['vercode'] in vc]
357 if len(app['builds']) != len(vercodes[appid]):
359 allvcs = [b['vercode'] for b in app['builds']]
360 for v in vercodes[appid]:
362 logging.critical("No such vercode %s for app %s" % (v, appid))
365 raise FDroidException("Found invalid vercodes for some apps")
370 def has_extension(filename, extension):
371 name, ext = os.path.splitext(filename)
372 ext = ext.lower()[1:]
373 return ext == extension
378 def clean_description(description):
379 'Remove unneeded newlines and spaces from a block of description text'
381 # this is split up by paragraph to make removing the newlines easier
382 for paragraph in re.split(r'\n\n', description):
383 paragraph = re.sub('\r', '', paragraph)
384 paragraph = re.sub('\n', ' ', paragraph)
385 paragraph = re.sub(' {2,}', ' ', paragraph)
386 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
387 returnstring += paragraph + '\n\n'
388 return returnstring.rstrip('\n')
391 def apknameinfo(filename):
393 filename = os.path.basename(filename)
394 if apk_regex is None:
395 apk_regex = re.compile(r"^(.+)_([0-9]+)\.apk$")
396 m = apk_regex.match(filename)
398 result = (m.group(1), m.group(2))
399 except AttributeError:
400 raise FDroidException("Invalid apk name: %s" % filename)
404 def getapkname(app, build):
405 return "%s_%s.apk" % (app['id'], build['vercode'])
408 def getsrcname(app, build):
409 return "%s_%s_src.tar.gz" % (app['id'], build['vercode'])
416 return app['Auto Name']
421 return '%s (%s)' % (app['Current Version'], app['Current Version Code'])
424 def getvcs(vcstype, remote, local):
426 return vcs_git(remote, local)
427 if vcstype == 'git-svn':
428 return vcs_gitsvn(remote, local)
430 return vcs_hg(remote, local)
432 return vcs_bzr(remote, local)
433 if vcstype == 'srclib':
434 if local != os.path.join('build', 'srclib', remote):
435 raise VCSException("Error: srclib paths are hard-coded!")
436 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
438 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
439 raise VCSException("Invalid vcs type " + vcstype)
442 def getsrclibvcs(name):
443 if name not in metadata.srclibs:
444 raise VCSException("Missing srclib " + name)
445 return metadata.srclibs[name]['Repo Type']
450 def __init__(self, remote, local):
452 # svn, git-svn and bzr may require auth
454 if self.repotype() in ('git-svn', 'bzr'):
456 if self.repotype == 'git-svn':
457 raise VCSException("Authentication is not supported for git-svn")
458 self.username, remote = remote.split('@')
459 if ':' not in self.username:
460 raise VCSException("Password required with username")
461 self.username, self.password = self.username.split(':')
465 self.clone_failed = False
466 self.refreshed = False
472 # Take the local repository to a clean version of the given revision, which
473 # is specificed in the VCS's native format. Beforehand, the repository can
474 # be dirty, or even non-existent. If the repository does already exist
475 # locally, it will be updated from the origin, but only once in the
476 # lifetime of the vcs object.
477 # None is acceptable for 'rev' if you know you are cloning a clean copy of
478 # the repo - otherwise it must specify a valid revision.
479 def gotorevision(self, rev, refresh=True):
481 if self.clone_failed:
482 raise VCSException("Downloading the repository already failed once, not trying again.")
484 # The .fdroidvcs-id file for a repo tells us what VCS type
485 # and remote that directory was created from, allowing us to drop it
486 # automatically if either of those things changes.
487 fdpath = os.path.join(self.local, '..',
488 '.fdroidvcs-' + os.path.basename(self.local))
489 cdata = self.repotype() + ' ' + self.remote
492 if os.path.exists(self.local):
493 if os.path.exists(fdpath):
494 with open(fdpath, 'r') as f:
495 fsdata = f.read().strip()
500 logging.info("Repository details for %s changed - deleting" % (
504 logging.info("Repository details for %s missing - deleting" % (
507 shutil.rmtree(self.local)
511 self.refreshed = True
514 self.gotorevisionx(rev)
515 except FDroidException, e:
518 # If necessary, write the .fdroidvcs file.
519 if writeback and not self.clone_failed:
520 with open(fdpath, 'w') as f:
526 # Derived classes need to implement this. It's called once basic checking
527 # has been performend.
528 def gotorevisionx(self, rev):
529 raise VCSException("This VCS type doesn't define gotorevisionx")
531 # Initialise and update submodules
532 def initsubmodules(self):
533 raise VCSException('Submodules not supported for this vcs type')
535 # Get a list of all known tags
537 if not self._gettags:
538 raise VCSException('gettags not supported for this vcs type')
540 for tag in self._gettags():
541 if re.match('[-A-Za-z0-9_. /]+$', tag):
545 def latesttags(self, tags, number):
546 """Get the most recent tags in a given list.
548 :param tags: a list of tags
549 :param number: the number to return
550 :returns: A list containing the most recent tags in the provided
551 list, up to the maximum number given.
553 raise VCSException('latesttags not supported for this vcs type')
555 # Get current commit reference (hash, revision, etc)
557 raise VCSException('getref not supported for this vcs type')
559 # Returns the srclib (name, path) used in setting up the current
570 # If the local directory exists, but is somehow not a git repository, git
571 # will traverse up the directory tree until it finds one that is (i.e.
572 # fdroidserver) and then we'll proceed to destroy it! This is called as
575 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
576 result = p.output.rstrip()
577 if not result.endswith(self.local):
578 raise VCSException('Repository mismatch')
580 def gotorevisionx(self, rev):
581 if not os.path.exists(self.local):
583 p = FDroidPopen(['git', 'clone', self.remote, self.local])
584 if p.returncode != 0:
585 self.clone_failed = True
586 raise VCSException("Git clone failed", p.output)
590 # Discard any working tree changes
591 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
592 'git', 'reset', '--hard'], cwd=self.local, output=False)
593 if p.returncode != 0:
594 raise VCSException("Git reset failed", p.output)
595 # Remove untracked files now, in case they're tracked in the target
596 # revision (it happens!)
597 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
598 'git', 'clean', '-dffx'], cwd=self.local, output=False)
599 if p.returncode != 0:
600 raise VCSException("Git clean failed", p.output)
601 if not self.refreshed:
602 # Get latest commits and tags from remote
603 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
604 if p.returncode != 0:
605 raise VCSException("Git fetch failed", p.output)
606 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
607 if p.returncode != 0:
608 raise VCSException("Git fetch failed", p.output)
609 # Recreate origin/HEAD as git clone would do it, in case it disappeared
610 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
611 if p.returncode != 0:
612 lines = p.output.splitlines()
613 if 'Multiple remote HEAD branches' not in lines[0]:
614 raise VCSException("Git remote set-head failed", p.output)
615 branch = lines[1].split(' ')[-1]
616 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
617 if p2.returncode != 0:
618 raise VCSException("Git remote set-head failed", p.output + '\n' + p2.output)
619 self.refreshed = True
620 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
621 # a github repo. Most of the time this is the same as origin/master.
622 rev = rev or 'origin/HEAD'
623 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
624 if p.returncode != 0:
625 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
626 # Get rid of any uncontrolled files left behind
627 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
628 if p.returncode != 0:
629 raise VCSException("Git clean failed", p.output)
631 def initsubmodules(self):
633 submfile = os.path.join(self.local, '.gitmodules')
634 if not os.path.isfile(submfile):
635 raise VCSException("No git submodules available")
637 # fix submodules not accessible without an account and public key auth
638 with open(submfile, 'r') as f:
639 lines = f.readlines()
640 with open(submfile, 'w') as f:
642 if 'git@github.com' in line:
643 line = line.replace('git@github.com:', 'https://github.com/')
646 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
647 if p.returncode != 0:
648 raise VCSException("Git submodule sync failed", p.output)
649 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
650 if p.returncode != 0:
651 raise VCSException("Git submodule update failed", p.output)
655 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
656 return p.output.splitlines()
658 def latesttags(self, tags, number):
663 ['git', 'show', '--format=format:%ct', '-s', tag],
664 cwd=self.local, output=False)
665 # Timestamp is on the last line. For a normal tag, it's the only
666 # line, but for annotated tags, the rest of the info precedes it.
667 ts = int(p.output.splitlines()[-1])
670 for _, t in sorted(tl)[-number:]:
675 class vcs_gitsvn(vcs):
680 # If the local directory exists, but is somehow not a git repository, git
681 # will traverse up the directory tree until it finds one that is (i.e.
682 # fdroidserver) and then we'll proceed to destory it! This is called as
685 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
686 result = p.output.rstrip()
687 if not result.endswith(self.local):
688 raise VCSException('Repository mismatch')
690 def gotorevisionx(self, rev):
691 if not os.path.exists(self.local):
693 gitsvn_args = ['git', 'svn', 'clone']
694 if ';' in self.remote:
695 remote_split = self.remote.split(';')
696 for i in remote_split[1:]:
697 if i.startswith('trunk='):
698 gitsvn_args.extend(['-T', i[6:]])
699 elif i.startswith('tags='):
700 gitsvn_args.extend(['-t', i[5:]])
701 elif i.startswith('branches='):
702 gitsvn_args.extend(['-b', i[9:]])
703 gitsvn_args.extend([remote_split[0], self.local])
704 p = FDroidPopen(gitsvn_args, output=False)
705 if p.returncode != 0:
706 self.clone_failed = True
707 raise VCSException("Git svn clone failed", p.output)
709 gitsvn_args.extend([self.remote, self.local])
710 p = FDroidPopen(gitsvn_args, output=False)
711 if p.returncode != 0:
712 self.clone_failed = True
713 raise VCSException("Git svn clone failed", p.output)
717 # Discard any working tree changes
718 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
719 if p.returncode != 0:
720 raise VCSException("Git reset failed", p.output)
721 # Remove untracked files now, in case they're tracked in the target
722 # revision (it happens!)
723 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
724 if p.returncode != 0:
725 raise VCSException("Git clean failed", p.output)
726 if not self.refreshed:
727 # Get new commits, branches and tags from repo
728 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
729 if p.returncode != 0:
730 raise VCSException("Git svn fetch failed")
731 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
732 if p.returncode != 0:
733 raise VCSException("Git svn rebase failed", p.output)
734 self.refreshed = True
736 rev = rev or 'master'
738 nospaces_rev = rev.replace(' ', '%20')
739 # Try finding a svn tag
740 for treeish in ['origin/', '']:
741 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
742 if p.returncode == 0:
744 if p.returncode != 0:
745 # No tag found, normal svn rev translation
746 # Translate svn rev into git format
747 rev_split = rev.split('/')
750 for treeish in ['origin/', '']:
751 if len(rev_split) > 1:
752 treeish += rev_split[0]
753 svn_rev = rev_split[1]
756 # if no branch is specified, then assume trunk (i.e. 'master' branch):
760 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
762 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
763 git_rev = p.output.rstrip()
765 if p.returncode == 0 and git_rev:
768 if p.returncode != 0 or not git_rev:
769 # Try a plain git checkout as a last resort
770 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
771 if p.returncode != 0:
772 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
774 # Check out the git rev equivalent to the svn rev
775 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
776 if p.returncode != 0:
777 raise VCSException("Git checkout of '%s' failed" % rev, p.output)
779 # Get rid of any uncontrolled files left behind
780 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
781 if p.returncode != 0:
782 raise VCSException("Git clean failed", p.output)
786 for treeish in ['origin/', '']:
787 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
793 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
794 if p.returncode != 0:
796 return p.output.strip()
804 def gotorevisionx(self, rev):
805 if not os.path.exists(self.local):
806 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
807 if p.returncode != 0:
808 self.clone_failed = True
809 raise VCSException("Hg clone failed", p.output)
811 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
812 if p.returncode != 0:
813 raise VCSException("Hg status failed", p.output)
814 for line in p.output.splitlines():
815 if not line.startswith('? '):
816 raise VCSException("Unexpected output from hg status -uS: " + line)
817 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
818 if not self.refreshed:
819 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
820 if p.returncode != 0:
821 raise VCSException("Hg pull failed", p.output)
822 self.refreshed = True
824 rev = rev or 'default'
827 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
828 if p.returncode != 0:
829 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
830 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
831 # Also delete untracked files, we have to enable purge extension for that:
832 if "'purge' is provided by the following extension" in p.output:
833 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
834 myfile.write("\n[extensions]\nhgext.purge=\n")
835 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
836 if p.returncode != 0:
837 raise VCSException("HG purge failed", p.output)
838 elif p.returncode != 0:
839 raise VCSException("HG purge failed", p.output)
842 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
843 return p.output.splitlines()[1:]
851 def gotorevisionx(self, rev):
852 if not os.path.exists(self.local):
853 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
854 if p.returncode != 0:
855 self.clone_failed = True
856 raise VCSException("Bzr branch failed", p.output)
858 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
859 if p.returncode != 0:
860 raise VCSException("Bzr revert failed", p.output)
861 if not self.refreshed:
862 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
863 if p.returncode != 0:
864 raise VCSException("Bzr update failed", p.output)
865 self.refreshed = True
867 revargs = list(['-r', rev] if rev else [])
868 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
869 if p.returncode != 0:
870 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
873 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
874 return [tag.split(' ')[0].strip() for tag in
875 p.output.splitlines()]
878 def unescape_string(string):
879 if string[0] == '"' and string[-1] == '"':
882 return string.replace("\\'", "'")
885 def retrieve_string(app_dir, string, xmlfiles=None):
890 os.path.join(app_dir, 'res'),
891 os.path.join(app_dir, 'src', 'main', 'res'),
893 for r, d, f in os.walk(res_dir):
894 if os.path.basename(r) == 'values':
895 xmlfiles += [os.path.join(r, x) for x in f if x.endswith('.xml')]
897 if not string.startswith('@string/'):
898 return unescape_string(string)
900 name = string[len('@string/'):]
902 for path in xmlfiles:
903 if not os.path.isfile(path):
905 xml = parse_xml(path)
906 element = xml.find('string[@name="' + name + '"]')
907 if element is not None and element.text is not None:
908 return retrieve_string(app_dir, element.text.encode('utf-8'), xmlfiles)
913 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
914 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
917 # Return list of existing files that will be used to find the highest vercode
918 def manifest_paths(app_dir, flavours):
920 possible_manifests = \
921 [os.path.join(app_dir, 'AndroidManifest.xml'),
922 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
923 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
924 os.path.join(app_dir, 'build.gradle')]
926 for flavour in flavours:
929 possible_manifests.append(
930 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
932 return [path for path in possible_manifests if os.path.isfile(path)]
935 # Retrieve the package name. Returns the name, or None if not found.
936 def fetch_real_name(app_dir, flavours):
937 for path in manifest_paths(app_dir, flavours):
938 if not has_extension(path, 'xml') or not os.path.isfile(path):
940 logging.debug("fetch_real_name: Checking manifest at " + path)
941 xml = parse_xml(path)
942 app = xml.find('application')
943 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
945 label = app.attrib["{http://schemas.android.com/apk/res/android}label"].encode('utf-8')
946 result = retrieve_string_singleline(app_dir, label)
948 result = result.strip()
953 def get_library_references(root_dir):
955 proppath = os.path.join(root_dir, 'project.properties')
956 if not os.path.isfile(proppath):
958 for line in file(proppath):
959 if not line.startswith('android.library.reference.'):
961 path = line.split('=')[1].strip()
962 relpath = os.path.join(root_dir, path)
963 if not os.path.isdir(relpath):
965 logging.debug("Found subproject at %s" % path)
966 libraries.append(path)
970 def ant_subprojects(root_dir):
971 subprojects = get_library_references(root_dir)
972 for subpath in subprojects:
973 subrelpath = os.path.join(root_dir, subpath)
974 for p in get_library_references(subrelpath):
975 relp = os.path.normpath(os.path.join(subpath, p))
976 if relp not in subprojects:
977 subprojects.insert(0, relp)
981 def remove_debuggable_flags(root_dir):
982 # Remove forced debuggable flags
983 logging.debug("Removing debuggable flags from %s" % root_dir)
984 for root, dirs, files in os.walk(root_dir):
985 if 'AndroidManifest.xml' in files:
986 regsub_file(r'android:debuggable="[^"]*"',
988 os.path.join(root, 'AndroidManifest.xml'))
991 # Extract some information from the AndroidManifest.xml at the given path.
992 # Returns (version, vercode, package), any or all of which might be None.
993 # All values returned are strings.
994 def parse_androidmanifests(paths, ignoreversions=None):
997 return (None, None, None)
999 vcsearch_g = re.compile(r'.*versionCode *=* *["\']*([0-9]+)["\']*').search
1000 vnsearch_g = re.compile(r'.*versionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1001 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1003 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1011 if not os.path.isfile(path):
1014 logging.debug("Parsing manifest at {0}".format(path))
1015 gradle = has_extension(path, 'gradle')
1021 for line in file(path):
1022 # Grab first occurence of each to avoid running into
1023 # alternative flavours and builds.
1025 matches = psearch_g(line)
1027 package = matches.group(2)
1029 matches = vnsearch_g(line)
1031 version = matches.group(2)
1033 matches = vcsearch_g(line)
1035 vercode = matches.group(1)
1037 xml = parse_xml(path)
1038 if "package" in xml.attrib:
1039 package = xml.attrib["package"].encode('utf-8')
1040 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1041 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"].encode('utf-8')
1042 base_dir = os.path.dirname(path)
1043 version = retrieve_string_singleline(base_dir, version)
1044 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1045 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"].encode('utf-8')
1046 if string_is_integer(a):
1049 # Remember package name, may be defined separately from version+vercode
1051 package = max_package
1053 logging.debug("..got package={0}, version={1}, vercode={2}"
1054 .format(package, version, vercode))
1056 # Always grab the package name and version name in case they are not
1057 # together with the highest version code
1058 if max_package is None and package is not None:
1059 max_package = package
1060 if max_version is None and version is not None:
1061 max_version = version
1063 if max_vercode is None or (vercode is not None and vercode > max_vercode):
1064 if not ignoresearch or not ignoresearch(version):
1065 if version is not None:
1066 max_version = version
1067 if vercode is not None:
1068 max_vercode = vercode
1069 if package is not None:
1070 max_package = package
1072 max_version = "Ignore"
1074 if max_version is None:
1075 max_version = "Unknown"
1077 if max_package and not is_valid_package_name(max_package):
1078 raise FDroidException("Invalid package name {0}".format(max_package))
1080 return (max_version, max_vercode, max_package)
1083 def is_valid_package_name(name):
1084 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1087 class FDroidException(Exception):
1089 def __init__(self, value, detail=None):
1091 self.detail = detail
1093 def get_wikitext(self):
1094 ret = repr(self.value) + "\n"
1098 txt = self.detail[-8192:] if len(self.detail) > 8192 else self.detail
1106 ret += "\n==== detail begin ====\n%s\n==== detail end ====" % self.detail.strip()
1110 class VCSException(FDroidException):
1114 class BuildException(FDroidException):
1118 # Get the specified source library.
1119 # Returns the path to it. Normally this is the path to be used when referencing
1120 # it, which may be a subdirectory of the actual project. If you want the base
1121 # directory of the project, pass 'basepath=True'.
1122 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1123 raw=False, prepare=True, preponly=False, refresh=True):
1131 name, ref = spec.split('@')
1133 number, name = name.split(':', 1)
1135 name, subdir = name.split('/', 1)
1137 if name not in metadata.srclibs:
1138 raise VCSException('srclib ' + name + ' not found.')
1140 srclib = metadata.srclibs[name]
1142 sdir = os.path.join(srclib_dir, name)
1145 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1146 vcs.srclib = (name, number, sdir)
1148 vcs.gotorevision(ref, refresh)
1155 libdir = os.path.join(sdir, subdir)
1156 elif srclib["Subdir"]:
1157 for subdir in srclib["Subdir"]:
1158 libdir_candidate = os.path.join(sdir, subdir)
1159 if os.path.exists(libdir_candidate):
1160 libdir = libdir_candidate
1166 remove_signing_keys(sdir)
1167 remove_debuggable_flags(sdir)
1171 if srclib["Prepare"]:
1172 cmd = replace_config_vars(srclib["Prepare"], None)
1174 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1175 if p.returncode != 0:
1176 raise BuildException("Error running prepare command for srclib %s"
1182 return (name, number, libdir)
1185 # Prepare the source code for a particular build
1186 # 'vcs' - the appropriate vcs object for the application
1187 # 'app' - the application details from the metadata
1188 # 'build' - the build details from the metadata
1189 # 'build_dir' - the path to the build directory, usually
1191 # 'srclib_dir' - the path to the source libraries directory, usually
1193 # 'extlib_dir' - the path to the external libraries directory, usually
1195 # Returns the (root, srclibpaths) where:
1196 # 'root' is the root directory, which may be the same as 'build_dir' or may
1197 # be a subdirectory of it.
1198 # 'srclibpaths' is information on the srclibs being used
1199 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1201 # Optionally, the actual app source can be in a subdirectory
1203 root_dir = os.path.join(build_dir, build['subdir'])
1205 root_dir = build_dir
1207 # Get a working copy of the right revision
1208 logging.info("Getting source for revision " + build['commit'])
1209 vcs.gotorevision(build['commit'], refresh)
1211 # Initialise submodules if required
1212 if build['submodules']:
1213 logging.info("Initialising submodules")
1214 vcs.initsubmodules()
1216 # Check that a subdir (if we're using one) exists. This has to happen
1217 # after the checkout, since it might not exist elsewhere
1218 if not os.path.exists(root_dir):
1219 raise BuildException('Missing subdir ' + root_dir)
1221 # Run an init command if one is required
1223 cmd = replace_config_vars(build['init'], build)
1224 logging.info("Running 'init' commands in %s" % root_dir)
1226 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1227 if p.returncode != 0:
1228 raise BuildException("Error running init command for %s:%s" %
1229 (app['id'], build['version']), p.output)
1231 # Apply patches if any
1233 logging.info("Applying patches")
1234 for patch in build['patch']:
1235 patch = patch.strip()
1236 logging.info("Applying " + patch)
1237 patch_path = os.path.join('metadata', app['id'], patch)
1238 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1239 if p.returncode != 0:
1240 raise BuildException("Failed to apply patch %s" % patch_path)
1242 # Get required source libraries
1244 if build['srclibs']:
1245 logging.info("Collecting source libraries")
1246 for lib in build['srclibs']:
1247 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver, refresh=refresh))
1249 for name, number, libpath in srclibpaths:
1250 place_srclib(root_dir, int(number) if number else None, libpath)
1252 basesrclib = vcs.getsrclib()
1253 # If one was used for the main source, add that too.
1255 srclibpaths.append(basesrclib)
1257 # Update the local.properties file
1258 localprops = [os.path.join(build_dir, 'local.properties')]
1260 localprops += [os.path.join(root_dir, 'local.properties')]
1261 for path in localprops:
1263 if os.path.isfile(path):
1264 logging.info("Updating local.properties file at %s" % path)
1265 with open(path, 'r') as f:
1269 logging.info("Creating local.properties file at %s" % path)
1270 # Fix old-fashioned 'sdk-location' by copying
1271 # from sdk.dir, if necessary
1272 if build['oldsdkloc']:
1273 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1274 re.S | re.M).group(1)
1275 props += "sdk-location=%s\n" % sdkloc
1277 props += "sdk.dir=%s\n" % config['sdk_path']
1278 props += "sdk-location=%s\n" % config['sdk_path']
1279 if build['ndk_path']:
1281 props += "ndk.dir=%s\n" % build['ndk_path']
1282 props += "ndk-location=%s\n" % build['ndk_path']
1283 # Add java.encoding if necessary
1284 if build['encoding']:
1285 props += "java.encoding=%s\n" % build['encoding']
1286 with open(path, 'w') as f:
1290 if build['type'] == 'gradle':
1291 flavours = build['gradle']
1293 version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1294 gradlepluginver = None
1296 gradle_dirs = [root_dir]
1298 # Parent dir build.gradle
1299 parent_dir = os.path.normpath(os.path.join(root_dir, '..'))
1300 if parent_dir.startswith(build_dir):
1301 gradle_dirs.append(parent_dir)
1303 for dir_path in gradle_dirs:
1306 if not os.path.isdir(dir_path):
1308 for filename in os.listdir(dir_path):
1309 if not filename.endswith('.gradle'):
1311 path = os.path.join(dir_path, filename)
1312 if not os.path.isfile(path):
1314 for line in file(path):
1315 match = version_regex.match(line)
1317 gradlepluginver = match.group(1)
1321 build['gradlepluginver'] = LooseVersion(gradlepluginver)
1323 logging.warn("Could not fetch the gradle plugin version, defaulting to 0.11")
1324 build['gradlepluginver'] = LooseVersion('0.11')
1327 n = build["target"].split('-')[1]
1328 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1329 r'compileSdkVersion %s' % n,
1330 os.path.join(root_dir, 'build.gradle'))
1332 # Remove forced debuggable flags
1333 remove_debuggable_flags(root_dir)
1335 # Insert version code and number into the manifest if necessary
1336 if build['forceversion']:
1337 logging.info("Changing the version name")
1338 for path in manifest_paths(root_dir, flavours):
1339 if not os.path.isfile(path):
1341 if has_extension(path, 'xml'):
1342 regsub_file(r'android:versionName="[^"]*"',
1343 r'android:versionName="%s"' % build['version'],
1345 elif has_extension(path, 'gradle'):
1346 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1347 r"""\1versionName '%s'""" % build['version'],
1350 if build['forcevercode']:
1351 logging.info("Changing the version code")
1352 for path in manifest_paths(root_dir, flavours):
1353 if not os.path.isfile(path):
1355 if has_extension(path, 'xml'):
1356 regsub_file(r'android:versionCode="[^"]*"',
1357 r'android:versionCode="%s"' % build['vercode'],
1359 elif has_extension(path, 'gradle'):
1360 regsub_file(r'versionCode[ =]+[0-9]+',
1361 r'versionCode %s' % build['vercode'],
1364 # Delete unwanted files
1366 logging.info("Removing specified files")
1367 for part in getpaths(build_dir, build, 'rm'):
1368 dest = os.path.join(build_dir, part)
1369 logging.info("Removing {0}".format(part))
1370 if os.path.lexists(dest):
1371 if os.path.islink(dest):
1372 FDroidPopen(['unlink', dest], output=False)
1374 FDroidPopen(['rm', '-rf', dest], output=False)
1376 logging.info("...but it didn't exist")
1378 remove_signing_keys(build_dir)
1380 # Add required external libraries
1381 if build['extlibs']:
1382 logging.info("Collecting prebuilt libraries")
1383 libsdir = os.path.join(root_dir, 'libs')
1384 if not os.path.exists(libsdir):
1386 for lib in build['extlibs']:
1388 logging.info("...installing extlib {0}".format(lib))
1389 libf = os.path.basename(lib)
1390 libsrc = os.path.join(extlib_dir, lib)
1391 if not os.path.exists(libsrc):
1392 raise BuildException("Missing extlib file {0}".format(libsrc))
1393 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1395 # Run a pre-build command if one is required
1396 if build['prebuild']:
1397 logging.info("Running 'prebuild' commands in %s" % root_dir)
1399 cmd = replace_config_vars(build['prebuild'], build)
1401 # Substitute source library paths into prebuild commands
1402 for name, number, libpath in srclibpaths:
1403 libpath = os.path.relpath(libpath, root_dir)
1404 cmd = cmd.replace('$$' + name + '$$', libpath)
1406 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1407 if p.returncode != 0:
1408 raise BuildException("Error running prebuild command for %s:%s" %
1409 (app['id'], build['version']), p.output)
1411 # Generate (or update) the ant build file, build.xml...
1412 if build['update'] and build['update'] != ['no'] and build['type'] == 'ant':
1413 parms = ['android', 'update', 'lib-project']
1414 lparms = ['android', 'update', 'project']
1417 parms += ['-t', build['target']]
1418 lparms += ['-t', build['target']]
1419 if build['update'] == ['auto']:
1420 update_dirs = ant_subprojects(root_dir) + ['.']
1422 update_dirs = build['update']
1424 for d in update_dirs:
1425 subdir = os.path.join(root_dir, d)
1427 logging.debug("Updating main project")
1428 cmd = parms + ['-p', d]
1430 logging.debug("Updating subproject %s" % d)
1431 cmd = lparms + ['-p', d]
1432 p = SdkToolsPopen(cmd, cwd=root_dir)
1433 # Check to see whether an error was returned without a proper exit
1434 # code (this is the case for the 'no target set or target invalid'
1436 if p.returncode != 0 or p.output.startswith("Error: "):
1437 raise BuildException("Failed to update project at %s" % d, p.output)
1438 # Clean update dirs via ant
1440 logging.info("Cleaning subproject %s" % d)
1441 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1443 return (root_dir, srclibpaths)
1446 # Split and extend via globbing the paths from a field
1447 def getpaths(build_dir, build, field):
1449 for p in build[field]:
1451 full_path = os.path.join(build_dir, p)
1452 full_path = os.path.normpath(full_path)
1453 paths += [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1458 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1464 self.path = os.path.join('stats', 'known_apks.txt')
1466 if os.path.isfile(self.path):
1467 for line in file(self.path):
1468 t = line.rstrip().split(' ')
1470 self.apks[t[0]] = (t[1], None)
1472 self.apks[t[0]] = (t[1], time.strptime(t[2], '%Y-%m-%d'))
1473 self.changed = False
1475 def writeifchanged(self):
1476 if not self.changed:
1479 if not os.path.exists('stats'):
1483 for apk, app in self.apks.iteritems():
1485 line = apk + ' ' + appid
1487 line += ' ' + time.strftime('%Y-%m-%d', added)
1490 with open(self.path, 'w') as f:
1491 for line in sorted(lst, key=natural_key):
1492 f.write(line + '\n')
1494 # Record an apk (if it's new, otherwise does nothing)
1495 # Returns the date it was added.
1496 def recordapk(self, apk, app):
1497 if apk not in self.apks:
1498 self.apks[apk] = (app, time.gmtime(time.time()))
1500 _, added = self.apks[apk]
1503 # Look up information - given the 'apkname', returns (app id, date added/None).
1504 # Or returns None for an unknown apk.
1505 def getapp(self, apkname):
1506 if apkname in self.apks:
1507 return self.apks[apkname]
1510 # Get the most recent 'num' apps added to the repo, as a list of package ids
1511 # with the most recent first.
1512 def getlatest(self, num):
1514 for apk, app in self.apks.iteritems():
1518 if apps[appid] > added:
1522 sortedapps = sorted(apps.iteritems(), key=operator.itemgetter(1))[-num:]
1523 lst = [app for app, _ in sortedapps]
1528 def isApkDebuggable(apkfile, config):
1529 """Returns True if the given apk file is debuggable
1531 :param apkfile: full path to the apk to check"""
1533 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1535 if p.returncode != 0:
1536 logging.critical("Failed to get apk manifest information")
1538 for line in p.output.splitlines():
1539 if 'android:debuggable' in line and not line.endswith('0x0'):
1544 class AsynchronousFileReader(threading.Thread):
1547 Helper class to implement asynchronous reading of a file
1548 in a separate thread. Pushes read lines on a queue to
1549 be consumed in another thread.
1552 def __init__(self, fd, queue):
1553 assert isinstance(queue, Queue.Queue)
1554 assert callable(fd.readline)
1555 threading.Thread.__init__(self)
1560 '''The body of the tread: read lines and put them on the queue.'''
1561 for line in iter(self._fd.readline, ''):
1562 self._queue.put(line)
1565 '''Check whether there is no more content to expect.'''
1566 return not self.is_alive() and self._queue.empty()
1574 def SdkToolsPopen(commands, cwd=None, output=True):
1576 if cmd not in config:
1577 config[cmd] = find_sdk_tools_cmd(commands[0])
1578 return FDroidPopen([config[cmd]] + commands[1:],
1579 cwd=cwd, output=output)
1582 def FDroidPopen(commands, cwd=None, output=True):
1584 Run a command and capture the possibly huge output.
1586 :param commands: command and argument list like in subprocess.Popen
1587 :param cwd: optionally specifies a working directory
1588 :returns: A PopenResult.
1594 cwd = os.path.normpath(cwd)
1595 logging.debug("Directory: %s" % cwd)
1596 logging.debug("> %s" % ' '.join(commands))
1598 result = PopenResult()
1601 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=env,
1602 stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1604 raise BuildException("OSError while trying to execute " +
1605 ' '.join(commands) + ': ' + str(e))
1607 stdout_queue = Queue.Queue()
1608 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1609 stdout_reader.start()
1611 # Check the queue for output (until there is no more to get)
1612 while not stdout_reader.eof():
1613 while not stdout_queue.empty():
1614 line = stdout_queue.get()
1615 if output and options.verbose:
1616 # Output directly to console
1617 sys.stderr.write(line)
1619 result.output += line
1623 result.returncode = p.wait()
1627 def remove_signing_keys(build_dir):
1628 comment = re.compile(r'[ ]*//')
1629 signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1631 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1632 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1633 re.compile(r'.*variant\.outputFile = .*'),
1634 re.compile(r'.*output\.outputFile = .*'),
1635 re.compile(r'.*\.readLine\(.*'),
1637 for root, dirs, files in os.walk(build_dir):
1638 if 'build.gradle' in files:
1639 path = os.path.join(root, 'build.gradle')
1641 with open(path, "r") as o:
1642 lines = o.readlines()
1648 with open(path, "w") as o:
1649 while i < len(lines):
1652 while line.endswith('\\\n'):
1653 line = line.rstrip('\\\n') + lines[i]
1656 if comment.match(line):
1660 opened += line.count('{')
1661 opened -= line.count('}')
1664 if signing_configs.match(line):
1669 if any(s.match(line) for s in line_matches):
1677 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1680 'project.properties',
1682 'default.properties',
1683 'ant.properties', ]:
1684 if propfile in files:
1685 path = os.path.join(root, propfile)
1687 with open(path, "r") as o:
1688 lines = o.readlines()
1692 with open(path, "w") as o:
1694 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
1701 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
1704 def reset_env_path():
1705 global env, orig_path
1706 env['PATH'] = orig_path
1709 def add_to_env_path(path):
1711 paths = env['PATH'].split(os.pathsep)
1715 env['PATH'] = os.pathsep.join(paths)
1718 def replace_config_vars(cmd, build):
1720 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
1721 # env['ANDROID_NDK'] is set in build_local right before prepare_source
1722 cmd = cmd.replace('$$NDK$$', env['ANDROID_NDK'])
1723 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
1724 if build is not None:
1725 cmd = cmd.replace('$$COMMIT$$', build['commit'])
1726 cmd = cmd.replace('$$VERSION$$', build['version'])
1727 cmd = cmd.replace('$$VERCODE$$', build['vercode'])
1731 def place_srclib(root_dir, number, libpath):
1734 relpath = os.path.relpath(libpath, root_dir)
1735 proppath = os.path.join(root_dir, 'project.properties')
1738 if os.path.isfile(proppath):
1739 with open(proppath, "r") as o:
1740 lines = o.readlines()
1742 with open(proppath, "w") as o:
1745 if line.startswith('android.library.reference.%d=' % number):
1746 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1751 o.write('android.library.reference.%d=%s\n' % (number, relpath))
1754 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
1755 """Verify that two apks are the same
1757 One of the inputs is signed, the other is unsigned. The signature metadata
1758 is transferred from the signed to the unsigned apk, and then jarsigner is
1759 used to verify that the signature from the signed apk is also varlid for
1761 :param signed_apk: Path to a signed apk file
1762 :param unsigned_apk: Path to an unsigned apk file expected to match it
1763 :param tmp_dir: Path to directory for temporary files
1764 :returns: None if the verification is successful, otherwise a string
1765 describing what went wrong.
1767 sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA)')
1768 with ZipFile(signed_apk) as signed_apk_as_zip:
1769 meta_inf_files = ['META-INF/MANIFEST.MF']
1770 for f in signed_apk_as_zip.namelist():
1771 if sigfile.match(f):
1772 meta_inf_files.append(f)
1773 if len(meta_inf_files) < 3:
1774 return "Signature files missing from {0}".format(signed_apk)
1775 signed_apk_as_zip.extractall(tmp_dir, meta_inf_files)
1776 with ZipFile(unsigned_apk, mode='a') as unsigned_apk_as_zip:
1777 for meta_inf_file in meta_inf_files:
1778 unsigned_apk_as_zip.write(os.path.join(tmp_dir, meta_inf_file), arcname=meta_inf_file)
1780 if subprocess.call(['jarsigner', '-verify', unsigned_apk]) != 0:
1781 logging.info("...NOT verified - {0}".format(signed_apk))
1782 return compare_apks(signed_apk, unsigned_apk, tmp_dir)
1783 logging.info("...successfully verified")
1787 def compare_apks(apk1, apk2, tmp_dir):
1790 Returns None if the apk content is the same (apart from the signing key),
1791 otherwise a string describing what's different, or what went wrong when
1792 trying to do the comparison.
1795 badchars = re.compile('''[/ :;'"]''')
1796 apk1dir = os.path.join(tmp_dir, badchars.sub('_', apk1[0:-4])) # trim .apk
1797 apk2dir = os.path.join(tmp_dir, badchars.sub('_', apk2[0:-4])) # trim .apk
1798 for d in [apk1dir, apk2dir]:
1799 if os.path.exists(d):
1802 os.mkdir(os.path.join(d, 'jar-xf'))
1804 if subprocess.call(['jar', 'xf',
1805 os.path.abspath(apk1)],
1806 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
1807 return("Failed to unpack " + apk1)
1808 if subprocess.call(['jar', 'xf',
1809 os.path.abspath(apk2)],
1810 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
1811 return("Failed to unpack " + apk2)
1813 # try to find apktool in the path, if it hasn't been manually configed
1814 if 'apktool' not in config:
1815 tmp = find_command('apktool')
1817 config['apktool'] = tmp
1818 if 'apktool' in config:
1819 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
1821 return("Failed to unpack " + apk1)
1822 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
1824 return("Failed to unpack " + apk2)
1826 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
1827 lines = p.output.splitlines()
1828 if len(lines) != 1 or 'META-INF' not in lines[0]:
1829 meld = find_command('meld')
1830 if meld is not None:
1831 p = FDroidPopen(['meld', apk1dir, apk2dir], output=False)
1832 return("Unexpected diff output - " + p.output)
1834 # since everything verifies, delete the comparison to keep cruft down
1835 shutil.rmtree(apk1dir)
1836 shutil.rmtree(apk2dir)
1838 # If we get here, it seems like they're the same!
1842 def find_command(command):
1843 '''find the full path of a command, or None if it can't be found in the PATH'''
1846 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
1848 fpath, fname = os.path.split(command)
1853 for path in os.environ["PATH"].split(os.pathsep):
1854 path = path.strip('"')
1855 exe_file = os.path.join(path, command)
1856 if is_exe(exe_file):
1863 '''generate a random password for when generating keys'''
1864 h = hashlib.sha256()
1865 h.update(os.urandom(16)) # salt
1866 h.update(bytes(socket.getfqdn()))
1867 return h.digest().encode('base64').strip()
1870 def genkeystore(localconfig):
1871 '''Generate a new key with random passwords and add it to new keystore'''
1872 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
1873 keystoredir = os.path.dirname(localconfig['keystore'])
1874 if keystoredir is None or keystoredir == '':
1875 keystoredir = os.path.join(os.getcwd(), keystoredir)
1876 if not os.path.exists(keystoredir):
1877 os.makedirs(keystoredir, mode=0o700)
1879 write_password_file("keystorepass", localconfig['keystorepass'])
1880 write_password_file("keypass", localconfig['keypass'])
1881 p = FDroidPopen(['keytool', '-genkey',
1882 '-keystore', localconfig['keystore'],
1883 '-alias', localconfig['repo_keyalias'],
1884 '-keyalg', 'RSA', '-keysize', '4096',
1885 '-sigalg', 'SHA256withRSA',
1886 '-validity', '10000',
1887 '-storepass:file', config['keystorepassfile'],
1888 '-keypass:file', config['keypassfile'],
1889 '-dname', localconfig['keydname']])
1890 # TODO keypass should be sent via stdin
1891 if p.returncode != 0:
1892 raise BuildException("Failed to generate key", p.output)
1893 os.chmod(localconfig['keystore'], 0o0600)
1894 # now show the lovely key that was just generated
1895 p = FDroidPopen(['keytool', '-list', '-v',
1896 '-keystore', localconfig['keystore'],
1897 '-alias', localconfig['repo_keyalias'],
1898 '-storepass:file', config['keystorepassfile']])
1899 logging.info(p.output.strip() + '\n\n')
1902 def write_to_config(thisconfig, key, value=None):
1903 '''write a key/value to the local config.py'''
1905 origkey = key + '_orig'
1906 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
1907 with open('config.py', 'r') as f:
1909 pattern = '\n[\s#]*' + key + '\s*=\s*"[^"]*"'
1910 repl = '\n' + key + ' = "' + value + '"'
1911 data = re.sub(pattern, repl, data)
1912 # if this key is not in the file, append it
1913 if not re.match('\s*' + key + '\s*=\s*"', data):
1915 # make sure the file ends with a carraige return
1916 if not re.match('\n$', data):
1918 with open('config.py', 'w') as f:
1922 def parse_xml(path):
1923 return XMLElementTree.parse(path).getroot()
1926 def string_is_integer(string):
1934 def get_per_app_repos():
1935 '''per-app repos are dirs named with the packageName of a single app'''
1937 # Android packageNames are Java packages, they may contain uppercase or
1938 # lowercase letters ('A' through 'Z'), numbers, and underscores
1939 # ('_'). However, individual package name parts may only start with
1940 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
1941 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
1944 for root, dirs, files in os.walk(os.getcwd()):
1946 print 'checking', root, 'for', d
1947 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
1948 # standard parts of an fdroid repo, so never packageNames
1951 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):