3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, BuildException, VerificationException
57 from .asynchronousfilereader import AsynchronousFileReader
60 # A signature block file with a .DSA, .RSA, or .EC extension
61 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
62 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
63 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
65 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
74 'sdk_path': "$ANDROID_HOME",
79 'r12b': "$ANDROID_NDK",
86 'build_tools': "25.0.2",
87 'force_build_tools': False,
92 'accepted_formats': ['txt', 'yml'],
93 'sync_from_local_copy_dir': False,
94 'allow_disabled_algorithms': False,
95 'per_app_repos': False,
96 'make_current_version_link': True,
97 'current_version_name_source': 'Name',
98 'update_stats': False,
100 'stats_server': None,
102 'stats_to_carbon': False,
104 'build_server_always': False,
105 'keystore': 'keystore.jks',
106 'smartcardoptions': [],
116 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
117 'repo_name': "My First FDroid Repo Demo",
118 'repo_icon': "fdroid-icon.png",
119 'repo_description': '''
120 This is a repository of apps to be used with FDroid. Applications in this
121 repository are either official binaries built by the original application
122 developers, or are binaries built from source by the admin of f-droid.org
123 using the tools on https://gitlab.com/u/fdroid.
129 def setup_global_opts(parser):
130 parser.add_argument("-v", "--verbose", action="store_true", default=False,
131 help=_("Spew out even more information than normal"))
132 parser.add_argument("-q", "--quiet", action="store_true", default=False,
133 help=_("Restrict output to warnings and errors"))
136 def _add_java_paths_to_config(pathlist, thisconfig):
137 def path_version_key(s):
139 for u in re.split('[^0-9]+', s):
141 versionlist.append(int(u))
146 for d in sorted(pathlist, key=path_version_key):
147 if os.path.islink(d):
149 j = os.path.basename(d)
150 # the last one found will be the canonical one, so order appropriately
152 r'^1\.([6-9])\.0\.jdk$', # OSX
153 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
154 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
155 r'^jdk([6-9])-openjdk$', # Arch
156 r'^java-([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-jdk$', # Arch (oracle)
158 r'^java-1\.([6-9])\.0-.*$', # RedHat
159 r'^java-([6-9])-oracle$', # Debian WebUpd8
160 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
161 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
163 m = re.match(regex, j)
166 for p in [d, os.path.join(d, 'Contents', 'Home')]:
167 if os.path.exists(os.path.join(p, 'bin', 'javac')):
168 thisconfig['java_paths'][m.group(1)] = p
171 def fill_config_defaults(thisconfig):
172 for k, v in default_config.items():
173 if k not in thisconfig:
176 # Expand paths (~users and $vars)
177 def expand_path(path):
181 path = os.path.expanduser(path)
182 path = os.path.expandvars(path)
187 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
192 thisconfig[k + '_orig'] = v
194 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
195 if thisconfig['java_paths'] is None:
196 thisconfig['java_paths'] = dict()
198 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
199 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
200 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
201 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
202 if os.getenv('JAVA_HOME') is not None:
203 pathlist.append(os.getenv('JAVA_HOME'))
204 if os.getenv('PROGRAMFILES') is not None:
205 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
206 _add_java_paths_to_config(pathlist, thisconfig)
208 for java_version in ('7', '8', '9'):
209 if java_version not in thisconfig['java_paths']:
211 java_home = thisconfig['java_paths'][java_version]
212 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
213 if os.path.exists(jarsigner):
214 thisconfig['jarsigner'] = jarsigner
215 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
216 break # Java7 is preferred, so quit if found
218 for k in ['ndk_paths', 'java_paths']:
224 thisconfig[k][k2] = exp
225 thisconfig[k][k2 + '_orig'] = v
228 def regsub_file(pattern, repl, path):
229 with open(path, 'rb') as f:
231 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
232 with open(path, 'wb') as f:
236 def read_config(opts, config_file='config.py'):
237 """Read the repository config
239 The config is read from config_file, which is in the current
240 directory when any of the repo management commands are used. If
241 there is a local metadata file in the git repo, then config.py is
242 not required, just use defaults.
245 global config, options
247 if config is not None:
254 if os.path.isfile(config_file):
255 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
256 with io.open(config_file, "rb") as f:
257 code = compile(f.read(), config_file, 'exec')
258 exec(code, None, config)
260 logging.warning(_("No 'config.py' found, using defaults."))
262 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
264 if not type(config[k]) in (str, list, tuple):
266 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
269 # smartcardoptions must be a list since its command line args for Popen
270 if 'smartcardoptions' in config:
271 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
272 elif 'keystore' in config and config['keystore'] == 'NONE':
273 # keystore='NONE' means use smartcard, these are required defaults
274 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
275 'SunPKCS11-OpenSC', '-providerClass',
276 'sun.security.pkcs11.SunPKCS11',
277 '-providerArg', 'opensc-fdroid.cfg']
279 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
280 st = os.stat(config_file)
281 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
282 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
283 .format(config_file=config_file))
285 fill_config_defaults(config)
287 for k in ["repo_description", "archive_description"]:
289 config[k] = clean_description(config[k])
291 if 'serverwebroot' in config:
292 if isinstance(config['serverwebroot'], str):
293 roots = [config['serverwebroot']]
294 elif all(isinstance(item, str) for item in config['serverwebroot']):
295 roots = config['serverwebroot']
297 raise TypeError(_('only accepts strings, lists, and tuples'))
299 for rootstr in roots:
300 # since this is used with rsync, where trailing slashes have
301 # meaning, ensure there is always a trailing slash
302 if rootstr[-1] != '/':
304 rootlist.append(rootstr.replace('//', '/'))
305 config['serverwebroot'] = rootlist
307 if 'servergitmirrors' in config:
308 if isinstance(config['servergitmirrors'], str):
309 roots = [config['servergitmirrors']]
310 elif all(isinstance(item, str) for item in config['servergitmirrors']):
311 roots = config['servergitmirrors']
313 raise TypeError(_('only accepts strings, lists, and tuples'))
314 config['servergitmirrors'] = roots
319 def assert_config_keystore(config):
320 """Check weather keystore is configured correctly and raise exception if not."""
323 if 'repo_keyalias' not in config:
325 logging.critical(_("'repo_keyalias' not found in config.py!"))
326 if 'keystore' not in config:
328 logging.critical(_("'keystore' not found in config.py!"))
329 elif not os.path.exists(config['keystore']):
331 logging.critical("'" + config['keystore'] + "' does not exist!")
332 if 'keystorepass' not in config:
334 logging.critical(_("'keystorepass' not found in config.py!"))
335 if 'keypass' not in config:
337 logging.critical(_("'keypass' not found in config.py!"))
339 raise FDroidException("This command requires a signing key, " +
340 "you can create one using: fdroid update --create-key")
343 def find_sdk_tools_cmd(cmd):
344 '''find a working path to a tool from the Android SDK'''
347 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
348 # try to find a working path to this command, in all the recent possible paths
349 if 'build_tools' in config:
350 build_tools = os.path.join(config['sdk_path'], 'build-tools')
351 # if 'build_tools' was manually set and exists, check only that one
352 configed_build_tools = os.path.join(build_tools, config['build_tools'])
353 if os.path.exists(configed_build_tools):
354 tooldirs.append(configed_build_tools)
356 # no configed version, so hunt known paths for it
357 for f in sorted(os.listdir(build_tools), reverse=True):
358 if os.path.isdir(os.path.join(build_tools, f)):
359 tooldirs.append(os.path.join(build_tools, f))
360 tooldirs.append(build_tools)
361 sdk_tools = os.path.join(config['sdk_path'], 'tools')
362 if os.path.exists(sdk_tools):
363 tooldirs.append(sdk_tools)
364 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
365 if os.path.exists(sdk_platform_tools):
366 tooldirs.append(sdk_platform_tools)
367 tooldirs.append('/usr/bin')
369 path = os.path.join(d, cmd)
370 if os.path.isfile(path):
372 test_aapt_version(path)
374 # did not find the command, exit with error message
375 ensure_build_tools_exists(config)
378 def test_aapt_version(aapt):
379 '''Check whether the version of aapt is new enough'''
380 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
381 if output is None or output == '':
382 logging.error(_("'{path}' failed to execute!").format(path=aapt))
384 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
389 # the Debian package has the version string like "v0.2-23.0.2"
390 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
391 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
394 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
397 def test_sdk_exists(thisconfig):
398 if 'sdk_path' not in thisconfig:
399 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
400 test_aapt_version(thisconfig['aapt'])
403 logging.error(_("'sdk_path' not set in 'config.py'!"))
405 if thisconfig['sdk_path'] == default_config['sdk_path']:
406 logging.error(_('No Android SDK found!'))
407 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
408 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
410 if not os.path.exists(thisconfig['sdk_path']):
411 logging.critical(_("Android SDK path '{path}' does not exist!")
412 .format(path=thisconfig['sdk_path']))
414 if not os.path.isdir(thisconfig['sdk_path']):
415 logging.critical(_("Android SDK path '{path}' is not a directory!")
416 .format(path=thisconfig['sdk_path']))
418 for d in ['build-tools', 'platform-tools', 'tools']:
419 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
420 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
421 .format(path=thisconfig['sdk_path'], dirname=d))
426 def ensure_build_tools_exists(thisconfig):
427 if not test_sdk_exists(thisconfig):
428 raise FDroidException(_("Android SDK not found!"))
429 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
430 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
431 if not os.path.isdir(versioned_build_tools):
432 raise FDroidException(
433 _("Android build-tools path '{path}' does not exist!")
434 .format(path=versioned_build_tools))
437 def get_local_metadata_files():
438 '''get any metadata files local to an app's source repo
440 This tries to ignore anything that does not count as app metdata,
441 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
444 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
447 def read_pkg_args(args, allow_vercodes=False):
449 :param args: arguments in the form of multiple appid:[vc] strings
450 :returns: a dictionary with the set of vercodes specified for each package
458 if allow_vercodes and ':' in p:
459 package, vercode = p.split(':')
461 package, vercode = p, None
462 if package not in vercodes:
463 vercodes[package] = [vercode] if vercode else []
465 elif vercode and vercode not in vercodes[package]:
466 vercodes[package] += [vercode] if vercode else []
471 def read_app_args(args, allapps, allow_vercodes=False):
473 On top of what read_pkg_args does, this returns the whole app metadata, but
474 limiting the builds list to the builds matching the vercodes specified.
477 vercodes = read_pkg_args(args, allow_vercodes)
483 for appid, app in allapps.items():
484 if appid in vercodes:
487 if len(apps) != len(vercodes):
490 logging.critical(_("No such package: %s") % p)
491 raise FDroidException(_("Found invalid appids in arguments"))
493 raise FDroidException(_("No packages specified"))
496 for appid, app in apps.items():
500 app.builds = [b for b in app.builds if b.versionCode in vc]
501 if len(app.builds) != len(vercodes[appid]):
503 allvcs = [b.versionCode for b in app.builds]
504 for v in vercodes[appid]:
506 logging.critical(_("No such versionCode {versionCode} for app {appid}")
507 .format(versionCode=v, appid=appid))
510 raise FDroidException(_("Found invalid versionCodes for some apps"))
515 def get_extension(filename):
516 base, ext = os.path.splitext(filename)
519 return base, ext.lower()[1:]
522 def has_extension(filename, ext):
523 _ignored, f_ext = get_extension(filename)
527 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
530 def clean_description(description):
531 'Remove unneeded newlines and spaces from a block of description text'
533 # this is split up by paragraph to make removing the newlines easier
534 for paragraph in re.split(r'\n\n', description):
535 paragraph = re.sub('\r', '', paragraph)
536 paragraph = re.sub('\n', ' ', paragraph)
537 paragraph = re.sub(' {2,}', ' ', paragraph)
538 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
539 returnstring += paragraph + '\n\n'
540 return returnstring.rstrip('\n')
543 def publishednameinfo(filename):
544 filename = os.path.basename(filename)
545 m = publish_name_regex.match(filename)
547 result = (m.group(1), m.group(2))
548 except AttributeError:
549 raise FDroidException(_("Invalid name for published file: %s") % filename)
553 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
554 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
557 def apk_parse_release_filename(apkname):
558 """Parses the name of an APK file according the F-Droids APK naming
559 scheme and returns the tokens.
561 WARNING: Returned values don't necessarily represent the APKs actual
562 properties, the are just paresed from the file name.
564 :returns: A triplet containing (appid, versionCode, signer), where appid
565 should be the package name, versionCode should be the integer
566 represion of the APKs version and signer should be the first 7 hex
567 digists of the sha256 signing key fingerprint which was used to sign
570 m = apk_release_filename_with_sigfp.match(apkname)
572 return m.group('appid'), m.group('vercode'), m.group('sigfp')
573 m = apk_release_filename.match(apkname)
575 return m.group('appid'), m.group('vercode'), None
576 return None, None, None
579 def get_release_filename(app, build):
581 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
583 return "%s_%s.apk" % (app.id, build.versionCode)
586 def get_toolsversion_logname(app, build):
587 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
590 def getsrcname(app, build):
591 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
603 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
606 def get_build_dir(app):
607 '''get the dir that this app will be built in'''
609 if app.RepoType == 'srclib':
610 return os.path.join('build', 'srclib', app.Repo)
612 return os.path.join('build', app.id)
616 '''checkout code from VCS and return instance of vcs and the build dir'''
617 build_dir = get_build_dir(app)
619 # Set up vcs interface and make sure we have the latest code...
620 logging.debug("Getting {0} vcs interface for {1}"
621 .format(app.RepoType, app.Repo))
622 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
626 vcs = getvcs(app.RepoType, remote, build_dir)
628 return vcs, build_dir
631 def getvcs(vcstype, remote, local):
633 return vcs_git(remote, local)
634 if vcstype == 'git-svn':
635 return vcs_gitsvn(remote, local)
637 return vcs_hg(remote, local)
639 return vcs_bzr(remote, local)
640 if vcstype == 'srclib':
641 if local != os.path.join('build', 'srclib', remote):
642 raise VCSException("Error: srclib paths are hard-coded!")
643 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
645 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
646 raise VCSException("Invalid vcs type " + vcstype)
649 def getsrclibvcs(name):
650 if name not in fdroidserver.metadata.srclibs:
651 raise VCSException("Missing srclib " + name)
652 return fdroidserver.metadata.srclibs[name]['Repo Type']
657 def __init__(self, remote, local):
659 # svn, git-svn and bzr may require auth
661 if self.repotype() in ('git-svn', 'bzr'):
663 if self.repotype == 'git-svn':
664 raise VCSException("Authentication is not supported for git-svn")
665 self.username, remote = remote.split('@')
666 if ':' not in self.username:
667 raise VCSException(_("Password required with username"))
668 self.username, self.password = self.username.split(':')
672 self.clone_failed = False
673 self.refreshed = False
679 def clientversion(self):
680 versionstr = FDroidPopen(self.clientversioncmd()).output
681 return versionstr[0:versionstr.find('\n')]
683 def clientversioncmd(self):
686 def gotorevision(self, rev, refresh=True):
687 """Take the local repository to a clean version of the given
688 revision, which is specificed in the VCS's native
689 format. Beforehand, the repository can be dirty, or even
690 non-existent. If the repository does already exist locally, it
691 will be updated from the origin, but only once in the lifetime
692 of the vcs object. None is acceptable for 'rev' if you know
693 you are cloning a clean copy of the repo - otherwise it must
694 specify a valid revision.
697 if self.clone_failed:
698 raise VCSException(_("Downloading the repository already failed once, not trying again."))
700 # The .fdroidvcs-id file for a repo tells us what VCS type
701 # and remote that directory was created from, allowing us to drop it
702 # automatically if either of those things changes.
703 fdpath = os.path.join(self.local, '..',
704 '.fdroidvcs-' + os.path.basename(self.local))
705 fdpath = os.path.normpath(fdpath)
706 cdata = self.repotype() + ' ' + self.remote
709 if os.path.exists(self.local):
710 if os.path.exists(fdpath):
711 with open(fdpath, 'r') as f:
712 fsdata = f.read().strip()
717 logging.info("Repository details for %s changed - deleting" % (
721 logging.info("Repository details for %s missing - deleting" % (
724 shutil.rmtree(self.local)
728 self.refreshed = True
731 self.gotorevisionx(rev)
732 except FDroidException as e:
735 # If necessary, write the .fdroidvcs file.
736 if writeback and not self.clone_failed:
737 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
738 with open(fdpath, 'w+') as f:
744 def gotorevisionx(self, rev): # pylint: disable=unused-argument
745 """Derived classes need to implement this.
747 It's called once basic checking has been performed.
749 raise VCSException("This VCS type doesn't define gotorevisionx")
751 # Initialise and update submodules
752 def initsubmodules(self):
753 raise VCSException('Submodules not supported for this vcs type')
755 # Get a list of all known tags
757 if not self._gettags:
758 raise VCSException('gettags not supported for this vcs type')
760 for tag in self._gettags():
761 if re.match('[-A-Za-z0-9_. /]+$', tag):
765 def latesttags(self):
766 """Get a list of all the known tags, sorted from newest to oldest"""
767 raise VCSException('latesttags not supported for this vcs type')
770 """Get current commit reference (hash, revision, etc)"""
771 raise VCSException('getref not supported for this vcs type')
774 """Returns the srclib (name, path) used in setting up the current revision, or None."""
783 def clientversioncmd(self):
784 return ['git', '--version']
786 def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
787 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
789 While fetch/pull/clone respect the command line option flags,
790 it seems that submodule commands do not. They do seem to
791 follow whatever is in env vars, if the version of git is new
792 enough. So we just throw the kitchen sink at it to see what
799 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
800 git_config.append('-c')
801 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
802 git_config.append('-c')
803 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
804 git_config.append('-c')
805 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
806 # add helpful tricks supported in git >= 2.3
807 ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
808 git_config.append('-c')
809 git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10
811 'GIT_TERMINAL_PROMPT': '0',
812 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3
814 return FDroidPopen(['git', ] + git_config + gitargs,
815 envs=envs, cwd=cwd, output=output)
818 """If the local directory exists, but is somehow not a git repository,
819 git will traverse up the directory tree until it finds one
820 that is (i.e. fdroidserver) and then we'll proceed to destroy
821 it! This is called as a safety check.
825 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
826 result = p.output.rstrip()
827 if not result.endswith(self.local):
828 raise VCSException('Repository mismatch')
830 def gotorevisionx(self, rev):
831 if not os.path.exists(self.local):
833 p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
834 if p.returncode != 0:
835 self.clone_failed = True
836 raise VCSException("Git clone failed", p.output)
840 # Discard any working tree changes
841 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
842 'git', 'reset', '--hard'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException(_("Git reset failed"), p.output)
845 # Remove untracked files now, in case they're tracked in the target
846 # revision (it happens!)
847 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
848 'git', 'clean', '-dffx'], cwd=self.local, output=False)
849 if p.returncode != 0:
850 raise VCSException(_("Git clean failed"), p.output)
851 if not self.refreshed:
852 # Get latest commits and tags from remote
853 p = self.GitFetchFDroidPopen(['fetch', 'origin'])
854 if p.returncode != 0:
855 raise VCSException(_("Git fetch failed"), p.output)
856 p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
857 if p.returncode != 0:
858 raise VCSException(_("Git fetch failed"), p.output)
859 # Recreate origin/HEAD as git clone would do it, in case it disappeared
860 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
861 if p.returncode != 0:
862 lines = p.output.splitlines()
863 if 'Multiple remote HEAD branches' not in lines[0]:
864 raise VCSException(_("Git remote set-head failed"), p.output)
865 branch = lines[1].split(' ')[-1]
866 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
867 if p2.returncode != 0:
868 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
869 self.refreshed = True
870 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
871 # a github repo. Most of the time this is the same as origin/master.
872 rev = rev or 'origin/HEAD'
873 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
874 if p.returncode != 0:
875 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
876 # Get rid of any uncontrolled files left behind
877 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException(_("Git clean failed"), p.output)
881 def initsubmodules(self):
883 submfile = os.path.join(self.local, '.gitmodules')
884 if not os.path.isfile(submfile):
885 raise VCSException(_("No git submodules available"))
887 # fix submodules not accessible without an account and public key auth
888 with open(submfile, 'r') as f:
889 lines = f.readlines()
890 with open(submfile, 'w') as f:
892 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
893 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
896 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
897 if p.returncode != 0:
898 raise VCSException(_("Git submodule sync failed"), p.output)
899 p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
900 if p.returncode != 0:
901 raise VCSException(_("Git submodule update failed"), p.output)
905 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
906 return p.output.splitlines()
908 tag_format = re.compile(r'tag: ([^),]*)')
910 def latesttags(self):
912 p = FDroidPopen(['git', 'log', '--tags',
913 '--simplify-by-decoration', '--pretty=format:%d'],
914 cwd=self.local, output=False)
916 for line in p.output.splitlines():
917 for tag in self.tag_format.findall(line):
922 class vcs_gitsvn(vcs):
927 def clientversioncmd(self):
928 return ['git', 'svn', '--version']
931 """If the local directory exists, but is somehow not a git repository,
932 git will traverse up the directory tree until it finds one that
933 is (i.e. fdroidserver) and then we'll proceed to destory it!
934 This is called as a safety check.
937 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
938 result = p.output.rstrip()
939 if not result.endswith(self.local):
940 raise VCSException('Repository mismatch')
942 def gotorevisionx(self, rev):
943 if not os.path.exists(self.local):
945 gitsvn_args = ['git', 'svn', 'clone']
946 if ';' in self.remote:
947 remote_split = self.remote.split(';')
948 for i in remote_split[1:]:
949 if i.startswith('trunk='):
950 gitsvn_args.extend(['-T', i[6:]])
951 elif i.startswith('tags='):
952 gitsvn_args.extend(['-t', i[5:]])
953 elif i.startswith('branches='):
954 gitsvn_args.extend(['-b', i[9:]])
955 gitsvn_args.extend([remote_split[0], self.local])
956 p = FDroidPopen(gitsvn_args, output=False)
957 if p.returncode != 0:
958 self.clone_failed = True
959 raise VCSException("Git svn clone failed", p.output)
961 gitsvn_args.extend([self.remote, self.local])
962 p = FDroidPopen(gitsvn_args, output=False)
963 if p.returncode != 0:
964 self.clone_failed = True
965 raise VCSException("Git svn clone failed", p.output)
969 # Discard any working tree changes
970 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
971 if p.returncode != 0:
972 raise VCSException("Git reset failed", p.output)
973 # Remove untracked files now, in case they're tracked in the target
974 # revision (it happens!)
975 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
976 if p.returncode != 0:
977 raise VCSException("Git clean failed", p.output)
978 if not self.refreshed:
979 # Get new commits, branches and tags from repo
980 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
981 if p.returncode != 0:
982 raise VCSException("Git svn fetch failed")
983 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
984 if p.returncode != 0:
985 raise VCSException("Git svn rebase failed", p.output)
986 self.refreshed = True
988 rev = rev or 'master'
990 nospaces_rev = rev.replace(' ', '%20')
991 # Try finding a svn tag
992 for treeish in ['origin/', '']:
993 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
994 if p.returncode == 0:
996 if p.returncode != 0:
997 # No tag found, normal svn rev translation
998 # Translate svn rev into git format
999 rev_split = rev.split('/')
1002 for treeish in ['origin/', '']:
1003 if len(rev_split) > 1:
1004 treeish += rev_split[0]
1005 svn_rev = rev_split[1]
1008 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1012 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1014 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1015 git_rev = p.output.rstrip()
1017 if p.returncode == 0 and git_rev:
1020 if p.returncode != 0 or not git_rev:
1021 # Try a plain git checkout as a last resort
1022 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
1023 if p.returncode != 0:
1024 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1026 # Check out the git rev equivalent to the svn rev
1027 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
1028 if p.returncode != 0:
1029 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1031 # Get rid of any uncontrolled files left behind
1032 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
1033 if p.returncode != 0:
1034 raise VCSException(_("Git clean failed"), p.output)
1038 for treeish in ['origin/', '']:
1039 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1040 if os.path.isdir(d):
1041 return os.listdir(d)
1045 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1046 if p.returncode != 0:
1048 return p.output.strip()
1056 def clientversioncmd(self):
1057 return ['hg', '--version']
1059 def gotorevisionx(self, rev):
1060 if not os.path.exists(self.local):
1061 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
1062 if p.returncode != 0:
1063 self.clone_failed = True
1064 raise VCSException("Hg clone failed", p.output)
1066 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1067 if p.returncode != 0:
1068 raise VCSException("Hg status failed", p.output)
1069 for line in p.output.splitlines():
1070 if not line.startswith('? '):
1071 raise VCSException("Unexpected output from hg status -uS: " + line)
1072 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1073 if not self.refreshed:
1074 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
1075 if p.returncode != 0:
1076 raise VCSException("Hg pull failed", p.output)
1077 self.refreshed = True
1079 rev = rev or 'default'
1082 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1083 if p.returncode != 0:
1084 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1085 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1086 # Also delete untracked files, we have to enable purge extension for that:
1087 if "'purge' is provided by the following extension" in p.output:
1088 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1089 myfile.write("\n[extensions]\nhgext.purge=\n")
1090 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1091 if p.returncode != 0:
1092 raise VCSException("HG purge failed", p.output)
1093 elif p.returncode != 0:
1094 raise VCSException("HG purge failed", p.output)
1097 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1098 return p.output.splitlines()[1:]
1106 def clientversioncmd(self):
1107 return ['bzr', '--version']
1109 def gotorevisionx(self, rev):
1110 if not os.path.exists(self.local):
1111 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1112 if p.returncode != 0:
1113 self.clone_failed = True
1114 raise VCSException("Bzr branch failed", p.output)
1116 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1117 if p.returncode != 0:
1118 raise VCSException("Bzr revert failed", p.output)
1119 if not self.refreshed:
1120 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1121 if p.returncode != 0:
1122 raise VCSException("Bzr update failed", p.output)
1123 self.refreshed = True
1125 revargs = list(['-r', rev] if rev else [])
1126 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1127 if p.returncode != 0:
1128 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1131 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1132 return [tag.split(' ')[0].strip() for tag in
1133 p.output.splitlines()]
1136 def unescape_string(string):
1139 if string[0] == '"' and string[-1] == '"':
1142 return string.replace("\\'", "'")
1145 def retrieve_string(app_dir, string, xmlfiles=None):
1147 if not string.startswith('@string/'):
1148 return unescape_string(string)
1150 if xmlfiles is None:
1153 os.path.join(app_dir, 'res'),
1154 os.path.join(app_dir, 'src', 'main', 'res'),
1156 for root, dirs, files in os.walk(res_dir):
1157 if os.path.basename(root) == 'values':
1158 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1160 name = string[len('@string/'):]
1162 def element_content(element):
1163 if element.text is None:
1165 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1166 return s.decode('utf-8').strip()
1168 for path in xmlfiles:
1169 if not os.path.isfile(path):
1171 xml = parse_xml(path)
1172 element = xml.find('string[@name="' + name + '"]')
1173 if element is not None:
1174 content = element_content(element)
1175 return retrieve_string(app_dir, content, xmlfiles)
1180 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1181 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1184 def manifest_paths(app_dir, flavours):
1185 '''Return list of existing files that will be used to find the highest vercode'''
1187 possible_manifests = \
1188 [os.path.join(app_dir, 'AndroidManifest.xml'),
1189 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1190 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1191 os.path.join(app_dir, 'build.gradle')]
1193 for flavour in flavours:
1194 if flavour == 'yes':
1196 possible_manifests.append(
1197 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1199 return [path for path in possible_manifests if os.path.isfile(path)]
1202 def fetch_real_name(app_dir, flavours):
1203 '''Retrieve the package name. Returns the name, or None if not found.'''
1204 for path in manifest_paths(app_dir, flavours):
1205 if not has_extension(path, 'xml') or not os.path.isfile(path):
1207 logging.debug("fetch_real_name: Checking manifest at " + path)
1208 xml = parse_xml(path)
1209 app = xml.find('application')
1212 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1214 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1215 result = retrieve_string_singleline(app_dir, label)
1217 result = result.strip()
1222 def get_library_references(root_dir):
1224 proppath = os.path.join(root_dir, 'project.properties')
1225 if not os.path.isfile(proppath):
1227 with open(proppath, 'r', encoding='iso-8859-1') as f:
1229 if not line.startswith('android.library.reference.'):
1231 path = line.split('=')[1].strip()
1232 relpath = os.path.join(root_dir, path)
1233 if not os.path.isdir(relpath):
1235 logging.debug("Found subproject at %s" % path)
1236 libraries.append(path)
1240 def ant_subprojects(root_dir):
1241 subprojects = get_library_references(root_dir)
1242 for subpath in subprojects:
1243 subrelpath = os.path.join(root_dir, subpath)
1244 for p in get_library_references(subrelpath):
1245 relp = os.path.normpath(os.path.join(subpath, p))
1246 if relp not in subprojects:
1247 subprojects.insert(0, relp)
1251 def remove_debuggable_flags(root_dir):
1252 # Remove forced debuggable flags
1253 logging.debug("Removing debuggable flags from %s" % root_dir)
1254 for root, dirs, files in os.walk(root_dir):
1255 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1256 regsub_file(r'android:debuggable="[^"]*"',
1258 os.path.join(root, 'AndroidManifest.xml'))
1261 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1262 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1263 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1266 def app_matches_packagename(app, package):
1269 appid = app.UpdateCheckName or app.id
1270 if appid is None or appid == "Ignore":
1272 return appid == package
1275 def parse_androidmanifests(paths, app):
1277 Extract some information from the AndroidManifest.xml at the given path.
1278 Returns (version, vercode, package), any or all of which might be None.
1279 All values returned are strings.
1282 ignoreversions = app.UpdateCheckIgnore
1283 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1286 return (None, None, None)
1294 if not os.path.isfile(path):
1297 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1302 if has_extension(path, 'gradle'):
1303 with open(path, 'r') as f:
1305 if gradle_comment.match(line):
1307 # Grab first occurence of each to avoid running into
1308 # alternative flavours and builds.
1310 matches = psearch_g(line)
1312 s = matches.group(2)
1313 if app_matches_packagename(app, s):
1316 matches = vnsearch_g(line)
1318 version = matches.group(2)
1320 matches = vcsearch_g(line)
1322 vercode = matches.group(1)
1325 xml = parse_xml(path)
1326 if "package" in xml.attrib:
1327 s = xml.attrib["package"]
1328 if app_matches_packagename(app, s):
1330 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1331 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1332 base_dir = os.path.dirname(path)
1333 version = retrieve_string_singleline(base_dir, version)
1334 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1335 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1336 if string_is_integer(a):
1339 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1341 # Remember package name, may be defined separately from version+vercode
1343 package = max_package
1345 logging.debug("..got package={0}, version={1}, vercode={2}"
1346 .format(package, version, vercode))
1348 # Always grab the package name and version name in case they are not
1349 # together with the highest version code
1350 if max_package is None and package is not None:
1351 max_package = package
1352 if max_version is None and version is not None:
1353 max_version = version
1355 if vercode is not None \
1356 and (max_vercode is None or vercode > max_vercode):
1357 if not ignoresearch or not ignoresearch(version):
1358 if version is not None:
1359 max_version = version
1360 if vercode is not None:
1361 max_vercode = vercode
1362 if package is not None:
1363 max_package = package
1365 max_version = "Ignore"
1367 if max_version is None:
1368 max_version = "Unknown"
1370 if max_package and not is_valid_package_name(max_package):
1371 raise FDroidException(_("Invalid package name {0}").format(max_package))
1373 return (max_version, max_vercode, max_package)
1376 def is_valid_package_name(name):
1377 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1380 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1381 raw=False, prepare=True, preponly=False, refresh=True,
1383 """Get the specified source library.
1385 Returns the path to it. Normally this is the path to be used when
1386 referencing it, which may be a subdirectory of the actual project. If
1387 you want the base directory of the project, pass 'basepath=True'.
1396 name, ref = spec.split('@')
1398 number, name = name.split(':', 1)
1400 name, subdir = name.split('/', 1)
1402 if name not in fdroidserver.metadata.srclibs:
1403 raise VCSException('srclib ' + name + ' not found.')
1405 srclib = fdroidserver.metadata.srclibs[name]
1407 sdir = os.path.join(srclib_dir, name)
1410 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1411 vcs.srclib = (name, number, sdir)
1413 vcs.gotorevision(ref, refresh)
1420 libdir = os.path.join(sdir, subdir)
1421 elif srclib["Subdir"]:
1422 for subdir in srclib["Subdir"]:
1423 libdir_candidate = os.path.join(sdir, subdir)
1424 if os.path.exists(libdir_candidate):
1425 libdir = libdir_candidate
1431 remove_signing_keys(sdir)
1432 remove_debuggable_flags(sdir)
1436 if srclib["Prepare"]:
1437 cmd = replace_config_vars(srclib["Prepare"], build)
1439 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1440 if p.returncode != 0:
1441 raise BuildException("Error running prepare command for srclib %s"
1447 return (name, number, libdir)
1450 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1453 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1454 """ Prepare the source code for a particular build
1456 :param vcs: the appropriate vcs object for the application
1457 :param app: the application details from the metadata
1458 :param build: the build details from the metadata
1459 :param build_dir: the path to the build directory, usually 'build/app.id'
1460 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1461 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1463 Returns the (root, srclibpaths) where:
1464 :param root: is the root directory, which may be the same as 'build_dir' or may
1465 be a subdirectory of it.
1466 :param srclibpaths: is information on the srclibs being used
1469 # Optionally, the actual app source can be in a subdirectory
1471 root_dir = os.path.join(build_dir, build.subdir)
1473 root_dir = build_dir
1475 # Get a working copy of the right revision
1476 logging.info("Getting source for revision " + build.commit)
1477 vcs.gotorevision(build.commit, refresh)
1479 # Initialise submodules if required
1480 if build.submodules:
1481 logging.info(_("Initialising submodules"))
1482 vcs.initsubmodules()
1484 # Check that a subdir (if we're using one) exists. This has to happen
1485 # after the checkout, since it might not exist elsewhere
1486 if not os.path.exists(root_dir):
1487 raise BuildException('Missing subdir ' + root_dir)
1489 # Run an init command if one is required
1491 cmd = replace_config_vars(build.init, build)
1492 logging.info("Running 'init' commands in %s" % root_dir)
1494 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1495 if p.returncode != 0:
1496 raise BuildException("Error running init command for %s:%s" %
1497 (app.id, build.versionName), p.output)
1499 # Apply patches if any
1501 logging.info("Applying patches")
1502 for patch in build.patch:
1503 patch = patch.strip()
1504 logging.info("Applying " + patch)
1505 patch_path = os.path.join('metadata', app.id, patch)
1506 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1507 if p.returncode != 0:
1508 raise BuildException("Failed to apply patch %s" % patch_path)
1510 # Get required source libraries
1513 logging.info("Collecting source libraries")
1514 for lib in build.srclibs:
1515 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1516 refresh=refresh, build=build))
1518 for name, number, libpath in srclibpaths:
1519 place_srclib(root_dir, int(number) if number else None, libpath)
1521 basesrclib = vcs.getsrclib()
1522 # If one was used for the main source, add that too.
1524 srclibpaths.append(basesrclib)
1526 # Update the local.properties file
1527 localprops = [os.path.join(build_dir, 'local.properties')]
1529 parts = build.subdir.split(os.sep)
1532 cur = os.path.join(cur, d)
1533 localprops += [os.path.join(cur, 'local.properties')]
1534 for path in localprops:
1536 if os.path.isfile(path):
1537 logging.info("Updating local.properties file at %s" % path)
1538 with open(path, 'r', encoding='iso-8859-1') as f:
1542 logging.info("Creating local.properties file at %s" % path)
1543 # Fix old-fashioned 'sdk-location' by copying
1544 # from sdk.dir, if necessary
1546 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1547 re.S | re.M).group(1)
1548 props += "sdk-location=%s\n" % sdkloc
1550 props += "sdk.dir=%s\n" % config['sdk_path']
1551 props += "sdk-location=%s\n" % config['sdk_path']
1552 ndk_path = build.ndk_path()
1553 # if for any reason the path isn't valid or the directory
1554 # doesn't exist, some versions of Gradle will error with a
1555 # cryptic message (even if the NDK is not even necessary).
1556 # https://gitlab.com/fdroid/fdroidserver/issues/171
1557 if ndk_path and os.path.exists(ndk_path):
1559 props += "ndk.dir=%s\n" % ndk_path
1560 props += "ndk-location=%s\n" % ndk_path
1561 # Add java.encoding if necessary
1563 props += "java.encoding=%s\n" % build.encoding
1564 with open(path, 'w', encoding='iso-8859-1') as f:
1568 if build.build_method() == 'gradle':
1569 flavours = build.gradle
1572 n = build.target.split('-')[1]
1573 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1574 r'compileSdkVersion %s' % n,
1575 os.path.join(root_dir, 'build.gradle'))
1577 # Remove forced debuggable flags
1578 remove_debuggable_flags(root_dir)
1580 # Insert version code and number into the manifest if necessary
1581 if build.forceversion:
1582 logging.info("Changing the version name")
1583 for path in manifest_paths(root_dir, flavours):
1584 if not os.path.isfile(path):
1586 if has_extension(path, 'xml'):
1587 regsub_file(r'android:versionName="[^"]*"',
1588 r'android:versionName="%s"' % build.versionName,
1590 elif has_extension(path, 'gradle'):
1591 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1592 r"""\1versionName '%s'""" % build.versionName,
1595 if build.forcevercode:
1596 logging.info("Changing the version code")
1597 for path in manifest_paths(root_dir, flavours):
1598 if not os.path.isfile(path):
1600 if has_extension(path, 'xml'):
1601 regsub_file(r'android:versionCode="[^"]*"',
1602 r'android:versionCode="%s"' % build.versionCode,
1604 elif has_extension(path, 'gradle'):
1605 regsub_file(r'versionCode[ =]+[0-9]+',
1606 r'versionCode %s' % build.versionCode,
1609 # Delete unwanted files
1611 logging.info(_("Removing specified files"))
1612 for part in getpaths(build_dir, build.rm):
1613 dest = os.path.join(build_dir, part)
1614 logging.info("Removing {0}".format(part))
1615 if os.path.lexists(dest):
1616 # rmtree can only handle directories that are not symlinks, so catch anything else
1617 if not os.path.isdir(dest) or os.path.islink(dest):
1622 logging.info("...but it didn't exist")
1624 remove_signing_keys(build_dir)
1626 # Add required external libraries
1628 logging.info("Collecting prebuilt libraries")
1629 libsdir = os.path.join(root_dir, 'libs')
1630 if not os.path.exists(libsdir):
1632 for lib in build.extlibs:
1634 logging.info("...installing extlib {0}".format(lib))
1635 libf = os.path.basename(lib)
1636 libsrc = os.path.join(extlib_dir, lib)
1637 if not os.path.exists(libsrc):
1638 raise BuildException("Missing extlib file {0}".format(libsrc))
1639 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1641 # Run a pre-build command if one is required
1643 logging.info("Running 'prebuild' commands in %s" % root_dir)
1645 cmd = replace_config_vars(build.prebuild, build)
1647 # Substitute source library paths into prebuild commands
1648 for name, number, libpath in srclibpaths:
1649 libpath = os.path.relpath(libpath, root_dir)
1650 cmd = cmd.replace('$$' + name + '$$', libpath)
1652 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1653 if p.returncode != 0:
1654 raise BuildException("Error running prebuild command for %s:%s" %
1655 (app.id, build.versionName), p.output)
1657 # Generate (or update) the ant build file, build.xml...
1658 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1659 parms = ['android', 'update', 'lib-project']
1660 lparms = ['android', 'update', 'project']
1663 parms += ['-t', build.target]
1664 lparms += ['-t', build.target]
1665 if build.androidupdate:
1666 update_dirs = build.androidupdate
1668 update_dirs = ant_subprojects(root_dir) + ['.']
1670 for d in update_dirs:
1671 subdir = os.path.join(root_dir, d)
1673 logging.debug("Updating main project")
1674 cmd = parms + ['-p', d]
1676 logging.debug("Updating subproject %s" % d)
1677 cmd = lparms + ['-p', d]
1678 p = SdkToolsPopen(cmd, cwd=root_dir)
1679 # Check to see whether an error was returned without a proper exit
1680 # code (this is the case for the 'no target set or target invalid'
1682 if p.returncode != 0 or p.output.startswith("Error: "):
1683 raise BuildException("Failed to update project at %s" % d, p.output)
1684 # Clean update dirs via ant
1686 logging.info("Cleaning subproject %s" % d)
1687 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1689 return (root_dir, srclibpaths)
1692 def getpaths_map(build_dir, globpaths):
1693 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1697 full_path = os.path.join(build_dir, p)
1698 full_path = os.path.normpath(full_path)
1699 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1701 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1705 def getpaths(build_dir, globpaths):
1706 """Extend via globbing the paths from a field and return them as a set"""
1707 paths_map = getpaths_map(build_dir, globpaths)
1709 for k, v in paths_map.items():
1716 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1720 """permanent store of existing APKs with the date they were added
1722 This is currently the only way to permanently store the "updated"
1727 '''Load filename/date info about previously seen APKs
1729 Since the appid and date strings both will never have spaces,
1730 this is parsed as a list from the end to allow the filename to
1731 have any combo of spaces.
1734 self.path = os.path.join('stats', 'known_apks.txt')
1736 if os.path.isfile(self.path):
1737 with open(self.path, 'r', encoding='utf8') as f:
1739 t = line.rstrip().split(' ')
1741 self.apks[t[0]] = (t[1], None)
1744 date = datetime.strptime(t[-1], '%Y-%m-%d')
1745 filename = line[0:line.rfind(appid) - 1]
1746 self.apks[filename] = (appid, date)
1747 self.changed = False
1749 def writeifchanged(self):
1750 if not self.changed:
1753 if not os.path.exists('stats'):
1757 for apk, app in self.apks.items():
1759 line = apk + ' ' + appid
1761 line += ' ' + added.strftime('%Y-%m-%d')
1764 with open(self.path, 'w', encoding='utf8') as f:
1765 for line in sorted(lst, key=natural_key):
1766 f.write(line + '\n')
1768 def recordapk(self, apkName, app, default_date=None):
1770 Record an apk (if it's new, otherwise does nothing)
1771 Returns the date it was added as a datetime instance
1773 if apkName not in self.apks:
1774 if default_date is None:
1775 default_date = datetime.utcnow()
1776 self.apks[apkName] = (app, default_date)
1778 _ignored, added = self.apks[apkName]
1781 def getapp(self, apkname):
1782 """Look up information - given the 'apkname', returns (app id, date added/None).
1784 Or returns None for an unknown apk.
1786 if apkname in self.apks:
1787 return self.apks[apkname]
1790 def getlatest(self, num):
1791 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1793 for apk, app in self.apks.items():
1797 if apps[appid] > added:
1801 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1802 lst = [app for app, _ignored in sortedapps]
1807 def get_file_extension(filename):
1808 """get the normalized file extension, can be blank string but never None"""
1809 if isinstance(filename, bytes):
1810 filename = filename.decode('utf-8')
1811 return os.path.splitext(filename)[1].lower()[1:]
1814 def get_apk_debuggable_aapt(apkfile):
1815 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1817 if p.returncode != 0:
1818 raise FDroidException(_("Failed to get APK manifest information"))
1819 for line in p.output.splitlines():
1820 if 'android:debuggable' in line and not line.endswith('0x0'):
1825 def get_apk_debuggable_androguard(apkfile):
1827 from androguard.core.bytecodes.apk import APK
1829 raise FDroidException("androguard library is not installed and aapt not present")
1831 apkobject = APK(apkfile)
1832 if apkobject.is_valid_APK():
1833 debuggable = apkobject.get_element("application", "debuggable")
1834 if debuggable is not None:
1835 return bool(strtobool(debuggable))
1839 def isApkAndDebuggable(apkfile):
1840 """Returns True if the given file is an APK and is debuggable
1842 :param apkfile: full path to the apk to check"""
1844 if get_file_extension(apkfile) != 'apk':
1847 if SdkToolsPopen(['aapt', 'version'], output=False):
1848 return get_apk_debuggable_aapt(apkfile)
1850 return get_apk_debuggable_androguard(apkfile)
1853 def get_apk_id_aapt(apkfile):
1854 """Extrat identification information from APK using aapt.
1856 :param apkfile: path to an APK file.
1857 :returns: triplet (appid, version code, version name)
1859 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1860 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1861 for line in p.output.splitlines():
1864 return m.group('appid'), m.group('vercode'), m.group('vername')
1865 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1866 .format(apkfilename=apkfile))
1871 self.returncode = None
1875 def SdkToolsPopen(commands, cwd=None, output=True):
1877 if cmd not in config:
1878 config[cmd] = find_sdk_tools_cmd(commands[0])
1879 abscmd = config[cmd]
1881 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1883 test_aapt_version(config['aapt'])
1884 return FDroidPopen([abscmd] + commands[1:],
1885 cwd=cwd, output=output)
1888 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1890 Run a command and capture the possibly huge output as bytes.
1892 :param commands: command and argument list like in subprocess.Popen
1893 :param cwd: optionally specifies a working directory
1894 :param envs: a optional dictionary of environment variables and their values
1895 :returns: A PopenResult.
1900 set_FDroidPopen_env()
1902 process_env = env.copy()
1903 if envs is not None and len(envs) > 0:
1904 process_env.update(envs)
1907 cwd = os.path.normpath(cwd)
1908 logging.debug("Directory: %s" % cwd)
1909 logging.debug("> %s" % ' '.join(commands))
1911 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1912 result = PopenResult()
1915 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1916 stdout=subprocess.PIPE, stderr=stderr_param)
1917 except OSError as e:
1918 raise BuildException("OSError while trying to execute " +
1919 ' '.join(commands) + ': ' + str(e))
1921 if not stderr_to_stdout and options.verbose:
1922 stderr_queue = Queue()
1923 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1925 while not stderr_reader.eof():
1926 while not stderr_queue.empty():
1927 line = stderr_queue.get()
1928 sys.stderr.buffer.write(line)
1933 stdout_queue = Queue()
1934 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1937 # Check the queue for output (until there is no more to get)
1938 while not stdout_reader.eof():
1939 while not stdout_queue.empty():
1940 line = stdout_queue.get()
1941 if output and options.verbose:
1942 # Output directly to console
1943 sys.stderr.buffer.write(line)
1949 result.returncode = p.wait()
1950 result.output = buf.getvalue()
1952 # make sure all filestreams of the subprocess are closed
1953 for streamvar in ['stdin', 'stdout', 'stderr']:
1954 if hasattr(p, streamvar):
1955 stream = getattr(p, streamvar)
1961 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1963 Run a command and capture the possibly huge output as a str.
1965 :param commands: command and argument list like in subprocess.Popen
1966 :param cwd: optionally specifies a working directory
1967 :param envs: a optional dictionary of environment variables and their values
1968 :returns: A PopenResult.
1970 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1971 result.output = result.output.decode('utf-8', 'ignore')
1975 gradle_comment = re.compile(r'[ ]*//')
1976 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1977 gradle_line_matches = [
1978 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1979 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1980 re.compile(r'.*\.readLine\(.*'),
1984 def remove_signing_keys(build_dir):
1985 for root, dirs, files in os.walk(build_dir):
1986 if 'build.gradle' in files:
1987 path = os.path.join(root, 'build.gradle')
1989 with open(path, "r", encoding='utf8') as o:
1990 lines = o.readlines()
1996 with open(path, "w", encoding='utf8') as o:
1997 while i < len(lines):
2000 while line.endswith('\\\n'):
2001 line = line.rstrip('\\\n') + lines[i]
2004 if gradle_comment.match(line):
2009 opened += line.count('{')
2010 opened -= line.count('}')
2013 if gradle_signing_configs.match(line):
2018 if any(s.match(line) for s in gradle_line_matches):
2026 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2029 'project.properties',
2031 'default.properties',
2032 'ant.properties', ]:
2033 if propfile in files:
2034 path = os.path.join(root, propfile)
2036 with open(path, "r", encoding='iso-8859-1') as o:
2037 lines = o.readlines()
2041 with open(path, "w", encoding='iso-8859-1') as o:
2043 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2050 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2053 def set_FDroidPopen_env(build=None):
2055 set up the environment variables for the build environment
2057 There is only a weak standard, the variables used by gradle, so also set
2058 up the most commonly used environment variables for SDK and NDK. Also, if
2059 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2061 global env, orig_path
2065 orig_path = env['PATH']
2066 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2067 env[n] = config['sdk_path']
2068 for k, v in config['java_paths'].items():
2069 env['JAVA%s_HOME' % k] = v
2071 missinglocale = True
2072 for k, v in env.items():
2073 if k == 'LANG' and v != 'C':
2074 missinglocale = False
2076 missinglocale = False
2078 env['LANG'] = 'en_US.UTF-8'
2080 if build is not None:
2081 path = build.ndk_path()
2082 paths = orig_path.split(os.pathsep)
2083 if path not in paths:
2084 paths = [path] + paths
2085 env['PATH'] = os.pathsep.join(paths)
2086 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2087 env[n] = build.ndk_path()
2090 def replace_build_vars(cmd, build):
2091 cmd = cmd.replace('$$COMMIT$$', build.commit)
2092 cmd = cmd.replace('$$VERSION$$', build.versionName)
2093 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2097 def replace_config_vars(cmd, build):
2098 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2099 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2100 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2101 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2102 if build is not None:
2103 cmd = replace_build_vars(cmd, build)
2107 def place_srclib(root_dir, number, libpath):
2110 relpath = os.path.relpath(libpath, root_dir)
2111 proppath = os.path.join(root_dir, 'project.properties')
2114 if os.path.isfile(proppath):
2115 with open(proppath, "r", encoding='iso-8859-1') as o:
2116 lines = o.readlines()
2118 with open(proppath, "w", encoding='iso-8859-1') as o:
2121 if line.startswith('android.library.reference.%d=' % number):
2122 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2127 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2130 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2133 def signer_fingerprint_short(sig):
2134 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2136 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2137 for a given pkcs7 signature.
2139 :param sig: Contents of an APK signing certificate.
2140 :returns: shortened signing-key fingerprint.
2142 return signer_fingerprint(sig)[:7]
2145 def signer_fingerprint(sig):
2146 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2148 Extracts hexadecimal sha256 signing-key fingerprint string
2149 for a given pkcs7 signature.
2151 :param: Contents of an APK signature.
2152 :returns: shortened signature fingerprint.
2154 cert_encoded = get_certificate(sig)
2155 return hashlib.sha256(cert_encoded).hexdigest()
2158 def apk_signer_fingerprint(apk_path):
2159 """Obtain sha256 signing-key fingerprint for APK.
2161 Extracts hexadecimal sha256 signing-key fingerprint string
2164 :param apkpath: path to APK
2165 :returns: signature fingerprint
2168 with zipfile.ZipFile(apk_path, 'r') as apk:
2169 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2172 logging.error("Found no signing certificates on %s" % apk_path)
2175 logging.error("Found multiple signing certificates on %s" % apk_path)
2178 cert = apk.read(certs[0])
2179 return signer_fingerprint(cert)
2182 def apk_signer_fingerprint_short(apk_path):
2183 """Obtain shortened sha256 signing-key fingerprint for APK.
2185 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2186 for a given pkcs7 APK.
2188 :param apk_path: path to APK
2189 :returns: shortened signing-key fingerprint
2191 return apk_signer_fingerprint(apk_path)[:7]
2194 def metadata_get_sigdir(appid, vercode=None):
2195 """Get signature directory for app"""
2197 return os.path.join('metadata', appid, 'signatures', vercode)
2199 return os.path.join('metadata', appid, 'signatures')
2202 def metadata_find_developer_signature(appid, vercode=None):
2203 """Tires to find the developer signature for given appid.
2205 This picks the first signature file found in metadata an returns its
2208 :returns: sha256 signing key fingerprint of the developer signing key.
2209 None in case no signature can not be found."""
2211 # fetch list of dirs for all versions of signatures
2214 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2216 appsigdir = metadata_get_sigdir(appid)
2217 if os.path.isdir(appsigdir):
2218 numre = re.compile('[0-9]+')
2219 for ver in os.listdir(appsigdir):
2220 if numre.match(ver):
2221 appversigdir = os.path.join(appsigdir, ver)
2222 appversigdirs.append(appversigdir)
2224 for sigdir in appversigdirs:
2225 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2226 glob.glob(os.path.join(sigdir, '*.EC')) + \
2227 glob.glob(os.path.join(sigdir, '*.RSA'))
2229 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2231 with open(sig, 'rb') as f:
2232 return signer_fingerprint(f.read())
2236 def metadata_find_signing_files(appid, vercode):
2237 """Gets a list of singed manifests and signatures.
2239 :param appid: app id string
2240 :param vercode: app version code
2241 :returns: a list of triplets for each signing key with following paths:
2242 (signature_file, singed_file, manifest_file)
2245 sigdir = metadata_get_sigdir(appid, vercode)
2246 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2247 glob.glob(os.path.join(sigdir, '*.EC')) + \
2248 glob.glob(os.path.join(sigdir, '*.RSA'))
2249 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2251 sf = extre.sub('.SF', sig)
2252 if os.path.isfile(sf):
2253 mf = os.path.join(sigdir, 'MANIFEST.MF')
2254 if os.path.isfile(mf):
2255 ret.append((sig, sf, mf))
2259 def metadata_find_developer_signing_files(appid, vercode):
2260 """Get developer signature files for specified app from metadata.
2262 :returns: A triplet of paths for signing files from metadata:
2263 (signature_file, singed_file, manifest_file)
2265 allsigningfiles = metadata_find_signing_files(appid, vercode)
2266 if allsigningfiles and len(allsigningfiles) == 1:
2267 return allsigningfiles[0]
2272 def apk_strip_signatures(signed_apk, strip_manifest=False):
2273 """Removes signatures from APK.
2275 :param signed_apk: path to apk file.
2276 :param strip_manifest: when set to True also the manifest file will
2277 be removed from the APK.
2279 with tempfile.TemporaryDirectory() as tmpdir:
2280 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2281 os.rename(signed_apk, tmp_apk)
2282 with ZipFile(tmp_apk, 'r') as in_apk:
2283 with ZipFile(signed_apk, 'w') as out_apk:
2284 for info in in_apk.infolist():
2285 if not apk_sigfile.match(info.filename):
2287 if info.filename != 'META-INF/MANIFEST.MF':
2288 buf = in_apk.read(info.filename)
2289 out_apk.writestr(info, buf)
2291 buf = in_apk.read(info.filename)
2292 out_apk.writestr(info, buf)
2295 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2296 """Implats a signature from metadata into an APK.
2298 Note: this changes there supplied APK in place. So copy it if you
2299 need the original to be preserved.
2301 :param apkpath: location of the apk
2303 # get list of available signature files in metadata
2304 with tempfile.TemporaryDirectory() as tmpdir:
2305 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2306 with ZipFile(apkpath, 'r') as in_apk:
2307 with ZipFile(apkwithnewsig, 'w') as out_apk:
2308 for sig_file in [signaturefile, signedfile, manifest]:
2309 with open(sig_file, 'rb') as fp:
2311 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2312 info.compress_type = zipfile.ZIP_DEFLATED
2313 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2314 out_apk.writestr(info, buf)
2315 for info in in_apk.infolist():
2316 if not apk_sigfile.match(info.filename):
2317 if info.filename != 'META-INF/MANIFEST.MF':
2318 buf = in_apk.read(info.filename)
2319 out_apk.writestr(info, buf)
2321 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2322 if p.returncode != 0:
2323 raise BuildException("Failed to align application")
2326 def apk_extract_signatures(apkpath, outdir, manifest=True):
2327 """Extracts a signature files from APK and puts them into target directory.
2329 :param apkpath: location of the apk
2330 :param outdir: folder where the extracted signature files will be stored
2331 :param manifest: (optionally) disable extracting manifest file
2333 with ZipFile(apkpath, 'r') as in_apk:
2334 for f in in_apk.infolist():
2335 if apk_sigfile.match(f.filename) or \
2336 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2337 newpath = os.path.join(outdir, os.path.basename(f.filename))
2338 with open(newpath, 'wb') as out_file:
2339 out_file.write(in_apk.read(f.filename))
2342 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2343 """Verify that two apks are the same
2345 One of the inputs is signed, the other is unsigned. The signature metadata
2346 is transferred from the signed to the unsigned apk, and then jarsigner is
2347 used to verify that the signature from the signed apk is also varlid for
2348 the unsigned one. If the APK given as unsigned actually does have a
2349 signature, it will be stripped out and ignored.
2351 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2352 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2353 into AndroidManifest.xml, but that makes the build not reproducible. So
2354 instead they are included as separate files in the APK's META-INF/ folder.
2355 If those files exist in the signed APK, they will be part of the signature
2356 and need to also be included in the unsigned APK for it to validate.
2358 :param signed_apk: Path to a signed apk file
2359 :param unsigned_apk: Path to an unsigned apk file expected to match it
2360 :param tmp_dir: Path to directory for temporary files
2361 :returns: None if the verification is successful, otherwise a string
2362 describing what went wrong.
2365 if not os.path.isfile(signed_apk):
2366 return 'can not verify: file does not exists: {}'.format(signed_apk)
2368 if not os.path.isfile(unsigned_apk):
2369 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2371 with ZipFile(signed_apk, 'r') as signed:
2372 meta_inf_files = ['META-INF/MANIFEST.MF']
2373 for f in signed.namelist():
2374 if apk_sigfile.match(f) \
2375 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2376 meta_inf_files.append(f)
2377 if len(meta_inf_files) < 3:
2378 return "Signature files missing from {0}".format(signed_apk)
2380 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2381 with ZipFile(unsigned_apk, 'r') as unsigned:
2382 # only read the signature from the signed APK, everything else from unsigned
2383 with ZipFile(tmp_apk, 'w') as tmp:
2384 for filename in meta_inf_files:
2385 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2386 for info in unsigned.infolist():
2387 if info.filename in meta_inf_files:
2388 logging.warning('Ignoring %s from %s',
2389 info.filename, unsigned_apk)
2391 if info.filename in tmp.namelist():
2392 return "duplicate filename found: " + info.filename
2393 tmp.writestr(info, unsigned.read(info.filename))
2395 verified = verify_apk_signature(tmp_apk)
2398 logging.info("...NOT verified - {0}".format(tmp_apk))
2399 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2400 os.path.dirname(unsigned_apk))
2402 logging.info("...successfully verified")
2406 def verify_jar_signature(jar):
2407 """Verifies the signature of a given JAR file.
2409 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2410 this has to turn on -strict then check for result 4, since this
2411 does not expect the signature to be from a CA-signed certificate.
2413 :raises: VerificationException() if the JAR's signature could not be verified
2417 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2418 raise VerificationException(_("The repository's index could not be verified."))
2421 def verify_apk_signature(apk, min_sdk_version=None):
2422 """verify the signature on an APK
2424 Try to use apksigner whenever possible since jarsigner is very
2425 shitty: unsigned APKs pass as "verified"! Warning, this does
2426 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2428 :returns: boolean whether the APK was verified
2430 if set_command_in_config('apksigner'):
2431 args = [config['apksigner'], 'verify']
2433 args += ['--min-sdk-version=' + min_sdk_version]
2434 return subprocess.call(args + [apk]) == 0
2436 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2438 verify_jar_signature(apk)
2445 def verify_old_apk_signature(apk):
2446 """verify the signature on an archived APK, supporting deprecated algorithms
2448 F-Droid aims to keep every single binary that it ever published. Therefore,
2449 it needs to be able to verify APK signatures that include deprecated/removed
2450 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2452 jarsigner passes unsigned APKs as "verified"! So this has to turn
2453 on -strict then check for result 4.
2455 :returns: boolean whether the APK was verified
2458 _java_security = os.path.join(os.getcwd(), '.java.security')
2459 with open(_java_security, 'w') as fp:
2460 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2462 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2463 '-strict', '-verify', apk]) == 4
2466 apk_badchars = re.compile('''[/ :;'"]''')
2469 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2472 Returns None if the apk content is the same (apart from the signing key),
2473 otherwise a string describing what's different, or what went wrong when
2474 trying to do the comparison.
2480 absapk1 = os.path.abspath(apk1)
2481 absapk2 = os.path.abspath(apk2)
2483 if set_command_in_config('diffoscope'):
2484 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2485 htmlfile = logfilename + '.diffoscope.html'
2486 textfile = logfilename + '.diffoscope.txt'
2487 if subprocess.call([config['diffoscope'],
2488 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2489 '--html', htmlfile, '--text', textfile,
2490 absapk1, absapk2]) != 0:
2491 return("Failed to unpack " + apk1)
2493 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2494 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2495 for d in [apk1dir, apk2dir]:
2496 if os.path.exists(d):
2499 os.mkdir(os.path.join(d, 'jar-xf'))
2501 if subprocess.call(['jar', 'xf',
2502 os.path.abspath(apk1)],
2503 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2504 return("Failed to unpack " + apk1)
2505 if subprocess.call(['jar', 'xf',
2506 os.path.abspath(apk2)],
2507 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2508 return("Failed to unpack " + apk2)
2510 if set_command_in_config('apktool'):
2511 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2513 return("Failed to unpack " + apk1)
2514 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2516 return("Failed to unpack " + apk2)
2518 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2519 lines = p.output.splitlines()
2520 if len(lines) != 1 or 'META-INF' not in lines[0]:
2521 if set_command_in_config('meld'):
2522 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2523 return("Unexpected diff output - " + p.output)
2525 # since everything verifies, delete the comparison to keep cruft down
2526 shutil.rmtree(apk1dir)
2527 shutil.rmtree(apk2dir)
2529 # If we get here, it seems like they're the same!
2533 def set_command_in_config(command):
2534 '''Try to find specified command in the path, if it hasn't been
2535 manually set in config.py. If found, it is added to the config
2536 dict. The return value says whether the command is available.
2539 if command in config:
2542 tmp = find_command(command)
2544 config[command] = tmp
2549 def find_command(command):
2550 '''find the full path of a command, or None if it can't be found in the PATH'''
2553 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2555 fpath, fname = os.path.split(command)
2560 for path in os.environ["PATH"].split(os.pathsep):
2561 path = path.strip('"')
2562 exe_file = os.path.join(path, command)
2563 if is_exe(exe_file):
2570 '''generate a random password for when generating keys'''
2571 h = hashlib.sha256()
2572 h.update(os.urandom(16)) # salt
2573 h.update(socket.getfqdn().encode('utf-8'))
2574 passwd = base64.b64encode(h.digest()).strip()
2575 return passwd.decode('utf-8')
2578 def genkeystore(localconfig):
2580 Generate a new key with password provided in :param localconfig and add it to new keystore
2581 :return: hexed public key, public key fingerprint
2583 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2584 keystoredir = os.path.dirname(localconfig['keystore'])
2585 if keystoredir is None or keystoredir == '':
2586 keystoredir = os.path.join(os.getcwd(), keystoredir)
2587 if not os.path.exists(keystoredir):
2588 os.makedirs(keystoredir, mode=0o700)
2591 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2592 'FDROID_KEY_PASS': localconfig['keypass'],
2594 p = FDroidPopen([config['keytool'], '-genkey',
2595 '-keystore', localconfig['keystore'],
2596 '-alias', localconfig['repo_keyalias'],
2597 '-keyalg', 'RSA', '-keysize', '4096',
2598 '-sigalg', 'SHA256withRSA',
2599 '-validity', '10000',
2600 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2601 '-keypass:env', 'FDROID_KEY_PASS',
2602 '-dname', localconfig['keydname']], envs=env_vars)
2603 if p.returncode != 0:
2604 raise BuildException("Failed to generate key", p.output)
2605 os.chmod(localconfig['keystore'], 0o0600)
2606 if not options.quiet:
2607 # now show the lovely key that was just generated
2608 p = FDroidPopen([config['keytool'], '-list', '-v',
2609 '-keystore', localconfig['keystore'],
2610 '-alias', localconfig['repo_keyalias'],
2611 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2612 logging.info(p.output.strip() + '\n\n')
2613 # get the public key
2614 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2615 '-keystore', localconfig['keystore'],
2616 '-alias', localconfig['repo_keyalias'],
2617 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2618 + config['smartcardoptions'],
2619 envs=env_vars, output=False, stderr_to_stdout=False)
2620 if p.returncode != 0 or len(p.output) < 20:
2621 raise BuildException("Failed to get public key", p.output)
2623 fingerprint = get_cert_fingerprint(pubkey)
2624 return hexlify(pubkey), fingerprint
2627 def get_cert_fingerprint(pubkey):
2629 Generate a certificate fingerprint the same way keytool does it
2630 (but with slightly different formatting)
2632 digest = hashlib.sha256(pubkey).digest()
2633 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2634 return " ".join(ret)
2637 def get_certificate(certificate_file):
2639 Extracts a certificate from the given file.
2640 :param certificate_file: file bytes (as string) representing the certificate
2641 :return: A binary representation of the certificate's public key, or None in case of error
2643 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2644 if content.getComponentByName('contentType') != rfc2315.signedData:
2646 content = decoder.decode(content.getComponentByName('content'),
2647 asn1Spec=rfc2315.SignedData())[0]
2649 certificates = content.getComponentByName('certificates')
2650 cert = certificates[0].getComponentByName('certificate')
2652 logging.error("Certificates not found.")
2654 return encoder.encode(cert)
2657 def load_stats_fdroid_signing_key_fingerprints():
2658 """Load list of signing-key fingerprints stored by fdroid publish from file.
2660 :returns: list of dictionanryies containing the singing-key fingerprints.
2662 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2663 if not os.path.isfile(jar_file):
2665 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2666 p = FDroidPopen(cmd, output=False)
2667 if p.returncode != 4:
2668 raise FDroidException("Signature validation of '{}' failed! "
2669 "Please run publish again to rebuild this file.".format(jar_file))
2671 jar_sigkey = apk_signer_fingerprint(jar_file)
2672 repo_key_sig = config.get('repo_key_sha256')
2674 if jar_sigkey != repo_key_sig:
2675 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2677 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2678 config['repo_key_sha256'] = jar_sigkey
2679 write_to_config(config, 'repo_key_sha256')
2681 with zipfile.ZipFile(jar_file, 'r') as f:
2682 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2685 def write_to_config(thisconfig, key, value=None, config_file=None):
2686 '''write a key/value to the local config.py
2688 NOTE: only supports writing string variables.
2690 :param thisconfig: config dictionary
2691 :param key: variable name in config.py to be overwritten/added
2692 :param value: optional value to be written, instead of fetched
2693 from 'thisconfig' dictionary.
2696 origkey = key + '_orig'
2697 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2698 cfg = config_file if config_file else 'config.py'
2700 # load config file, create one if it doesn't exist
2701 if not os.path.exists(cfg):
2702 open(cfg, 'a').close()
2703 logging.info("Creating empty " + cfg)
2704 with open(cfg, 'r', encoding="utf-8") as f:
2705 lines = f.readlines()
2707 # make sure the file ends with a carraige return
2709 if not lines[-1].endswith('\n'):
2712 # regex for finding and replacing python string variable
2713 # definitions/initializations
2714 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2715 repl = key + ' = "' + value + '"'
2716 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2717 repl2 = key + " = '" + value + "'"
2719 # If we replaced this line once, we make sure won't be a
2720 # second instance of this line for this key in the document.
2723 with open(cfg, 'w', encoding="utf-8") as f:
2725 if pattern.match(line) or pattern2.match(line):
2727 line = pattern.sub(repl, line)
2728 line = pattern2.sub(repl2, line)
2739 def parse_xml(path):
2740 return XMLElementTree.parse(path).getroot()
2743 def string_is_integer(string):
2751 def local_rsync(options, fromdir, todir):
2752 '''Rsync method for local to local copying of things
2754 This is an rsync wrapper with all the settings for safe use within
2755 the various fdroidserver use cases. This uses stricter rsync
2756 checking on all files since people using offline mode are already
2757 prioritizing security above ease and speed.
2760 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2761 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2762 if not options.no_checksum:
2763 rsyncargs.append('--checksum')
2765 rsyncargs += ['--verbose']
2767 rsyncargs += ['--quiet']
2768 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2769 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2770 raise FDroidException()
2773 def get_per_app_repos():
2774 '''per-app repos are dirs named with the packageName of a single app'''
2776 # Android packageNames are Java packages, they may contain uppercase or
2777 # lowercase letters ('A' through 'Z'), numbers, and underscores
2778 # ('_'). However, individual package name parts may only start with
2779 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2780 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2783 for root, dirs, files in os.walk(os.getcwd()):
2785 print('checking', root, 'for', d)
2786 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2787 # standard parts of an fdroid repo, so never packageNames
2790 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2796 def is_repo_file(filename):
2797 '''Whether the file in a repo is a build product to be delivered to users'''
2798 if isinstance(filename, str):
2799 filename = filename.encode('utf-8', errors="surrogateescape")
2800 return os.path.isfile(filename) \
2801 and not filename.endswith(b'.asc') \
2802 and not filename.endswith(b'.sig') \
2803 and os.path.basename(filename) not in [
2805 b'index_unsigned.jar',
2814 def get_examples_dir():
2815 '''Return the dir where the fdroidserver example files are available'''
2817 tmp = os.path.dirname(sys.argv[0])
2818 if os.path.basename(tmp) == 'bin':
2819 egg_links = glob.glob(os.path.join(tmp, '..',
2820 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2822 # installed from local git repo
2823 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2826 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2827 if not os.path.exists(examplesdir): # use UNIX layout
2828 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2830 # we're running straight out of the git repo
2831 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2832 examplesdir = prefix + '/examples'