3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
61 # A signature block file with a .DSA, .RSA, or .EC extension
62 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
63 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
64 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
66 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
75 'sdk_path': "$ANDROID_HOME",
80 'r12b': "$ANDROID_NDK",
87 'build_tools': "25.0.2",
88 'force_build_tools': False,
93 'accepted_formats': ['txt', 'yml'],
94 'sync_from_local_copy_dir': False,
95 'allow_disabled_algorithms': False,
96 'per_app_repos': False,
97 'make_current_version_link': True,
98 'current_version_name_source': 'Name',
99 'update_stats': False,
101 'stats_server': None,
103 'stats_to_carbon': False,
105 'build_server_always': False,
106 'keystore': 'keystore.jks',
107 'smartcardoptions': [],
117 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
118 'repo_name': "My First FDroid Repo Demo",
119 'repo_icon': "fdroid-icon.png",
120 'repo_description': '''
121 This is a repository of apps to be used with FDroid. Applications in this
122 repository are either official binaries built by the original application
123 developers, or are binaries built from source by the admin of f-droid.org
124 using the tools on https://gitlab.com/u/fdroid.
130 def setup_global_opts(parser):
131 parser.add_argument("-v", "--verbose", action="store_true", default=False,
132 help=_("Spew out even more information than normal"))
133 parser.add_argument("-q", "--quiet", action="store_true", default=False,
134 help=_("Restrict output to warnings and errors"))
137 def _add_java_paths_to_config(pathlist, thisconfig):
138 def path_version_key(s):
140 for u in re.split('[^0-9]+', s):
142 versionlist.append(int(u))
147 for d in sorted(pathlist, key=path_version_key):
148 if os.path.islink(d):
150 j = os.path.basename(d)
151 # the last one found will be the canonical one, so order appropriately
153 r'^1\.([6-9])\.0\.jdk$', # OSX
154 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
155 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
156 r'^jdk([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-jdk$', # Arch (oracle)
159 r'^java-1\.([6-9])\.0-.*$', # RedHat
160 r'^java-([6-9])-oracle$', # Debian WebUpd8
161 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
162 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
164 m = re.match(regex, j)
167 for p in [d, os.path.join(d, 'Contents', 'Home')]:
168 if os.path.exists(os.path.join(p, 'bin', 'javac')):
169 thisconfig['java_paths'][m.group(1)] = p
172 def fill_config_defaults(thisconfig):
173 for k, v in default_config.items():
174 if k not in thisconfig:
177 # Expand paths (~users and $vars)
178 def expand_path(path):
182 path = os.path.expanduser(path)
183 path = os.path.expandvars(path)
188 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
193 thisconfig[k + '_orig'] = v
195 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
196 if thisconfig['java_paths'] is None:
197 thisconfig['java_paths'] = dict()
199 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
200 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
201 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
202 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
203 if os.getenv('JAVA_HOME') is not None:
204 pathlist.append(os.getenv('JAVA_HOME'))
205 if os.getenv('PROGRAMFILES') is not None:
206 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
207 _add_java_paths_to_config(pathlist, thisconfig)
209 for java_version in ('7', '8', '9'):
210 if java_version not in thisconfig['java_paths']:
212 java_home = thisconfig['java_paths'][java_version]
213 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
214 if os.path.exists(jarsigner):
215 thisconfig['jarsigner'] = jarsigner
216 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
217 break # Java7 is preferred, so quit if found
219 for k in ['ndk_paths', 'java_paths']:
225 thisconfig[k][k2] = exp
226 thisconfig[k][k2 + '_orig'] = v
229 def regsub_file(pattern, repl, path):
230 with open(path, 'rb') as f:
232 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
233 with open(path, 'wb') as f:
237 def read_config(opts, config_file='config.py'):
238 """Read the repository config
240 The config is read from config_file, which is in the current
241 directory when any of the repo management commands are used. If
242 there is a local metadata file in the git repo, then config.py is
243 not required, just use defaults.
246 global config, options
248 if config is not None:
255 if os.path.isfile(config_file):
256 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
257 with io.open(config_file, "rb") as f:
258 code = compile(f.read(), config_file, 'exec')
259 exec(code, None, config)
261 logging.warning(_("No 'config.py' found, using defaults."))
263 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
265 if not type(config[k]) in (str, list, tuple):
267 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
270 # smartcardoptions must be a list since its command line args for Popen
271 if 'smartcardoptions' in config:
272 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
273 elif 'keystore' in config and config['keystore'] == 'NONE':
274 # keystore='NONE' means use smartcard, these are required defaults
275 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
276 'SunPKCS11-OpenSC', '-providerClass',
277 'sun.security.pkcs11.SunPKCS11',
278 '-providerArg', 'opensc-fdroid.cfg']
280 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
281 st = os.stat(config_file)
282 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
283 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
284 .format(config_file=config_file))
286 fill_config_defaults(config)
288 for k in ["repo_description", "archive_description"]:
290 config[k] = clean_description(config[k])
292 if 'serverwebroot' in config:
293 if isinstance(config['serverwebroot'], str):
294 roots = [config['serverwebroot']]
295 elif all(isinstance(item, str) for item in config['serverwebroot']):
296 roots = config['serverwebroot']
298 raise TypeError(_('only accepts strings, lists, and tuples'))
300 for rootstr in roots:
301 # since this is used with rsync, where trailing slashes have
302 # meaning, ensure there is always a trailing slash
303 if rootstr[-1] != '/':
305 rootlist.append(rootstr.replace('//', '/'))
306 config['serverwebroot'] = rootlist
308 if 'servergitmirrors' in config:
309 if isinstance(config['servergitmirrors'], str):
310 roots = [config['servergitmirrors']]
311 elif all(isinstance(item, str) for item in config['servergitmirrors']):
312 roots = config['servergitmirrors']
314 raise TypeError(_('only accepts strings, lists, and tuples'))
315 config['servergitmirrors'] = roots
320 def assert_config_keystore(config):
321 """Check weather keystore is configured correctly and raise exception if not."""
324 if 'repo_keyalias' not in config:
326 logging.critical(_("'repo_keyalias' not found in config.py!"))
327 if 'keystore' not in config:
329 logging.critical(_("'keystore' not found in config.py!"))
330 elif not os.path.exists(config['keystore']):
332 logging.critical("'" + config['keystore'] + "' does not exist!")
333 if 'keystorepass' not in config:
335 logging.critical(_("'keystorepass' not found in config.py!"))
336 if 'keypass' not in config:
338 logging.critical(_("'keypass' not found in config.py!"))
340 raise FDroidException("This command requires a signing key, " +
341 "you can create one using: fdroid update --create-key")
344 def find_sdk_tools_cmd(cmd):
345 '''find a working path to a tool from the Android SDK'''
348 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
349 # try to find a working path to this command, in all the recent possible paths
350 if 'build_tools' in config:
351 build_tools = os.path.join(config['sdk_path'], 'build-tools')
352 # if 'build_tools' was manually set and exists, check only that one
353 configed_build_tools = os.path.join(build_tools, config['build_tools'])
354 if os.path.exists(configed_build_tools):
355 tooldirs.append(configed_build_tools)
357 # no configed version, so hunt known paths for it
358 for f in sorted(os.listdir(build_tools), reverse=True):
359 if os.path.isdir(os.path.join(build_tools, f)):
360 tooldirs.append(os.path.join(build_tools, f))
361 tooldirs.append(build_tools)
362 sdk_tools = os.path.join(config['sdk_path'], 'tools')
363 if os.path.exists(sdk_tools):
364 tooldirs.append(sdk_tools)
365 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
366 if os.path.exists(sdk_platform_tools):
367 tooldirs.append(sdk_platform_tools)
368 tooldirs.append('/usr/bin')
370 path = os.path.join(d, cmd)
371 if os.path.isfile(path):
373 test_aapt_version(path)
375 # did not find the command, exit with error message
376 ensure_build_tools_exists(config)
379 def test_aapt_version(aapt):
380 '''Check whether the version of aapt is new enough'''
381 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
382 if output is None or output == '':
383 logging.error(_("'{path}' failed to execute!").format(path=aapt))
385 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
390 # the Debian package has the version string like "v0.2-23.0.2"
393 if LooseVersion(bugfix) < LooseVersion('24.0.0'):
395 elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2964546'):
398 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-24.0.0 or newer!")
401 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
404 def test_sdk_exists(thisconfig):
405 if 'sdk_path' not in thisconfig:
406 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
407 test_aapt_version(thisconfig['aapt'])
410 logging.error(_("'sdk_path' not set in 'config.py'!"))
412 if thisconfig['sdk_path'] == default_config['sdk_path']:
413 logging.error(_('No Android SDK found!'))
414 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
415 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
417 if not os.path.exists(thisconfig['sdk_path']):
418 logging.critical(_("Android SDK path '{path}' does not exist!")
419 .format(path=thisconfig['sdk_path']))
421 if not os.path.isdir(thisconfig['sdk_path']):
422 logging.critical(_("Android SDK path '{path}' is not a directory!")
423 .format(path=thisconfig['sdk_path']))
425 for d in ['build-tools', 'platform-tools', 'tools']:
426 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
427 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
428 .format(path=thisconfig['sdk_path'], dirname=d))
433 def ensure_build_tools_exists(thisconfig):
434 if not test_sdk_exists(thisconfig):
435 raise FDroidException(_("Android SDK not found!"))
436 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
437 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
438 if not os.path.isdir(versioned_build_tools):
439 raise FDroidException(
440 _("Android build-tools path '{path}' does not exist!")
441 .format(path=versioned_build_tools))
444 def get_local_metadata_files():
445 '''get any metadata files local to an app's source repo
447 This tries to ignore anything that does not count as app metdata,
448 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
451 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
454 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
456 :param appids: arguments in the form of multiple appid:[vc] strings
457 :returns: a dictionary with the set of vercodes specified for each package
460 if not appid_versionCode_pairs:
463 for p in appid_versionCode_pairs:
464 if allow_vercodes and ':' in p:
465 package, vercode = p.split(':')
467 package, vercode = p, None
468 if package not in vercodes:
469 vercodes[package] = [vercode] if vercode else []
471 elif vercode and vercode not in vercodes[package]:
472 vercodes[package] += [vercode] if vercode else []
477 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
478 """Build a list of App instances for processing
480 On top of what read_pkg_args does, this returns the whole app
481 metadata, but limiting the builds list to the builds matching the
482 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
483 all App and Build instances are returned.
487 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
493 for appid, app in allapps.items():
494 if appid in vercodes:
497 if len(apps) != len(vercodes):
500 logging.critical(_("No such package: %s") % p)
501 raise FDroidException(_("Found invalid appids in arguments"))
503 raise FDroidException(_("No packages specified"))
506 for appid, app in apps.items():
510 app.builds = [b for b in app.builds if b.versionCode in vc]
511 if len(app.builds) != len(vercodes[appid]):
513 allvcs = [b.versionCode for b in app.builds]
514 for v in vercodes[appid]:
516 logging.critical(_("No such versionCode {versionCode} for app {appid}")
517 .format(versionCode=v, appid=appid))
520 raise FDroidException(_("Found invalid versionCodes for some apps"))
525 def get_extension(filename):
526 base, ext = os.path.splitext(filename)
529 return base, ext.lower()[1:]
532 def has_extension(filename, ext):
533 _ignored, f_ext = get_extension(filename)
537 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
540 def clean_description(description):
541 'Remove unneeded newlines and spaces from a block of description text'
543 # this is split up by paragraph to make removing the newlines easier
544 for paragraph in re.split(r'\n\n', description):
545 paragraph = re.sub('\r', '', paragraph)
546 paragraph = re.sub('\n', ' ', paragraph)
547 paragraph = re.sub(' {2,}', ' ', paragraph)
548 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
549 returnstring += paragraph + '\n\n'
550 return returnstring.rstrip('\n')
553 def publishednameinfo(filename):
554 filename = os.path.basename(filename)
555 m = publish_name_regex.match(filename)
557 result = (m.group(1), m.group(2))
558 except AttributeError:
559 raise FDroidException(_("Invalid name for published file: %s") % filename)
563 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
564 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
567 def apk_parse_release_filename(apkname):
568 """Parses the name of an APK file according the F-Droids APK naming
569 scheme and returns the tokens.
571 WARNING: Returned values don't necessarily represent the APKs actual
572 properties, the are just paresed from the file name.
574 :returns: A triplet containing (appid, versionCode, signer), where appid
575 should be the package name, versionCode should be the integer
576 represion of the APKs version and signer should be the first 7 hex
577 digists of the sha256 signing key fingerprint which was used to sign
580 m = apk_release_filename_with_sigfp.match(apkname)
582 return m.group('appid'), m.group('vercode'), m.group('sigfp')
583 m = apk_release_filename.match(apkname)
585 return m.group('appid'), m.group('vercode'), None
586 return None, None, None
589 def get_release_filename(app, build):
591 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
593 return "%s_%s.apk" % (app.id, build.versionCode)
596 def get_toolsversion_logname(app, build):
597 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
600 def getsrcname(app, build):
601 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
613 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
616 def get_build_dir(app):
617 '''get the dir that this app will be built in'''
619 if app.RepoType == 'srclib':
620 return os.path.join('build', 'srclib', app.Repo)
622 return os.path.join('build', app.id)
626 '''checkout code from VCS and return instance of vcs and the build dir'''
627 build_dir = get_build_dir(app)
629 # Set up vcs interface and make sure we have the latest code...
630 logging.debug("Getting {0} vcs interface for {1}"
631 .format(app.RepoType, app.Repo))
632 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
636 vcs = getvcs(app.RepoType, remote, build_dir)
638 return vcs, build_dir
641 def getvcs(vcstype, remote, local):
643 return vcs_git(remote, local)
644 if vcstype == 'git-svn':
645 return vcs_gitsvn(remote, local)
647 return vcs_hg(remote, local)
649 return vcs_bzr(remote, local)
650 if vcstype == 'srclib':
651 if local != os.path.join('build', 'srclib', remote):
652 raise VCSException("Error: srclib paths are hard-coded!")
653 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
655 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
656 raise VCSException("Invalid vcs type " + vcstype)
659 def getsrclibvcs(name):
660 if name not in fdroidserver.metadata.srclibs:
661 raise VCSException("Missing srclib " + name)
662 return fdroidserver.metadata.srclibs[name]['Repo Type']
667 def __init__(self, remote, local):
669 # svn, git-svn and bzr may require auth
671 if self.repotype() in ('git-svn', 'bzr'):
673 if self.repotype == 'git-svn':
674 raise VCSException("Authentication is not supported for git-svn")
675 self.username, remote = remote.split('@')
676 if ':' not in self.username:
677 raise VCSException(_("Password required with username"))
678 self.username, self.password = self.username.split(':')
682 self.clone_failed = False
683 self.refreshed = False
689 def clientversion(self):
690 versionstr = FDroidPopen(self.clientversioncmd()).output
691 return versionstr[0:versionstr.find('\n')]
693 def clientversioncmd(self):
696 def gotorevision(self, rev, refresh=True):
697 """Take the local repository to a clean version of the given
698 revision, which is specificed in the VCS's native
699 format. Beforehand, the repository can be dirty, or even
700 non-existent. If the repository does already exist locally, it
701 will be updated from the origin, but only once in the lifetime
702 of the vcs object. None is acceptable for 'rev' if you know
703 you are cloning a clean copy of the repo - otherwise it must
704 specify a valid revision.
707 if self.clone_failed:
708 raise VCSException(_("Downloading the repository already failed once, not trying again."))
710 # The .fdroidvcs-id file for a repo tells us what VCS type
711 # and remote that directory was created from, allowing us to drop it
712 # automatically if either of those things changes.
713 fdpath = os.path.join(self.local, '..',
714 '.fdroidvcs-' + os.path.basename(self.local))
715 fdpath = os.path.normpath(fdpath)
716 cdata = self.repotype() + ' ' + self.remote
719 if os.path.exists(self.local):
720 if os.path.exists(fdpath):
721 with open(fdpath, 'r') as f:
722 fsdata = f.read().strip()
727 logging.info("Repository details for %s changed - deleting" % (
731 logging.info("Repository details for %s missing - deleting" % (
734 shutil.rmtree(self.local)
738 self.refreshed = True
741 self.gotorevisionx(rev)
742 except FDroidException as e:
745 # If necessary, write the .fdroidvcs file.
746 if writeback and not self.clone_failed:
747 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
748 with open(fdpath, 'w+') as f:
754 def gotorevisionx(self, rev): # pylint: disable=unused-argument
755 """Derived classes need to implement this.
757 It's called once basic checking has been performed.
759 raise VCSException("This VCS type doesn't define gotorevisionx")
761 # Initialise and update submodules
762 def initsubmodules(self):
763 raise VCSException('Submodules not supported for this vcs type')
765 # Get a list of all known tags
767 if not self._gettags:
768 raise VCSException('gettags not supported for this vcs type')
770 for tag in self._gettags():
771 if re.match('[-A-Za-z0-9_. /]+$', tag):
775 def latesttags(self):
776 """Get a list of all the known tags, sorted from newest to oldest"""
777 raise VCSException('latesttags not supported for this vcs type')
780 """Get current commit reference (hash, revision, etc)"""
781 raise VCSException('getref not supported for this vcs type')
784 """Returns the srclib (name, path) used in setting up the current revision, or None."""
793 def clientversioncmd(self):
794 return ['git', '--version']
796 def git(self, args, envs=dict(), cwd=None, output=True):
797 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
799 While fetch/pull/clone respect the command line option flags,
800 it seems that submodule commands do not. They do seem to
801 follow whatever is in env vars, if the version of git is new
802 enough. So we just throw the kitchen sink at it to see what
805 Also, because of CVE-2017-1000117, block all SSH URLs.
808 # supported in git >= 2.3
810 '-c', 'core.sshCommand=false',
811 '-c', 'url.https://.insteadOf=ssh://',
813 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
814 git_config.append('-c')
815 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
816 git_config.append('-c')
817 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
818 git_config.append('-c')
819 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
821 'GIT_TERMINAL_PROMPT': '0',
822 'GIT_SSH': 'false', # for git < 2.3
824 return FDroidPopen(['git', ] + git_config + args,
825 envs=envs, cwd=cwd, output=output)
828 """If the local directory exists, but is somehow not a git repository,
829 git will traverse up the directory tree until it finds one
830 that is (i.e. fdroidserver) and then we'll proceed to destroy
831 it! This is called as a safety check.
835 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
836 result = p.output.rstrip()
837 if not result.endswith(self.local):
838 raise VCSException('Repository mismatch')
840 def gotorevisionx(self, rev):
841 if not os.path.exists(self.local):
843 p = self.git(['clone', self.remote, self.local])
844 if p.returncode != 0:
845 self.clone_failed = True
846 raise VCSException("Git clone failed", p.output)
850 # Discard any working tree changes
851 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
852 'git', 'reset', '--hard'], cwd=self.local, output=False)
853 if p.returncode != 0:
854 raise VCSException(_("Git reset failed"), p.output)
855 # Remove untracked files now, in case they're tracked in the target
856 # revision (it happens!)
857 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
858 'git', 'clean', '-dffx'], cwd=self.local, output=False)
859 if p.returncode != 0:
860 raise VCSException(_("Git clean failed"), p.output)
861 if not self.refreshed:
862 # Get latest commits and tags from remote
863 p = self.git(['fetch', 'origin'], cwd=self.local)
864 if p.returncode != 0:
865 raise VCSException(_("Git fetch failed"), p.output)
866 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
867 if p.returncode != 0:
868 raise VCSException(_("Git fetch failed"), p.output)
869 # Recreate origin/HEAD as git clone would do it, in case it disappeared
870 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
871 if p.returncode != 0:
872 lines = p.output.splitlines()
873 if 'Multiple remote HEAD branches' not in lines[0]:
874 raise VCSException(_("Git remote set-head failed"), p.output)
875 branch = lines[1].split(' ')[-1]
876 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
877 if p2.returncode != 0:
878 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
879 self.refreshed = True
880 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
881 # a github repo. Most of the time this is the same as origin/master.
882 rev = rev or 'origin/HEAD'
883 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
884 if p.returncode != 0:
885 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
886 # Get rid of any uncontrolled files left behind
887 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
888 if p.returncode != 0:
889 raise VCSException(_("Git clean failed"), p.output)
891 def initsubmodules(self):
893 submfile = os.path.join(self.local, '.gitmodules')
894 if not os.path.isfile(submfile):
895 raise NoSubmodulesException(_("No git submodules available"))
897 # fix submodules not accessible without an account and public key auth
898 with open(submfile, 'r') as f:
899 lines = f.readlines()
900 with open(submfile, 'w') as f:
902 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
903 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
906 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
907 if p.returncode != 0:
908 raise VCSException(_("Git submodule sync failed"), p.output)
909 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
910 if p.returncode != 0:
911 raise VCSException(_("Git submodule update failed"), p.output)
915 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
916 return p.output.splitlines()
918 tag_format = re.compile(r'tag: ([^),]*)')
920 def latesttags(self):
922 p = FDroidPopen(['git', 'log', '--tags',
923 '--simplify-by-decoration', '--pretty=format:%d'],
924 cwd=self.local, output=False)
926 for line in p.output.splitlines():
927 for tag in self.tag_format.findall(line):
932 class vcs_gitsvn(vcs):
937 def clientversioncmd(self):
938 return ['git', 'svn', '--version']
941 """If the local directory exists, but is somehow not a git repository,
942 git will traverse up the directory tree until it finds one that
943 is (i.e. fdroidserver) and then we'll proceed to destory it!
944 This is called as a safety check.
947 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
948 result = p.output.rstrip()
949 if not result.endswith(self.local):
950 raise VCSException('Repository mismatch')
952 def git(self, args, envs=dict(), cwd=None, output=True):
953 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
955 # CVE-2017-1000117 block all SSH URLs (supported in git >= 2.3)
956 config = ['-c', 'core.sshCommand=false']
958 'GIT_TERMINAL_PROMPT': '0',
959 'GIT_SSH': 'false', # for git < 2.3
962 return FDroidPopen(['git', ] + config + args,
963 envs=envs, cwd=cwd, output=output)
965 def gotorevisionx(self, rev):
966 if not os.path.exists(self.local):
968 gitsvn_args = ['svn', 'clone']
969 if ';' in self.remote:
970 remote_split = self.remote.split(';')
971 for i in remote_split[1:]:
972 if i.startswith('trunk='):
973 gitsvn_args.extend(['-T', i[6:]])
974 elif i.startswith('tags='):
975 gitsvn_args.extend(['-t', i[5:]])
976 elif i.startswith('branches='):
977 gitsvn_args.extend(['-b', i[9:]])
978 gitsvn_args.extend([remote_split[0], self.local])
979 p = self.git(gitsvn_args, output=False)
980 if p.returncode != 0:
981 self.clone_failed = True
982 raise VCSException("Git svn clone failed", p.output)
984 gitsvn_args.extend([self.remote, self.local])
985 p = self.git(gitsvn_args, output=False)
986 if p.returncode != 0:
987 self.clone_failed = True
988 raise VCSException("Git svn clone failed", p.output)
992 # Discard any working tree changes
993 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
994 if p.returncode != 0:
995 raise VCSException("Git reset failed", p.output)
996 # Remove untracked files now, in case they're tracked in the target
997 # revision (it happens!)
998 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
999 if p.returncode != 0:
1000 raise VCSException("Git clean failed", p.output)
1001 if not self.refreshed:
1002 # Get new commits, branches and tags from repo
1003 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
1004 if p.returncode != 0:
1005 raise VCSException("Git svn fetch failed")
1006 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1007 if p.returncode != 0:
1008 raise VCSException("Git svn rebase failed", p.output)
1009 self.refreshed = True
1011 rev = rev or 'master'
1013 nospaces_rev = rev.replace(' ', '%20')
1014 # Try finding a svn tag
1015 for treeish in ['origin/', '']:
1016 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1017 if p.returncode == 0:
1019 if p.returncode != 0:
1020 # No tag found, normal svn rev translation
1021 # Translate svn rev into git format
1022 rev_split = rev.split('/')
1025 for treeish in ['origin/', '']:
1026 if len(rev_split) > 1:
1027 treeish += rev_split[0]
1028 svn_rev = rev_split[1]
1031 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1035 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1037 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1038 git_rev = p.output.rstrip()
1040 if p.returncode == 0 and git_rev:
1043 if p.returncode != 0 or not git_rev:
1044 # Try a plain git checkout as a last resort
1045 p = self.git(['checkout', rev], cwd=self.local, output=False)
1046 if p.returncode != 0:
1047 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1049 # Check out the git rev equivalent to the svn rev
1050 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1051 if p.returncode != 0:
1052 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1054 # Get rid of any uncontrolled files left behind
1055 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1056 if p.returncode != 0:
1057 raise VCSException(_("Git clean failed"), p.output)
1061 for treeish in ['origin/', '']:
1062 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1063 if os.path.isdir(d):
1064 return os.listdir(d)
1068 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1069 if p.returncode != 0:
1071 return p.output.strip()
1079 def clientversioncmd(self):
1080 return ['hg', '--version']
1082 def gotorevisionx(self, rev):
1083 if not os.path.exists(self.local):
1084 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', self.remote, self.local], output=False)
1085 if p.returncode != 0:
1086 self.clone_failed = True
1087 raise VCSException("Hg clone failed", p.output)
1089 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1090 if p.returncode != 0:
1091 raise VCSException("Hg status failed", p.output)
1092 for line in p.output.splitlines():
1093 if not line.startswith('? '):
1094 raise VCSException("Unexpected output from hg status -uS: " + line)
1095 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1096 if not self.refreshed:
1097 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1098 if p.returncode != 0:
1099 raise VCSException("Hg pull failed", p.output)
1100 self.refreshed = True
1102 rev = rev or 'default'
1105 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1106 if p.returncode != 0:
1107 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1108 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1109 # Also delete untracked files, we have to enable purge extension for that:
1110 if "'purge' is provided by the following extension" in p.output:
1111 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1112 myfile.write("\n[extensions]\nhgext.purge=\n")
1113 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1114 if p.returncode != 0:
1115 raise VCSException("HG purge failed", p.output)
1116 elif p.returncode != 0:
1117 raise VCSException("HG purge failed", p.output)
1120 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1121 return p.output.splitlines()[1:]
1129 def clientversioncmd(self):
1130 return ['bzr', '--version']
1132 def bzr(self, args, envs=dict(), cwd=None, output=True):
1133 '''Prevent bzr from ever using SSH to avoid security vulns'''
1137 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1139 def gotorevisionx(self, rev):
1140 if not os.path.exists(self.local):
1141 p = self.bzr(['branch', self.remote, self.local], output=False)
1142 if p.returncode != 0:
1143 self.clone_failed = True
1144 raise VCSException("Bzr branch failed", p.output)
1146 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1147 if p.returncode != 0:
1148 raise VCSException("Bzr revert failed", p.output)
1149 if not self.refreshed:
1150 p = self.bzr(['pull'], cwd=self.local, output=False)
1151 if p.returncode != 0:
1152 raise VCSException("Bzr update failed", p.output)
1153 self.refreshed = True
1155 revargs = list(['-r', rev] if rev else [])
1156 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1157 if p.returncode != 0:
1158 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1161 p = self.bzr(['tags'], cwd=self.local, output=False)
1162 return [tag.split(' ')[0].strip() for tag in
1163 p.output.splitlines()]
1166 def unescape_string(string):
1169 if string[0] == '"' and string[-1] == '"':
1172 return string.replace("\\'", "'")
1175 def retrieve_string(app_dir, string, xmlfiles=None):
1177 if not string.startswith('@string/'):
1178 return unescape_string(string)
1180 if xmlfiles is None:
1183 os.path.join(app_dir, 'res'),
1184 os.path.join(app_dir, 'src', 'main', 'res'),
1186 for root, dirs, files in os.walk(res_dir):
1187 if os.path.basename(root) == 'values':
1188 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1190 name = string[len('@string/'):]
1192 def element_content(element):
1193 if element.text is None:
1195 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1196 return s.decode('utf-8').strip()
1198 for path in xmlfiles:
1199 if not os.path.isfile(path):
1201 xml = parse_xml(path)
1202 element = xml.find('string[@name="' + name + '"]')
1203 if element is not None:
1204 content = element_content(element)
1205 return retrieve_string(app_dir, content, xmlfiles)
1210 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1211 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1214 def manifest_paths(app_dir, flavours):
1215 '''Return list of existing files that will be used to find the highest vercode'''
1217 possible_manifests = \
1218 [os.path.join(app_dir, 'AndroidManifest.xml'),
1219 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1220 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1221 os.path.join(app_dir, 'build.gradle')]
1223 for flavour in flavours:
1224 if flavour == 'yes':
1226 possible_manifests.append(
1227 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1229 return [path for path in possible_manifests if os.path.isfile(path)]
1232 def fetch_real_name(app_dir, flavours):
1233 '''Retrieve the package name. Returns the name, or None if not found.'''
1234 for path in manifest_paths(app_dir, flavours):
1235 if not has_extension(path, 'xml') or not os.path.isfile(path):
1237 logging.debug("fetch_real_name: Checking manifest at " + path)
1238 xml = parse_xml(path)
1239 app = xml.find('application')
1242 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1244 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1245 result = retrieve_string_singleline(app_dir, label)
1247 result = result.strip()
1252 def get_library_references(root_dir):
1254 proppath = os.path.join(root_dir, 'project.properties')
1255 if not os.path.isfile(proppath):
1257 with open(proppath, 'r', encoding='iso-8859-1') as f:
1259 if not line.startswith('android.library.reference.'):
1261 path = line.split('=')[1].strip()
1262 relpath = os.path.join(root_dir, path)
1263 if not os.path.isdir(relpath):
1265 logging.debug("Found subproject at %s" % path)
1266 libraries.append(path)
1270 def ant_subprojects(root_dir):
1271 subprojects = get_library_references(root_dir)
1272 for subpath in subprojects:
1273 subrelpath = os.path.join(root_dir, subpath)
1274 for p in get_library_references(subrelpath):
1275 relp = os.path.normpath(os.path.join(subpath, p))
1276 if relp not in subprojects:
1277 subprojects.insert(0, relp)
1281 def remove_debuggable_flags(root_dir):
1282 # Remove forced debuggable flags
1283 logging.debug("Removing debuggable flags from %s" % root_dir)
1284 for root, dirs, files in os.walk(root_dir):
1285 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1286 regsub_file(r'android:debuggable="[^"]*"',
1288 os.path.join(root, 'AndroidManifest.xml'))
1291 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1292 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1293 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1296 def app_matches_packagename(app, package):
1299 appid = app.UpdateCheckName or app.id
1300 if appid is None or appid == "Ignore":
1302 return appid == package
1305 def parse_androidmanifests(paths, app):
1307 Extract some information from the AndroidManifest.xml at the given path.
1308 Returns (version, vercode, package), any or all of which might be None.
1309 All values returned are strings.
1312 ignoreversions = app.UpdateCheckIgnore
1313 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1316 return (None, None, None)
1324 if not os.path.isfile(path):
1327 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1333 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1334 flavour = app.builds[-1].gradle[-1]
1336 if has_extension(path, 'gradle'):
1337 # first try to get version name and code from correct flavour
1338 with open(path, 'r') as f:
1339 buildfile = f.read()
1341 regex_string = r"" + flavour + "[^}]*?{.*?}"
1342 search = re.compile(regex_string, re.DOTALL)
1343 result = search.search(buildfile)
1345 if result is not None:
1346 resultgroup = result.group()
1349 matches = psearch_g(resultgroup)
1351 s = matches.group(2)
1352 if app_matches_packagename(app, s):
1355 matches = vnsearch_g(resultgroup)
1357 version = matches.group(2)
1359 matches = vcsearch_g(resultgroup)
1361 vercode = matches.group(1)
1363 # fall back to parse file line by line
1364 with open(path, 'r') as f:
1366 if gradle_comment.match(line):
1368 # Grab first occurence of each to avoid running into
1369 # alternative flavours and builds.
1371 matches = psearch_g(line)
1373 s = matches.group(2)
1374 if app_matches_packagename(app, s):
1377 matches = vnsearch_g(line)
1379 version = matches.group(2)
1381 matches = vcsearch_g(line)
1383 vercode = matches.group(1)
1386 xml = parse_xml(path)
1387 if "package" in xml.attrib:
1388 s = xml.attrib["package"]
1389 if app_matches_packagename(app, s):
1391 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1392 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1393 base_dir = os.path.dirname(path)
1394 version = retrieve_string_singleline(base_dir, version)
1395 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1396 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1397 if string_is_integer(a):
1400 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1402 # Remember package name, may be defined separately from version+vercode
1404 package = max_package
1406 logging.debug("..got package={0}, version={1}, vercode={2}"
1407 .format(package, version, vercode))
1409 # Always grab the package name and version name in case they are not
1410 # together with the highest version code
1411 if max_package is None and package is not None:
1412 max_package = package
1413 if max_version is None and version is not None:
1414 max_version = version
1416 if vercode is not None \
1417 and (max_vercode is None or vercode > max_vercode):
1418 if not ignoresearch or not ignoresearch(version):
1419 if version is not None:
1420 max_version = version
1421 if vercode is not None:
1422 max_vercode = vercode
1423 if package is not None:
1424 max_package = package
1426 max_version = "Ignore"
1428 if max_version is None:
1429 max_version = "Unknown"
1431 if max_package and not is_valid_package_name(max_package):
1432 raise FDroidException(_("Invalid package name {0}").format(max_package))
1434 return (max_version, max_vercode, max_package)
1437 def is_valid_package_name(name):
1438 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1441 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1442 raw=False, prepare=True, preponly=False, refresh=True,
1444 """Get the specified source library.
1446 Returns the path to it. Normally this is the path to be used when
1447 referencing it, which may be a subdirectory of the actual project. If
1448 you want the base directory of the project, pass 'basepath=True'.
1457 name, ref = spec.split('@')
1459 number, name = name.split(':', 1)
1461 name, subdir = name.split('/', 1)
1463 if name not in fdroidserver.metadata.srclibs:
1464 raise VCSException('srclib ' + name + ' not found.')
1466 srclib = fdroidserver.metadata.srclibs[name]
1468 sdir = os.path.join(srclib_dir, name)
1471 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1472 vcs.srclib = (name, number, sdir)
1474 vcs.gotorevision(ref, refresh)
1481 libdir = os.path.join(sdir, subdir)
1482 elif srclib["Subdir"]:
1483 for subdir in srclib["Subdir"]:
1484 libdir_candidate = os.path.join(sdir, subdir)
1485 if os.path.exists(libdir_candidate):
1486 libdir = libdir_candidate
1492 remove_signing_keys(sdir)
1493 remove_debuggable_flags(sdir)
1497 if srclib["Prepare"]:
1498 cmd = replace_config_vars(srclib["Prepare"], build)
1500 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1501 if p.returncode != 0:
1502 raise BuildException("Error running prepare command for srclib %s"
1508 return (name, number, libdir)
1511 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1514 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1515 """ Prepare the source code for a particular build
1517 :param vcs: the appropriate vcs object for the application
1518 :param app: the application details from the metadata
1519 :param build: the build details from the metadata
1520 :param build_dir: the path to the build directory, usually 'build/app.id'
1521 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1522 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1524 Returns the (root, srclibpaths) where:
1525 :param root: is the root directory, which may be the same as 'build_dir' or may
1526 be a subdirectory of it.
1527 :param srclibpaths: is information on the srclibs being used
1530 # Optionally, the actual app source can be in a subdirectory
1532 root_dir = os.path.join(build_dir, build.subdir)
1534 root_dir = build_dir
1536 # Get a working copy of the right revision
1537 logging.info("Getting source for revision " + build.commit)
1538 vcs.gotorevision(build.commit, refresh)
1540 # Initialise submodules if required
1541 if build.submodules:
1542 logging.info(_("Initialising submodules"))
1543 vcs.initsubmodules()
1545 # Check that a subdir (if we're using one) exists. This has to happen
1546 # after the checkout, since it might not exist elsewhere
1547 if not os.path.exists(root_dir):
1548 raise BuildException('Missing subdir ' + root_dir)
1550 # Run an init command if one is required
1552 cmd = replace_config_vars(build.init, build)
1553 logging.info("Running 'init' commands in %s" % root_dir)
1555 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1556 if p.returncode != 0:
1557 raise BuildException("Error running init command for %s:%s" %
1558 (app.id, build.versionName), p.output)
1560 # Apply patches if any
1562 logging.info("Applying patches")
1563 for patch in build.patch:
1564 patch = patch.strip()
1565 logging.info("Applying " + patch)
1566 patch_path = os.path.join('metadata', app.id, patch)
1567 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1568 if p.returncode != 0:
1569 raise BuildException("Failed to apply patch %s" % patch_path)
1571 # Get required source libraries
1574 logging.info("Collecting source libraries")
1575 for lib in build.srclibs:
1576 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1577 refresh=refresh, build=build))
1579 for name, number, libpath in srclibpaths:
1580 place_srclib(root_dir, int(number) if number else None, libpath)
1582 basesrclib = vcs.getsrclib()
1583 # If one was used for the main source, add that too.
1585 srclibpaths.append(basesrclib)
1587 # Update the local.properties file
1588 localprops = [os.path.join(build_dir, 'local.properties')]
1590 parts = build.subdir.split(os.sep)
1593 cur = os.path.join(cur, d)
1594 localprops += [os.path.join(cur, 'local.properties')]
1595 for path in localprops:
1597 if os.path.isfile(path):
1598 logging.info("Updating local.properties file at %s" % path)
1599 with open(path, 'r', encoding='iso-8859-1') as f:
1603 logging.info("Creating local.properties file at %s" % path)
1604 # Fix old-fashioned 'sdk-location' by copying
1605 # from sdk.dir, if necessary
1607 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1608 re.S | re.M).group(1)
1609 props += "sdk-location=%s\n" % sdkloc
1611 props += "sdk.dir=%s\n" % config['sdk_path']
1612 props += "sdk-location=%s\n" % config['sdk_path']
1613 ndk_path = build.ndk_path()
1614 # if for any reason the path isn't valid or the directory
1615 # doesn't exist, some versions of Gradle will error with a
1616 # cryptic message (even if the NDK is not even necessary).
1617 # https://gitlab.com/fdroid/fdroidserver/issues/171
1618 if ndk_path and os.path.exists(ndk_path):
1620 props += "ndk.dir=%s\n" % ndk_path
1621 props += "ndk-location=%s\n" % ndk_path
1622 # Add java.encoding if necessary
1624 props += "java.encoding=%s\n" % build.encoding
1625 with open(path, 'w', encoding='iso-8859-1') as f:
1629 if build.build_method() == 'gradle':
1630 flavours = build.gradle
1633 n = build.target.split('-')[1]
1634 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1635 r'compileSdkVersion %s' % n,
1636 os.path.join(root_dir, 'build.gradle'))
1638 # Remove forced debuggable flags
1639 remove_debuggable_flags(root_dir)
1641 # Insert version code and number into the manifest if necessary
1642 if build.forceversion:
1643 logging.info("Changing the version name")
1644 for path in manifest_paths(root_dir, flavours):
1645 if not os.path.isfile(path):
1647 if has_extension(path, 'xml'):
1648 regsub_file(r'android:versionName="[^"]*"',
1649 r'android:versionName="%s"' % build.versionName,
1651 elif has_extension(path, 'gradle'):
1652 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1653 r"""\1versionName '%s'""" % build.versionName,
1656 if build.forcevercode:
1657 logging.info("Changing the version code")
1658 for path in manifest_paths(root_dir, flavours):
1659 if not os.path.isfile(path):
1661 if has_extension(path, 'xml'):
1662 regsub_file(r'android:versionCode="[^"]*"',
1663 r'android:versionCode="%s"' % build.versionCode,
1665 elif has_extension(path, 'gradle'):
1666 regsub_file(r'versionCode[ =]+[0-9]+',
1667 r'versionCode %s' % build.versionCode,
1670 # Delete unwanted files
1672 logging.info(_("Removing specified files"))
1673 for part in getpaths(build_dir, build.rm):
1674 dest = os.path.join(build_dir, part)
1675 logging.info("Removing {0}".format(part))
1676 if os.path.lexists(dest):
1677 # rmtree can only handle directories that are not symlinks, so catch anything else
1678 if not os.path.isdir(dest) or os.path.islink(dest):
1683 logging.info("...but it didn't exist")
1685 remove_signing_keys(build_dir)
1687 # Add required external libraries
1689 logging.info("Collecting prebuilt libraries")
1690 libsdir = os.path.join(root_dir, 'libs')
1691 if not os.path.exists(libsdir):
1693 for lib in build.extlibs:
1695 logging.info("...installing extlib {0}".format(lib))
1696 libf = os.path.basename(lib)
1697 libsrc = os.path.join(extlib_dir, lib)
1698 if not os.path.exists(libsrc):
1699 raise BuildException("Missing extlib file {0}".format(libsrc))
1700 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1702 # Run a pre-build command if one is required
1704 logging.info("Running 'prebuild' commands in %s" % root_dir)
1706 cmd = replace_config_vars(build.prebuild, build)
1708 # Substitute source library paths into prebuild commands
1709 for name, number, libpath in srclibpaths:
1710 libpath = os.path.relpath(libpath, root_dir)
1711 cmd = cmd.replace('$$' + name + '$$', libpath)
1713 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1714 if p.returncode != 0:
1715 raise BuildException("Error running prebuild command for %s:%s" %
1716 (app.id, build.versionName), p.output)
1718 # Generate (or update) the ant build file, build.xml...
1719 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1720 parms = ['android', 'update', 'lib-project']
1721 lparms = ['android', 'update', 'project']
1724 parms += ['-t', build.target]
1725 lparms += ['-t', build.target]
1726 if build.androidupdate:
1727 update_dirs = build.androidupdate
1729 update_dirs = ant_subprojects(root_dir) + ['.']
1731 for d in update_dirs:
1732 subdir = os.path.join(root_dir, d)
1734 logging.debug("Updating main project")
1735 cmd = parms + ['-p', d]
1737 logging.debug("Updating subproject %s" % d)
1738 cmd = lparms + ['-p', d]
1739 p = SdkToolsPopen(cmd, cwd=root_dir)
1740 # Check to see whether an error was returned without a proper exit
1741 # code (this is the case for the 'no target set or target invalid'
1743 if p.returncode != 0 or p.output.startswith("Error: "):
1744 raise BuildException("Failed to update project at %s" % d, p.output)
1745 # Clean update dirs via ant
1747 logging.info("Cleaning subproject %s" % d)
1748 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1750 return (root_dir, srclibpaths)
1753 def getpaths_map(build_dir, globpaths):
1754 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1758 full_path = os.path.join(build_dir, p)
1759 full_path = os.path.normpath(full_path)
1760 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1762 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1766 def getpaths(build_dir, globpaths):
1767 """Extend via globbing the paths from a field and return them as a set"""
1768 paths_map = getpaths_map(build_dir, globpaths)
1770 for k, v in paths_map.items():
1777 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1780 def check_system_clock(dt_obj, path):
1781 """Check if system clock is updated based on provided date
1783 If an APK has files newer than the system time, suggest updating
1784 the system clock. This is useful for offline systems, used for
1785 signing, which do not have another source of clock sync info. It
1786 has to be more than 24 hours newer because ZIP/APK files do not
1790 checkdt = dt_obj - timedelta(1)
1791 if datetime.today() < checkdt:
1792 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1793 + '\n' + _('Set clock to that time using:') + '\n'
1794 + 'sudo date -s "' + str(dt_obj) + '"')
1798 """permanent store of existing APKs with the date they were added
1800 This is currently the only way to permanently store the "updated"
1805 '''Load filename/date info about previously seen APKs
1807 Since the appid and date strings both will never have spaces,
1808 this is parsed as a list from the end to allow the filename to
1809 have any combo of spaces.
1812 self.path = os.path.join('stats', 'known_apks.txt')
1814 if os.path.isfile(self.path):
1815 with open(self.path, 'r', encoding='utf8') as f:
1817 t = line.rstrip().split(' ')
1819 self.apks[t[0]] = (t[1], None)
1822 date = datetime.strptime(t[-1], '%Y-%m-%d')
1823 filename = line[0:line.rfind(appid) - 1]
1824 self.apks[filename] = (appid, date)
1825 check_system_clock(date, self.path)
1826 self.changed = False
1828 def writeifchanged(self):
1829 if not self.changed:
1832 if not os.path.exists('stats'):
1836 for apk, app in self.apks.items():
1838 line = apk + ' ' + appid
1840 line += ' ' + added.strftime('%Y-%m-%d')
1843 with open(self.path, 'w', encoding='utf8') as f:
1844 for line in sorted(lst, key=natural_key):
1845 f.write(line + '\n')
1847 def recordapk(self, apkName, app, default_date=None):
1849 Record an apk (if it's new, otherwise does nothing)
1850 Returns the date it was added as a datetime instance
1852 if apkName not in self.apks:
1853 if default_date is None:
1854 default_date = datetime.utcnow()
1855 self.apks[apkName] = (app, default_date)
1857 _ignored, added = self.apks[apkName]
1860 def getapp(self, apkname):
1861 """Look up information - given the 'apkname', returns (app id, date added/None).
1863 Or returns None for an unknown apk.
1865 if apkname in self.apks:
1866 return self.apks[apkname]
1869 def getlatest(self, num):
1870 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1872 for apk, app in self.apks.items():
1876 if apps[appid] > added:
1880 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1881 lst = [app for app, _ignored in sortedapps]
1886 def get_file_extension(filename):
1887 """get the normalized file extension, can be blank string but never None"""
1888 if isinstance(filename, bytes):
1889 filename = filename.decode('utf-8')
1890 return os.path.splitext(filename)[1].lower()[1:]
1893 def get_apk_debuggable_aapt(apkfile):
1894 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1896 if p.returncode != 0:
1897 raise FDroidException(_("Failed to get APK manifest information"))
1898 for line in p.output.splitlines():
1899 if 'android:debuggable' in line and not line.endswith('0x0'):
1904 def get_apk_debuggable_androguard(apkfile):
1906 from androguard.core.bytecodes.apk import APK
1908 raise FDroidException("androguard library is not installed and aapt not present")
1910 apkobject = APK(apkfile)
1911 if apkobject.is_valid_APK():
1912 debuggable = apkobject.get_element("application", "debuggable")
1913 if debuggable is not None:
1914 return bool(strtobool(debuggable))
1918 def isApkAndDebuggable(apkfile):
1919 """Returns True if the given file is an APK and is debuggable
1921 :param apkfile: full path to the apk to check"""
1923 if get_file_extension(apkfile) != 'apk':
1926 if SdkToolsPopen(['aapt', 'version'], output=False):
1927 return get_apk_debuggable_aapt(apkfile)
1929 return get_apk_debuggable_androguard(apkfile)
1932 def get_apk_id_aapt(apkfile):
1933 """Extrat identification information from APK using aapt.
1935 :param apkfile: path to an APK file.
1936 :returns: triplet (appid, version code, version name)
1938 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1939 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1940 for line in p.output.splitlines():
1943 return m.group('appid'), m.group('vercode'), m.group('vername')
1944 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1945 .format(apkfilename=apkfile))
1948 def get_minSdkVersion_aapt(apkfile):
1949 """Extract the minimum supported Android SDK from an APK using aapt
1951 :param apkfile: path to an APK file.
1952 :returns: the integer representing the SDK version
1954 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1955 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1956 for line in p.output.splitlines():
1959 return int(m.group(1))
1960 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1961 .format(apkfilename=apkfile))
1966 self.returncode = None
1970 def SdkToolsPopen(commands, cwd=None, output=True):
1972 if cmd not in config:
1973 config[cmd] = find_sdk_tools_cmd(commands[0])
1974 abscmd = config[cmd]
1976 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1978 test_aapt_version(config['aapt'])
1979 return FDroidPopen([abscmd] + commands[1:],
1980 cwd=cwd, output=output)
1983 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1985 Run a command and capture the possibly huge output as bytes.
1987 :param commands: command and argument list like in subprocess.Popen
1988 :param cwd: optionally specifies a working directory
1989 :param envs: a optional dictionary of environment variables and their values
1990 :returns: A PopenResult.
1995 set_FDroidPopen_env()
1997 process_env = env.copy()
1998 if envs is not None and len(envs) > 0:
1999 process_env.update(envs)
2002 cwd = os.path.normpath(cwd)
2003 logging.debug("Directory: %s" % cwd)
2004 logging.debug("> %s" % ' '.join(commands))
2006 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2007 result = PopenResult()
2010 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2011 stdout=subprocess.PIPE, stderr=stderr_param)
2012 except OSError as e:
2013 raise BuildException("OSError while trying to execute " +
2014 ' '.join(commands) + ': ' + str(e))
2016 # TODO are these AsynchronousFileReader threads always exiting?
2017 if not stderr_to_stdout and options.verbose:
2018 stderr_queue = Queue()
2019 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2021 while not stderr_reader.eof():
2022 while not stderr_queue.empty():
2023 line = stderr_queue.get()
2024 sys.stderr.buffer.write(line)
2029 stdout_queue = Queue()
2030 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2033 # Check the queue for output (until there is no more to get)
2034 while not stdout_reader.eof():
2035 while not stdout_queue.empty():
2036 line = stdout_queue.get()
2037 if output and options.verbose:
2038 # Output directly to console
2039 sys.stderr.buffer.write(line)
2045 result.returncode = p.wait()
2046 result.output = buf.getvalue()
2048 # make sure all filestreams of the subprocess are closed
2049 for streamvar in ['stdin', 'stdout', 'stderr']:
2050 if hasattr(p, streamvar):
2051 stream = getattr(p, streamvar)
2057 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2059 Run a command and capture the possibly huge output as a str.
2061 :param commands: command and argument list like in subprocess.Popen
2062 :param cwd: optionally specifies a working directory
2063 :param envs: a optional dictionary of environment variables and their values
2064 :returns: A PopenResult.
2066 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2067 result.output = result.output.decode('utf-8', 'ignore')
2071 gradle_comment = re.compile(r'[ ]*//')
2072 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2073 gradle_line_matches = [
2074 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2075 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2076 re.compile(r'.*\.readLine\(.*'),
2080 def remove_signing_keys(build_dir):
2081 for root, dirs, files in os.walk(build_dir):
2082 if 'build.gradle' in files:
2083 path = os.path.join(root, 'build.gradle')
2085 with open(path, "r", encoding='utf8') as o:
2086 lines = o.readlines()
2092 with open(path, "w", encoding='utf8') as o:
2093 while i < len(lines):
2096 while line.endswith('\\\n'):
2097 line = line.rstrip('\\\n') + lines[i]
2100 if gradle_comment.match(line):
2105 opened += line.count('{')
2106 opened -= line.count('}')
2109 if gradle_signing_configs.match(line):
2114 if any(s.match(line) for s in gradle_line_matches):
2122 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2125 'project.properties',
2127 'default.properties',
2128 'ant.properties', ]:
2129 if propfile in files:
2130 path = os.path.join(root, propfile)
2132 with open(path, "r", encoding='iso-8859-1') as o:
2133 lines = o.readlines()
2137 with open(path, "w", encoding='iso-8859-1') as o:
2139 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2146 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2149 def set_FDroidPopen_env(build=None):
2151 set up the environment variables for the build environment
2153 There is only a weak standard, the variables used by gradle, so also set
2154 up the most commonly used environment variables for SDK and NDK. Also, if
2155 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2157 global env, orig_path
2161 orig_path = env['PATH']
2162 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2163 env[n] = config['sdk_path']
2164 for k, v in config['java_paths'].items():
2165 env['JAVA%s_HOME' % k] = v
2167 missinglocale = True
2168 for k, v in env.items():
2169 if k == 'LANG' and v != 'C':
2170 missinglocale = False
2172 missinglocale = False
2174 env['LANG'] = 'en_US.UTF-8'
2176 if build is not None:
2177 path = build.ndk_path()
2178 paths = orig_path.split(os.pathsep)
2179 if path not in paths:
2180 paths = [path] + paths
2181 env['PATH'] = os.pathsep.join(paths)
2182 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2183 env[n] = build.ndk_path()
2186 def replace_build_vars(cmd, build):
2187 cmd = cmd.replace('$$COMMIT$$', build.commit)
2188 cmd = cmd.replace('$$VERSION$$', build.versionName)
2189 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2193 def replace_config_vars(cmd, build):
2194 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2195 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2196 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2197 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2198 if build is not None:
2199 cmd = replace_build_vars(cmd, build)
2203 def place_srclib(root_dir, number, libpath):
2206 relpath = os.path.relpath(libpath, root_dir)
2207 proppath = os.path.join(root_dir, 'project.properties')
2210 if os.path.isfile(proppath):
2211 with open(proppath, "r", encoding='iso-8859-1') as o:
2212 lines = o.readlines()
2214 with open(proppath, "w", encoding='iso-8859-1') as o:
2217 if line.startswith('android.library.reference.%d=' % number):
2218 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2223 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2226 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2229 def signer_fingerprint_short(sig):
2230 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2232 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2233 for a given pkcs7 signature.
2235 :param sig: Contents of an APK signing certificate.
2236 :returns: shortened signing-key fingerprint.
2238 return signer_fingerprint(sig)[:7]
2241 def signer_fingerprint(sig):
2242 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2244 Extracts hexadecimal sha256 signing-key fingerprint string
2245 for a given pkcs7 signature.
2247 :param: Contents of an APK signature.
2248 :returns: shortened signature fingerprint.
2250 cert_encoded = get_certificate(sig)
2251 return hashlib.sha256(cert_encoded).hexdigest()
2254 def apk_signer_fingerprint(apk_path):
2255 """Obtain sha256 signing-key fingerprint for APK.
2257 Extracts hexadecimal sha256 signing-key fingerprint string
2260 :param apkpath: path to APK
2261 :returns: signature fingerprint
2264 with zipfile.ZipFile(apk_path, 'r') as apk:
2265 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2268 logging.error("Found no signing certificates on %s" % apk_path)
2271 logging.error("Found multiple signing certificates on %s" % apk_path)
2274 cert = apk.read(certs[0])
2275 return signer_fingerprint(cert)
2278 def apk_signer_fingerprint_short(apk_path):
2279 """Obtain shortened sha256 signing-key fingerprint for APK.
2281 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2282 for a given pkcs7 APK.
2284 :param apk_path: path to APK
2285 :returns: shortened signing-key fingerprint
2287 return apk_signer_fingerprint(apk_path)[:7]
2290 def metadata_get_sigdir(appid, vercode=None):
2291 """Get signature directory for app"""
2293 return os.path.join('metadata', appid, 'signatures', vercode)
2295 return os.path.join('metadata', appid, 'signatures')
2298 def metadata_find_developer_signature(appid, vercode=None):
2299 """Tires to find the developer signature for given appid.
2301 This picks the first signature file found in metadata an returns its
2304 :returns: sha256 signing key fingerprint of the developer signing key.
2305 None in case no signature can not be found."""
2307 # fetch list of dirs for all versions of signatures
2310 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2312 appsigdir = metadata_get_sigdir(appid)
2313 if os.path.isdir(appsigdir):
2314 numre = re.compile('[0-9]+')
2315 for ver in os.listdir(appsigdir):
2316 if numre.match(ver):
2317 appversigdir = os.path.join(appsigdir, ver)
2318 appversigdirs.append(appversigdir)
2320 for sigdir in appversigdirs:
2321 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2322 glob.glob(os.path.join(sigdir, '*.EC')) + \
2323 glob.glob(os.path.join(sigdir, '*.RSA'))
2325 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2327 with open(sig, 'rb') as f:
2328 return signer_fingerprint(f.read())
2332 def metadata_find_signing_files(appid, vercode):
2333 """Gets a list of singed manifests and signatures.
2335 :param appid: app id string
2336 :param vercode: app version code
2337 :returns: a list of triplets for each signing key with following paths:
2338 (signature_file, singed_file, manifest_file)
2341 sigdir = metadata_get_sigdir(appid, vercode)
2342 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2343 glob.glob(os.path.join(sigdir, '*.EC')) + \
2344 glob.glob(os.path.join(sigdir, '*.RSA'))
2345 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2347 sf = extre.sub('.SF', sig)
2348 if os.path.isfile(sf):
2349 mf = os.path.join(sigdir, 'MANIFEST.MF')
2350 if os.path.isfile(mf):
2351 ret.append((sig, sf, mf))
2355 def metadata_find_developer_signing_files(appid, vercode):
2356 """Get developer signature files for specified app from metadata.
2358 :returns: A triplet of paths for signing files from metadata:
2359 (signature_file, singed_file, manifest_file)
2361 allsigningfiles = metadata_find_signing_files(appid, vercode)
2362 if allsigningfiles and len(allsigningfiles) == 1:
2363 return allsigningfiles[0]
2368 def apk_strip_signatures(signed_apk, strip_manifest=False):
2369 """Removes signatures from APK.
2371 :param signed_apk: path to apk file.
2372 :param strip_manifest: when set to True also the manifest file will
2373 be removed from the APK.
2375 with tempfile.TemporaryDirectory() as tmpdir:
2376 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2377 shutil.move(signed_apk, tmp_apk)
2378 with ZipFile(tmp_apk, 'r') as in_apk:
2379 with ZipFile(signed_apk, 'w') as out_apk:
2380 for info in in_apk.infolist():
2381 if not apk_sigfile.match(info.filename):
2383 if info.filename != 'META-INF/MANIFEST.MF':
2384 buf = in_apk.read(info.filename)
2385 out_apk.writestr(info, buf)
2387 buf = in_apk.read(info.filename)
2388 out_apk.writestr(info, buf)
2391 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2392 """Implats a signature from metadata into an APK.
2394 Note: this changes there supplied APK in place. So copy it if you
2395 need the original to be preserved.
2397 :param apkpath: location of the apk
2399 # get list of available signature files in metadata
2400 with tempfile.TemporaryDirectory() as tmpdir:
2401 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2402 with ZipFile(apkpath, 'r') as in_apk:
2403 with ZipFile(apkwithnewsig, 'w') as out_apk:
2404 for sig_file in [signaturefile, signedfile, manifest]:
2405 with open(sig_file, 'rb') as fp:
2407 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2408 info.compress_type = zipfile.ZIP_DEFLATED
2409 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2410 out_apk.writestr(info, buf)
2411 for info in in_apk.infolist():
2412 if not apk_sigfile.match(info.filename):
2413 if info.filename != 'META-INF/MANIFEST.MF':
2414 buf = in_apk.read(info.filename)
2415 out_apk.writestr(info, buf)
2417 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2418 if p.returncode != 0:
2419 raise BuildException("Failed to align application")
2422 def apk_extract_signatures(apkpath, outdir, manifest=True):
2423 """Extracts a signature files from APK and puts them into target directory.
2425 :param apkpath: location of the apk
2426 :param outdir: folder where the extracted signature files will be stored
2427 :param manifest: (optionally) disable extracting manifest file
2429 with ZipFile(apkpath, 'r') as in_apk:
2430 for f in in_apk.infolist():
2431 if apk_sigfile.match(f.filename) or \
2432 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2433 newpath = os.path.join(outdir, os.path.basename(f.filename))
2434 with open(newpath, 'wb') as out_file:
2435 out_file.write(in_apk.read(f.filename))
2438 def sign_apk(unsigned_path, signed_path, keyalias):
2439 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2441 android-18 (4.3) finally added support for reasonable hash
2442 algorithms, like SHA-256, before then, the only options were MD5
2443 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2444 older Android versions, and is therefore safe to do so.
2446 https://issuetracker.google.com/issues/36956587
2447 https://android-review.googlesource.com/c/platform/libcore/+/44491
2451 if get_minSdkVersion_aapt(unsigned_path) < 18:
2452 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2454 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA256']
2456 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2457 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2458 '-keypass:env', 'FDROID_KEY_PASS']
2459 + signature_algorithm + [unsigned_path, keyalias],
2461 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2462 'FDROID_KEY_PASS': config['keypass'], })
2463 if p.returncode != 0:
2464 raise BuildException(_("Failed to sign application"), p.output)
2466 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2467 if p.returncode != 0:
2468 raise BuildException(_("Failed to zipalign application"))
2469 os.remove(unsigned_path)
2472 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2473 """Verify that two apks are the same
2475 One of the inputs is signed, the other is unsigned. The signature metadata
2476 is transferred from the signed to the unsigned apk, and then jarsigner is
2477 used to verify that the signature from the signed apk is also varlid for
2478 the unsigned one. If the APK given as unsigned actually does have a
2479 signature, it will be stripped out and ignored.
2481 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2482 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2483 into AndroidManifest.xml, but that makes the build not reproducible. So
2484 instead they are included as separate files in the APK's META-INF/ folder.
2485 If those files exist in the signed APK, they will be part of the signature
2486 and need to also be included in the unsigned APK for it to validate.
2488 :param signed_apk: Path to a signed apk file
2489 :param unsigned_apk: Path to an unsigned apk file expected to match it
2490 :param tmp_dir: Path to directory for temporary files
2491 :returns: None if the verification is successful, otherwise a string
2492 describing what went wrong.
2495 if not os.path.isfile(signed_apk):
2496 return 'can not verify: file does not exists: {}'.format(signed_apk)
2498 if not os.path.isfile(unsigned_apk):
2499 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2501 with ZipFile(signed_apk, 'r') as signed:
2502 meta_inf_files = ['META-INF/MANIFEST.MF']
2503 for f in signed.namelist():
2504 if apk_sigfile.match(f) \
2505 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2506 meta_inf_files.append(f)
2507 if len(meta_inf_files) < 3:
2508 return "Signature files missing from {0}".format(signed_apk)
2510 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2511 with ZipFile(unsigned_apk, 'r') as unsigned:
2512 # only read the signature from the signed APK, everything else from unsigned
2513 with ZipFile(tmp_apk, 'w') as tmp:
2514 for filename in meta_inf_files:
2515 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2516 for info in unsigned.infolist():
2517 if info.filename in meta_inf_files:
2518 logging.warning('Ignoring %s from %s',
2519 info.filename, unsigned_apk)
2521 if info.filename in tmp.namelist():
2522 return "duplicate filename found: " + info.filename
2523 tmp.writestr(info, unsigned.read(info.filename))
2525 verified = verify_apk_signature(tmp_apk)
2528 logging.info("...NOT verified - {0}".format(tmp_apk))
2529 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2530 os.path.dirname(unsigned_apk))
2532 logging.info("...successfully verified")
2536 def verify_jar_signature(jar):
2537 """Verifies the signature of a given JAR file.
2539 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2540 this has to turn on -strict then check for result 4, since this
2541 does not expect the signature to be from a CA-signed certificate.
2543 :raises: VerificationException() if the JAR's signature could not be verified
2547 error = _('JAR signature failed to verify: {path}').format(path=jar)
2549 output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar],
2550 stderr=subprocess.STDOUT)
2551 raise VerificationException(error + '\n' + output.decode('utf-8'))
2552 except subprocess.CalledProcessError as e:
2553 if e.returncode == 4:
2554 logging.debug(_('JAR signature verified: {path}').format(path=jar))
2556 raise VerificationException(error + '\n' + e.output.decode('utf-8'))
2559 def verify_apk_signature(apk, min_sdk_version=None):
2560 """verify the signature on an APK
2562 Try to use apksigner whenever possible since jarsigner is very
2563 shitty: unsigned APKs pass as "verified"! Warning, this does
2564 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2566 :returns: boolean whether the APK was verified
2568 if set_command_in_config('apksigner'):
2569 args = [config['apksigner'], 'verify']
2571 args += ['--min-sdk-version=' + min_sdk_version]
2573 args += ['--verbose']
2575 output = subprocess.check_output(args + [apk])
2577 logging.debug(apk + ': ' + output.decode('utf-8'))
2579 except subprocess.CalledProcessError as e:
2580 logging.error('\n' + apk + ': ' + e.output.decode('utf-8'))
2582 if not config.get('jarsigner_warning_displayed'):
2583 config['jarsigner_warning_displayed'] = True
2584 logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner"))
2586 verify_jar_signature(apk)
2588 except Exception as e:
2593 def verify_old_apk_signature(apk):
2594 """verify the signature on an archived APK, supporting deprecated algorithms
2596 F-Droid aims to keep every single binary that it ever published. Therefore,
2597 it needs to be able to verify APK signatures that include deprecated/removed
2598 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2600 jarsigner passes unsigned APKs as "verified"! So this has to turn
2601 on -strict then check for result 4.
2603 :returns: boolean whether the APK was verified
2606 _java_security = os.path.join(os.getcwd(), '.java.security')
2607 with open(_java_security, 'w') as fp:
2608 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2612 config['jarsigner'],
2613 '-J-Djava.security.properties=' + _java_security,
2614 '-strict', '-verify', apk
2616 output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
2617 except subprocess.CalledProcessError as e:
2618 if e.returncode != 4:
2621 logging.debug(_('JAR signature verified: {path}').format(path=apk))
2624 logging.error(_('Old APK signature failed to verify: {path}').format(path=apk)
2625 + '\n' + output.decode('utf-8'))
2629 apk_badchars = re.compile('''[/ :;'"]''')
2632 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2635 Returns None if the apk content is the same (apart from the signing key),
2636 otherwise a string describing what's different, or what went wrong when
2637 trying to do the comparison.
2643 absapk1 = os.path.abspath(apk1)
2644 absapk2 = os.path.abspath(apk2)
2646 if set_command_in_config('diffoscope'):
2647 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2648 htmlfile = logfilename + '.diffoscope.html'
2649 textfile = logfilename + '.diffoscope.txt'
2650 if subprocess.call([config['diffoscope'],
2651 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2652 '--html', htmlfile, '--text', textfile,
2653 absapk1, absapk2]) != 0:
2654 return("Failed to unpack " + apk1)
2656 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2657 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2658 for d in [apk1dir, apk2dir]:
2659 if os.path.exists(d):
2662 os.mkdir(os.path.join(d, 'jar-xf'))
2664 if subprocess.call(['jar', 'xf',
2665 os.path.abspath(apk1)],
2666 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2667 return("Failed to unpack " + apk1)
2668 if subprocess.call(['jar', 'xf',
2669 os.path.abspath(apk2)],
2670 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2671 return("Failed to unpack " + apk2)
2673 if set_command_in_config('apktool'):
2674 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2676 return("Failed to unpack " + apk1)
2677 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2679 return("Failed to unpack " + apk2)
2681 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2682 lines = p.output.splitlines()
2683 if len(lines) != 1 or 'META-INF' not in lines[0]:
2684 if set_command_in_config('meld'):
2685 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2686 return("Unexpected diff output - " + p.output)
2688 # since everything verifies, delete the comparison to keep cruft down
2689 shutil.rmtree(apk1dir)
2690 shutil.rmtree(apk2dir)
2692 # If we get here, it seems like they're the same!
2696 def set_command_in_config(command):
2697 '''Try to find specified command in the path, if it hasn't been
2698 manually set in config.py. If found, it is added to the config
2699 dict. The return value says whether the command is available.
2702 if command in config:
2705 tmp = find_command(command)
2707 config[command] = tmp
2712 def find_command(command):
2713 '''find the full path of a command, or None if it can't be found in the PATH'''
2716 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2718 fpath, fname = os.path.split(command)
2723 for path in os.environ["PATH"].split(os.pathsep):
2724 path = path.strip('"')
2725 exe_file = os.path.join(path, command)
2726 if is_exe(exe_file):
2733 '''generate a random password for when generating keys'''
2734 h = hashlib.sha256()
2735 h.update(os.urandom(16)) # salt
2736 h.update(socket.getfqdn().encode('utf-8'))
2737 passwd = base64.b64encode(h.digest()).strip()
2738 return passwd.decode('utf-8')
2741 def genkeystore(localconfig):
2743 Generate a new key with password provided in :param localconfig and add it to new keystore
2744 :return: hexed public key, public key fingerprint
2746 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2747 keystoredir = os.path.dirname(localconfig['keystore'])
2748 if keystoredir is None or keystoredir == '':
2749 keystoredir = os.path.join(os.getcwd(), keystoredir)
2750 if not os.path.exists(keystoredir):
2751 os.makedirs(keystoredir, mode=0o700)
2754 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2755 'FDROID_KEY_PASS': localconfig['keypass'],
2757 p = FDroidPopen([config['keytool'], '-genkey',
2758 '-keystore', localconfig['keystore'],
2759 '-alias', localconfig['repo_keyalias'],
2760 '-keyalg', 'RSA', '-keysize', '4096',
2761 '-sigalg', 'SHA256withRSA',
2762 '-validity', '10000',
2763 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2764 '-keypass:env', 'FDROID_KEY_PASS',
2765 '-dname', localconfig['keydname']], envs=env_vars)
2766 if p.returncode != 0:
2767 raise BuildException("Failed to generate key", p.output)
2768 os.chmod(localconfig['keystore'], 0o0600)
2769 if not options.quiet:
2770 # now show the lovely key that was just generated
2771 p = FDroidPopen([config['keytool'], '-list', '-v',
2772 '-keystore', localconfig['keystore'],
2773 '-alias', localconfig['repo_keyalias'],
2774 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2775 logging.info(p.output.strip() + '\n\n')
2776 # get the public key
2777 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2778 '-keystore', localconfig['keystore'],
2779 '-alias', localconfig['repo_keyalias'],
2780 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2781 + config['smartcardoptions'],
2782 envs=env_vars, output=False, stderr_to_stdout=False)
2783 if p.returncode != 0 or len(p.output) < 20:
2784 raise BuildException("Failed to get public key", p.output)
2786 fingerprint = get_cert_fingerprint(pubkey)
2787 return hexlify(pubkey), fingerprint
2790 def get_cert_fingerprint(pubkey):
2792 Generate a certificate fingerprint the same way keytool does it
2793 (but with slightly different formatting)
2795 digest = hashlib.sha256(pubkey).digest()
2796 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2797 return " ".join(ret)
2800 def get_certificate(certificate_file):
2802 Extracts a certificate from the given file.
2803 :param certificate_file: file bytes (as string) representing the certificate
2804 :return: A binary representation of the certificate's public key, or None in case of error
2806 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2807 if content.getComponentByName('contentType') != rfc2315.signedData:
2809 content = decoder.decode(content.getComponentByName('content'),
2810 asn1Spec=rfc2315.SignedData())[0]
2812 certificates = content.getComponentByName('certificates')
2813 cert = certificates[0].getComponentByName('certificate')
2815 logging.error("Certificates not found.")
2817 return encoder.encode(cert)
2820 def load_stats_fdroid_signing_key_fingerprints():
2821 """Load list of signing-key fingerprints stored by fdroid publish from file.
2823 :returns: list of dictionanryies containing the singing-key fingerprints.
2825 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2826 if not os.path.isfile(jar_file):
2828 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2829 p = FDroidPopen(cmd, output=False)
2830 if p.returncode != 4:
2831 raise FDroidException("Signature validation of '{}' failed! "
2832 "Please run publish again to rebuild this file.".format(jar_file))
2834 jar_sigkey = apk_signer_fingerprint(jar_file)
2835 repo_key_sig = config.get('repo_key_sha256')
2837 if jar_sigkey != repo_key_sig:
2838 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2840 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2841 config['repo_key_sha256'] = jar_sigkey
2842 write_to_config(config, 'repo_key_sha256')
2844 with zipfile.ZipFile(jar_file, 'r') as f:
2845 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2848 def write_to_config(thisconfig, key, value=None, config_file=None):
2849 '''write a key/value to the local config.py
2851 NOTE: only supports writing string variables.
2853 :param thisconfig: config dictionary
2854 :param key: variable name in config.py to be overwritten/added
2855 :param value: optional value to be written, instead of fetched
2856 from 'thisconfig' dictionary.
2859 origkey = key + '_orig'
2860 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2861 cfg = config_file if config_file else 'config.py'
2863 # load config file, create one if it doesn't exist
2864 if not os.path.exists(cfg):
2865 open(cfg, 'a').close()
2866 logging.info("Creating empty " + cfg)
2867 with open(cfg, 'r', encoding="utf-8") as f:
2868 lines = f.readlines()
2870 # make sure the file ends with a carraige return
2872 if not lines[-1].endswith('\n'):
2875 # regex for finding and replacing python string variable
2876 # definitions/initializations
2877 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2878 repl = key + ' = "' + value + '"'
2879 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2880 repl2 = key + " = '" + value + "'"
2882 # If we replaced this line once, we make sure won't be a
2883 # second instance of this line for this key in the document.
2886 with open(cfg, 'w', encoding="utf-8") as f:
2888 if pattern.match(line) or pattern2.match(line):
2890 line = pattern.sub(repl, line)
2891 line = pattern2.sub(repl2, line)
2902 def parse_xml(path):
2903 return XMLElementTree.parse(path).getroot()
2906 def string_is_integer(string):
2914 def local_rsync(options, fromdir, todir):
2915 '''Rsync method for local to local copying of things
2917 This is an rsync wrapper with all the settings for safe use within
2918 the various fdroidserver use cases. This uses stricter rsync
2919 checking on all files since people using offline mode are already
2920 prioritizing security above ease and speed.
2923 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2924 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2925 if not options.no_checksum:
2926 rsyncargs.append('--checksum')
2928 rsyncargs += ['--verbose']
2930 rsyncargs += ['--quiet']
2931 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2932 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2933 raise FDroidException()
2936 def get_per_app_repos():
2937 '''per-app repos are dirs named with the packageName of a single app'''
2939 # Android packageNames are Java packages, they may contain uppercase or
2940 # lowercase letters ('A' through 'Z'), numbers, and underscores
2941 # ('_'). However, individual package name parts may only start with
2942 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2943 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2946 for root, dirs, files in os.walk(os.getcwd()):
2948 print('checking', root, 'for', d)
2949 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2950 # standard parts of an fdroid repo, so never packageNames
2953 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2959 def is_repo_file(filename):
2960 '''Whether the file in a repo is a build product to be delivered to users'''
2961 if isinstance(filename, str):
2962 filename = filename.encode('utf-8', errors="surrogateescape")
2963 return os.path.isfile(filename) \
2964 and not filename.endswith(b'.asc') \
2965 and not filename.endswith(b'.sig') \
2966 and os.path.basename(filename) not in [
2968 b'index_unsigned.jar',
2977 def get_examples_dir():
2978 '''Return the dir where the fdroidserver example files are available'''
2980 tmp = os.path.dirname(sys.argv[0])
2981 if os.path.basename(tmp) == 'bin':
2982 egg_links = glob.glob(os.path.join(tmp, '..',
2983 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2985 # installed from local git repo
2986 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2989 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2990 if not os.path.exists(examplesdir): # use UNIX layout
2991 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2993 # we're running straight out of the git repo
2994 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2995 examplesdir = prefix + '/examples'