3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
61 # A signature block file with a .DSA, .RSA, or .EC extension
62 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
63 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
64 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
66 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
75 'sdk_path': "$ANDROID_HOME",
80 'r12b': "$ANDROID_NDK",
87 'build_tools': "25.0.2",
88 'force_build_tools': False,
93 'accepted_formats': ['txt', 'yml'],
94 'sync_from_local_copy_dir': False,
95 'allow_disabled_algorithms': False,
96 'per_app_repos': False,
97 'make_current_version_link': True,
98 'current_version_name_source': 'Name',
99 'update_stats': False,
101 'stats_server': None,
103 'stats_to_carbon': False,
105 'build_server_always': False,
106 'keystore': 'keystore.jks',
107 'smartcardoptions': [],
117 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
118 'repo_name': "My First FDroid Repo Demo",
119 'repo_icon': "fdroid-icon.png",
120 'repo_description': '''
121 This is a repository of apps to be used with FDroid. Applications in this
122 repository are either official binaries built by the original application
123 developers, or are binaries built from source by the admin of f-droid.org
124 using the tools on https://gitlab.com/u/fdroid.
130 def setup_global_opts(parser):
131 parser.add_argument("-v", "--verbose", action="store_true", default=False,
132 help=_("Spew out even more information than normal"))
133 parser.add_argument("-q", "--quiet", action="store_true", default=False,
134 help=_("Restrict output to warnings and errors"))
137 def _add_java_paths_to_config(pathlist, thisconfig):
138 def path_version_key(s):
140 for u in re.split('[^0-9]+', s):
142 versionlist.append(int(u))
147 for d in sorted(pathlist, key=path_version_key):
148 if os.path.islink(d):
150 j = os.path.basename(d)
151 # the last one found will be the canonical one, so order appropriately
153 r'^1\.([6-9])\.0\.jdk$', # OSX
154 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
155 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
156 r'^jdk([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-jdk$', # Arch (oracle)
159 r'^java-1\.([6-9])\.0-.*$', # RedHat
160 r'^java-([6-9])-oracle$', # Debian WebUpd8
161 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
162 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
164 m = re.match(regex, j)
167 for p in [d, os.path.join(d, 'Contents', 'Home')]:
168 if os.path.exists(os.path.join(p, 'bin', 'javac')):
169 thisconfig['java_paths'][m.group(1)] = p
172 def fill_config_defaults(thisconfig):
173 for k, v in default_config.items():
174 if k not in thisconfig:
177 # Expand paths (~users and $vars)
178 def expand_path(path):
182 path = os.path.expanduser(path)
183 path = os.path.expandvars(path)
188 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
193 thisconfig[k + '_orig'] = v
195 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
196 if thisconfig['java_paths'] is None:
197 thisconfig['java_paths'] = dict()
199 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
200 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
201 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
202 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
203 if os.getenv('JAVA_HOME') is not None:
204 pathlist.append(os.getenv('JAVA_HOME'))
205 if os.getenv('PROGRAMFILES') is not None:
206 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
207 _add_java_paths_to_config(pathlist, thisconfig)
209 for java_version in ('7', '8', '9'):
210 if java_version not in thisconfig['java_paths']:
212 java_home = thisconfig['java_paths'][java_version]
213 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
214 if os.path.exists(jarsigner):
215 thisconfig['jarsigner'] = jarsigner
216 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
217 break # Java7 is preferred, so quit if found
219 for k in ['ndk_paths', 'java_paths']:
225 thisconfig[k][k2] = exp
226 thisconfig[k][k2 + '_orig'] = v
229 def regsub_file(pattern, repl, path):
230 with open(path, 'rb') as f:
232 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
233 with open(path, 'wb') as f:
237 def read_config(opts, config_file='config.py'):
238 """Read the repository config
240 The config is read from config_file, which is in the current
241 directory when any of the repo management commands are used. If
242 there is a local metadata file in the git repo, then config.py is
243 not required, just use defaults.
246 global config, options
248 if config is not None:
255 if os.path.isfile(config_file):
256 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
257 with io.open(config_file, "rb") as f:
258 code = compile(f.read(), config_file, 'exec')
259 exec(code, None, config)
261 logging.warning(_("No 'config.py' found, using defaults."))
263 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
265 if not type(config[k]) in (str, list, tuple):
267 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
270 # smartcardoptions must be a list since its command line args for Popen
271 if 'smartcardoptions' in config:
272 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
273 elif 'keystore' in config and config['keystore'] == 'NONE':
274 # keystore='NONE' means use smartcard, these are required defaults
275 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
276 'SunPKCS11-OpenSC', '-providerClass',
277 'sun.security.pkcs11.SunPKCS11',
278 '-providerArg', 'opensc-fdroid.cfg']
280 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
281 st = os.stat(config_file)
282 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
283 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
284 .format(config_file=config_file))
286 fill_config_defaults(config)
288 for k in ["repo_description", "archive_description"]:
290 config[k] = clean_description(config[k])
292 if 'serverwebroot' in config:
293 if isinstance(config['serverwebroot'], str):
294 roots = [config['serverwebroot']]
295 elif all(isinstance(item, str) for item in config['serverwebroot']):
296 roots = config['serverwebroot']
298 raise TypeError(_('only accepts strings, lists, and tuples'))
300 for rootstr in roots:
301 # since this is used with rsync, where trailing slashes have
302 # meaning, ensure there is always a trailing slash
303 if rootstr[-1] != '/':
305 rootlist.append(rootstr.replace('//', '/'))
306 config['serverwebroot'] = rootlist
308 if 'servergitmirrors' in config:
309 if isinstance(config['servergitmirrors'], str):
310 roots = [config['servergitmirrors']]
311 elif all(isinstance(item, str) for item in config['servergitmirrors']):
312 roots = config['servergitmirrors']
314 raise TypeError(_('only accepts strings, lists, and tuples'))
315 config['servergitmirrors'] = roots
320 def assert_config_keystore(config):
321 """Check weather keystore is configured correctly and raise exception if not."""
324 if 'repo_keyalias' not in config:
326 logging.critical(_("'repo_keyalias' not found in config.py!"))
327 if 'keystore' not in config:
329 logging.critical(_("'keystore' not found in config.py!"))
330 elif not os.path.exists(config['keystore']):
332 logging.critical("'" + config['keystore'] + "' does not exist!")
333 if 'keystorepass' not in config:
335 logging.critical(_("'keystorepass' not found in config.py!"))
336 if 'keypass' not in config:
338 logging.critical(_("'keypass' not found in config.py!"))
340 raise FDroidException("This command requires a signing key, " +
341 "you can create one using: fdroid update --create-key")
344 def find_sdk_tools_cmd(cmd):
345 '''find a working path to a tool from the Android SDK'''
348 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
349 # try to find a working path to this command, in all the recent possible paths
350 if 'build_tools' in config:
351 build_tools = os.path.join(config['sdk_path'], 'build-tools')
352 # if 'build_tools' was manually set and exists, check only that one
353 configed_build_tools = os.path.join(build_tools, config['build_tools'])
354 if os.path.exists(configed_build_tools):
355 tooldirs.append(configed_build_tools)
357 # no configed version, so hunt known paths for it
358 for f in sorted(os.listdir(build_tools), reverse=True):
359 if os.path.isdir(os.path.join(build_tools, f)):
360 tooldirs.append(os.path.join(build_tools, f))
361 tooldirs.append(build_tools)
362 sdk_tools = os.path.join(config['sdk_path'], 'tools')
363 if os.path.exists(sdk_tools):
364 tooldirs.append(sdk_tools)
365 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
366 if os.path.exists(sdk_platform_tools):
367 tooldirs.append(sdk_platform_tools)
368 tooldirs.append('/usr/bin')
370 path = os.path.join(d, cmd)
371 if os.path.isfile(path):
373 test_aapt_version(path)
375 # did not find the command, exit with error message
376 ensure_build_tools_exists(config)
379 def test_aapt_version(aapt):
380 '''Check whether the version of aapt is new enough'''
381 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
382 if output is None or output == '':
383 logging.error(_("'{path}' failed to execute!").format(path=aapt))
385 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
390 # the Debian package has the version string like "v0.2-23.0.2"
391 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
392 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
395 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
398 def test_sdk_exists(thisconfig):
399 if 'sdk_path' not in thisconfig:
400 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
401 test_aapt_version(thisconfig['aapt'])
404 logging.error(_("'sdk_path' not set in 'config.py'!"))
406 if thisconfig['sdk_path'] == default_config['sdk_path']:
407 logging.error(_('No Android SDK found!'))
408 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
409 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
411 if not os.path.exists(thisconfig['sdk_path']):
412 logging.critical(_("Android SDK path '{path}' does not exist!")
413 .format(path=thisconfig['sdk_path']))
415 if not os.path.isdir(thisconfig['sdk_path']):
416 logging.critical(_("Android SDK path '{path}' is not a directory!")
417 .format(path=thisconfig['sdk_path']))
419 for d in ['build-tools', 'platform-tools', 'tools']:
420 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
421 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
422 .format(path=thisconfig['sdk_path'], dirname=d))
427 def ensure_build_tools_exists(thisconfig):
428 if not test_sdk_exists(thisconfig):
429 raise FDroidException(_("Android SDK not found!"))
430 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
431 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
432 if not os.path.isdir(versioned_build_tools):
433 raise FDroidException(
434 _("Android build-tools path '{path}' does not exist!")
435 .format(path=versioned_build_tools))
438 def get_local_metadata_files():
439 '''get any metadata files local to an app's source repo
441 This tries to ignore anything that does not count as app metdata,
442 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
445 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
448 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
450 :param appids: arguments in the form of multiple appid:[vc] strings
451 :returns: a dictionary with the set of vercodes specified for each package
454 if not appid_versionCode_pairs:
457 for p in appid_versionCode_pairs:
458 if allow_vercodes and ':' in p:
459 package, vercode = p.split(':')
461 package, vercode = p, None
462 if package not in vercodes:
463 vercodes[package] = [vercode] if vercode else []
465 elif vercode and vercode not in vercodes[package]:
466 vercodes[package] += [vercode] if vercode else []
471 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
472 """Build a list of App instances for processing
474 On top of what read_pkg_args does, this returns the whole app
475 metadata, but limiting the builds list to the builds matching the
476 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
477 all App and Build instances are returned.
481 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
487 for appid, app in allapps.items():
488 if appid in vercodes:
491 if len(apps) != len(vercodes):
494 logging.critical(_("No such package: %s") % p)
495 raise FDroidException(_("Found invalid appids in arguments"))
497 raise FDroidException(_("No packages specified"))
500 for appid, app in apps.items():
504 app.builds = [b for b in app.builds if b.versionCode in vc]
505 if len(app.builds) != len(vercodes[appid]):
507 allvcs = [b.versionCode for b in app.builds]
508 for v in vercodes[appid]:
510 logging.critical(_("No such versionCode {versionCode} for app {appid}")
511 .format(versionCode=v, appid=appid))
514 raise FDroidException(_("Found invalid versionCodes for some apps"))
519 def get_extension(filename):
520 base, ext = os.path.splitext(filename)
523 return base, ext.lower()[1:]
526 def has_extension(filename, ext):
527 _ignored, f_ext = get_extension(filename)
531 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
534 def clean_description(description):
535 'Remove unneeded newlines and spaces from a block of description text'
537 # this is split up by paragraph to make removing the newlines easier
538 for paragraph in re.split(r'\n\n', description):
539 paragraph = re.sub('\r', '', paragraph)
540 paragraph = re.sub('\n', ' ', paragraph)
541 paragraph = re.sub(' {2,}', ' ', paragraph)
542 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
543 returnstring += paragraph + '\n\n'
544 return returnstring.rstrip('\n')
547 def publishednameinfo(filename):
548 filename = os.path.basename(filename)
549 m = publish_name_regex.match(filename)
551 result = (m.group(1), m.group(2))
552 except AttributeError:
553 raise FDroidException(_("Invalid name for published file: %s") % filename)
557 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
558 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
561 def apk_parse_release_filename(apkname):
562 """Parses the name of an APK file according the F-Droids APK naming
563 scheme and returns the tokens.
565 WARNING: Returned values don't necessarily represent the APKs actual
566 properties, the are just paresed from the file name.
568 :returns: A triplet containing (appid, versionCode, signer), where appid
569 should be the package name, versionCode should be the integer
570 represion of the APKs version and signer should be the first 7 hex
571 digists of the sha256 signing key fingerprint which was used to sign
574 m = apk_release_filename_with_sigfp.match(apkname)
576 return m.group('appid'), m.group('vercode'), m.group('sigfp')
577 m = apk_release_filename.match(apkname)
579 return m.group('appid'), m.group('vercode'), None
580 return None, None, None
583 def get_release_filename(app, build):
585 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
587 return "%s_%s.apk" % (app.id, build.versionCode)
590 def get_toolsversion_logname(app, build):
591 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
594 def getsrcname(app, build):
595 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
607 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
610 def get_build_dir(app):
611 '''get the dir that this app will be built in'''
613 if app.RepoType == 'srclib':
614 return os.path.join('build', 'srclib', app.Repo)
616 return os.path.join('build', app.id)
620 '''checkout code from VCS and return instance of vcs and the build dir'''
621 build_dir = get_build_dir(app)
623 # Set up vcs interface and make sure we have the latest code...
624 logging.debug("Getting {0} vcs interface for {1}"
625 .format(app.RepoType, app.Repo))
626 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
630 vcs = getvcs(app.RepoType, remote, build_dir)
632 return vcs, build_dir
635 def getvcs(vcstype, remote, local):
637 return vcs_git(remote, local)
638 if vcstype == 'git-svn':
639 return vcs_gitsvn(remote, local)
641 return vcs_hg(remote, local)
643 return vcs_bzr(remote, local)
644 if vcstype == 'srclib':
645 if local != os.path.join('build', 'srclib', remote):
646 raise VCSException("Error: srclib paths are hard-coded!")
647 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
649 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
650 raise VCSException("Invalid vcs type " + vcstype)
653 def getsrclibvcs(name):
654 if name not in fdroidserver.metadata.srclibs:
655 raise VCSException("Missing srclib " + name)
656 return fdroidserver.metadata.srclibs[name]['Repo Type']
661 def __init__(self, remote, local):
663 # svn, git-svn and bzr may require auth
665 if self.repotype() in ('git-svn', 'bzr'):
667 if self.repotype == 'git-svn':
668 raise VCSException("Authentication is not supported for git-svn")
669 self.username, remote = remote.split('@')
670 if ':' not in self.username:
671 raise VCSException(_("Password required with username"))
672 self.username, self.password = self.username.split(':')
676 self.clone_failed = False
677 self.refreshed = False
683 def clientversion(self):
684 versionstr = FDroidPopen(self.clientversioncmd()).output
685 return versionstr[0:versionstr.find('\n')]
687 def clientversioncmd(self):
690 def gotorevision(self, rev, refresh=True):
691 """Take the local repository to a clean version of the given
692 revision, which is specificed in the VCS's native
693 format. Beforehand, the repository can be dirty, or even
694 non-existent. If the repository does already exist locally, it
695 will be updated from the origin, but only once in the lifetime
696 of the vcs object. None is acceptable for 'rev' if you know
697 you are cloning a clean copy of the repo - otherwise it must
698 specify a valid revision.
701 if self.clone_failed:
702 raise VCSException(_("Downloading the repository already failed once, not trying again."))
704 # The .fdroidvcs-id file for a repo tells us what VCS type
705 # and remote that directory was created from, allowing us to drop it
706 # automatically if either of those things changes.
707 fdpath = os.path.join(self.local, '..',
708 '.fdroidvcs-' + os.path.basename(self.local))
709 fdpath = os.path.normpath(fdpath)
710 cdata = self.repotype() + ' ' + self.remote
713 if os.path.exists(self.local):
714 if os.path.exists(fdpath):
715 with open(fdpath, 'r') as f:
716 fsdata = f.read().strip()
721 logging.info("Repository details for %s changed - deleting" % (
725 logging.info("Repository details for %s missing - deleting" % (
728 shutil.rmtree(self.local)
732 self.refreshed = True
735 self.gotorevisionx(rev)
736 except FDroidException as e:
739 # If necessary, write the .fdroidvcs file.
740 if writeback and not self.clone_failed:
741 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
742 with open(fdpath, 'w+') as f:
748 def gotorevisionx(self, rev): # pylint: disable=unused-argument
749 """Derived classes need to implement this.
751 It's called once basic checking has been performed.
753 raise VCSException("This VCS type doesn't define gotorevisionx")
755 # Initialise and update submodules
756 def initsubmodules(self):
757 raise VCSException('Submodules not supported for this vcs type')
759 # Get a list of all known tags
761 if not self._gettags:
762 raise VCSException('gettags not supported for this vcs type')
764 for tag in self._gettags():
765 if re.match('[-A-Za-z0-9_. /]+$', tag):
769 def latesttags(self):
770 """Get a list of all the known tags, sorted from newest to oldest"""
771 raise VCSException('latesttags not supported for this vcs type')
774 """Get current commit reference (hash, revision, etc)"""
775 raise VCSException('getref not supported for this vcs type')
778 """Returns the srclib (name, path) used in setting up the current revision, or None."""
787 def clientversioncmd(self):
788 return ['git', '--version']
790 def git(self, args, envs=dict(), cwd=None, output=True):
791 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
793 While fetch/pull/clone respect the command line option flags,
794 it seems that submodule commands do not. They do seem to
795 follow whatever is in env vars, if the version of git is new
796 enough. So we just throw the kitchen sink at it to see what
799 Also, because of CVE-2017-1000117, block all SSH URLs.
802 # supported in git >= 2.3
804 '-c', 'core.sshCommand=false',
805 '-c', 'url.https://.insteadOf=ssh://',
807 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
808 git_config.append('-c')
809 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
810 git_config.append('-c')
811 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
812 git_config.append('-c')
813 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
815 'GIT_TERMINAL_PROMPT': '0',
816 'GIT_SSH': 'false', # for git < 2.3
818 return FDroidPopen(['git', ] + git_config + args,
819 envs=envs, cwd=cwd, output=output)
822 """If the local directory exists, but is somehow not a git repository,
823 git will traverse up the directory tree until it finds one
824 that is (i.e. fdroidserver) and then we'll proceed to destroy
825 it! This is called as a safety check.
829 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
830 result = p.output.rstrip()
831 if not result.endswith(self.local):
832 raise VCSException('Repository mismatch')
834 def gotorevisionx(self, rev):
835 if not os.path.exists(self.local):
837 p = self.git(['clone', self.remote, self.local])
838 if p.returncode != 0:
839 self.clone_failed = True
840 raise VCSException("Git clone failed", p.output)
844 # Discard any working tree changes
845 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
846 'git', 'reset', '--hard'], cwd=self.local, output=False)
847 if p.returncode != 0:
848 raise VCSException(_("Git reset failed"), p.output)
849 # Remove untracked files now, in case they're tracked in the target
850 # revision (it happens!)
851 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
852 'git', 'clean', '-dffx'], cwd=self.local, output=False)
853 if p.returncode != 0:
854 raise VCSException(_("Git clean failed"), p.output)
855 if not self.refreshed:
856 # Get latest commits and tags from remote
857 p = self.git(['fetch', 'origin'], cwd=self.local)
858 if p.returncode != 0:
859 raise VCSException(_("Git fetch failed"), p.output)
860 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
861 if p.returncode != 0:
862 raise VCSException(_("Git fetch failed"), p.output)
863 # Recreate origin/HEAD as git clone would do it, in case it disappeared
864 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
865 if p.returncode != 0:
866 lines = p.output.splitlines()
867 if 'Multiple remote HEAD branches' not in lines[0]:
868 raise VCSException(_("Git remote set-head failed"), p.output)
869 branch = lines[1].split(' ')[-1]
870 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
871 if p2.returncode != 0:
872 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
873 self.refreshed = True
874 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
875 # a github repo. Most of the time this is the same as origin/master.
876 rev = rev or 'origin/HEAD'
877 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
880 # Get rid of any uncontrolled files left behind
881 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 raise VCSException(_("Git clean failed"), p.output)
885 def initsubmodules(self):
887 submfile = os.path.join(self.local, '.gitmodules')
888 if not os.path.isfile(submfile):
889 raise NoSubmodulesException(_("No git submodules available"))
891 # fix submodules not accessible without an account and public key auth
892 with open(submfile, 'r') as f:
893 lines = f.readlines()
894 with open(submfile, 'w') as f:
896 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
897 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
900 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
901 if p.returncode != 0:
902 raise VCSException(_("Git submodule sync failed"), p.output)
903 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
904 if p.returncode != 0:
905 raise VCSException(_("Git submodule update failed"), p.output)
909 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
910 return p.output.splitlines()
912 tag_format = re.compile(r'tag: ([^),]*)')
914 def latesttags(self):
916 p = FDroidPopen(['git', 'log', '--tags',
917 '--simplify-by-decoration', '--pretty=format:%d'],
918 cwd=self.local, output=False)
920 for line in p.output.splitlines():
921 for tag in self.tag_format.findall(line):
926 class vcs_gitsvn(vcs):
931 def clientversioncmd(self):
932 return ['git', 'svn', '--version']
935 """If the local directory exists, but is somehow not a git repository,
936 git will traverse up the directory tree until it finds one that
937 is (i.e. fdroidserver) and then we'll proceed to destory it!
938 This is called as a safety check.
941 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
942 result = p.output.rstrip()
943 if not result.endswith(self.local):
944 raise VCSException('Repository mismatch')
946 def git(self, args, envs=dict(), cwd=None, output=True):
947 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
949 # CVE-2017-1000117 block all SSH URLs (supported in git >= 2.3)
950 config = ['-c', 'core.sshCommand=false']
952 'GIT_TERMINAL_PROMPT': '0',
953 'GIT_SSH': 'false', # for git < 2.3
956 return FDroidPopen(['git', ] + config + args,
957 envs=envs, cwd=cwd, output=output)
959 def gotorevisionx(self, rev):
960 if not os.path.exists(self.local):
962 gitsvn_args = ['svn', 'clone']
963 if ';' in self.remote:
964 remote_split = self.remote.split(';')
965 for i in remote_split[1:]:
966 if i.startswith('trunk='):
967 gitsvn_args.extend(['-T', i[6:]])
968 elif i.startswith('tags='):
969 gitsvn_args.extend(['-t', i[5:]])
970 elif i.startswith('branches='):
971 gitsvn_args.extend(['-b', i[9:]])
972 gitsvn_args.extend([remote_split[0], self.local])
973 p = self.git(gitsvn_args, output=False)
974 if p.returncode != 0:
975 self.clone_failed = True
976 raise VCSException("Git svn clone failed", p.output)
978 gitsvn_args.extend([self.remote, self.local])
979 p = self.git(gitsvn_args, output=False)
980 if p.returncode != 0:
981 self.clone_failed = True
982 raise VCSException("Git svn clone failed", p.output)
986 # Discard any working tree changes
987 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
988 if p.returncode != 0:
989 raise VCSException("Git reset failed", p.output)
990 # Remove untracked files now, in case they're tracked in the target
991 # revision (it happens!)
992 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
993 if p.returncode != 0:
994 raise VCSException("Git clean failed", p.output)
995 if not self.refreshed:
996 # Get new commits, branches and tags from repo
997 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
998 if p.returncode != 0:
999 raise VCSException("Git svn fetch failed")
1000 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1001 if p.returncode != 0:
1002 raise VCSException("Git svn rebase failed", p.output)
1003 self.refreshed = True
1005 rev = rev or 'master'
1007 nospaces_rev = rev.replace(' ', '%20')
1008 # Try finding a svn tag
1009 for treeish in ['origin/', '']:
1010 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1011 if p.returncode == 0:
1013 if p.returncode != 0:
1014 # No tag found, normal svn rev translation
1015 # Translate svn rev into git format
1016 rev_split = rev.split('/')
1019 for treeish in ['origin/', '']:
1020 if len(rev_split) > 1:
1021 treeish += rev_split[0]
1022 svn_rev = rev_split[1]
1025 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1029 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1031 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1032 git_rev = p.output.rstrip()
1034 if p.returncode == 0 and git_rev:
1037 if p.returncode != 0 or not git_rev:
1038 # Try a plain git checkout as a last resort
1039 p = self.git(['checkout', rev], cwd=self.local, output=False)
1040 if p.returncode != 0:
1041 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1043 # Check out the git rev equivalent to the svn rev
1044 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1045 if p.returncode != 0:
1046 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1048 # Get rid of any uncontrolled files left behind
1049 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1050 if p.returncode != 0:
1051 raise VCSException(_("Git clean failed"), p.output)
1055 for treeish in ['origin/', '']:
1056 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1057 if os.path.isdir(d):
1058 return os.listdir(d)
1062 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1063 if p.returncode != 0:
1065 return p.output.strip()
1073 def clientversioncmd(self):
1074 return ['hg', '--version']
1076 def gotorevisionx(self, rev):
1077 if not os.path.exists(self.local):
1078 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', self.remote, self.local], output=False)
1079 if p.returncode != 0:
1080 self.clone_failed = True
1081 raise VCSException("Hg clone failed", p.output)
1083 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1084 if p.returncode != 0:
1085 raise VCSException("Hg status failed", p.output)
1086 for line in p.output.splitlines():
1087 if not line.startswith('? '):
1088 raise VCSException("Unexpected output from hg status -uS: " + line)
1089 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1090 if not self.refreshed:
1091 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1092 if p.returncode != 0:
1093 raise VCSException("Hg pull failed", p.output)
1094 self.refreshed = True
1096 rev = rev or 'default'
1099 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1100 if p.returncode != 0:
1101 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1102 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1103 # Also delete untracked files, we have to enable purge extension for that:
1104 if "'purge' is provided by the following extension" in p.output:
1105 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1106 myfile.write("\n[extensions]\nhgext.purge=\n")
1107 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1108 if p.returncode != 0:
1109 raise VCSException("HG purge failed", p.output)
1110 elif p.returncode != 0:
1111 raise VCSException("HG purge failed", p.output)
1114 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1115 return p.output.splitlines()[1:]
1123 def clientversioncmd(self):
1124 return ['bzr', '--version']
1126 def bzr(self, args, envs=dict(), cwd=None, output=True):
1127 '''Prevent bzr from ever using SSH to avoid security vulns'''
1131 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1133 def gotorevisionx(self, rev):
1134 if not os.path.exists(self.local):
1135 p = self.bzr(['branch', self.remote, self.local], output=False)
1136 if p.returncode != 0:
1137 self.clone_failed = True
1138 raise VCSException("Bzr branch failed", p.output)
1140 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1141 if p.returncode != 0:
1142 raise VCSException("Bzr revert failed", p.output)
1143 if not self.refreshed:
1144 p = self.bzr(['pull'], cwd=self.local, output=False)
1145 if p.returncode != 0:
1146 raise VCSException("Bzr update failed", p.output)
1147 self.refreshed = True
1149 revargs = list(['-r', rev] if rev else [])
1150 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1151 if p.returncode != 0:
1152 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1155 p = self.bzr(['tags'], cwd=self.local, output=False)
1156 return [tag.split(' ')[0].strip() for tag in
1157 p.output.splitlines()]
1160 def unescape_string(string):
1163 if string[0] == '"' and string[-1] == '"':
1166 return string.replace("\\'", "'")
1169 def retrieve_string(app_dir, string, xmlfiles=None):
1171 if not string.startswith('@string/'):
1172 return unescape_string(string)
1174 if xmlfiles is None:
1177 os.path.join(app_dir, 'res'),
1178 os.path.join(app_dir, 'src', 'main', 'res'),
1180 for root, dirs, files in os.walk(res_dir):
1181 if os.path.basename(root) == 'values':
1182 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1184 name = string[len('@string/'):]
1186 def element_content(element):
1187 if element.text is None:
1189 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1190 return s.decode('utf-8').strip()
1192 for path in xmlfiles:
1193 if not os.path.isfile(path):
1195 xml = parse_xml(path)
1196 element = xml.find('string[@name="' + name + '"]')
1197 if element is not None:
1198 content = element_content(element)
1199 return retrieve_string(app_dir, content, xmlfiles)
1204 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1205 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1208 def manifest_paths(app_dir, flavours):
1209 '''Return list of existing files that will be used to find the highest vercode'''
1211 possible_manifests = \
1212 [os.path.join(app_dir, 'AndroidManifest.xml'),
1213 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1214 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1215 os.path.join(app_dir, 'build.gradle')]
1217 for flavour in flavours:
1218 if flavour == 'yes':
1220 possible_manifests.append(
1221 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1223 return [path for path in possible_manifests if os.path.isfile(path)]
1226 def fetch_real_name(app_dir, flavours):
1227 '''Retrieve the package name. Returns the name, or None if not found.'''
1228 for path in manifest_paths(app_dir, flavours):
1229 if not has_extension(path, 'xml') or not os.path.isfile(path):
1231 logging.debug("fetch_real_name: Checking manifest at " + path)
1232 xml = parse_xml(path)
1233 app = xml.find('application')
1236 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1238 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1239 result = retrieve_string_singleline(app_dir, label)
1241 result = result.strip()
1246 def get_library_references(root_dir):
1248 proppath = os.path.join(root_dir, 'project.properties')
1249 if not os.path.isfile(proppath):
1251 with open(proppath, 'r', encoding='iso-8859-1') as f:
1253 if not line.startswith('android.library.reference.'):
1255 path = line.split('=')[1].strip()
1256 relpath = os.path.join(root_dir, path)
1257 if not os.path.isdir(relpath):
1259 logging.debug("Found subproject at %s" % path)
1260 libraries.append(path)
1264 def ant_subprojects(root_dir):
1265 subprojects = get_library_references(root_dir)
1266 for subpath in subprojects:
1267 subrelpath = os.path.join(root_dir, subpath)
1268 for p in get_library_references(subrelpath):
1269 relp = os.path.normpath(os.path.join(subpath, p))
1270 if relp not in subprojects:
1271 subprojects.insert(0, relp)
1275 def remove_debuggable_flags(root_dir):
1276 # Remove forced debuggable flags
1277 logging.debug("Removing debuggable flags from %s" % root_dir)
1278 for root, dirs, files in os.walk(root_dir):
1279 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1280 regsub_file(r'android:debuggable="[^"]*"',
1282 os.path.join(root, 'AndroidManifest.xml'))
1285 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1286 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1287 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1290 def app_matches_packagename(app, package):
1293 appid = app.UpdateCheckName or app.id
1294 if appid is None or appid == "Ignore":
1296 return appid == package
1299 def parse_androidmanifests(paths, app):
1301 Extract some information from the AndroidManifest.xml at the given path.
1302 Returns (version, vercode, package), any or all of which might be None.
1303 All values returned are strings.
1306 ignoreversions = app.UpdateCheckIgnore
1307 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1310 return (None, None, None)
1318 if not os.path.isfile(path):
1321 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1327 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1328 flavour = app.builds[-1].gradle[-1]
1330 if has_extension(path, 'gradle'):
1331 # first try to get version name and code from correct flavour
1332 with open(path, 'r') as f:
1333 buildfile = f.read()
1335 regex_string = r"" + flavour + ".*?}"
1336 search = re.compile(regex_string, re.DOTALL)
1337 result = search.search(buildfile)
1339 if result is not None:
1340 resultgroup = result.group()
1343 matches = psearch_g(resultgroup)
1345 s = matches.group(2)
1346 if app_matches_packagename(app, s):
1349 matches = vnsearch_g(resultgroup)
1351 version = matches.group(2)
1353 matches = vcsearch_g(resultgroup)
1355 vercode = matches.group(1)
1357 # fall back to parse file line by line
1358 with open(path, 'r') as f:
1360 if gradle_comment.match(line):
1362 # Grab first occurence of each to avoid running into
1363 # alternative flavours and builds.
1365 matches = psearch_g(line)
1367 s = matches.group(2)
1368 if app_matches_packagename(app, s):
1371 matches = vnsearch_g(line)
1373 version = matches.group(2)
1375 matches = vcsearch_g(line)
1377 vercode = matches.group(1)
1380 xml = parse_xml(path)
1381 if "package" in xml.attrib:
1382 s = xml.attrib["package"]
1383 if app_matches_packagename(app, s):
1385 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1386 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1387 base_dir = os.path.dirname(path)
1388 version = retrieve_string_singleline(base_dir, version)
1389 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1390 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1391 if string_is_integer(a):
1394 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1396 # Remember package name, may be defined separately from version+vercode
1398 package = max_package
1400 logging.debug("..got package={0}, version={1}, vercode={2}"
1401 .format(package, version, vercode))
1403 # Always grab the package name and version name in case they are not
1404 # together with the highest version code
1405 if max_package is None and package is not None:
1406 max_package = package
1407 if max_version is None and version is not None:
1408 max_version = version
1410 if vercode is not None \
1411 and (max_vercode is None or vercode > max_vercode):
1412 if not ignoresearch or not ignoresearch(version):
1413 if version is not None:
1414 max_version = version
1415 if vercode is not None:
1416 max_vercode = vercode
1417 if package is not None:
1418 max_package = package
1420 max_version = "Ignore"
1422 if max_version is None:
1423 max_version = "Unknown"
1425 if max_package and not is_valid_package_name(max_package):
1426 raise FDroidException(_("Invalid package name {0}").format(max_package))
1428 return (max_version, max_vercode, max_package)
1431 def is_valid_package_name(name):
1432 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1435 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1436 raw=False, prepare=True, preponly=False, refresh=True,
1438 """Get the specified source library.
1440 Returns the path to it. Normally this is the path to be used when
1441 referencing it, which may be a subdirectory of the actual project. If
1442 you want the base directory of the project, pass 'basepath=True'.
1451 name, ref = spec.split('@')
1453 number, name = name.split(':', 1)
1455 name, subdir = name.split('/', 1)
1457 if name not in fdroidserver.metadata.srclibs:
1458 raise VCSException('srclib ' + name + ' not found.')
1460 srclib = fdroidserver.metadata.srclibs[name]
1462 sdir = os.path.join(srclib_dir, name)
1465 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1466 vcs.srclib = (name, number, sdir)
1468 vcs.gotorevision(ref, refresh)
1475 libdir = os.path.join(sdir, subdir)
1476 elif srclib["Subdir"]:
1477 for subdir in srclib["Subdir"]:
1478 libdir_candidate = os.path.join(sdir, subdir)
1479 if os.path.exists(libdir_candidate):
1480 libdir = libdir_candidate
1486 remove_signing_keys(sdir)
1487 remove_debuggable_flags(sdir)
1491 if srclib["Prepare"]:
1492 cmd = replace_config_vars(srclib["Prepare"], build)
1494 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1495 if p.returncode != 0:
1496 raise BuildException("Error running prepare command for srclib %s"
1502 return (name, number, libdir)
1505 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1508 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1509 """ Prepare the source code for a particular build
1511 :param vcs: the appropriate vcs object for the application
1512 :param app: the application details from the metadata
1513 :param build: the build details from the metadata
1514 :param build_dir: the path to the build directory, usually 'build/app.id'
1515 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1516 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1518 Returns the (root, srclibpaths) where:
1519 :param root: is the root directory, which may be the same as 'build_dir' or may
1520 be a subdirectory of it.
1521 :param srclibpaths: is information on the srclibs being used
1524 # Optionally, the actual app source can be in a subdirectory
1526 root_dir = os.path.join(build_dir, build.subdir)
1528 root_dir = build_dir
1530 # Get a working copy of the right revision
1531 logging.info("Getting source for revision " + build.commit)
1532 vcs.gotorevision(build.commit, refresh)
1534 # Initialise submodules if required
1535 if build.submodules:
1536 logging.info(_("Initialising submodules"))
1537 vcs.initsubmodules()
1539 # Check that a subdir (if we're using one) exists. This has to happen
1540 # after the checkout, since it might not exist elsewhere
1541 if not os.path.exists(root_dir):
1542 raise BuildException('Missing subdir ' + root_dir)
1544 # Run an init command if one is required
1546 cmd = replace_config_vars(build.init, build)
1547 logging.info("Running 'init' commands in %s" % root_dir)
1549 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1550 if p.returncode != 0:
1551 raise BuildException("Error running init command for %s:%s" %
1552 (app.id, build.versionName), p.output)
1554 # Apply patches if any
1556 logging.info("Applying patches")
1557 for patch in build.patch:
1558 patch = patch.strip()
1559 logging.info("Applying " + patch)
1560 patch_path = os.path.join('metadata', app.id, patch)
1561 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1562 if p.returncode != 0:
1563 raise BuildException("Failed to apply patch %s" % patch_path)
1565 # Get required source libraries
1568 logging.info("Collecting source libraries")
1569 for lib in build.srclibs:
1570 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1571 refresh=refresh, build=build))
1573 for name, number, libpath in srclibpaths:
1574 place_srclib(root_dir, int(number) if number else None, libpath)
1576 basesrclib = vcs.getsrclib()
1577 # If one was used for the main source, add that too.
1579 srclibpaths.append(basesrclib)
1581 # Update the local.properties file
1582 localprops = [os.path.join(build_dir, 'local.properties')]
1584 parts = build.subdir.split(os.sep)
1587 cur = os.path.join(cur, d)
1588 localprops += [os.path.join(cur, 'local.properties')]
1589 for path in localprops:
1591 if os.path.isfile(path):
1592 logging.info("Updating local.properties file at %s" % path)
1593 with open(path, 'r', encoding='iso-8859-1') as f:
1597 logging.info("Creating local.properties file at %s" % path)
1598 # Fix old-fashioned 'sdk-location' by copying
1599 # from sdk.dir, if necessary
1601 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1602 re.S | re.M).group(1)
1603 props += "sdk-location=%s\n" % sdkloc
1605 props += "sdk.dir=%s\n" % config['sdk_path']
1606 props += "sdk-location=%s\n" % config['sdk_path']
1607 ndk_path = build.ndk_path()
1608 # if for any reason the path isn't valid or the directory
1609 # doesn't exist, some versions of Gradle will error with a
1610 # cryptic message (even if the NDK is not even necessary).
1611 # https://gitlab.com/fdroid/fdroidserver/issues/171
1612 if ndk_path and os.path.exists(ndk_path):
1614 props += "ndk.dir=%s\n" % ndk_path
1615 props += "ndk-location=%s\n" % ndk_path
1616 # Add java.encoding if necessary
1618 props += "java.encoding=%s\n" % build.encoding
1619 with open(path, 'w', encoding='iso-8859-1') as f:
1623 if build.build_method() == 'gradle':
1624 flavours = build.gradle
1627 n = build.target.split('-')[1]
1628 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1629 r'compileSdkVersion %s' % n,
1630 os.path.join(root_dir, 'build.gradle'))
1632 # Remove forced debuggable flags
1633 remove_debuggable_flags(root_dir)
1635 # Insert version code and number into the manifest if necessary
1636 if build.forceversion:
1637 logging.info("Changing the version name")
1638 for path in manifest_paths(root_dir, flavours):
1639 if not os.path.isfile(path):
1641 if has_extension(path, 'xml'):
1642 regsub_file(r'android:versionName="[^"]*"',
1643 r'android:versionName="%s"' % build.versionName,
1645 elif has_extension(path, 'gradle'):
1646 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1647 r"""\1versionName '%s'""" % build.versionName,
1650 if build.forcevercode:
1651 logging.info("Changing the version code")
1652 for path in manifest_paths(root_dir, flavours):
1653 if not os.path.isfile(path):
1655 if has_extension(path, 'xml'):
1656 regsub_file(r'android:versionCode="[^"]*"',
1657 r'android:versionCode="%s"' % build.versionCode,
1659 elif has_extension(path, 'gradle'):
1660 regsub_file(r'versionCode[ =]+[0-9]+',
1661 r'versionCode %s' % build.versionCode,
1664 # Delete unwanted files
1666 logging.info(_("Removing specified files"))
1667 for part in getpaths(build_dir, build.rm):
1668 dest = os.path.join(build_dir, part)
1669 logging.info("Removing {0}".format(part))
1670 if os.path.lexists(dest):
1671 # rmtree can only handle directories that are not symlinks, so catch anything else
1672 if not os.path.isdir(dest) or os.path.islink(dest):
1677 logging.info("...but it didn't exist")
1679 remove_signing_keys(build_dir)
1681 # Add required external libraries
1683 logging.info("Collecting prebuilt libraries")
1684 libsdir = os.path.join(root_dir, 'libs')
1685 if not os.path.exists(libsdir):
1687 for lib in build.extlibs:
1689 logging.info("...installing extlib {0}".format(lib))
1690 libf = os.path.basename(lib)
1691 libsrc = os.path.join(extlib_dir, lib)
1692 if not os.path.exists(libsrc):
1693 raise BuildException("Missing extlib file {0}".format(libsrc))
1694 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1696 # Run a pre-build command if one is required
1698 logging.info("Running 'prebuild' commands in %s" % root_dir)
1700 cmd = replace_config_vars(build.prebuild, build)
1702 # Substitute source library paths into prebuild commands
1703 for name, number, libpath in srclibpaths:
1704 libpath = os.path.relpath(libpath, root_dir)
1705 cmd = cmd.replace('$$' + name + '$$', libpath)
1707 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1708 if p.returncode != 0:
1709 raise BuildException("Error running prebuild command for %s:%s" %
1710 (app.id, build.versionName), p.output)
1712 # Generate (or update) the ant build file, build.xml...
1713 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1714 parms = ['android', 'update', 'lib-project']
1715 lparms = ['android', 'update', 'project']
1718 parms += ['-t', build.target]
1719 lparms += ['-t', build.target]
1720 if build.androidupdate:
1721 update_dirs = build.androidupdate
1723 update_dirs = ant_subprojects(root_dir) + ['.']
1725 for d in update_dirs:
1726 subdir = os.path.join(root_dir, d)
1728 logging.debug("Updating main project")
1729 cmd = parms + ['-p', d]
1731 logging.debug("Updating subproject %s" % d)
1732 cmd = lparms + ['-p', d]
1733 p = SdkToolsPopen(cmd, cwd=root_dir)
1734 # Check to see whether an error was returned without a proper exit
1735 # code (this is the case for the 'no target set or target invalid'
1737 if p.returncode != 0 or p.output.startswith("Error: "):
1738 raise BuildException("Failed to update project at %s" % d, p.output)
1739 # Clean update dirs via ant
1741 logging.info("Cleaning subproject %s" % d)
1742 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1744 return (root_dir, srclibpaths)
1747 def getpaths_map(build_dir, globpaths):
1748 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1752 full_path = os.path.join(build_dir, p)
1753 full_path = os.path.normpath(full_path)
1754 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1756 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1760 def getpaths(build_dir, globpaths):
1761 """Extend via globbing the paths from a field and return them as a set"""
1762 paths_map = getpaths_map(build_dir, globpaths)
1764 for k, v in paths_map.items():
1771 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1774 def check_system_clock(dt_obj, path):
1775 """Check if system clock is updated based on provided date
1777 If an APK has files newer than the system time, suggest updating
1778 the system clock. This is useful for offline systems, used for
1779 signing, which do not have another source of clock sync info. It
1780 has to be more than 24 hours newer because ZIP/APK files do not
1784 checkdt = dt_obj - timedelta(1)
1785 if datetime.today() < checkdt:
1786 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1787 + '\n' + _('Set clock to that time using:') + '\n'
1788 + 'sudo date -s "' + str(dt_obj) + '"')
1792 """permanent store of existing APKs with the date they were added
1794 This is currently the only way to permanently store the "updated"
1799 '''Load filename/date info about previously seen APKs
1801 Since the appid and date strings both will never have spaces,
1802 this is parsed as a list from the end to allow the filename to
1803 have any combo of spaces.
1806 self.path = os.path.join('stats', 'known_apks.txt')
1808 if os.path.isfile(self.path):
1809 with open(self.path, 'r', encoding='utf8') as f:
1811 t = line.rstrip().split(' ')
1813 self.apks[t[0]] = (t[1], None)
1816 date = datetime.strptime(t[-1], '%Y-%m-%d')
1817 filename = line[0:line.rfind(appid) - 1]
1818 self.apks[filename] = (appid, date)
1819 check_system_clock(date, self.path)
1820 self.changed = False
1822 def writeifchanged(self):
1823 if not self.changed:
1826 if not os.path.exists('stats'):
1830 for apk, app in self.apks.items():
1832 line = apk + ' ' + appid
1834 line += ' ' + added.strftime('%Y-%m-%d')
1837 with open(self.path, 'w', encoding='utf8') as f:
1838 for line in sorted(lst, key=natural_key):
1839 f.write(line + '\n')
1841 def recordapk(self, apkName, app, default_date=None):
1843 Record an apk (if it's new, otherwise does nothing)
1844 Returns the date it was added as a datetime instance
1846 if apkName not in self.apks:
1847 if default_date is None:
1848 default_date = datetime.utcnow()
1849 self.apks[apkName] = (app, default_date)
1851 _ignored, added = self.apks[apkName]
1854 def getapp(self, apkname):
1855 """Look up information - given the 'apkname', returns (app id, date added/None).
1857 Or returns None for an unknown apk.
1859 if apkname in self.apks:
1860 return self.apks[apkname]
1863 def getlatest(self, num):
1864 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1866 for apk, app in self.apks.items():
1870 if apps[appid] > added:
1874 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1875 lst = [app for app, _ignored in sortedapps]
1880 def get_file_extension(filename):
1881 """get the normalized file extension, can be blank string but never None"""
1882 if isinstance(filename, bytes):
1883 filename = filename.decode('utf-8')
1884 return os.path.splitext(filename)[1].lower()[1:]
1887 def get_apk_debuggable_aapt(apkfile):
1888 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1890 if p.returncode != 0:
1891 raise FDroidException(_("Failed to get APK manifest information"))
1892 for line in p.output.splitlines():
1893 if 'android:debuggable' in line and not line.endswith('0x0'):
1898 def get_apk_debuggable_androguard(apkfile):
1900 from androguard.core.bytecodes.apk import APK
1902 raise FDroidException("androguard library is not installed and aapt not present")
1904 apkobject = APK(apkfile)
1905 if apkobject.is_valid_APK():
1906 debuggable = apkobject.get_element("application", "debuggable")
1907 if debuggable is not None:
1908 return bool(strtobool(debuggable))
1912 def isApkAndDebuggable(apkfile):
1913 """Returns True if the given file is an APK and is debuggable
1915 :param apkfile: full path to the apk to check"""
1917 if get_file_extension(apkfile) != 'apk':
1920 if SdkToolsPopen(['aapt', 'version'], output=False):
1921 return get_apk_debuggable_aapt(apkfile)
1923 return get_apk_debuggable_androguard(apkfile)
1926 def get_apk_id_aapt(apkfile):
1927 """Extrat identification information from APK using aapt.
1929 :param apkfile: path to an APK file.
1930 :returns: triplet (appid, version code, version name)
1932 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1933 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1934 for line in p.output.splitlines():
1937 return m.group('appid'), m.group('vercode'), m.group('vername')
1938 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1939 .format(apkfilename=apkfile))
1942 def get_minSdkVersion_aapt(apkfile):
1943 """Extract the minimum supported Android SDK from an APK using aapt
1945 :param apkfile: path to an APK file.
1946 :returns: the integer representing the SDK version
1948 r = re.compile(r"^sdkVersion:'([0-9]+)'")
1949 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1950 for line in p.output.splitlines():
1953 return int(m.group(1))
1954 raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"')
1955 .format(apkfilename=apkfile))
1960 self.returncode = None
1964 def SdkToolsPopen(commands, cwd=None, output=True):
1966 if cmd not in config:
1967 config[cmd] = find_sdk_tools_cmd(commands[0])
1968 abscmd = config[cmd]
1970 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1972 test_aapt_version(config['aapt'])
1973 return FDroidPopen([abscmd] + commands[1:],
1974 cwd=cwd, output=output)
1977 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1979 Run a command and capture the possibly huge output as bytes.
1981 :param commands: command and argument list like in subprocess.Popen
1982 :param cwd: optionally specifies a working directory
1983 :param envs: a optional dictionary of environment variables and their values
1984 :returns: A PopenResult.
1989 set_FDroidPopen_env()
1991 process_env = env.copy()
1992 if envs is not None and len(envs) > 0:
1993 process_env.update(envs)
1996 cwd = os.path.normpath(cwd)
1997 logging.debug("Directory: %s" % cwd)
1998 logging.debug("> %s" % ' '.join(commands))
2000 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
2001 result = PopenResult()
2004 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
2005 stdout=subprocess.PIPE, stderr=stderr_param)
2006 except OSError as e:
2007 raise BuildException("OSError while trying to execute " +
2008 ' '.join(commands) + ': ' + str(e))
2010 # TODO are these AsynchronousFileReader threads always exiting?
2011 if not stderr_to_stdout and options.verbose:
2012 stderr_queue = Queue()
2013 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
2015 while not stderr_reader.eof():
2016 while not stderr_queue.empty():
2017 line = stderr_queue.get()
2018 sys.stderr.buffer.write(line)
2023 stdout_queue = Queue()
2024 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2027 # Check the queue for output (until there is no more to get)
2028 while not stdout_reader.eof():
2029 while not stdout_queue.empty():
2030 line = stdout_queue.get()
2031 if output and options.verbose:
2032 # Output directly to console
2033 sys.stderr.buffer.write(line)
2039 result.returncode = p.wait()
2040 result.output = buf.getvalue()
2042 # make sure all filestreams of the subprocess are closed
2043 for streamvar in ['stdin', 'stdout', 'stderr']:
2044 if hasattr(p, streamvar):
2045 stream = getattr(p, streamvar)
2051 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2053 Run a command and capture the possibly huge output as a str.
2055 :param commands: command and argument list like in subprocess.Popen
2056 :param cwd: optionally specifies a working directory
2057 :param envs: a optional dictionary of environment variables and their values
2058 :returns: A PopenResult.
2060 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2061 result.output = result.output.decode('utf-8', 'ignore')
2065 gradle_comment = re.compile(r'[ ]*//')
2066 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2067 gradle_line_matches = [
2068 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2069 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2070 re.compile(r'.*\.readLine\(.*'),
2074 def remove_signing_keys(build_dir):
2075 for root, dirs, files in os.walk(build_dir):
2076 if 'build.gradle' in files:
2077 path = os.path.join(root, 'build.gradle')
2079 with open(path, "r", encoding='utf8') as o:
2080 lines = o.readlines()
2086 with open(path, "w", encoding='utf8') as o:
2087 while i < len(lines):
2090 while line.endswith('\\\n'):
2091 line = line.rstrip('\\\n') + lines[i]
2094 if gradle_comment.match(line):
2099 opened += line.count('{')
2100 opened -= line.count('}')
2103 if gradle_signing_configs.match(line):
2108 if any(s.match(line) for s in gradle_line_matches):
2116 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2119 'project.properties',
2121 'default.properties',
2122 'ant.properties', ]:
2123 if propfile in files:
2124 path = os.path.join(root, propfile)
2126 with open(path, "r", encoding='iso-8859-1') as o:
2127 lines = o.readlines()
2131 with open(path, "w", encoding='iso-8859-1') as o:
2133 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2140 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2143 def set_FDroidPopen_env(build=None):
2145 set up the environment variables for the build environment
2147 There is only a weak standard, the variables used by gradle, so also set
2148 up the most commonly used environment variables for SDK and NDK. Also, if
2149 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2151 global env, orig_path
2155 orig_path = env['PATH']
2156 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2157 env[n] = config['sdk_path']
2158 for k, v in config['java_paths'].items():
2159 env['JAVA%s_HOME' % k] = v
2161 missinglocale = True
2162 for k, v in env.items():
2163 if k == 'LANG' and v != 'C':
2164 missinglocale = False
2166 missinglocale = False
2168 env['LANG'] = 'en_US.UTF-8'
2170 if build is not None:
2171 path = build.ndk_path()
2172 paths = orig_path.split(os.pathsep)
2173 if path not in paths:
2174 paths = [path] + paths
2175 env['PATH'] = os.pathsep.join(paths)
2176 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2177 env[n] = build.ndk_path()
2180 def replace_build_vars(cmd, build):
2181 cmd = cmd.replace('$$COMMIT$$', build.commit)
2182 cmd = cmd.replace('$$VERSION$$', build.versionName)
2183 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2187 def replace_config_vars(cmd, build):
2188 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2189 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2190 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2191 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2192 if build is not None:
2193 cmd = replace_build_vars(cmd, build)
2197 def place_srclib(root_dir, number, libpath):
2200 relpath = os.path.relpath(libpath, root_dir)
2201 proppath = os.path.join(root_dir, 'project.properties')
2204 if os.path.isfile(proppath):
2205 with open(proppath, "r", encoding='iso-8859-1') as o:
2206 lines = o.readlines()
2208 with open(proppath, "w", encoding='iso-8859-1') as o:
2211 if line.startswith('android.library.reference.%d=' % number):
2212 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2217 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2220 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2223 def signer_fingerprint_short(sig):
2224 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2226 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2227 for a given pkcs7 signature.
2229 :param sig: Contents of an APK signing certificate.
2230 :returns: shortened signing-key fingerprint.
2232 return signer_fingerprint(sig)[:7]
2235 def signer_fingerprint(sig):
2236 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2238 Extracts hexadecimal sha256 signing-key fingerprint string
2239 for a given pkcs7 signature.
2241 :param: Contents of an APK signature.
2242 :returns: shortened signature fingerprint.
2244 cert_encoded = get_certificate(sig)
2245 return hashlib.sha256(cert_encoded).hexdigest()
2248 def apk_signer_fingerprint(apk_path):
2249 """Obtain sha256 signing-key fingerprint for APK.
2251 Extracts hexadecimal sha256 signing-key fingerprint string
2254 :param apkpath: path to APK
2255 :returns: signature fingerprint
2258 with zipfile.ZipFile(apk_path, 'r') as apk:
2259 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2262 logging.error("Found no signing certificates on %s" % apk_path)
2265 logging.error("Found multiple signing certificates on %s" % apk_path)
2268 cert = apk.read(certs[0])
2269 return signer_fingerprint(cert)
2272 def apk_signer_fingerprint_short(apk_path):
2273 """Obtain shortened sha256 signing-key fingerprint for APK.
2275 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2276 for a given pkcs7 APK.
2278 :param apk_path: path to APK
2279 :returns: shortened signing-key fingerprint
2281 return apk_signer_fingerprint(apk_path)[:7]
2284 def metadata_get_sigdir(appid, vercode=None):
2285 """Get signature directory for app"""
2287 return os.path.join('metadata', appid, 'signatures', vercode)
2289 return os.path.join('metadata', appid, 'signatures')
2292 def metadata_find_developer_signature(appid, vercode=None):
2293 """Tires to find the developer signature for given appid.
2295 This picks the first signature file found in metadata an returns its
2298 :returns: sha256 signing key fingerprint of the developer signing key.
2299 None in case no signature can not be found."""
2301 # fetch list of dirs for all versions of signatures
2304 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2306 appsigdir = metadata_get_sigdir(appid)
2307 if os.path.isdir(appsigdir):
2308 numre = re.compile('[0-9]+')
2309 for ver in os.listdir(appsigdir):
2310 if numre.match(ver):
2311 appversigdir = os.path.join(appsigdir, ver)
2312 appversigdirs.append(appversigdir)
2314 for sigdir in appversigdirs:
2315 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2316 glob.glob(os.path.join(sigdir, '*.EC')) + \
2317 glob.glob(os.path.join(sigdir, '*.RSA'))
2319 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2321 with open(sig, 'rb') as f:
2322 return signer_fingerprint(f.read())
2326 def metadata_find_signing_files(appid, vercode):
2327 """Gets a list of singed manifests and signatures.
2329 :param appid: app id string
2330 :param vercode: app version code
2331 :returns: a list of triplets for each signing key with following paths:
2332 (signature_file, singed_file, manifest_file)
2335 sigdir = metadata_get_sigdir(appid, vercode)
2336 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2337 glob.glob(os.path.join(sigdir, '*.EC')) + \
2338 glob.glob(os.path.join(sigdir, '*.RSA'))
2339 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2341 sf = extre.sub('.SF', sig)
2342 if os.path.isfile(sf):
2343 mf = os.path.join(sigdir, 'MANIFEST.MF')
2344 if os.path.isfile(mf):
2345 ret.append((sig, sf, mf))
2349 def metadata_find_developer_signing_files(appid, vercode):
2350 """Get developer signature files for specified app from metadata.
2352 :returns: A triplet of paths for signing files from metadata:
2353 (signature_file, singed_file, manifest_file)
2355 allsigningfiles = metadata_find_signing_files(appid, vercode)
2356 if allsigningfiles and len(allsigningfiles) == 1:
2357 return allsigningfiles[0]
2362 def apk_strip_signatures(signed_apk, strip_manifest=False):
2363 """Removes signatures from APK.
2365 :param signed_apk: path to apk file.
2366 :param strip_manifest: when set to True also the manifest file will
2367 be removed from the APK.
2369 with tempfile.TemporaryDirectory() as tmpdir:
2370 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2371 shutil.move(signed_apk, tmp_apk)
2372 with ZipFile(tmp_apk, 'r') as in_apk:
2373 with ZipFile(signed_apk, 'w') as out_apk:
2374 for info in in_apk.infolist():
2375 if not apk_sigfile.match(info.filename):
2377 if info.filename != 'META-INF/MANIFEST.MF':
2378 buf = in_apk.read(info.filename)
2379 out_apk.writestr(info, buf)
2381 buf = in_apk.read(info.filename)
2382 out_apk.writestr(info, buf)
2385 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2386 """Implats a signature from metadata into an APK.
2388 Note: this changes there supplied APK in place. So copy it if you
2389 need the original to be preserved.
2391 :param apkpath: location of the apk
2393 # get list of available signature files in metadata
2394 with tempfile.TemporaryDirectory() as tmpdir:
2395 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2396 with ZipFile(apkpath, 'r') as in_apk:
2397 with ZipFile(apkwithnewsig, 'w') as out_apk:
2398 for sig_file in [signaturefile, signedfile, manifest]:
2399 with open(sig_file, 'rb') as fp:
2401 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2402 info.compress_type = zipfile.ZIP_DEFLATED
2403 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2404 out_apk.writestr(info, buf)
2405 for info in in_apk.infolist():
2406 if not apk_sigfile.match(info.filename):
2407 if info.filename != 'META-INF/MANIFEST.MF':
2408 buf = in_apk.read(info.filename)
2409 out_apk.writestr(info, buf)
2411 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2412 if p.returncode != 0:
2413 raise BuildException("Failed to align application")
2416 def apk_extract_signatures(apkpath, outdir, manifest=True):
2417 """Extracts a signature files from APK and puts them into target directory.
2419 :param apkpath: location of the apk
2420 :param outdir: folder where the extracted signature files will be stored
2421 :param manifest: (optionally) disable extracting manifest file
2423 with ZipFile(apkpath, 'r') as in_apk:
2424 for f in in_apk.infolist():
2425 if apk_sigfile.match(f.filename) or \
2426 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2427 newpath = os.path.join(outdir, os.path.basename(f.filename))
2428 with open(newpath, 'wb') as out_file:
2429 out_file.write(in_apk.read(f.filename))
2432 def sign_apk(unsigned_path, signed_path, keyalias):
2433 """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned
2435 android-18 (4.3) finally added support for reasonable hash
2436 algorithms, like SHA-256, before then, the only options were MD5
2437 and SHA1 :-/ This aims to use SHA-256 when the APK does not target
2438 older Android versions, and is therefore safe to do so.
2440 https://issuetracker.google.com/issues/36956587
2441 https://android-review.googlesource.com/c/platform/libcore/+/44491
2445 if get_minSdkVersion_aapt(unsigned_path) < 18:
2446 signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1']
2448 signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA256']
2450 p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'],
2451 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2452 '-keypass:env', 'FDROID_KEY_PASS']
2453 + signature_algorithm + [unsigned_path, keyalias],
2455 'FDROID_KEY_STORE_PASS': config['keystorepass'],
2456 'FDROID_KEY_PASS': config['keypass'], })
2457 if p.returncode != 0:
2458 raise BuildException(_("Failed to sign application"), p.output)
2460 p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path])
2461 if p.returncode != 0:
2462 raise BuildException(_("Failed to zipalign application"))
2463 os.remove(unsigned_path)
2466 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2467 """Verify that two apks are the same
2469 One of the inputs is signed, the other is unsigned. The signature metadata
2470 is transferred from the signed to the unsigned apk, and then jarsigner is
2471 used to verify that the signature from the signed apk is also varlid for
2472 the unsigned one. If the APK given as unsigned actually does have a
2473 signature, it will be stripped out and ignored.
2475 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2476 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2477 into AndroidManifest.xml, but that makes the build not reproducible. So
2478 instead they are included as separate files in the APK's META-INF/ folder.
2479 If those files exist in the signed APK, they will be part of the signature
2480 and need to also be included in the unsigned APK for it to validate.
2482 :param signed_apk: Path to a signed apk file
2483 :param unsigned_apk: Path to an unsigned apk file expected to match it
2484 :param tmp_dir: Path to directory for temporary files
2485 :returns: None if the verification is successful, otherwise a string
2486 describing what went wrong.
2489 if not os.path.isfile(signed_apk):
2490 return 'can not verify: file does not exists: {}'.format(signed_apk)
2492 if not os.path.isfile(unsigned_apk):
2493 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2495 with ZipFile(signed_apk, 'r') as signed:
2496 meta_inf_files = ['META-INF/MANIFEST.MF']
2497 for f in signed.namelist():
2498 if apk_sigfile.match(f) \
2499 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2500 meta_inf_files.append(f)
2501 if len(meta_inf_files) < 3:
2502 return "Signature files missing from {0}".format(signed_apk)
2504 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2505 with ZipFile(unsigned_apk, 'r') as unsigned:
2506 # only read the signature from the signed APK, everything else from unsigned
2507 with ZipFile(tmp_apk, 'w') as tmp:
2508 for filename in meta_inf_files:
2509 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2510 for info in unsigned.infolist():
2511 if info.filename in meta_inf_files:
2512 logging.warning('Ignoring %s from %s',
2513 info.filename, unsigned_apk)
2515 if info.filename in tmp.namelist():
2516 return "duplicate filename found: " + info.filename
2517 tmp.writestr(info, unsigned.read(info.filename))
2519 verified = verify_apk_signature(tmp_apk)
2522 logging.info("...NOT verified - {0}".format(tmp_apk))
2523 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2524 os.path.dirname(unsigned_apk))
2526 logging.info("...successfully verified")
2530 def verify_jar_signature(jar):
2531 """Verifies the signature of a given JAR file.
2533 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2534 this has to turn on -strict then check for result 4, since this
2535 does not expect the signature to be from a CA-signed certificate.
2537 :raises: VerificationException() if the JAR's signature could not be verified
2541 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2542 raise VerificationException(_("The repository's index could not be verified."))
2545 def verify_apk_signature(apk, min_sdk_version=None):
2546 """verify the signature on an APK
2548 Try to use apksigner whenever possible since jarsigner is very
2549 shitty: unsigned APKs pass as "verified"! Warning, this does
2550 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2552 :returns: boolean whether the APK was verified
2554 if set_command_in_config('apksigner'):
2555 args = [config['apksigner'], 'verify']
2557 args += ['--min-sdk-version=' + min_sdk_version]
2558 return subprocess.call(args + [apk]) == 0
2560 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2562 verify_jar_signature(apk)
2569 def verify_old_apk_signature(apk):
2570 """verify the signature on an archived APK, supporting deprecated algorithms
2572 F-Droid aims to keep every single binary that it ever published. Therefore,
2573 it needs to be able to verify APK signatures that include deprecated/removed
2574 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2576 jarsigner passes unsigned APKs as "verified"! So this has to turn
2577 on -strict then check for result 4.
2579 :returns: boolean whether the APK was verified
2582 _java_security = os.path.join(os.getcwd(), '.java.security')
2583 with open(_java_security, 'w') as fp:
2584 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2586 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2587 '-strict', '-verify', apk]) == 4
2590 apk_badchars = re.compile('''[/ :;'"]''')
2593 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2596 Returns None if the apk content is the same (apart from the signing key),
2597 otherwise a string describing what's different, or what went wrong when
2598 trying to do the comparison.
2604 absapk1 = os.path.abspath(apk1)
2605 absapk2 = os.path.abspath(apk2)
2607 if set_command_in_config('diffoscope'):
2608 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2609 htmlfile = logfilename + '.diffoscope.html'
2610 textfile = logfilename + '.diffoscope.txt'
2611 if subprocess.call([config['diffoscope'],
2612 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2613 '--html', htmlfile, '--text', textfile,
2614 absapk1, absapk2]) != 0:
2615 return("Failed to unpack " + apk1)
2617 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2618 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2619 for d in [apk1dir, apk2dir]:
2620 if os.path.exists(d):
2623 os.mkdir(os.path.join(d, 'jar-xf'))
2625 if subprocess.call(['jar', 'xf',
2626 os.path.abspath(apk1)],
2627 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2628 return("Failed to unpack " + apk1)
2629 if subprocess.call(['jar', 'xf',
2630 os.path.abspath(apk2)],
2631 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2632 return("Failed to unpack " + apk2)
2634 if set_command_in_config('apktool'):
2635 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2637 return("Failed to unpack " + apk1)
2638 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2640 return("Failed to unpack " + apk2)
2642 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2643 lines = p.output.splitlines()
2644 if len(lines) != 1 or 'META-INF' not in lines[0]:
2645 if set_command_in_config('meld'):
2646 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2647 return("Unexpected diff output - " + p.output)
2649 # since everything verifies, delete the comparison to keep cruft down
2650 shutil.rmtree(apk1dir)
2651 shutil.rmtree(apk2dir)
2653 # If we get here, it seems like they're the same!
2657 def set_command_in_config(command):
2658 '''Try to find specified command in the path, if it hasn't been
2659 manually set in config.py. If found, it is added to the config
2660 dict. The return value says whether the command is available.
2663 if command in config:
2666 tmp = find_command(command)
2668 config[command] = tmp
2673 def find_command(command):
2674 '''find the full path of a command, or None if it can't be found in the PATH'''
2677 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2679 fpath, fname = os.path.split(command)
2684 for path in os.environ["PATH"].split(os.pathsep):
2685 path = path.strip('"')
2686 exe_file = os.path.join(path, command)
2687 if is_exe(exe_file):
2694 '''generate a random password for when generating keys'''
2695 h = hashlib.sha256()
2696 h.update(os.urandom(16)) # salt
2697 h.update(socket.getfqdn().encode('utf-8'))
2698 passwd = base64.b64encode(h.digest()).strip()
2699 return passwd.decode('utf-8')
2702 def genkeystore(localconfig):
2704 Generate a new key with password provided in :param localconfig and add it to new keystore
2705 :return: hexed public key, public key fingerprint
2707 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2708 keystoredir = os.path.dirname(localconfig['keystore'])
2709 if keystoredir is None or keystoredir == '':
2710 keystoredir = os.path.join(os.getcwd(), keystoredir)
2711 if not os.path.exists(keystoredir):
2712 os.makedirs(keystoredir, mode=0o700)
2715 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2716 'FDROID_KEY_PASS': localconfig['keypass'],
2718 p = FDroidPopen([config['keytool'], '-genkey',
2719 '-keystore', localconfig['keystore'],
2720 '-alias', localconfig['repo_keyalias'],
2721 '-keyalg', 'RSA', '-keysize', '4096',
2722 '-sigalg', 'SHA256withRSA',
2723 '-validity', '10000',
2724 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2725 '-keypass:env', 'FDROID_KEY_PASS',
2726 '-dname', localconfig['keydname']], envs=env_vars)
2727 if p.returncode != 0:
2728 raise BuildException("Failed to generate key", p.output)
2729 os.chmod(localconfig['keystore'], 0o0600)
2730 if not options.quiet:
2731 # now show the lovely key that was just generated
2732 p = FDroidPopen([config['keytool'], '-list', '-v',
2733 '-keystore', localconfig['keystore'],
2734 '-alias', localconfig['repo_keyalias'],
2735 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2736 logging.info(p.output.strip() + '\n\n')
2737 # get the public key
2738 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2739 '-keystore', localconfig['keystore'],
2740 '-alias', localconfig['repo_keyalias'],
2741 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2742 + config['smartcardoptions'],
2743 envs=env_vars, output=False, stderr_to_stdout=False)
2744 if p.returncode != 0 or len(p.output) < 20:
2745 raise BuildException("Failed to get public key", p.output)
2747 fingerprint = get_cert_fingerprint(pubkey)
2748 return hexlify(pubkey), fingerprint
2751 def get_cert_fingerprint(pubkey):
2753 Generate a certificate fingerprint the same way keytool does it
2754 (but with slightly different formatting)
2756 digest = hashlib.sha256(pubkey).digest()
2757 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2758 return " ".join(ret)
2761 def get_certificate(certificate_file):
2763 Extracts a certificate from the given file.
2764 :param certificate_file: file bytes (as string) representing the certificate
2765 :return: A binary representation of the certificate's public key, or None in case of error
2767 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2768 if content.getComponentByName('contentType') != rfc2315.signedData:
2770 content = decoder.decode(content.getComponentByName('content'),
2771 asn1Spec=rfc2315.SignedData())[0]
2773 certificates = content.getComponentByName('certificates')
2774 cert = certificates[0].getComponentByName('certificate')
2776 logging.error("Certificates not found.")
2778 return encoder.encode(cert)
2781 def load_stats_fdroid_signing_key_fingerprints():
2782 """Load list of signing-key fingerprints stored by fdroid publish from file.
2784 :returns: list of dictionanryies containing the singing-key fingerprints.
2786 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2787 if not os.path.isfile(jar_file):
2789 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2790 p = FDroidPopen(cmd, output=False)
2791 if p.returncode != 4:
2792 raise FDroidException("Signature validation of '{}' failed! "
2793 "Please run publish again to rebuild this file.".format(jar_file))
2795 jar_sigkey = apk_signer_fingerprint(jar_file)
2796 repo_key_sig = config.get('repo_key_sha256')
2798 if jar_sigkey != repo_key_sig:
2799 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2801 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2802 config['repo_key_sha256'] = jar_sigkey
2803 write_to_config(config, 'repo_key_sha256')
2805 with zipfile.ZipFile(jar_file, 'r') as f:
2806 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2809 def write_to_config(thisconfig, key, value=None, config_file=None):
2810 '''write a key/value to the local config.py
2812 NOTE: only supports writing string variables.
2814 :param thisconfig: config dictionary
2815 :param key: variable name in config.py to be overwritten/added
2816 :param value: optional value to be written, instead of fetched
2817 from 'thisconfig' dictionary.
2820 origkey = key + '_orig'
2821 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2822 cfg = config_file if config_file else 'config.py'
2824 # load config file, create one if it doesn't exist
2825 if not os.path.exists(cfg):
2826 open(cfg, 'a').close()
2827 logging.info("Creating empty " + cfg)
2828 with open(cfg, 'r', encoding="utf-8") as f:
2829 lines = f.readlines()
2831 # make sure the file ends with a carraige return
2833 if not lines[-1].endswith('\n'):
2836 # regex for finding and replacing python string variable
2837 # definitions/initializations
2838 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2839 repl = key + ' = "' + value + '"'
2840 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2841 repl2 = key + " = '" + value + "'"
2843 # If we replaced this line once, we make sure won't be a
2844 # second instance of this line for this key in the document.
2847 with open(cfg, 'w', encoding="utf-8") as f:
2849 if pattern.match(line) or pattern2.match(line):
2851 line = pattern.sub(repl, line)
2852 line = pattern2.sub(repl2, line)
2863 def parse_xml(path):
2864 return XMLElementTree.parse(path).getroot()
2867 def string_is_integer(string):
2875 def local_rsync(options, fromdir, todir):
2876 '''Rsync method for local to local copying of things
2878 This is an rsync wrapper with all the settings for safe use within
2879 the various fdroidserver use cases. This uses stricter rsync
2880 checking on all files since people using offline mode are already
2881 prioritizing security above ease and speed.
2884 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2885 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2886 if not options.no_checksum:
2887 rsyncargs.append('--checksum')
2889 rsyncargs += ['--verbose']
2891 rsyncargs += ['--quiet']
2892 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2893 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2894 raise FDroidException()
2897 def get_per_app_repos():
2898 '''per-app repos are dirs named with the packageName of a single app'''
2900 # Android packageNames are Java packages, they may contain uppercase or
2901 # lowercase letters ('A' through 'Z'), numbers, and underscores
2902 # ('_'). However, individual package name parts may only start with
2903 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2904 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2907 for root, dirs, files in os.walk(os.getcwd()):
2909 print('checking', root, 'for', d)
2910 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2911 # standard parts of an fdroid repo, so never packageNames
2914 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2920 def is_repo_file(filename):
2921 '''Whether the file in a repo is a build product to be delivered to users'''
2922 if isinstance(filename, str):
2923 filename = filename.encode('utf-8', errors="surrogateescape")
2924 return os.path.isfile(filename) \
2925 and not filename.endswith(b'.asc') \
2926 and not filename.endswith(b'.sig') \
2927 and os.path.basename(filename) not in [
2929 b'index_unsigned.jar',
2938 def get_examples_dir():
2939 '''Return the dir where the fdroidserver example files are available'''
2941 tmp = os.path.dirname(sys.argv[0])
2942 if os.path.basename(tmp) == 'bin':
2943 egg_links = glob.glob(os.path.join(tmp, '..',
2944 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2946 # installed from local git repo
2947 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2950 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2951 if not os.path.exists(examplesdir): # use UNIX layout
2952 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2954 # we're running straight out of the git repo
2955 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2956 examplesdir = prefix + '/examples'