3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
61 # A signature block file with a .DSA, .RSA, or .EC extension
62 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
63 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
64 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
66 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
75 'sdk_path': "$ANDROID_HOME",
80 'r12b': "$ANDROID_NDK",
87 'build_tools': "25.0.2",
88 'force_build_tools': False,
93 'accepted_formats': ['txt', 'yml'],
94 'sync_from_local_copy_dir': False,
95 'allow_disabled_algorithms': False,
96 'per_app_repos': False,
97 'make_current_version_link': True,
98 'current_version_name_source': 'Name',
99 'update_stats': False,
101 'stats_server': None,
103 'stats_to_carbon': False,
105 'build_server_always': False,
106 'keystore': 'keystore.jks',
107 'smartcardoptions': [],
117 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
118 'repo_name': "My First FDroid Repo Demo",
119 'repo_icon': "fdroid-icon.png",
120 'repo_description': '''
121 This is a repository of apps to be used with FDroid. Applications in this
122 repository are either official binaries built by the original application
123 developers, or are binaries built from source by the admin of f-droid.org
124 using the tools on https://gitlab.com/u/fdroid.
130 def setup_global_opts(parser):
131 parser.add_argument("-v", "--verbose", action="store_true", default=False,
132 help=_("Spew out even more information than normal"))
133 parser.add_argument("-q", "--quiet", action="store_true", default=False,
134 help=_("Restrict output to warnings and errors"))
137 def _add_java_paths_to_config(pathlist, thisconfig):
138 def path_version_key(s):
140 for u in re.split('[^0-9]+', s):
142 versionlist.append(int(u))
147 for d in sorted(pathlist, key=path_version_key):
148 if os.path.islink(d):
150 j = os.path.basename(d)
151 # the last one found will be the canonical one, so order appropriately
153 r'^1\.([6-9])\.0\.jdk$', # OSX
154 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
155 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
156 r'^jdk([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-jdk$', # Arch (oracle)
159 r'^java-1\.([6-9])\.0-.*$', # RedHat
160 r'^java-([6-9])-oracle$', # Debian WebUpd8
161 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
162 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
164 m = re.match(regex, j)
167 for p in [d, os.path.join(d, 'Contents', 'Home')]:
168 if os.path.exists(os.path.join(p, 'bin', 'javac')):
169 thisconfig['java_paths'][m.group(1)] = p
172 def fill_config_defaults(thisconfig):
173 for k, v in default_config.items():
174 if k not in thisconfig:
177 # Expand paths (~users and $vars)
178 def expand_path(path):
182 path = os.path.expanduser(path)
183 path = os.path.expandvars(path)
188 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
193 thisconfig[k + '_orig'] = v
195 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
196 if thisconfig['java_paths'] is None:
197 thisconfig['java_paths'] = dict()
199 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
200 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
201 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
202 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
203 if os.getenv('JAVA_HOME') is not None:
204 pathlist.append(os.getenv('JAVA_HOME'))
205 if os.getenv('PROGRAMFILES') is not None:
206 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
207 _add_java_paths_to_config(pathlist, thisconfig)
209 for java_version in ('7', '8', '9'):
210 if java_version not in thisconfig['java_paths']:
212 java_home = thisconfig['java_paths'][java_version]
213 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
214 if os.path.exists(jarsigner):
215 thisconfig['jarsigner'] = jarsigner
216 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
217 break # Java7 is preferred, so quit if found
219 for k in ['ndk_paths', 'java_paths']:
225 thisconfig[k][k2] = exp
226 thisconfig[k][k2 + '_orig'] = v
229 def regsub_file(pattern, repl, path):
230 with open(path, 'rb') as f:
232 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
233 with open(path, 'wb') as f:
237 def read_config(opts, config_file='config.py'):
238 """Read the repository config
240 The config is read from config_file, which is in the current
241 directory when any of the repo management commands are used. If
242 there is a local metadata file in the git repo, then config.py is
243 not required, just use defaults.
246 global config, options
248 if config is not None:
255 if os.path.isfile(config_file):
256 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
257 with io.open(config_file, "rb") as f:
258 code = compile(f.read(), config_file, 'exec')
259 exec(code, None, config)
261 logging.warning(_("No 'config.py' found, using defaults."))
263 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
265 if not type(config[k]) in (str, list, tuple):
267 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
270 # smartcardoptions must be a list since its command line args for Popen
271 if 'smartcardoptions' in config:
272 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
273 elif 'keystore' in config and config['keystore'] == 'NONE':
274 # keystore='NONE' means use smartcard, these are required defaults
275 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
276 'SunPKCS11-OpenSC', '-providerClass',
277 'sun.security.pkcs11.SunPKCS11',
278 '-providerArg', 'opensc-fdroid.cfg']
280 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
281 st = os.stat(config_file)
282 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
283 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
284 .format(config_file=config_file))
286 fill_config_defaults(config)
288 for k in ["repo_description", "archive_description"]:
290 config[k] = clean_description(config[k])
292 if 'serverwebroot' in config:
293 if isinstance(config['serverwebroot'], str):
294 roots = [config['serverwebroot']]
295 elif all(isinstance(item, str) for item in config['serverwebroot']):
296 roots = config['serverwebroot']
298 raise TypeError(_('only accepts strings, lists, and tuples'))
300 for rootstr in roots:
301 # since this is used with rsync, where trailing slashes have
302 # meaning, ensure there is always a trailing slash
303 if rootstr[-1] != '/':
305 rootlist.append(rootstr.replace('//', '/'))
306 config['serverwebroot'] = rootlist
308 if 'servergitmirrors' in config:
309 if isinstance(config['servergitmirrors'], str):
310 roots = [config['servergitmirrors']]
311 elif all(isinstance(item, str) for item in config['servergitmirrors']):
312 roots = config['servergitmirrors']
314 raise TypeError(_('only accepts strings, lists, and tuples'))
315 config['servergitmirrors'] = roots
320 def assert_config_keystore(config):
321 """Check weather keystore is configured correctly and raise exception if not."""
324 if 'repo_keyalias' not in config:
326 logging.critical(_("'repo_keyalias' not found in config.py!"))
327 if 'keystore' not in config:
329 logging.critical(_("'keystore' not found in config.py!"))
330 elif not os.path.exists(config['keystore']):
332 logging.critical("'" + config['keystore'] + "' does not exist!")
333 if 'keystorepass' not in config:
335 logging.critical(_("'keystorepass' not found in config.py!"))
336 if 'keypass' not in config:
338 logging.critical(_("'keypass' not found in config.py!"))
340 raise FDroidException("This command requires a signing key, " +
341 "you can create one using: fdroid update --create-key")
344 def find_sdk_tools_cmd(cmd):
345 '''find a working path to a tool from the Android SDK'''
348 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
349 # try to find a working path to this command, in all the recent possible paths
350 if 'build_tools' in config:
351 build_tools = os.path.join(config['sdk_path'], 'build-tools')
352 # if 'build_tools' was manually set and exists, check only that one
353 configed_build_tools = os.path.join(build_tools, config['build_tools'])
354 if os.path.exists(configed_build_tools):
355 tooldirs.append(configed_build_tools)
357 # no configed version, so hunt known paths for it
358 for f in sorted(os.listdir(build_tools), reverse=True):
359 if os.path.isdir(os.path.join(build_tools, f)):
360 tooldirs.append(os.path.join(build_tools, f))
361 tooldirs.append(build_tools)
362 sdk_tools = os.path.join(config['sdk_path'], 'tools')
363 if os.path.exists(sdk_tools):
364 tooldirs.append(sdk_tools)
365 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
366 if os.path.exists(sdk_platform_tools):
367 tooldirs.append(sdk_platform_tools)
368 tooldirs.append('/usr/bin')
370 path = os.path.join(d, cmd)
371 if os.path.isfile(path):
373 test_aapt_version(path)
375 # did not find the command, exit with error message
376 ensure_build_tools_exists(config)
379 def test_aapt_version(aapt):
380 '''Check whether the version of aapt is new enough'''
381 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
382 if output is None or output == '':
383 logging.error(_("'{path}' failed to execute!").format(path=aapt))
385 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
390 # the Debian package has the version string like "v0.2-23.0.2"
391 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
392 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
395 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
398 def test_sdk_exists(thisconfig):
399 if 'sdk_path' not in thisconfig:
400 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
401 test_aapt_version(thisconfig['aapt'])
404 logging.error(_("'sdk_path' not set in 'config.py'!"))
406 if thisconfig['sdk_path'] == default_config['sdk_path']:
407 logging.error(_('No Android SDK found!'))
408 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
409 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
411 if not os.path.exists(thisconfig['sdk_path']):
412 logging.critical(_("Android SDK path '{path}' does not exist!")
413 .format(path=thisconfig['sdk_path']))
415 if not os.path.isdir(thisconfig['sdk_path']):
416 logging.critical(_("Android SDK path '{path}' is not a directory!")
417 .format(path=thisconfig['sdk_path']))
419 for d in ['build-tools', 'platform-tools', 'tools']:
420 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
421 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
422 .format(path=thisconfig['sdk_path'], dirname=d))
427 def ensure_build_tools_exists(thisconfig):
428 if not test_sdk_exists(thisconfig):
429 raise FDroidException(_("Android SDK not found!"))
430 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
431 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
432 if not os.path.isdir(versioned_build_tools):
433 raise FDroidException(
434 _("Android build-tools path '{path}' does not exist!")
435 .format(path=versioned_build_tools))
438 def get_local_metadata_files():
439 '''get any metadata files local to an app's source repo
441 This tries to ignore anything that does not count as app metdata,
442 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
445 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
448 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
450 :param appids: arguments in the form of multiple appid:[vc] strings
451 :returns: a dictionary with the set of vercodes specified for each package
454 if not appid_versionCode_pairs:
457 for p in appid_versionCode_pairs:
458 if allow_vercodes and ':' in p:
459 package, vercode = p.split(':')
461 package, vercode = p, None
462 if package not in vercodes:
463 vercodes[package] = [vercode] if vercode else []
465 elif vercode and vercode not in vercodes[package]:
466 vercodes[package] += [vercode] if vercode else []
471 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
472 """Build a list of App instances for processing
474 On top of what read_pkg_args does, this returns the whole app
475 metadata, but limiting the builds list to the builds matching the
476 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
477 all App and Build instances are returned.
481 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
487 for appid, app in allapps.items():
488 if appid in vercodes:
491 if len(apps) != len(vercodes):
494 logging.critical(_("No such package: %s") % p)
495 raise FDroidException(_("Found invalid appids in arguments"))
497 raise FDroidException(_("No packages specified"))
500 for appid, app in apps.items():
504 app.builds = [b for b in app.builds if b.versionCode in vc]
505 if len(app.builds) != len(vercodes[appid]):
507 allvcs = [b.versionCode for b in app.builds]
508 for v in vercodes[appid]:
510 logging.critical(_("No such versionCode {versionCode} for app {appid}")
511 .format(versionCode=v, appid=appid))
514 raise FDroidException(_("Found invalid versionCodes for some apps"))
519 def get_extension(filename):
520 base, ext = os.path.splitext(filename)
523 return base, ext.lower()[1:]
526 def has_extension(filename, ext):
527 _ignored, f_ext = get_extension(filename)
531 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
534 def clean_description(description):
535 'Remove unneeded newlines and spaces from a block of description text'
537 # this is split up by paragraph to make removing the newlines easier
538 for paragraph in re.split(r'\n\n', description):
539 paragraph = re.sub('\r', '', paragraph)
540 paragraph = re.sub('\n', ' ', paragraph)
541 paragraph = re.sub(' {2,}', ' ', paragraph)
542 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
543 returnstring += paragraph + '\n\n'
544 return returnstring.rstrip('\n')
547 def publishednameinfo(filename):
548 filename = os.path.basename(filename)
549 m = publish_name_regex.match(filename)
551 result = (m.group(1), m.group(2))
552 except AttributeError:
553 raise FDroidException(_("Invalid name for published file: %s") % filename)
557 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
558 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
561 def apk_parse_release_filename(apkname):
562 """Parses the name of an APK file according the F-Droids APK naming
563 scheme and returns the tokens.
565 WARNING: Returned values don't necessarily represent the APKs actual
566 properties, the are just paresed from the file name.
568 :returns: A triplet containing (appid, versionCode, signer), where appid
569 should be the package name, versionCode should be the integer
570 represion of the APKs version and signer should be the first 7 hex
571 digists of the sha256 signing key fingerprint which was used to sign
574 m = apk_release_filename_with_sigfp.match(apkname)
576 return m.group('appid'), m.group('vercode'), m.group('sigfp')
577 m = apk_release_filename.match(apkname)
579 return m.group('appid'), m.group('vercode'), None
580 return None, None, None
583 def get_release_filename(app, build):
585 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
587 return "%s_%s.apk" % (app.id, build.versionCode)
590 def get_toolsversion_logname(app, build):
591 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
594 def getsrcname(app, build):
595 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
607 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
610 def get_build_dir(app):
611 '''get the dir that this app will be built in'''
613 if app.RepoType == 'srclib':
614 return os.path.join('build', 'srclib', app.Repo)
616 return os.path.join('build', app.id)
620 '''checkout code from VCS and return instance of vcs and the build dir'''
621 build_dir = get_build_dir(app)
623 # Set up vcs interface and make sure we have the latest code...
624 logging.debug("Getting {0} vcs interface for {1}"
625 .format(app.RepoType, app.Repo))
626 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
630 vcs = getvcs(app.RepoType, remote, build_dir)
632 return vcs, build_dir
635 def getvcs(vcstype, remote, local):
637 return vcs_git(remote, local)
638 if vcstype == 'git-svn':
639 return vcs_gitsvn(remote, local)
641 return vcs_hg(remote, local)
643 return vcs_bzr(remote, local)
644 if vcstype == 'srclib':
645 if local != os.path.join('build', 'srclib', remote):
646 raise VCSException("Error: srclib paths are hard-coded!")
647 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
649 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
650 raise VCSException("Invalid vcs type " + vcstype)
653 def getsrclibvcs(name):
654 if name not in fdroidserver.metadata.srclibs:
655 raise VCSException("Missing srclib " + name)
656 return fdroidserver.metadata.srclibs[name]['Repo Type']
661 def __init__(self, remote, local):
663 # svn, git-svn and bzr may require auth
665 if self.repotype() in ('git-svn', 'bzr'):
667 if self.repotype == 'git-svn':
668 raise VCSException("Authentication is not supported for git-svn")
669 self.username, remote = remote.split('@')
670 if ':' not in self.username:
671 raise VCSException(_("Password required with username"))
672 self.username, self.password = self.username.split(':')
676 self.clone_failed = False
677 self.refreshed = False
683 def clientversion(self):
684 versionstr = FDroidPopen(self.clientversioncmd()).output
685 return versionstr[0:versionstr.find('\n')]
687 def clientversioncmd(self):
690 def gotorevision(self, rev, refresh=True):
691 """Take the local repository to a clean version of the given
692 revision, which is specificed in the VCS's native
693 format. Beforehand, the repository can be dirty, or even
694 non-existent. If the repository does already exist locally, it
695 will be updated from the origin, but only once in the lifetime
696 of the vcs object. None is acceptable for 'rev' if you know
697 you are cloning a clean copy of the repo - otherwise it must
698 specify a valid revision.
701 if self.clone_failed:
702 raise VCSException(_("Downloading the repository already failed once, not trying again."))
704 # The .fdroidvcs-id file for a repo tells us what VCS type
705 # and remote that directory was created from, allowing us to drop it
706 # automatically if either of those things changes.
707 fdpath = os.path.join(self.local, '..',
708 '.fdroidvcs-' + os.path.basename(self.local))
709 fdpath = os.path.normpath(fdpath)
710 cdata = self.repotype() + ' ' + self.remote
713 if os.path.exists(self.local):
714 if os.path.exists(fdpath):
715 with open(fdpath, 'r') as f:
716 fsdata = f.read().strip()
721 logging.info("Repository details for %s changed - deleting" % (
725 logging.info("Repository details for %s missing - deleting" % (
728 shutil.rmtree(self.local)
732 self.refreshed = True
735 self.gotorevisionx(rev)
736 except FDroidException as e:
739 # If necessary, write the .fdroidvcs file.
740 if writeback and not self.clone_failed:
741 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
742 with open(fdpath, 'w+') as f:
748 def gotorevisionx(self, rev): # pylint: disable=unused-argument
749 """Derived classes need to implement this.
751 It's called once basic checking has been performed.
753 raise VCSException("This VCS type doesn't define gotorevisionx")
755 # Initialise and update submodules
756 def initsubmodules(self):
757 raise VCSException('Submodules not supported for this vcs type')
759 # Get a list of all known tags
761 if not self._gettags:
762 raise VCSException('gettags not supported for this vcs type')
764 for tag in self._gettags():
765 if re.match('[-A-Za-z0-9_. /]+$', tag):
769 def latesttags(self):
770 """Get a list of all the known tags, sorted from newest to oldest"""
771 raise VCSException('latesttags not supported for this vcs type')
774 """Get current commit reference (hash, revision, etc)"""
775 raise VCSException('getref not supported for this vcs type')
778 """Returns the srclib (name, path) used in setting up the current revision, or None."""
787 def clientversioncmd(self):
788 return ['git', '--version']
790 def git(self, args, envs=dict(), cwd=None, output=True):
791 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
793 While fetch/pull/clone respect the command line option flags,
794 it seems that submodule commands do not. They do seem to
795 follow whatever is in env vars, if the version of git is new
796 enough. So we just throw the kitchen sink at it to see what
799 Also, because of CVE-2017-1000117, block all SSH URLs.
802 # supported in git >= 2.3
804 '-c', 'core.sshCommand=false',
805 '-c', 'url.https://.insteadOf=ssh://',
807 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
808 git_config.append('-c')
809 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
810 git_config.append('-c')
811 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
812 git_config.append('-c')
813 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
815 'GIT_TERMINAL_PROMPT': '0',
816 'GIT_SSH': 'false', # for git < 2.3
818 return FDroidPopen(['git', ] + git_config + args,
819 envs=envs, cwd=cwd, output=output)
822 """If the local directory exists, but is somehow not a git repository,
823 git will traverse up the directory tree until it finds one
824 that is (i.e. fdroidserver) and then we'll proceed to destroy
825 it! This is called as a safety check.
829 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
830 result = p.output.rstrip()
831 if not result.endswith(self.local):
832 raise VCSException('Repository mismatch')
834 def gotorevisionx(self, rev):
835 if not os.path.exists(self.local):
837 p = self.git(['clone', self.remote, self.local])
838 if p.returncode != 0:
839 self.clone_failed = True
840 raise VCSException("Git clone failed", p.output)
844 # Discard any working tree changes
845 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
846 'git', 'reset', '--hard'], cwd=self.local, output=False)
847 if p.returncode != 0:
848 raise VCSException(_("Git reset failed"), p.output)
849 # Remove untracked files now, in case they're tracked in the target
850 # revision (it happens!)
851 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
852 'git', 'clean', '-dffx'], cwd=self.local, output=False)
853 if p.returncode != 0:
854 raise VCSException(_("Git clean failed"), p.output)
855 if not self.refreshed:
856 # Get latest commits and tags from remote
857 p = self.git(['fetch', 'origin'], cwd=self.local)
858 if p.returncode != 0:
859 raise VCSException(_("Git fetch failed"), p.output)
860 p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
861 if p.returncode != 0:
862 raise VCSException(_("Git fetch failed"), p.output)
863 # Recreate origin/HEAD as git clone would do it, in case it disappeared
864 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
865 if p.returncode != 0:
866 lines = p.output.splitlines()
867 if 'Multiple remote HEAD branches' not in lines[0]:
868 raise VCSException(_("Git remote set-head failed"), p.output)
869 branch = lines[1].split(' ')[-1]
870 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
871 if p2.returncode != 0:
872 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
873 self.refreshed = True
874 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
875 # a github repo. Most of the time this is the same as origin/master.
876 rev = rev or 'origin/HEAD'
877 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
880 # Get rid of any uncontrolled files left behind
881 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
882 if p.returncode != 0:
883 raise VCSException(_("Git clean failed"), p.output)
885 def initsubmodules(self):
887 submfile = os.path.join(self.local, '.gitmodules')
888 if not os.path.isfile(submfile):
889 raise NoSubmodulesException(_("No git submodules available"))
891 # fix submodules not accessible without an account and public key auth
892 with open(submfile, 'r') as f:
893 lines = f.readlines()
894 with open(submfile, 'w') as f:
896 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
897 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
900 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
901 if p.returncode != 0:
902 raise VCSException(_("Git submodule sync failed"), p.output)
903 p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
904 if p.returncode != 0:
905 raise VCSException(_("Git submodule update failed"), p.output)
909 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
910 return p.output.splitlines()
912 tag_format = re.compile(r'tag: ([^),]*)')
914 def latesttags(self):
916 p = FDroidPopen(['git', 'log', '--tags',
917 '--simplify-by-decoration', '--pretty=format:%d'],
918 cwd=self.local, output=False)
920 for line in p.output.splitlines():
921 for tag in self.tag_format.findall(line):
926 class vcs_gitsvn(vcs):
931 def clientversioncmd(self):
932 return ['git', 'svn', '--version']
935 """If the local directory exists, but is somehow not a git repository,
936 git will traverse up the directory tree until it finds one that
937 is (i.e. fdroidserver) and then we'll proceed to destory it!
938 This is called as a safety check.
941 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
942 result = p.output.rstrip()
943 if not result.endswith(self.local):
944 raise VCSException('Repository mismatch')
946 def git(self, args, envs=dict(), cwd=None, output=True):
947 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
949 # CVE-2017-1000117 block all SSH URLs (supported in git >= 2.3)
950 config = ['-c', 'core.sshCommand=false']
952 'GIT_TERMINAL_PROMPT': '0',
953 'GIT_SSH': 'false', # for git < 2.3
956 return FDroidPopen(['git', ] + config + args,
957 envs=envs, cwd=cwd, output=output)
959 def gotorevisionx(self, rev):
960 if not os.path.exists(self.local):
962 gitsvn_args = ['svn', 'clone']
963 if ';' in self.remote:
964 remote_split = self.remote.split(';')
965 for i in remote_split[1:]:
966 if i.startswith('trunk='):
967 gitsvn_args.extend(['-T', i[6:]])
968 elif i.startswith('tags='):
969 gitsvn_args.extend(['-t', i[5:]])
970 elif i.startswith('branches='):
971 gitsvn_args.extend(['-b', i[9:]])
972 gitsvn_args.extend([remote_split[0], self.local])
973 p = self.git(gitsvn_args, output=False)
974 if p.returncode != 0:
975 self.clone_failed = True
976 raise VCSException("Git svn clone failed", p.output)
978 gitsvn_args.extend([self.remote, self.local])
979 p = self.git(gitsvn_args, output=False)
980 if p.returncode != 0:
981 self.clone_failed = True
982 raise VCSException("Git svn clone failed", p.output)
986 # Discard any working tree changes
987 p = self.git(['reset', '--hard'], cwd=self.local, output=False)
988 if p.returncode != 0:
989 raise VCSException("Git reset failed", p.output)
990 # Remove untracked files now, in case they're tracked in the target
991 # revision (it happens!)
992 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
993 if p.returncode != 0:
994 raise VCSException("Git clean failed", p.output)
995 if not self.refreshed:
996 # Get new commits, branches and tags from repo
997 p = self.git(['svn', 'fetch'], cwd=self.local, output=False)
998 if p.returncode != 0:
999 raise VCSException("Git svn fetch failed")
1000 p = self.git(['svn', 'rebase'], cwd=self.local, output=False)
1001 if p.returncode != 0:
1002 raise VCSException("Git svn rebase failed", p.output)
1003 self.refreshed = True
1005 rev = rev or 'master'
1007 nospaces_rev = rev.replace(' ', '%20')
1008 # Try finding a svn tag
1009 for treeish in ['origin/', '']:
1010 p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
1011 if p.returncode == 0:
1013 if p.returncode != 0:
1014 # No tag found, normal svn rev translation
1015 # Translate svn rev into git format
1016 rev_split = rev.split('/')
1019 for treeish in ['origin/', '']:
1020 if len(rev_split) > 1:
1021 treeish += rev_split[0]
1022 svn_rev = rev_split[1]
1025 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1029 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1031 p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1032 git_rev = p.output.rstrip()
1034 if p.returncode == 0 and git_rev:
1037 if p.returncode != 0 or not git_rev:
1038 # Try a plain git checkout as a last resort
1039 p = self.git(['checkout', rev], cwd=self.local, output=False)
1040 if p.returncode != 0:
1041 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1043 # Check out the git rev equivalent to the svn rev
1044 p = self.git(['checkout', git_rev], cwd=self.local, output=False)
1045 if p.returncode != 0:
1046 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1048 # Get rid of any uncontrolled files left behind
1049 p = self.git(['clean', '-dffx'], cwd=self.local, output=False)
1050 if p.returncode != 0:
1051 raise VCSException(_("Git clean failed"), p.output)
1055 for treeish in ['origin/', '']:
1056 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1057 if os.path.isdir(d):
1058 return os.listdir(d)
1062 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1063 if p.returncode != 0:
1065 return p.output.strip()
1073 def clientversioncmd(self):
1074 return ['hg', '--version']
1076 def gotorevisionx(self, rev):
1077 if not os.path.exists(self.local):
1078 p = FDroidPopen(['hg', 'clone', '--ssh', 'false', self.remote, self.local], output=False)
1079 if p.returncode != 0:
1080 self.clone_failed = True
1081 raise VCSException("Hg clone failed", p.output)
1083 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1084 if p.returncode != 0:
1085 raise VCSException("Hg status failed", p.output)
1086 for line in p.output.splitlines():
1087 if not line.startswith('? '):
1088 raise VCSException("Unexpected output from hg status -uS: " + line)
1089 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1090 if not self.refreshed:
1091 p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False)
1092 if p.returncode != 0:
1093 raise VCSException("Hg pull failed", p.output)
1094 self.refreshed = True
1096 rev = rev or 'default'
1099 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1100 if p.returncode != 0:
1101 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1102 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1103 # Also delete untracked files, we have to enable purge extension for that:
1104 if "'purge' is provided by the following extension" in p.output:
1105 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1106 myfile.write("\n[extensions]\nhgext.purge=\n")
1107 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1108 if p.returncode != 0:
1109 raise VCSException("HG purge failed", p.output)
1110 elif p.returncode != 0:
1111 raise VCSException("HG purge failed", p.output)
1114 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1115 return p.output.splitlines()[1:]
1123 def clientversioncmd(self):
1124 return ['bzr', '--version']
1126 def bzr(self, args, envs=dict(), cwd=None, output=True):
1127 '''Prevent bzr from ever using SSH to avoid security vulns'''
1131 return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output)
1133 def gotorevisionx(self, rev):
1134 if not os.path.exists(self.local):
1135 p = self.bzr(['branch', self.remote, self.local], output=False)
1136 if p.returncode != 0:
1137 self.clone_failed = True
1138 raise VCSException("Bzr branch failed", p.output)
1140 p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1141 if p.returncode != 0:
1142 raise VCSException("Bzr revert failed", p.output)
1143 if not self.refreshed:
1144 p = self.bzr(['pull'], cwd=self.local, output=False)
1145 if p.returncode != 0:
1146 raise VCSException("Bzr update failed", p.output)
1147 self.refreshed = True
1149 revargs = list(['-r', rev] if rev else [])
1150 p = self.bzr(['revert'] + revargs, cwd=self.local, output=False)
1151 if p.returncode != 0:
1152 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1155 p = self.bzr(['tags'], cwd=self.local, output=False)
1156 return [tag.split(' ')[0].strip() for tag in
1157 p.output.splitlines()]
1160 def unescape_string(string):
1163 if string[0] == '"' and string[-1] == '"':
1166 return string.replace("\\'", "'")
1169 def retrieve_string(app_dir, string, xmlfiles=None):
1171 if not string.startswith('@string/'):
1172 return unescape_string(string)
1174 if xmlfiles is None:
1177 os.path.join(app_dir, 'res'),
1178 os.path.join(app_dir, 'src', 'main', 'res'),
1180 for root, dirs, files in os.walk(res_dir):
1181 if os.path.basename(root) == 'values':
1182 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1184 name = string[len('@string/'):]
1186 def element_content(element):
1187 if element.text is None:
1189 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1190 return s.decode('utf-8').strip()
1192 for path in xmlfiles:
1193 if not os.path.isfile(path):
1195 xml = parse_xml(path)
1196 element = xml.find('string[@name="' + name + '"]')
1197 if element is not None:
1198 content = element_content(element)
1199 return retrieve_string(app_dir, content, xmlfiles)
1204 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1205 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1208 def manifest_paths(app_dir, flavours):
1209 '''Return list of existing files that will be used to find the highest vercode'''
1211 possible_manifests = \
1212 [os.path.join(app_dir, 'AndroidManifest.xml'),
1213 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1214 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1215 os.path.join(app_dir, 'build.gradle')]
1217 for flavour in flavours:
1218 if flavour == 'yes':
1220 possible_manifests.append(
1221 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1223 return [path for path in possible_manifests if os.path.isfile(path)]
1226 def fetch_real_name(app_dir, flavours):
1227 '''Retrieve the package name. Returns the name, or None if not found.'''
1228 for path in manifest_paths(app_dir, flavours):
1229 if not has_extension(path, 'xml') or not os.path.isfile(path):
1231 logging.debug("fetch_real_name: Checking manifest at " + path)
1232 xml = parse_xml(path)
1233 app = xml.find('application')
1236 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1238 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1239 result = retrieve_string_singleline(app_dir, label)
1241 result = result.strip()
1246 def get_library_references(root_dir):
1248 proppath = os.path.join(root_dir, 'project.properties')
1249 if not os.path.isfile(proppath):
1251 with open(proppath, 'r', encoding='iso-8859-1') as f:
1253 if not line.startswith('android.library.reference.'):
1255 path = line.split('=')[1].strip()
1256 relpath = os.path.join(root_dir, path)
1257 if not os.path.isdir(relpath):
1259 logging.debug("Found subproject at %s" % path)
1260 libraries.append(path)
1264 def ant_subprojects(root_dir):
1265 subprojects = get_library_references(root_dir)
1266 for subpath in subprojects:
1267 subrelpath = os.path.join(root_dir, subpath)
1268 for p in get_library_references(subrelpath):
1269 relp = os.path.normpath(os.path.join(subpath, p))
1270 if relp not in subprojects:
1271 subprojects.insert(0, relp)
1275 def remove_debuggable_flags(root_dir):
1276 # Remove forced debuggable flags
1277 logging.debug("Removing debuggable flags from %s" % root_dir)
1278 for root, dirs, files in os.walk(root_dir):
1279 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1280 regsub_file(r'android:debuggable="[^"]*"',
1282 os.path.join(root, 'AndroidManifest.xml'))
1285 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1286 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1287 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1290 def app_matches_packagename(app, package):
1293 appid = app.UpdateCheckName or app.id
1294 if appid is None or appid == "Ignore":
1296 return appid == package
1299 def parse_androidmanifests(paths, app):
1301 Extract some information from the AndroidManifest.xml at the given path.
1302 Returns (version, vercode, package), any or all of which might be None.
1303 All values returned are strings.
1306 ignoreversions = app.UpdateCheckIgnore
1307 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1310 return (None, None, None)
1318 if not os.path.isfile(path):
1321 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1327 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1328 flavour = app.builds[-1].gradle[-1]
1330 if has_extension(path, 'gradle'):
1331 # first try to get version name and code from correct flavour
1332 with open(path, 'r') as f:
1333 buildfile = f.read()
1335 regex_string = r"" + flavour + ".*?}"
1336 search = re.compile(regex_string, re.DOTALL)
1337 result = search.search(buildfile)
1339 if result is not None:
1340 resultgroup = result.group()
1343 matches = psearch_g(resultgroup)
1345 s = matches.group(2)
1346 if app_matches_packagename(app, s):
1349 matches = vnsearch_g(resultgroup)
1351 version = matches.group(2)
1353 matches = vcsearch_g(resultgroup)
1355 vercode = matches.group(1)
1357 # fall back to parse file line by line
1358 with open(path, 'r') as f:
1360 if gradle_comment.match(line):
1362 # Grab first occurence of each to avoid running into
1363 # alternative flavours and builds.
1365 matches = psearch_g(line)
1367 s = matches.group(2)
1368 if app_matches_packagename(app, s):
1371 matches = vnsearch_g(line)
1373 version = matches.group(2)
1375 matches = vcsearch_g(line)
1377 vercode = matches.group(1)
1380 xml = parse_xml(path)
1381 if "package" in xml.attrib:
1382 s = xml.attrib["package"]
1383 if app_matches_packagename(app, s):
1385 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1386 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1387 base_dir = os.path.dirname(path)
1388 version = retrieve_string_singleline(base_dir, version)
1389 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1390 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1391 if string_is_integer(a):
1394 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1396 # Remember package name, may be defined separately from version+vercode
1398 package = max_package
1400 logging.debug("..got package={0}, version={1}, vercode={2}"
1401 .format(package, version, vercode))
1403 # Always grab the package name and version name in case they are not
1404 # together with the highest version code
1405 if max_package is None and package is not None:
1406 max_package = package
1407 if max_version is None and version is not None:
1408 max_version = version
1410 if vercode is not None \
1411 and (max_vercode is None or vercode > max_vercode):
1412 if not ignoresearch or not ignoresearch(version):
1413 if version is not None:
1414 max_version = version
1415 if vercode is not None:
1416 max_vercode = vercode
1417 if package is not None:
1418 max_package = package
1420 max_version = "Ignore"
1422 if max_version is None:
1423 max_version = "Unknown"
1425 if max_package and not is_valid_package_name(max_package):
1426 raise FDroidException(_("Invalid package name {0}").format(max_package))
1428 return (max_version, max_vercode, max_package)
1431 def is_valid_package_name(name):
1432 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1435 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1436 raw=False, prepare=True, preponly=False, refresh=True,
1438 """Get the specified source library.
1440 Returns the path to it. Normally this is the path to be used when
1441 referencing it, which may be a subdirectory of the actual project. If
1442 you want the base directory of the project, pass 'basepath=True'.
1451 name, ref = spec.split('@')
1453 number, name = name.split(':', 1)
1455 name, subdir = name.split('/', 1)
1457 if name not in fdroidserver.metadata.srclibs:
1458 raise VCSException('srclib ' + name + ' not found.')
1460 srclib = fdroidserver.metadata.srclibs[name]
1462 sdir = os.path.join(srclib_dir, name)
1465 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1466 vcs.srclib = (name, number, sdir)
1468 vcs.gotorevision(ref, refresh)
1475 libdir = os.path.join(sdir, subdir)
1476 elif srclib["Subdir"]:
1477 for subdir in srclib["Subdir"]:
1478 libdir_candidate = os.path.join(sdir, subdir)
1479 if os.path.exists(libdir_candidate):
1480 libdir = libdir_candidate
1486 remove_signing_keys(sdir)
1487 remove_debuggable_flags(sdir)
1491 if srclib["Prepare"]:
1492 cmd = replace_config_vars(srclib["Prepare"], build)
1494 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1495 if p.returncode != 0:
1496 raise BuildException("Error running prepare command for srclib %s"
1502 return (name, number, libdir)
1505 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1508 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1509 """ Prepare the source code for a particular build
1511 :param vcs: the appropriate vcs object for the application
1512 :param app: the application details from the metadata
1513 :param build: the build details from the metadata
1514 :param build_dir: the path to the build directory, usually 'build/app.id'
1515 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1516 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1518 Returns the (root, srclibpaths) where:
1519 :param root: is the root directory, which may be the same as 'build_dir' or may
1520 be a subdirectory of it.
1521 :param srclibpaths: is information on the srclibs being used
1524 # Optionally, the actual app source can be in a subdirectory
1526 root_dir = os.path.join(build_dir, build.subdir)
1528 root_dir = build_dir
1530 # Get a working copy of the right revision
1531 logging.info("Getting source for revision " + build.commit)
1532 vcs.gotorevision(build.commit, refresh)
1534 # Initialise submodules if required
1535 if build.submodules:
1536 logging.info(_("Initialising submodules"))
1537 vcs.initsubmodules()
1539 # Check that a subdir (if we're using one) exists. This has to happen
1540 # after the checkout, since it might not exist elsewhere
1541 if not os.path.exists(root_dir):
1542 raise BuildException('Missing subdir ' + root_dir)
1544 # Run an init command if one is required
1546 cmd = replace_config_vars(build.init, build)
1547 logging.info("Running 'init' commands in %s" % root_dir)
1549 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1550 if p.returncode != 0:
1551 raise BuildException("Error running init command for %s:%s" %
1552 (app.id, build.versionName), p.output)
1554 # Apply patches if any
1556 logging.info("Applying patches")
1557 for patch in build.patch:
1558 patch = patch.strip()
1559 logging.info("Applying " + patch)
1560 patch_path = os.path.join('metadata', app.id, patch)
1561 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1562 if p.returncode != 0:
1563 raise BuildException("Failed to apply patch %s" % patch_path)
1565 # Get required source libraries
1568 logging.info("Collecting source libraries")
1569 for lib in build.srclibs:
1570 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1571 refresh=refresh, build=build))
1573 for name, number, libpath in srclibpaths:
1574 place_srclib(root_dir, int(number) if number else None, libpath)
1576 basesrclib = vcs.getsrclib()
1577 # If one was used for the main source, add that too.
1579 srclibpaths.append(basesrclib)
1581 # Update the local.properties file
1582 localprops = [os.path.join(build_dir, 'local.properties')]
1584 parts = build.subdir.split(os.sep)
1587 cur = os.path.join(cur, d)
1588 localprops += [os.path.join(cur, 'local.properties')]
1589 for path in localprops:
1591 if os.path.isfile(path):
1592 logging.info("Updating local.properties file at %s" % path)
1593 with open(path, 'r', encoding='iso-8859-1') as f:
1597 logging.info("Creating local.properties file at %s" % path)
1598 # Fix old-fashioned 'sdk-location' by copying
1599 # from sdk.dir, if necessary
1601 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1602 re.S | re.M).group(1)
1603 props += "sdk-location=%s\n" % sdkloc
1605 props += "sdk.dir=%s\n" % config['sdk_path']
1606 props += "sdk-location=%s\n" % config['sdk_path']
1607 ndk_path = build.ndk_path()
1608 # if for any reason the path isn't valid or the directory
1609 # doesn't exist, some versions of Gradle will error with a
1610 # cryptic message (even if the NDK is not even necessary).
1611 # https://gitlab.com/fdroid/fdroidserver/issues/171
1612 if ndk_path and os.path.exists(ndk_path):
1614 props += "ndk.dir=%s\n" % ndk_path
1615 props += "ndk-location=%s\n" % ndk_path
1616 # Add java.encoding if necessary
1618 props += "java.encoding=%s\n" % build.encoding
1619 with open(path, 'w', encoding='iso-8859-1') as f:
1623 if build.build_method() == 'gradle':
1624 flavours = build.gradle
1627 n = build.target.split('-')[1]
1628 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1629 r'compileSdkVersion %s' % n,
1630 os.path.join(root_dir, 'build.gradle'))
1632 # Remove forced debuggable flags
1633 remove_debuggable_flags(root_dir)
1635 # Insert version code and number into the manifest if necessary
1636 if build.forceversion:
1637 logging.info("Changing the version name")
1638 for path in manifest_paths(root_dir, flavours):
1639 if not os.path.isfile(path):
1641 if has_extension(path, 'xml'):
1642 regsub_file(r'android:versionName="[^"]*"',
1643 r'android:versionName="%s"' % build.versionName,
1645 elif has_extension(path, 'gradle'):
1646 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1647 r"""\1versionName '%s'""" % build.versionName,
1650 if build.forcevercode:
1651 logging.info("Changing the version code")
1652 for path in manifest_paths(root_dir, flavours):
1653 if not os.path.isfile(path):
1655 if has_extension(path, 'xml'):
1656 regsub_file(r'android:versionCode="[^"]*"',
1657 r'android:versionCode="%s"' % build.versionCode,
1659 elif has_extension(path, 'gradle'):
1660 regsub_file(r'versionCode[ =]+[0-9]+',
1661 r'versionCode %s' % build.versionCode,
1664 # Delete unwanted files
1666 logging.info(_("Removing specified files"))
1667 for part in getpaths(build_dir, build.rm):
1668 dest = os.path.join(build_dir, part)
1669 logging.info("Removing {0}".format(part))
1670 if os.path.lexists(dest):
1671 # rmtree can only handle directories that are not symlinks, so catch anything else
1672 if not os.path.isdir(dest) or os.path.islink(dest):
1677 logging.info("...but it didn't exist")
1679 remove_signing_keys(build_dir)
1681 # Add required external libraries
1683 logging.info("Collecting prebuilt libraries")
1684 libsdir = os.path.join(root_dir, 'libs')
1685 if not os.path.exists(libsdir):
1687 for lib in build.extlibs:
1689 logging.info("...installing extlib {0}".format(lib))
1690 libf = os.path.basename(lib)
1691 libsrc = os.path.join(extlib_dir, lib)
1692 if not os.path.exists(libsrc):
1693 raise BuildException("Missing extlib file {0}".format(libsrc))
1694 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1696 # Run a pre-build command if one is required
1698 logging.info("Running 'prebuild' commands in %s" % root_dir)
1700 cmd = replace_config_vars(build.prebuild, build)
1702 # Substitute source library paths into prebuild commands
1703 for name, number, libpath in srclibpaths:
1704 libpath = os.path.relpath(libpath, root_dir)
1705 cmd = cmd.replace('$$' + name + '$$', libpath)
1707 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1708 if p.returncode != 0:
1709 raise BuildException("Error running prebuild command for %s:%s" %
1710 (app.id, build.versionName), p.output)
1712 # Generate (or update) the ant build file, build.xml...
1713 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1714 parms = ['android', 'update', 'lib-project']
1715 lparms = ['android', 'update', 'project']
1718 parms += ['-t', build.target]
1719 lparms += ['-t', build.target]
1720 if build.androidupdate:
1721 update_dirs = build.androidupdate
1723 update_dirs = ant_subprojects(root_dir) + ['.']
1725 for d in update_dirs:
1726 subdir = os.path.join(root_dir, d)
1728 logging.debug("Updating main project")
1729 cmd = parms + ['-p', d]
1731 logging.debug("Updating subproject %s" % d)
1732 cmd = lparms + ['-p', d]
1733 p = SdkToolsPopen(cmd, cwd=root_dir)
1734 # Check to see whether an error was returned without a proper exit
1735 # code (this is the case for the 'no target set or target invalid'
1737 if p.returncode != 0 or p.output.startswith("Error: "):
1738 raise BuildException("Failed to update project at %s" % d, p.output)
1739 # Clean update dirs via ant
1741 logging.info("Cleaning subproject %s" % d)
1742 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1744 return (root_dir, srclibpaths)
1747 def getpaths_map(build_dir, globpaths):
1748 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1752 full_path = os.path.join(build_dir, p)
1753 full_path = os.path.normpath(full_path)
1754 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1756 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1760 def getpaths(build_dir, globpaths):
1761 """Extend via globbing the paths from a field and return them as a set"""
1762 paths_map = getpaths_map(build_dir, globpaths)
1764 for k, v in paths_map.items():
1771 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1774 def check_system_clock(dt_obj, path):
1775 """Check if system clock is updated based on provided date
1777 If an APK has files newer than the system time, suggest updating
1778 the system clock. This is useful for offline systems, used for
1779 signing, which do not have another source of clock sync info. It
1780 has to be more than 24 hours newer because ZIP/APK files do not
1784 checkdt = dt_obj - timedelta(1)
1785 if datetime.today() < checkdt:
1786 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1787 + '\n' + _('Set clock to that time using:') + '\n'
1788 + 'sudo date -s "' + str(dt_obj) + '"')
1792 """permanent store of existing APKs with the date they were added
1794 This is currently the only way to permanently store the "updated"
1799 '''Load filename/date info about previously seen APKs
1801 Since the appid and date strings both will never have spaces,
1802 this is parsed as a list from the end to allow the filename to
1803 have any combo of spaces.
1806 self.path = os.path.join('stats', 'known_apks.txt')
1808 if os.path.isfile(self.path):
1809 with open(self.path, 'r', encoding='utf8') as f:
1811 t = line.rstrip().split(' ')
1813 self.apks[t[0]] = (t[1], None)
1816 date = datetime.strptime(t[-1], '%Y-%m-%d')
1817 filename = line[0:line.rfind(appid) - 1]
1818 self.apks[filename] = (appid, date)
1819 check_system_clock(date, self.path)
1820 self.changed = False
1822 def writeifchanged(self):
1823 if not self.changed:
1826 if not os.path.exists('stats'):
1830 for apk, app in self.apks.items():
1832 line = apk + ' ' + appid
1834 line += ' ' + added.strftime('%Y-%m-%d')
1837 with open(self.path, 'w', encoding='utf8') as f:
1838 for line in sorted(lst, key=natural_key):
1839 f.write(line + '\n')
1841 def recordapk(self, apkName, app, default_date=None):
1843 Record an apk (if it's new, otherwise does nothing)
1844 Returns the date it was added as a datetime instance
1846 if apkName not in self.apks:
1847 if default_date is None:
1848 default_date = datetime.utcnow()
1849 self.apks[apkName] = (app, default_date)
1851 _ignored, added = self.apks[apkName]
1854 def getapp(self, apkname):
1855 """Look up information - given the 'apkname', returns (app id, date added/None).
1857 Or returns None for an unknown apk.
1859 if apkname in self.apks:
1860 return self.apks[apkname]
1863 def getlatest(self, num):
1864 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1866 for apk, app in self.apks.items():
1870 if apps[appid] > added:
1874 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1875 lst = [app for app, _ignored in sortedapps]
1880 def get_file_extension(filename):
1881 """get the normalized file extension, can be blank string but never None"""
1882 if isinstance(filename, bytes):
1883 filename = filename.decode('utf-8')
1884 return os.path.splitext(filename)[1].lower()[1:]
1887 def get_apk_debuggable_aapt(apkfile):
1888 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1890 if p.returncode != 0:
1891 raise FDroidException(_("Failed to get APK manifest information"))
1892 for line in p.output.splitlines():
1893 if 'android:debuggable' in line and not line.endswith('0x0'):
1898 def get_apk_debuggable_androguard(apkfile):
1900 from androguard.core.bytecodes.apk import APK
1902 raise FDroidException("androguard library is not installed and aapt not present")
1904 apkobject = APK(apkfile)
1905 if apkobject.is_valid_APK():
1906 debuggable = apkobject.get_element("application", "debuggable")
1907 if debuggable is not None:
1908 return bool(strtobool(debuggable))
1912 def isApkAndDebuggable(apkfile):
1913 """Returns True if the given file is an APK and is debuggable
1915 :param apkfile: full path to the apk to check"""
1917 if get_file_extension(apkfile) != 'apk':
1920 if SdkToolsPopen(['aapt', 'version'], output=False):
1921 return get_apk_debuggable_aapt(apkfile)
1923 return get_apk_debuggable_androguard(apkfile)
1926 def get_apk_id_aapt(apkfile):
1927 """Extrat identification information from APK using aapt.
1929 :param apkfile: path to an APK file.
1930 :returns: triplet (appid, version code, version name)
1932 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1933 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1934 for line in p.output.splitlines():
1937 return m.group('appid'), m.group('vercode'), m.group('vername')
1938 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1939 .format(apkfilename=apkfile))
1944 self.returncode = None
1948 def SdkToolsPopen(commands, cwd=None, output=True):
1950 if cmd not in config:
1951 config[cmd] = find_sdk_tools_cmd(commands[0])
1952 abscmd = config[cmd]
1954 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1956 test_aapt_version(config['aapt'])
1957 return FDroidPopen([abscmd] + commands[1:],
1958 cwd=cwd, output=output)
1961 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1963 Run a command and capture the possibly huge output as bytes.
1965 :param commands: command and argument list like in subprocess.Popen
1966 :param cwd: optionally specifies a working directory
1967 :param envs: a optional dictionary of environment variables and their values
1968 :returns: A PopenResult.
1973 set_FDroidPopen_env()
1975 process_env = env.copy()
1976 if envs is not None and len(envs) > 0:
1977 process_env.update(envs)
1980 cwd = os.path.normpath(cwd)
1981 logging.debug("Directory: %s" % cwd)
1982 logging.debug("> %s" % ' '.join(commands))
1984 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1985 result = PopenResult()
1988 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1989 stdout=subprocess.PIPE, stderr=stderr_param)
1990 except OSError as e:
1991 raise BuildException("OSError while trying to execute " +
1992 ' '.join(commands) + ': ' + str(e))
1994 # TODO are these AsynchronousFileReader threads always exiting?
1995 if not stderr_to_stdout and options.verbose:
1996 stderr_queue = Queue()
1997 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1999 while not stderr_reader.eof():
2000 while not stderr_queue.empty():
2001 line = stderr_queue.get()
2002 sys.stderr.buffer.write(line)
2007 stdout_queue = Queue()
2008 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
2011 # Check the queue for output (until there is no more to get)
2012 while not stdout_reader.eof():
2013 while not stdout_queue.empty():
2014 line = stdout_queue.get()
2015 if output and options.verbose:
2016 # Output directly to console
2017 sys.stderr.buffer.write(line)
2023 result.returncode = p.wait()
2024 result.output = buf.getvalue()
2026 # make sure all filestreams of the subprocess are closed
2027 for streamvar in ['stdin', 'stdout', 'stderr']:
2028 if hasattr(p, streamvar):
2029 stream = getattr(p, streamvar)
2035 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2037 Run a command and capture the possibly huge output as a str.
2039 :param commands: command and argument list like in subprocess.Popen
2040 :param cwd: optionally specifies a working directory
2041 :param envs: a optional dictionary of environment variables and their values
2042 :returns: A PopenResult.
2044 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2045 result.output = result.output.decode('utf-8', 'ignore')
2049 gradle_comment = re.compile(r'[ ]*//')
2050 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2051 gradle_line_matches = [
2052 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2053 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2054 re.compile(r'.*\.readLine\(.*'),
2058 def remove_signing_keys(build_dir):
2059 for root, dirs, files in os.walk(build_dir):
2060 if 'build.gradle' in files:
2061 path = os.path.join(root, 'build.gradle')
2063 with open(path, "r", encoding='utf8') as o:
2064 lines = o.readlines()
2070 with open(path, "w", encoding='utf8') as o:
2071 while i < len(lines):
2074 while line.endswith('\\\n'):
2075 line = line.rstrip('\\\n') + lines[i]
2078 if gradle_comment.match(line):
2083 opened += line.count('{')
2084 opened -= line.count('}')
2087 if gradle_signing_configs.match(line):
2092 if any(s.match(line) for s in gradle_line_matches):
2100 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2103 'project.properties',
2105 'default.properties',
2106 'ant.properties', ]:
2107 if propfile in files:
2108 path = os.path.join(root, propfile)
2110 with open(path, "r", encoding='iso-8859-1') as o:
2111 lines = o.readlines()
2115 with open(path, "w", encoding='iso-8859-1') as o:
2117 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2124 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2127 def set_FDroidPopen_env(build=None):
2129 set up the environment variables for the build environment
2131 There is only a weak standard, the variables used by gradle, so also set
2132 up the most commonly used environment variables for SDK and NDK. Also, if
2133 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2135 global env, orig_path
2139 orig_path = env['PATH']
2140 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2141 env[n] = config['sdk_path']
2142 for k, v in config['java_paths'].items():
2143 env['JAVA%s_HOME' % k] = v
2145 missinglocale = True
2146 for k, v in env.items():
2147 if k == 'LANG' and v != 'C':
2148 missinglocale = False
2150 missinglocale = False
2152 env['LANG'] = 'en_US.UTF-8'
2154 if build is not None:
2155 path = build.ndk_path()
2156 paths = orig_path.split(os.pathsep)
2157 if path not in paths:
2158 paths = [path] + paths
2159 env['PATH'] = os.pathsep.join(paths)
2160 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2161 env[n] = build.ndk_path()
2164 def replace_build_vars(cmd, build):
2165 cmd = cmd.replace('$$COMMIT$$', build.commit)
2166 cmd = cmd.replace('$$VERSION$$', build.versionName)
2167 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2171 def replace_config_vars(cmd, build):
2172 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2173 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2174 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2175 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2176 if build is not None:
2177 cmd = replace_build_vars(cmd, build)
2181 def place_srclib(root_dir, number, libpath):
2184 relpath = os.path.relpath(libpath, root_dir)
2185 proppath = os.path.join(root_dir, 'project.properties')
2188 if os.path.isfile(proppath):
2189 with open(proppath, "r", encoding='iso-8859-1') as o:
2190 lines = o.readlines()
2192 with open(proppath, "w", encoding='iso-8859-1') as o:
2195 if line.startswith('android.library.reference.%d=' % number):
2196 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2201 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2204 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2207 def signer_fingerprint_short(sig):
2208 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2210 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2211 for a given pkcs7 signature.
2213 :param sig: Contents of an APK signing certificate.
2214 :returns: shortened signing-key fingerprint.
2216 return signer_fingerprint(sig)[:7]
2219 def signer_fingerprint(sig):
2220 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2222 Extracts hexadecimal sha256 signing-key fingerprint string
2223 for a given pkcs7 signature.
2225 :param: Contents of an APK signature.
2226 :returns: shortened signature fingerprint.
2228 cert_encoded = get_certificate(sig)
2229 return hashlib.sha256(cert_encoded).hexdigest()
2232 def apk_signer_fingerprint(apk_path):
2233 """Obtain sha256 signing-key fingerprint for APK.
2235 Extracts hexadecimal sha256 signing-key fingerprint string
2238 :param apkpath: path to APK
2239 :returns: signature fingerprint
2242 with zipfile.ZipFile(apk_path, 'r') as apk:
2243 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2246 logging.error("Found no signing certificates on %s" % apk_path)
2249 logging.error("Found multiple signing certificates on %s" % apk_path)
2252 cert = apk.read(certs[0])
2253 return signer_fingerprint(cert)
2256 def apk_signer_fingerprint_short(apk_path):
2257 """Obtain shortened sha256 signing-key fingerprint for APK.
2259 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2260 for a given pkcs7 APK.
2262 :param apk_path: path to APK
2263 :returns: shortened signing-key fingerprint
2265 return apk_signer_fingerprint(apk_path)[:7]
2268 def metadata_get_sigdir(appid, vercode=None):
2269 """Get signature directory for app"""
2271 return os.path.join('metadata', appid, 'signatures', vercode)
2273 return os.path.join('metadata', appid, 'signatures')
2276 def metadata_find_developer_signature(appid, vercode=None):
2277 """Tires to find the developer signature for given appid.
2279 This picks the first signature file found in metadata an returns its
2282 :returns: sha256 signing key fingerprint of the developer signing key.
2283 None in case no signature can not be found."""
2285 # fetch list of dirs for all versions of signatures
2288 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2290 appsigdir = metadata_get_sigdir(appid)
2291 if os.path.isdir(appsigdir):
2292 numre = re.compile('[0-9]+')
2293 for ver in os.listdir(appsigdir):
2294 if numre.match(ver):
2295 appversigdir = os.path.join(appsigdir, ver)
2296 appversigdirs.append(appversigdir)
2298 for sigdir in appversigdirs:
2299 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2300 glob.glob(os.path.join(sigdir, '*.EC')) + \
2301 glob.glob(os.path.join(sigdir, '*.RSA'))
2303 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2305 with open(sig, 'rb') as f:
2306 return signer_fingerprint(f.read())
2310 def metadata_find_signing_files(appid, vercode):
2311 """Gets a list of singed manifests and signatures.
2313 :param appid: app id string
2314 :param vercode: app version code
2315 :returns: a list of triplets for each signing key with following paths:
2316 (signature_file, singed_file, manifest_file)
2319 sigdir = metadata_get_sigdir(appid, vercode)
2320 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2321 glob.glob(os.path.join(sigdir, '*.EC')) + \
2322 glob.glob(os.path.join(sigdir, '*.RSA'))
2323 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2325 sf = extre.sub('.SF', sig)
2326 if os.path.isfile(sf):
2327 mf = os.path.join(sigdir, 'MANIFEST.MF')
2328 if os.path.isfile(mf):
2329 ret.append((sig, sf, mf))
2333 def metadata_find_developer_signing_files(appid, vercode):
2334 """Get developer signature files for specified app from metadata.
2336 :returns: A triplet of paths for signing files from metadata:
2337 (signature_file, singed_file, manifest_file)
2339 allsigningfiles = metadata_find_signing_files(appid, vercode)
2340 if allsigningfiles and len(allsigningfiles) == 1:
2341 return allsigningfiles[0]
2346 def apk_strip_signatures(signed_apk, strip_manifest=False):
2347 """Removes signatures from APK.
2349 :param signed_apk: path to apk file.
2350 :param strip_manifest: when set to True also the manifest file will
2351 be removed from the APK.
2353 with tempfile.TemporaryDirectory() as tmpdir:
2354 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2355 os.rename(signed_apk, tmp_apk)
2356 with ZipFile(tmp_apk, 'r') as in_apk:
2357 with ZipFile(signed_apk, 'w') as out_apk:
2358 for info in in_apk.infolist():
2359 if not apk_sigfile.match(info.filename):
2361 if info.filename != 'META-INF/MANIFEST.MF':
2362 buf = in_apk.read(info.filename)
2363 out_apk.writestr(info, buf)
2365 buf = in_apk.read(info.filename)
2366 out_apk.writestr(info, buf)
2369 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2370 """Implats a signature from metadata into an APK.
2372 Note: this changes there supplied APK in place. So copy it if you
2373 need the original to be preserved.
2375 :param apkpath: location of the apk
2377 # get list of available signature files in metadata
2378 with tempfile.TemporaryDirectory() as tmpdir:
2379 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2380 with ZipFile(apkpath, 'r') as in_apk:
2381 with ZipFile(apkwithnewsig, 'w') as out_apk:
2382 for sig_file in [signaturefile, signedfile, manifest]:
2383 with open(sig_file, 'rb') as fp:
2385 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2386 info.compress_type = zipfile.ZIP_DEFLATED
2387 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2388 out_apk.writestr(info, buf)
2389 for info in in_apk.infolist():
2390 if not apk_sigfile.match(info.filename):
2391 if info.filename != 'META-INF/MANIFEST.MF':
2392 buf = in_apk.read(info.filename)
2393 out_apk.writestr(info, buf)
2395 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2396 if p.returncode != 0:
2397 raise BuildException("Failed to align application")
2400 def apk_extract_signatures(apkpath, outdir, manifest=True):
2401 """Extracts a signature files from APK and puts them into target directory.
2403 :param apkpath: location of the apk
2404 :param outdir: folder where the extracted signature files will be stored
2405 :param manifest: (optionally) disable extracting manifest file
2407 with ZipFile(apkpath, 'r') as in_apk:
2408 for f in in_apk.infolist():
2409 if apk_sigfile.match(f.filename) or \
2410 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2411 newpath = os.path.join(outdir, os.path.basename(f.filename))
2412 with open(newpath, 'wb') as out_file:
2413 out_file.write(in_apk.read(f.filename))
2416 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2417 """Verify that two apks are the same
2419 One of the inputs is signed, the other is unsigned. The signature metadata
2420 is transferred from the signed to the unsigned apk, and then jarsigner is
2421 used to verify that the signature from the signed apk is also varlid for
2422 the unsigned one. If the APK given as unsigned actually does have a
2423 signature, it will be stripped out and ignored.
2425 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2426 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2427 into AndroidManifest.xml, but that makes the build not reproducible. So
2428 instead they are included as separate files in the APK's META-INF/ folder.
2429 If those files exist in the signed APK, they will be part of the signature
2430 and need to also be included in the unsigned APK for it to validate.
2432 :param signed_apk: Path to a signed apk file
2433 :param unsigned_apk: Path to an unsigned apk file expected to match it
2434 :param tmp_dir: Path to directory for temporary files
2435 :returns: None if the verification is successful, otherwise a string
2436 describing what went wrong.
2439 if not os.path.isfile(signed_apk):
2440 return 'can not verify: file does not exists: {}'.format(signed_apk)
2442 if not os.path.isfile(unsigned_apk):
2443 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2445 with ZipFile(signed_apk, 'r') as signed:
2446 meta_inf_files = ['META-INF/MANIFEST.MF']
2447 for f in signed.namelist():
2448 if apk_sigfile.match(f) \
2449 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2450 meta_inf_files.append(f)
2451 if len(meta_inf_files) < 3:
2452 return "Signature files missing from {0}".format(signed_apk)
2454 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2455 with ZipFile(unsigned_apk, 'r') as unsigned:
2456 # only read the signature from the signed APK, everything else from unsigned
2457 with ZipFile(tmp_apk, 'w') as tmp:
2458 for filename in meta_inf_files:
2459 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2460 for info in unsigned.infolist():
2461 if info.filename in meta_inf_files:
2462 logging.warning('Ignoring %s from %s',
2463 info.filename, unsigned_apk)
2465 if info.filename in tmp.namelist():
2466 return "duplicate filename found: " + info.filename
2467 tmp.writestr(info, unsigned.read(info.filename))
2469 verified = verify_apk_signature(tmp_apk)
2472 logging.info("...NOT verified - {0}".format(tmp_apk))
2473 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2474 os.path.dirname(unsigned_apk))
2476 logging.info("...successfully verified")
2480 def verify_jar_signature(jar):
2481 """Verifies the signature of a given JAR file.
2483 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2484 this has to turn on -strict then check for result 4, since this
2485 does not expect the signature to be from a CA-signed certificate.
2487 :raises: VerificationException() if the JAR's signature could not be verified
2491 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2492 raise VerificationException(_("The repository's index could not be verified."))
2495 def verify_apk_signature(apk, min_sdk_version=None):
2496 """verify the signature on an APK
2498 Try to use apksigner whenever possible since jarsigner is very
2499 shitty: unsigned APKs pass as "verified"! Warning, this does
2500 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2502 :returns: boolean whether the APK was verified
2504 if set_command_in_config('apksigner'):
2505 args = [config['apksigner'], 'verify']
2507 args += ['--min-sdk-version=' + min_sdk_version]
2508 return subprocess.call(args + [apk]) == 0
2510 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2512 verify_jar_signature(apk)
2519 def verify_old_apk_signature(apk):
2520 """verify the signature on an archived APK, supporting deprecated algorithms
2522 F-Droid aims to keep every single binary that it ever published. Therefore,
2523 it needs to be able to verify APK signatures that include deprecated/removed
2524 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2526 jarsigner passes unsigned APKs as "verified"! So this has to turn
2527 on -strict then check for result 4.
2529 :returns: boolean whether the APK was verified
2532 _java_security = os.path.join(os.getcwd(), '.java.security')
2533 with open(_java_security, 'w') as fp:
2534 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2536 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2537 '-strict', '-verify', apk]) == 4
2540 apk_badchars = re.compile('''[/ :;'"]''')
2543 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2546 Returns None if the apk content is the same (apart from the signing key),
2547 otherwise a string describing what's different, or what went wrong when
2548 trying to do the comparison.
2554 absapk1 = os.path.abspath(apk1)
2555 absapk2 = os.path.abspath(apk2)
2557 if set_command_in_config('diffoscope'):
2558 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2559 htmlfile = logfilename + '.diffoscope.html'
2560 textfile = logfilename + '.diffoscope.txt'
2561 if subprocess.call([config['diffoscope'],
2562 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2563 '--html', htmlfile, '--text', textfile,
2564 absapk1, absapk2]) != 0:
2565 return("Failed to unpack " + apk1)
2567 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2568 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2569 for d in [apk1dir, apk2dir]:
2570 if os.path.exists(d):
2573 os.mkdir(os.path.join(d, 'jar-xf'))
2575 if subprocess.call(['jar', 'xf',
2576 os.path.abspath(apk1)],
2577 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2578 return("Failed to unpack " + apk1)
2579 if subprocess.call(['jar', 'xf',
2580 os.path.abspath(apk2)],
2581 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2582 return("Failed to unpack " + apk2)
2584 if set_command_in_config('apktool'):
2585 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2587 return("Failed to unpack " + apk1)
2588 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2590 return("Failed to unpack " + apk2)
2592 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2593 lines = p.output.splitlines()
2594 if len(lines) != 1 or 'META-INF' not in lines[0]:
2595 if set_command_in_config('meld'):
2596 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2597 return("Unexpected diff output - " + p.output)
2599 # since everything verifies, delete the comparison to keep cruft down
2600 shutil.rmtree(apk1dir)
2601 shutil.rmtree(apk2dir)
2603 # If we get here, it seems like they're the same!
2607 def set_command_in_config(command):
2608 '''Try to find specified command in the path, if it hasn't been
2609 manually set in config.py. If found, it is added to the config
2610 dict. The return value says whether the command is available.
2613 if command in config:
2616 tmp = find_command(command)
2618 config[command] = tmp
2623 def find_command(command):
2624 '''find the full path of a command, or None if it can't be found in the PATH'''
2627 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2629 fpath, fname = os.path.split(command)
2634 for path in os.environ["PATH"].split(os.pathsep):
2635 path = path.strip('"')
2636 exe_file = os.path.join(path, command)
2637 if is_exe(exe_file):
2644 '''generate a random password for when generating keys'''
2645 h = hashlib.sha256()
2646 h.update(os.urandom(16)) # salt
2647 h.update(socket.getfqdn().encode('utf-8'))
2648 passwd = base64.b64encode(h.digest()).strip()
2649 return passwd.decode('utf-8')
2652 def genkeystore(localconfig):
2654 Generate a new key with password provided in :param localconfig and add it to new keystore
2655 :return: hexed public key, public key fingerprint
2657 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2658 keystoredir = os.path.dirname(localconfig['keystore'])
2659 if keystoredir is None or keystoredir == '':
2660 keystoredir = os.path.join(os.getcwd(), keystoredir)
2661 if not os.path.exists(keystoredir):
2662 os.makedirs(keystoredir, mode=0o700)
2665 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2666 'FDROID_KEY_PASS': localconfig['keypass'],
2668 p = FDroidPopen([config['keytool'], '-genkey',
2669 '-keystore', localconfig['keystore'],
2670 '-alias', localconfig['repo_keyalias'],
2671 '-keyalg', 'RSA', '-keysize', '4096',
2672 '-sigalg', 'SHA256withRSA',
2673 '-validity', '10000',
2674 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2675 '-keypass:env', 'FDROID_KEY_PASS',
2676 '-dname', localconfig['keydname']], envs=env_vars)
2677 if p.returncode != 0:
2678 raise BuildException("Failed to generate key", p.output)
2679 os.chmod(localconfig['keystore'], 0o0600)
2680 if not options.quiet:
2681 # now show the lovely key that was just generated
2682 p = FDroidPopen([config['keytool'], '-list', '-v',
2683 '-keystore', localconfig['keystore'],
2684 '-alias', localconfig['repo_keyalias'],
2685 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2686 logging.info(p.output.strip() + '\n\n')
2687 # get the public key
2688 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2689 '-keystore', localconfig['keystore'],
2690 '-alias', localconfig['repo_keyalias'],
2691 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2692 + config['smartcardoptions'],
2693 envs=env_vars, output=False, stderr_to_stdout=False)
2694 if p.returncode != 0 or len(p.output) < 20:
2695 raise BuildException("Failed to get public key", p.output)
2697 fingerprint = get_cert_fingerprint(pubkey)
2698 return hexlify(pubkey), fingerprint
2701 def get_cert_fingerprint(pubkey):
2703 Generate a certificate fingerprint the same way keytool does it
2704 (but with slightly different formatting)
2706 digest = hashlib.sha256(pubkey).digest()
2707 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2708 return " ".join(ret)
2711 def get_certificate(certificate_file):
2713 Extracts a certificate from the given file.
2714 :param certificate_file: file bytes (as string) representing the certificate
2715 :return: A binary representation of the certificate's public key, or None in case of error
2717 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2718 if content.getComponentByName('contentType') != rfc2315.signedData:
2720 content = decoder.decode(content.getComponentByName('content'),
2721 asn1Spec=rfc2315.SignedData())[0]
2723 certificates = content.getComponentByName('certificates')
2724 cert = certificates[0].getComponentByName('certificate')
2726 logging.error("Certificates not found.")
2728 return encoder.encode(cert)
2731 def load_stats_fdroid_signing_key_fingerprints():
2732 """Load list of signing-key fingerprints stored by fdroid publish from file.
2734 :returns: list of dictionanryies containing the singing-key fingerprints.
2736 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2737 if not os.path.isfile(jar_file):
2739 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2740 p = FDroidPopen(cmd, output=False)
2741 if p.returncode != 4:
2742 raise FDroidException("Signature validation of '{}' failed! "
2743 "Please run publish again to rebuild this file.".format(jar_file))
2745 jar_sigkey = apk_signer_fingerprint(jar_file)
2746 repo_key_sig = config.get('repo_key_sha256')
2748 if jar_sigkey != repo_key_sig:
2749 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2751 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2752 config['repo_key_sha256'] = jar_sigkey
2753 write_to_config(config, 'repo_key_sha256')
2755 with zipfile.ZipFile(jar_file, 'r') as f:
2756 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2759 def write_to_config(thisconfig, key, value=None, config_file=None):
2760 '''write a key/value to the local config.py
2762 NOTE: only supports writing string variables.
2764 :param thisconfig: config dictionary
2765 :param key: variable name in config.py to be overwritten/added
2766 :param value: optional value to be written, instead of fetched
2767 from 'thisconfig' dictionary.
2770 origkey = key + '_orig'
2771 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2772 cfg = config_file if config_file else 'config.py'
2774 # load config file, create one if it doesn't exist
2775 if not os.path.exists(cfg):
2776 open(cfg, 'a').close()
2777 logging.info("Creating empty " + cfg)
2778 with open(cfg, 'r', encoding="utf-8") as f:
2779 lines = f.readlines()
2781 # make sure the file ends with a carraige return
2783 if not lines[-1].endswith('\n'):
2786 # regex for finding and replacing python string variable
2787 # definitions/initializations
2788 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2789 repl = key + ' = "' + value + '"'
2790 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2791 repl2 = key + " = '" + value + "'"
2793 # If we replaced this line once, we make sure won't be a
2794 # second instance of this line for this key in the document.
2797 with open(cfg, 'w', encoding="utf-8") as f:
2799 if pattern.match(line) or pattern2.match(line):
2801 line = pattern.sub(repl, line)
2802 line = pattern2.sub(repl2, line)
2813 def parse_xml(path):
2814 return XMLElementTree.parse(path).getroot()
2817 def string_is_integer(string):
2825 def local_rsync(options, fromdir, todir):
2826 '''Rsync method for local to local copying of things
2828 This is an rsync wrapper with all the settings for safe use within
2829 the various fdroidserver use cases. This uses stricter rsync
2830 checking on all files since people using offline mode are already
2831 prioritizing security above ease and speed.
2834 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2835 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2836 if not options.no_checksum:
2837 rsyncargs.append('--checksum')
2839 rsyncargs += ['--verbose']
2841 rsyncargs += ['--quiet']
2842 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2843 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2844 raise FDroidException()
2847 def get_per_app_repos():
2848 '''per-app repos are dirs named with the packageName of a single app'''
2850 # Android packageNames are Java packages, they may contain uppercase or
2851 # lowercase letters ('A' through 'Z'), numbers, and underscores
2852 # ('_'). However, individual package name parts may only start with
2853 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2854 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2857 for root, dirs, files in os.walk(os.getcwd()):
2859 print('checking', root, 'for', d)
2860 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2861 # standard parts of an fdroid repo, so never packageNames
2864 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2870 def is_repo_file(filename):
2871 '''Whether the file in a repo is a build product to be delivered to users'''
2872 if isinstance(filename, str):
2873 filename = filename.encode('utf-8', errors="surrogateescape")
2874 return os.path.isfile(filename) \
2875 and not filename.endswith(b'.asc') \
2876 and not filename.endswith(b'.sig') \
2877 and os.path.basename(filename) not in [
2879 b'index_unsigned.jar',
2888 def get_examples_dir():
2889 '''Return the dir where the fdroidserver example files are available'''
2891 tmp = os.path.dirname(sys.argv[0])
2892 if os.path.basename(tmp) == 'bin':
2893 egg_links = glob.glob(os.path.join(tmp, '..',
2894 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2896 # installed from local git repo
2897 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2900 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2901 if not os.path.exists(examplesdir): # use UNIX layout
2902 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2904 # we're running straight out of the git repo
2905 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2906 examplesdir = prefix + '/examples'