3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, BuildException, VerificationException
57 from .asynchronousfilereader import AsynchronousFileReader
60 # A signature block file with a .DSA, .RSA, or .EC extension
61 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
62 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
63 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
65 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
74 'sdk_path': "$ANDROID_HOME",
79 'r12b': "$ANDROID_NDK",
85 'build_tools': "25.0.2",
86 'force_build_tools': False,
91 'accepted_formats': ['txt', 'yml'],
92 'sync_from_local_copy_dir': False,
93 'allow_disabled_algorithms': False,
94 'per_app_repos': False,
95 'make_current_version_link': True,
96 'current_version_name_source': 'Name',
97 'update_stats': False,
101 'stats_to_carbon': False,
103 'build_server_always': False,
104 'keystore': 'keystore.jks',
105 'smartcardoptions': [],
115 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
116 'repo_name': "My First FDroid Repo Demo",
117 'repo_icon': "fdroid-icon.png",
118 'repo_description': '''
119 This is a repository of apps to be used with FDroid. Applications in this
120 repository are either official binaries built by the original application
121 developers, or are binaries built from source by the admin of f-droid.org
122 using the tools on https://gitlab.com/u/fdroid.
128 def setup_global_opts(parser):
129 parser.add_argument("-v", "--verbose", action="store_true", default=False,
130 help=_("Spew out even more information than normal"))
131 parser.add_argument("-q", "--quiet", action="store_true", default=False,
132 help=_("Restrict output to warnings and errors"))
135 def fill_config_defaults(thisconfig):
136 for k, v in default_config.items():
137 if k not in thisconfig:
140 # Expand paths (~users and $vars)
141 def expand_path(path):
145 path = os.path.expanduser(path)
146 path = os.path.expandvars(path)
151 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
156 thisconfig[k + '_orig'] = v
158 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
159 if thisconfig['java_paths'] is None:
160 thisconfig['java_paths'] = dict()
162 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
163 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
164 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
165 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
166 if os.getenv('JAVA_HOME') is not None:
167 pathlist.append(os.getenv('JAVA_HOME'))
168 if os.getenv('PROGRAMFILES') is not None:
169 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
170 for d in sorted(pathlist):
171 if os.path.islink(d):
173 j = os.path.basename(d)
174 # the last one found will be the canonical one, so order appropriately
176 r'^1\.([6-9])\.0\.jdk$', # OSX
177 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
178 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
179 r'^jdk([6-9])-openjdk$', # Arch
180 r'^java-([6-9])-openjdk$', # Arch
181 r'^java-([6-9])-jdk$', # Arch (oracle)
182 r'^java-1\.([6-9])\.0-.*$', # RedHat
183 r'^java-([6-9])-oracle$', # Debian WebUpd8
184 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
185 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
187 m = re.match(regex, j)
190 for p in [d, os.path.join(d, 'Contents', 'Home')]:
191 if os.path.exists(os.path.join(p, 'bin', 'javac')):
192 thisconfig['java_paths'][m.group(1)] = p
194 for java_version in ('7', '8', '9'):
195 if java_version not in thisconfig['java_paths']:
197 java_home = thisconfig['java_paths'][java_version]
198 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
199 if os.path.exists(jarsigner):
200 thisconfig['jarsigner'] = jarsigner
201 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
202 break # Java7 is preferred, so quit if found
204 for k in ['ndk_paths', 'java_paths']:
210 thisconfig[k][k2] = exp
211 thisconfig[k][k2 + '_orig'] = v
214 def regsub_file(pattern, repl, path):
215 with open(path, 'rb') as f:
217 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
218 with open(path, 'wb') as f:
222 def read_config(opts, config_file='config.py'):
223 """Read the repository config
225 The config is read from config_file, which is in the current
226 directory when any of the repo management commands are used. If
227 there is a local metadata file in the git repo, then config.py is
228 not required, just use defaults.
231 global config, options
233 if config is not None:
240 if os.path.isfile(config_file):
241 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
242 with io.open(config_file, "rb") as f:
243 code = compile(f.read(), config_file, 'exec')
244 exec(code, None, config)
246 logging.warning(_("No 'config.py' found, using defaults."))
248 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
250 if not type(config[k]) in (str, list, tuple):
252 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
255 # smartcardoptions must be a list since its command line args for Popen
256 if 'smartcardoptions' in config:
257 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
258 elif 'keystore' in config and config['keystore'] == 'NONE':
259 # keystore='NONE' means use smartcard, these are required defaults
260 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
261 'SunPKCS11-OpenSC', '-providerClass',
262 'sun.security.pkcs11.SunPKCS11',
263 '-providerArg', 'opensc-fdroid.cfg']
265 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
266 st = os.stat(config_file)
267 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
268 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
269 .format(config_file=config_file))
271 fill_config_defaults(config)
273 for k in ["repo_description", "archive_description"]:
275 config[k] = clean_description(config[k])
277 if 'serverwebroot' in config:
278 if isinstance(config['serverwebroot'], str):
279 roots = [config['serverwebroot']]
280 elif all(isinstance(item, str) for item in config['serverwebroot']):
281 roots = config['serverwebroot']
283 raise TypeError(_('only accepts strings, lists, and tuples'))
285 for rootstr in roots:
286 # since this is used with rsync, where trailing slashes have
287 # meaning, ensure there is always a trailing slash
288 if rootstr[-1] != '/':
290 rootlist.append(rootstr.replace('//', '/'))
291 config['serverwebroot'] = rootlist
293 if 'servergitmirrors' in config:
294 if isinstance(config['servergitmirrors'], str):
295 roots = [config['servergitmirrors']]
296 elif all(isinstance(item, str) for item in config['servergitmirrors']):
297 roots = config['servergitmirrors']
299 raise TypeError(_('only accepts strings, lists, and tuples'))
300 config['servergitmirrors'] = roots
305 def assert_config_keystore(config):
306 """Check weather keystore is configured correctly and raise exception if not."""
309 if 'repo_keyalias' not in config:
311 logging.critical(_("'repo_keyalias' not found in config.py!"))
312 if 'keystore' not in config:
314 logging.critical(_("'keystore' not found in config.py!"))
315 elif not os.path.exists(config['keystore']):
317 logging.critical("'" + config['keystore'] + "' does not exist!")
318 if 'keystorepass' not in config:
320 logging.critical(_("'keystorepass' not found in config.py!"))
321 if 'keypass' not in config:
323 logging.critical(_("'keypass' not found in config.py!"))
325 raise FDroidException("This command requires a signing key, " +
326 "you can create one using: fdroid update --create-key")
329 def find_sdk_tools_cmd(cmd):
330 '''find a working path to a tool from the Android SDK'''
333 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
334 # try to find a working path to this command, in all the recent possible paths
335 if 'build_tools' in config:
336 build_tools = os.path.join(config['sdk_path'], 'build-tools')
337 # if 'build_tools' was manually set and exists, check only that one
338 configed_build_tools = os.path.join(build_tools, config['build_tools'])
339 if os.path.exists(configed_build_tools):
340 tooldirs.append(configed_build_tools)
342 # no configed version, so hunt known paths for it
343 for f in sorted(os.listdir(build_tools), reverse=True):
344 if os.path.isdir(os.path.join(build_tools, f)):
345 tooldirs.append(os.path.join(build_tools, f))
346 tooldirs.append(build_tools)
347 sdk_tools = os.path.join(config['sdk_path'], 'tools')
348 if os.path.exists(sdk_tools):
349 tooldirs.append(sdk_tools)
350 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
351 if os.path.exists(sdk_platform_tools):
352 tooldirs.append(sdk_platform_tools)
353 tooldirs.append('/usr/bin')
355 path = os.path.join(d, cmd)
356 if os.path.isfile(path):
358 test_aapt_version(path)
360 # did not find the command, exit with error message
361 ensure_build_tools_exists(config)
364 def test_aapt_version(aapt):
365 '''Check whether the version of aapt is new enough'''
366 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
367 if output is None or output == '':
368 logging.error(_("'{path}' failed to execute!").format(path=aapt))
370 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
375 # the Debian package has the version string like "v0.2-23.0.2"
376 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
377 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
380 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
383 def test_sdk_exists(thisconfig):
384 if 'sdk_path' not in thisconfig:
385 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
386 test_aapt_version(thisconfig['aapt'])
389 logging.error(_("'sdk_path' not set in 'config.py'!"))
391 if thisconfig['sdk_path'] == default_config['sdk_path']:
392 logging.error(_('No Android SDK found!'))
393 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
394 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
396 if not os.path.exists(thisconfig['sdk_path']):
397 logging.critical(_("Android SDK path '{path}' does not exist!")
398 .format(path=thisconfig['sdk_path']))
400 if not os.path.isdir(thisconfig['sdk_path']):
401 logging.critical(_("Android SDK path '{path}' is not a directory!")
402 .format(path=thisconfig['sdk_path']))
404 for d in ['build-tools', 'platform-tools', 'tools']:
405 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
406 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
407 .format(path=thisconfig['sdk_path'], dirname=d))
412 def ensure_build_tools_exists(thisconfig):
413 if not test_sdk_exists(thisconfig):
414 raise FDroidException(_("Android SDK not found!"))
415 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
416 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
417 if not os.path.isdir(versioned_build_tools):
418 raise FDroidException(
419 _("Android build-tools path '{path}' does not exist!")
420 .format(path=versioned_build_tools))
423 def get_local_metadata_files():
424 '''get any metadata files local to an app's source repo
426 This tries to ignore anything that does not count as app metdata,
427 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
430 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
433 def read_pkg_args(args, allow_vercodes=False):
435 :param args: arguments in the form of multiple appid:[vc] strings
436 :returns: a dictionary with the set of vercodes specified for each package
444 if allow_vercodes and ':' in p:
445 package, vercode = p.split(':')
447 package, vercode = p, None
448 if package not in vercodes:
449 vercodes[package] = [vercode] if vercode else []
451 elif vercode and vercode not in vercodes[package]:
452 vercodes[package] += [vercode] if vercode else []
457 def read_app_args(args, allapps, allow_vercodes=False):
459 On top of what read_pkg_args does, this returns the whole app metadata, but
460 limiting the builds list to the builds matching the vercodes specified.
463 vercodes = read_pkg_args(args, allow_vercodes)
469 for appid, app in allapps.items():
470 if appid in vercodes:
473 if len(apps) != len(vercodes):
476 logging.critical(_("No such package: %s") % p)
477 raise FDroidException(_("Found invalid appids in arguments"))
479 raise FDroidException(_("No packages specified"))
482 for appid, app in apps.items():
486 app.builds = [b for b in app.builds if b.versionCode in vc]
487 if len(app.builds) != len(vercodes[appid]):
489 allvcs = [b.versionCode for b in app.builds]
490 for v in vercodes[appid]:
492 logging.critical(_("No such versionCode {versionCode} for app {appid}")
493 .format(versionCode=v, appid=appid))
496 raise FDroidException(_("Found invalid versionCodes for some apps"))
501 def get_extension(filename):
502 base, ext = os.path.splitext(filename)
505 return base, ext.lower()[1:]
508 def has_extension(filename, ext):
509 _, f_ext = get_extension(filename)
513 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
516 def clean_description(description):
517 'Remove unneeded newlines and spaces from a block of description text'
519 # this is split up by paragraph to make removing the newlines easier
520 for paragraph in re.split(r'\n\n', description):
521 paragraph = re.sub('\r', '', paragraph)
522 paragraph = re.sub('\n', ' ', paragraph)
523 paragraph = re.sub(' {2,}', ' ', paragraph)
524 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
525 returnstring += paragraph + '\n\n'
526 return returnstring.rstrip('\n')
529 def publishednameinfo(filename):
530 filename = os.path.basename(filename)
531 m = publish_name_regex.match(filename)
533 result = (m.group(1), m.group(2))
534 except AttributeError:
535 raise FDroidException(_("Invalid name for published file: %s") % filename)
539 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
540 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
543 def apk_parse_release_filename(apkname):
544 """Parses the name of an APK file according the F-Droids APK naming
545 scheme and returns the tokens.
547 WARNING: Returned values don't necessarily represent the APKs actual
548 properties, the are just paresed from the file name.
550 :returns: A triplet containing (appid, versionCode, signer), where appid
551 should be the package name, versionCode should be the integer
552 represion of the APKs version and signer should be the first 7 hex
553 digists of the sha256 signing key fingerprint which was used to sign
556 m = apk_release_filename_with_sigfp.match(apkname)
558 return m.group('appid'), m.group('vercode'), m.group('sigfp')
559 m = apk_release_filename.match(apkname)
561 return m.group('appid'), m.group('vercode'), None
562 return None, None, None
565 def get_release_filename(app, build):
567 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
569 return "%s_%s.apk" % (app.id, build.versionCode)
572 def get_toolsversion_logname(app, build):
573 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
576 def getsrcname(app, build):
577 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
589 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
592 def get_build_dir(app):
593 '''get the dir that this app will be built in'''
595 if app.RepoType == 'srclib':
596 return os.path.join('build', 'srclib', app.Repo)
598 return os.path.join('build', app.id)
602 '''checkout code from VCS and return instance of vcs and the build dir'''
603 build_dir = get_build_dir(app)
605 # Set up vcs interface and make sure we have the latest code...
606 logging.debug("Getting {0} vcs interface for {1}"
607 .format(app.RepoType, app.Repo))
608 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
612 vcs = getvcs(app.RepoType, remote, build_dir)
613 logging.debug("Using %s" % vcs.clientversion())
615 return vcs, build_dir
618 def getvcs(vcstype, remote, local):
620 return vcs_git(remote, local)
621 if vcstype == 'git-svn':
622 return vcs_gitsvn(remote, local)
624 return vcs_hg(remote, local)
626 return vcs_bzr(remote, local)
627 if vcstype == 'srclib':
628 if local != os.path.join('build', 'srclib', remote):
629 raise VCSException("Error: srclib paths are hard-coded!")
630 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
632 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
633 raise VCSException("Invalid vcs type " + vcstype)
636 def getsrclibvcs(name):
637 if name not in fdroidserver.metadata.srclibs:
638 raise VCSException("Missing srclib " + name)
639 return fdroidserver.metadata.srclibs[name]['Repo Type']
644 def __init__(self, remote, local):
646 # svn, git-svn and bzr may require auth
648 if self.repotype() in ('git-svn', 'bzr'):
650 if self.repotype == 'git-svn':
651 raise VCSException("Authentication is not supported for git-svn")
652 self.username, remote = remote.split('@')
653 if ':' not in self.username:
654 raise VCSException(_("Password required with username"))
655 self.username, self.password = self.username.split(':')
659 self.clone_failed = False
660 self.refreshed = False
666 def clientversion(self):
667 versionstr = FDroidPopen(self.clientversioncmd()).output
668 return versionstr[0:versionstr.find('\n')]
670 def clientversioncmd(self):
673 def gotorevision(self, rev, refresh=True):
674 """Take the local repository to a clean version of the given
675 revision, which is specificed in the VCS's native
676 format. Beforehand, the repository can be dirty, or even
677 non-existent. If the repository does already exist locally, it
678 will be updated from the origin, but only once in the lifetime
679 of the vcs object. None is acceptable for 'rev' if you know
680 you are cloning a clean copy of the repo - otherwise it must
681 specify a valid revision.
684 if self.clone_failed:
685 raise VCSException(_("Downloading the repository already failed once, not trying again."))
687 # The .fdroidvcs-id file for a repo tells us what VCS type
688 # and remote that directory was created from, allowing us to drop it
689 # automatically if either of those things changes.
690 fdpath = os.path.join(self.local, '..',
691 '.fdroidvcs-' + os.path.basename(self.local))
692 fdpath = os.path.normpath(fdpath)
693 cdata = self.repotype() + ' ' + self.remote
696 if os.path.exists(self.local):
697 if os.path.exists(fdpath):
698 with open(fdpath, 'r') as f:
699 fsdata = f.read().strip()
704 logging.info("Repository details for %s changed - deleting" % (
708 logging.info("Repository details for %s missing - deleting" % (
711 shutil.rmtree(self.local)
715 self.refreshed = True
718 self.gotorevisionx(rev)
719 except FDroidException as e:
722 # If necessary, write the .fdroidvcs file.
723 if writeback and not self.clone_failed:
724 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
725 with open(fdpath, 'w+') as f:
731 def gotorevisionx(self, rev): # pylint: disable=unused-argument
732 """Derived classes need to implement this.
734 It's called once basic checking has been performed.
736 raise VCSException("This VCS type doesn't define gotorevisionx")
738 # Initialise and update submodules
739 def initsubmodules(self):
740 raise VCSException('Submodules not supported for this vcs type')
742 # Get a list of all known tags
744 if not self._gettags:
745 raise VCSException('gettags not supported for this vcs type')
747 for tag in self._gettags():
748 if re.match('[-A-Za-z0-9_. /]+$', tag):
752 def latesttags(self):
753 """Get a list of all the known tags, sorted from newest to oldest"""
754 raise VCSException('latesttags not supported for this vcs type')
757 """Get current commit reference (hash, revision, etc)"""
758 raise VCSException('getref not supported for this vcs type')
761 """Returns the srclib (name, path) used in setting up the current revision, or None."""
770 def clientversioncmd(self):
771 return ['git', '--version']
774 """If the local directory exists, but is somehow not a git repository,
775 git will traverse up the directory tree until it finds one
776 that is (i.e. fdroidserver) and then we'll proceed to destroy
777 it! This is called as a safety check.
781 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
782 result = p.output.rstrip()
783 if not result.endswith(self.local):
784 raise VCSException('Repository mismatch')
786 def gotorevisionx(self, rev):
787 if not os.path.exists(self.local):
789 p = FDroidPopen(['git', 'clone', self.remote, self.local])
790 if p.returncode != 0:
791 self.clone_failed = True
792 raise VCSException("Git clone failed", p.output)
796 # Discard any working tree changes
797 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
798 'git', 'reset', '--hard'], cwd=self.local, output=False)
799 if p.returncode != 0:
800 raise VCSException(_("Git reset failed"), p.output)
801 # Remove untracked files now, in case they're tracked in the target
802 # revision (it happens!)
803 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
804 'git', 'clean', '-dffx'], cwd=self.local, output=False)
805 if p.returncode != 0:
806 raise VCSException(_("Git clean failed"), p.output)
807 if not self.refreshed:
808 # Get latest commits and tags from remote
809 p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
810 if p.returncode != 0:
811 raise VCSException(_("Git fetch failed"), p.output)
812 p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
813 if p.returncode != 0:
814 raise VCSException(_("Git fetch failed"), p.output)
815 # Recreate origin/HEAD as git clone would do it, in case it disappeared
816 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
817 if p.returncode != 0:
818 lines = p.output.splitlines()
819 if 'Multiple remote HEAD branches' not in lines[0]:
820 raise VCSException(_("Git remote set-head failed"), p.output)
821 branch = lines[1].split(' ')[-1]
822 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
823 if p2.returncode != 0:
824 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
825 self.refreshed = True
826 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
827 # a github repo. Most of the time this is the same as origin/master.
828 rev = rev or 'origin/HEAD'
829 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
830 if p.returncode != 0:
831 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
832 # Get rid of any uncontrolled files left behind
833 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
834 if p.returncode != 0:
835 raise VCSException(_("Git clean failed"), p.output)
837 def initsubmodules(self):
839 submfile = os.path.join(self.local, '.gitmodules')
840 if not os.path.isfile(submfile):
841 raise VCSException(_("No git submodules available"))
843 # fix submodules not accessible without an account and public key auth
844 with open(submfile, 'r') as f:
845 lines = f.readlines()
846 with open(submfile, 'w') as f:
848 if 'git@github.com' in line:
849 line = line.replace('git@github.com:', 'https://github.com/')
850 if 'git@gitlab.com' in line:
851 line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
854 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
855 if p.returncode != 0:
856 raise VCSException(_("Git submodule sync failed"), p.output)
857 p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
858 if p.returncode != 0:
859 raise VCSException(_("Git submodule update failed"), p.output)
863 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
864 return p.output.splitlines()
866 tag_format = re.compile(r'tag: ([^),]*)')
868 def latesttags(self):
870 p = FDroidPopen(['git', 'log', '--tags',
871 '--simplify-by-decoration', '--pretty=format:%d'],
872 cwd=self.local, output=False)
874 for line in p.output.splitlines():
875 for tag in self.tag_format.findall(line):
880 class vcs_gitsvn(vcs):
885 def clientversioncmd(self):
886 return ['git', 'svn', '--version']
889 """If the local directory exists, but is somehow not a git repository,
890 git will traverse up the directory tree until it finds one that
891 is (i.e. fdroidserver) and then we'll proceed to destory it!
892 This is called as a safety check.
895 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
896 result = p.output.rstrip()
897 if not result.endswith(self.local):
898 raise VCSException('Repository mismatch')
900 def gotorevisionx(self, rev):
901 if not os.path.exists(self.local):
903 gitsvn_args = ['git', 'svn', 'clone']
904 if ';' in self.remote:
905 remote_split = self.remote.split(';')
906 for i in remote_split[1:]:
907 if i.startswith('trunk='):
908 gitsvn_args.extend(['-T', i[6:]])
909 elif i.startswith('tags='):
910 gitsvn_args.extend(['-t', i[5:]])
911 elif i.startswith('branches='):
912 gitsvn_args.extend(['-b', i[9:]])
913 gitsvn_args.extend([remote_split[0], self.local])
914 p = FDroidPopen(gitsvn_args, output=False)
915 if p.returncode != 0:
916 self.clone_failed = True
917 raise VCSException("Git svn clone failed", p.output)
919 gitsvn_args.extend([self.remote, self.local])
920 p = FDroidPopen(gitsvn_args, output=False)
921 if p.returncode != 0:
922 self.clone_failed = True
923 raise VCSException("Git svn clone failed", p.output)
927 # Discard any working tree changes
928 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
929 if p.returncode != 0:
930 raise VCSException("Git reset failed", p.output)
931 # Remove untracked files now, in case they're tracked in the target
932 # revision (it happens!)
933 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
934 if p.returncode != 0:
935 raise VCSException("Git clean failed", p.output)
936 if not self.refreshed:
937 # Get new commits, branches and tags from repo
938 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
939 if p.returncode != 0:
940 raise VCSException("Git svn fetch failed")
941 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
942 if p.returncode != 0:
943 raise VCSException("Git svn rebase failed", p.output)
944 self.refreshed = True
946 rev = rev or 'master'
948 nospaces_rev = rev.replace(' ', '%20')
949 # Try finding a svn tag
950 for treeish in ['origin/', '']:
951 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
952 if p.returncode == 0:
954 if p.returncode != 0:
955 # No tag found, normal svn rev translation
956 # Translate svn rev into git format
957 rev_split = rev.split('/')
960 for treeish in ['origin/', '']:
961 if len(rev_split) > 1:
962 treeish += rev_split[0]
963 svn_rev = rev_split[1]
966 # if no branch is specified, then assume trunk (i.e. 'master' branch):
970 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
972 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
973 git_rev = p.output.rstrip()
975 if p.returncode == 0 and git_rev:
978 if p.returncode != 0 or not git_rev:
979 # Try a plain git checkout as a last resort
980 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
981 if p.returncode != 0:
982 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
984 # Check out the git rev equivalent to the svn rev
985 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
989 # Get rid of any uncontrolled files left behind
990 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
991 if p.returncode != 0:
992 raise VCSException(_("Git clean failed"), p.output)
996 for treeish in ['origin/', '']:
997 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1003 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1004 if p.returncode != 0:
1006 return p.output.strip()
1014 def clientversioncmd(self):
1015 return ['hg', '--version']
1017 def gotorevisionx(self, rev):
1018 if not os.path.exists(self.local):
1019 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
1020 if p.returncode != 0:
1021 self.clone_failed = True
1022 raise VCSException("Hg clone failed", p.output)
1024 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("Hg status failed", p.output)
1027 for line in p.output.splitlines():
1028 if not line.startswith('? '):
1029 raise VCSException("Unexpected output from hg status -uS: " + line)
1030 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1031 if not self.refreshed:
1032 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
1033 if p.returncode != 0:
1034 raise VCSException("Hg pull failed", p.output)
1035 self.refreshed = True
1037 rev = rev or 'default'
1040 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1041 if p.returncode != 0:
1042 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1043 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1044 # Also delete untracked files, we have to enable purge extension for that:
1045 if "'purge' is provided by the following extension" in p.output:
1046 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1047 myfile.write("\n[extensions]\nhgext.purge=\n")
1048 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1049 if p.returncode != 0:
1050 raise VCSException("HG purge failed", p.output)
1051 elif p.returncode != 0:
1052 raise VCSException("HG purge failed", p.output)
1055 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1056 return p.output.splitlines()[1:]
1064 def clientversioncmd(self):
1065 return ['bzr', '--version']
1067 def gotorevisionx(self, rev):
1068 if not os.path.exists(self.local):
1069 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1070 if p.returncode != 0:
1071 self.clone_failed = True
1072 raise VCSException("Bzr branch failed", p.output)
1074 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1075 if p.returncode != 0:
1076 raise VCSException("Bzr revert failed", p.output)
1077 if not self.refreshed:
1078 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1079 if p.returncode != 0:
1080 raise VCSException("Bzr update failed", p.output)
1081 self.refreshed = True
1083 revargs = list(['-r', rev] if rev else [])
1084 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1085 if p.returncode != 0:
1086 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1089 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1090 return [tag.split(' ')[0].strip() for tag in
1091 p.output.splitlines()]
1094 def unescape_string(string):
1097 if string[0] == '"' and string[-1] == '"':
1100 return string.replace("\\'", "'")
1103 def retrieve_string(app_dir, string, xmlfiles=None):
1105 if not string.startswith('@string/'):
1106 return unescape_string(string)
1108 if xmlfiles is None:
1111 os.path.join(app_dir, 'res'),
1112 os.path.join(app_dir, 'src', 'main', 'res'),
1114 for root, dirs, files in os.walk(res_dir):
1115 if os.path.basename(root) == 'values':
1116 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1118 name = string[len('@string/'):]
1120 def element_content(element):
1121 if element.text is None:
1123 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1124 return s.decode('utf-8').strip()
1126 for path in xmlfiles:
1127 if not os.path.isfile(path):
1129 xml = parse_xml(path)
1130 element = xml.find('string[@name="' + name + '"]')
1131 if element is not None:
1132 content = element_content(element)
1133 return retrieve_string(app_dir, content, xmlfiles)
1138 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1139 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1142 def manifest_paths(app_dir, flavours):
1143 '''Return list of existing files that will be used to find the highest vercode'''
1145 possible_manifests = \
1146 [os.path.join(app_dir, 'AndroidManifest.xml'),
1147 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1148 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1149 os.path.join(app_dir, 'build.gradle')]
1151 for flavour in flavours:
1152 if flavour == 'yes':
1154 possible_manifests.append(
1155 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1157 return [path for path in possible_manifests if os.path.isfile(path)]
1160 def fetch_real_name(app_dir, flavours):
1161 '''Retrieve the package name. Returns the name, or None if not found.'''
1162 for path in manifest_paths(app_dir, flavours):
1163 if not has_extension(path, 'xml') or not os.path.isfile(path):
1165 logging.debug("fetch_real_name: Checking manifest at " + path)
1166 xml = parse_xml(path)
1167 app = xml.find('application')
1170 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1172 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1173 result = retrieve_string_singleline(app_dir, label)
1175 result = result.strip()
1180 def get_library_references(root_dir):
1182 proppath = os.path.join(root_dir, 'project.properties')
1183 if not os.path.isfile(proppath):
1185 with open(proppath, 'r', encoding='iso-8859-1') as f:
1187 if not line.startswith('android.library.reference.'):
1189 path = line.split('=')[1].strip()
1190 relpath = os.path.join(root_dir, path)
1191 if not os.path.isdir(relpath):
1193 logging.debug("Found subproject at %s" % path)
1194 libraries.append(path)
1198 def ant_subprojects(root_dir):
1199 subprojects = get_library_references(root_dir)
1200 for subpath in subprojects:
1201 subrelpath = os.path.join(root_dir, subpath)
1202 for p in get_library_references(subrelpath):
1203 relp = os.path.normpath(os.path.join(subpath, p))
1204 if relp not in subprojects:
1205 subprojects.insert(0, relp)
1209 def remove_debuggable_flags(root_dir):
1210 # Remove forced debuggable flags
1211 logging.debug("Removing debuggable flags from %s" % root_dir)
1212 for root, dirs, files in os.walk(root_dir):
1213 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1214 regsub_file(r'android:debuggable="[^"]*"',
1216 os.path.join(root, 'AndroidManifest.xml'))
1219 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1220 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1221 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1224 def app_matches_packagename(app, package):
1227 appid = app.UpdateCheckName or app.id
1228 if appid is None or appid == "Ignore":
1230 return appid == package
1233 def parse_androidmanifests(paths, app):
1235 Extract some information from the AndroidManifest.xml at the given path.
1236 Returns (version, vercode, package), any or all of which might be None.
1237 All values returned are strings.
1240 ignoreversions = app.UpdateCheckIgnore
1241 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1244 return (None, None, None)
1252 if not os.path.isfile(path):
1255 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1260 if has_extension(path, 'gradle'):
1261 with open(path, 'r') as f:
1263 if gradle_comment.match(line):
1265 # Grab first occurence of each to avoid running into
1266 # alternative flavours and builds.
1268 matches = psearch_g(line)
1270 s = matches.group(2)
1271 if app_matches_packagename(app, s):
1274 matches = vnsearch_g(line)
1276 version = matches.group(2)
1278 matches = vcsearch_g(line)
1280 vercode = matches.group(1)
1283 xml = parse_xml(path)
1284 if "package" in xml.attrib:
1285 s = xml.attrib["package"]
1286 if app_matches_packagename(app, s):
1288 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1289 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1290 base_dir = os.path.dirname(path)
1291 version = retrieve_string_singleline(base_dir, version)
1292 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1293 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1294 if string_is_integer(a):
1297 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1299 # Remember package name, may be defined separately from version+vercode
1301 package = max_package
1303 logging.debug("..got package={0}, version={1}, vercode={2}"
1304 .format(package, version, vercode))
1306 # Always grab the package name and version name in case they are not
1307 # together with the highest version code
1308 if max_package is None and package is not None:
1309 max_package = package
1310 if max_version is None and version is not None:
1311 max_version = version
1313 if vercode is not None \
1314 and (max_vercode is None or vercode > max_vercode):
1315 if not ignoresearch or not ignoresearch(version):
1316 if version is not None:
1317 max_version = version
1318 if vercode is not None:
1319 max_vercode = vercode
1320 if package is not None:
1321 max_package = package
1323 max_version = "Ignore"
1325 if max_version is None:
1326 max_version = "Unknown"
1328 if max_package and not is_valid_package_name(max_package):
1329 raise FDroidException(_("Invalid package name {0}").format(max_package))
1331 return (max_version, max_vercode, max_package)
1334 def is_valid_package_name(name):
1335 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1338 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1339 raw=False, prepare=True, preponly=False, refresh=True,
1341 """Get the specified source library.
1343 Returns the path to it. Normally this is the path to be used when
1344 referencing it, which may be a subdirectory of the actual project. If
1345 you want the base directory of the project, pass 'basepath=True'.
1354 name, ref = spec.split('@')
1356 number, name = name.split(':', 1)
1358 name, subdir = name.split('/', 1)
1360 if name not in fdroidserver.metadata.srclibs:
1361 raise VCSException('srclib ' + name + ' not found.')
1363 srclib = fdroidserver.metadata.srclibs[name]
1365 sdir = os.path.join(srclib_dir, name)
1368 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1369 vcs.srclib = (name, number, sdir)
1371 vcs.gotorevision(ref, refresh)
1378 libdir = os.path.join(sdir, subdir)
1379 elif srclib["Subdir"]:
1380 for subdir in srclib["Subdir"]:
1381 libdir_candidate = os.path.join(sdir, subdir)
1382 if os.path.exists(libdir_candidate):
1383 libdir = libdir_candidate
1389 remove_signing_keys(sdir)
1390 remove_debuggable_flags(sdir)
1394 if srclib["Prepare"]:
1395 cmd = replace_config_vars(srclib["Prepare"], build)
1397 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1398 if p.returncode != 0:
1399 raise BuildException("Error running prepare command for srclib %s"
1405 return (name, number, libdir)
1408 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1411 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1412 """ Prepare the source code for a particular build
1414 :param vcs: the appropriate vcs object for the application
1415 :param app: the application details from the metadata
1416 :param build: the build details from the metadata
1417 :param build_dir: the path to the build directory, usually 'build/app.id'
1418 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1419 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1421 Returns the (root, srclibpaths) where:
1422 :param root: is the root directory, which may be the same as 'build_dir' or may
1423 be a subdirectory of it.
1424 :param srclibpaths: is information on the srclibs being used
1427 # Optionally, the actual app source can be in a subdirectory
1429 root_dir = os.path.join(build_dir, build.subdir)
1431 root_dir = build_dir
1433 # Get a working copy of the right revision
1434 logging.info("Getting source for revision " + build.commit)
1435 vcs.gotorevision(build.commit, refresh)
1437 # Initialise submodules if required
1438 if build.submodules:
1439 logging.info(_("Initialising submodules"))
1440 vcs.initsubmodules()
1442 # Check that a subdir (if we're using one) exists. This has to happen
1443 # after the checkout, since it might not exist elsewhere
1444 if not os.path.exists(root_dir):
1445 raise BuildException('Missing subdir ' + root_dir)
1447 # Run an init command if one is required
1449 cmd = replace_config_vars(build.init, build)
1450 logging.info("Running 'init' commands in %s" % root_dir)
1452 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1453 if p.returncode != 0:
1454 raise BuildException("Error running init command for %s:%s" %
1455 (app.id, build.versionName), p.output)
1457 # Apply patches if any
1459 logging.info("Applying patches")
1460 for patch in build.patch:
1461 patch = patch.strip()
1462 logging.info("Applying " + patch)
1463 patch_path = os.path.join('metadata', app.id, patch)
1464 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1465 if p.returncode != 0:
1466 raise BuildException("Failed to apply patch %s" % patch_path)
1468 # Get required source libraries
1471 logging.info("Collecting source libraries")
1472 for lib in build.srclibs:
1473 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1474 refresh=refresh, build=build))
1476 for name, number, libpath in srclibpaths:
1477 place_srclib(root_dir, int(number) if number else None, libpath)
1479 basesrclib = vcs.getsrclib()
1480 # If one was used for the main source, add that too.
1482 srclibpaths.append(basesrclib)
1484 # Update the local.properties file
1485 localprops = [os.path.join(build_dir, 'local.properties')]
1487 parts = build.subdir.split(os.sep)
1490 cur = os.path.join(cur, d)
1491 localprops += [os.path.join(cur, 'local.properties')]
1492 for path in localprops:
1494 if os.path.isfile(path):
1495 logging.info("Updating local.properties file at %s" % path)
1496 with open(path, 'r', encoding='iso-8859-1') as f:
1500 logging.info("Creating local.properties file at %s" % path)
1501 # Fix old-fashioned 'sdk-location' by copying
1502 # from sdk.dir, if necessary
1504 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1505 re.S | re.M).group(1)
1506 props += "sdk-location=%s\n" % sdkloc
1508 props += "sdk.dir=%s\n" % config['sdk_path']
1509 props += "sdk-location=%s\n" % config['sdk_path']
1510 ndk_path = build.ndk_path()
1511 # if for any reason the path isn't valid or the directory
1512 # doesn't exist, some versions of Gradle will error with a
1513 # cryptic message (even if the NDK is not even necessary).
1514 # https://gitlab.com/fdroid/fdroidserver/issues/171
1515 if ndk_path and os.path.exists(ndk_path):
1517 props += "ndk.dir=%s\n" % ndk_path
1518 props += "ndk-location=%s\n" % ndk_path
1519 # Add java.encoding if necessary
1521 props += "java.encoding=%s\n" % build.encoding
1522 with open(path, 'w', encoding='iso-8859-1') as f:
1526 if build.build_method() == 'gradle':
1527 flavours = build.gradle
1530 n = build.target.split('-')[1]
1531 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1532 r'compileSdkVersion %s' % n,
1533 os.path.join(root_dir, 'build.gradle'))
1535 # Remove forced debuggable flags
1536 remove_debuggable_flags(root_dir)
1538 # Insert version code and number into the manifest if necessary
1539 if build.forceversion:
1540 logging.info("Changing the version name")
1541 for path in manifest_paths(root_dir, flavours):
1542 if not os.path.isfile(path):
1544 if has_extension(path, 'xml'):
1545 regsub_file(r'android:versionName="[^"]*"',
1546 r'android:versionName="%s"' % build.versionName,
1548 elif has_extension(path, 'gradle'):
1549 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1550 r"""\1versionName '%s'""" % build.versionName,
1553 if build.forcevercode:
1554 logging.info("Changing the version code")
1555 for path in manifest_paths(root_dir, flavours):
1556 if not os.path.isfile(path):
1558 if has_extension(path, 'xml'):
1559 regsub_file(r'android:versionCode="[^"]*"',
1560 r'android:versionCode="%s"' % build.versionCode,
1562 elif has_extension(path, 'gradle'):
1563 regsub_file(r'versionCode[ =]+[0-9]+',
1564 r'versionCode %s' % build.versionCode,
1567 # Delete unwanted files
1569 logging.info(_("Removing specified files"))
1570 for part in getpaths(build_dir, build.rm):
1571 dest = os.path.join(build_dir, part)
1572 logging.info("Removing {0}".format(part))
1573 if os.path.lexists(dest):
1574 if os.path.islink(dest):
1575 FDroidPopen(['unlink', dest], output=False)
1577 FDroidPopen(['rm', '-rf', dest], output=False)
1579 logging.info("...but it didn't exist")
1581 remove_signing_keys(build_dir)
1583 # Add required external libraries
1585 logging.info("Collecting prebuilt libraries")
1586 libsdir = os.path.join(root_dir, 'libs')
1587 if not os.path.exists(libsdir):
1589 for lib in build.extlibs:
1591 logging.info("...installing extlib {0}".format(lib))
1592 libf = os.path.basename(lib)
1593 libsrc = os.path.join(extlib_dir, lib)
1594 if not os.path.exists(libsrc):
1595 raise BuildException("Missing extlib file {0}".format(libsrc))
1596 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1598 # Run a pre-build command if one is required
1600 logging.info("Running 'prebuild' commands in %s" % root_dir)
1602 cmd = replace_config_vars(build.prebuild, build)
1604 # Substitute source library paths into prebuild commands
1605 for name, number, libpath in srclibpaths:
1606 libpath = os.path.relpath(libpath, root_dir)
1607 cmd = cmd.replace('$$' + name + '$$', libpath)
1609 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1610 if p.returncode != 0:
1611 raise BuildException("Error running prebuild command for %s:%s" %
1612 (app.id, build.versionName), p.output)
1614 # Generate (or update) the ant build file, build.xml...
1615 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1616 parms = ['android', 'update', 'lib-project']
1617 lparms = ['android', 'update', 'project']
1620 parms += ['-t', build.target]
1621 lparms += ['-t', build.target]
1622 if build.androidupdate:
1623 update_dirs = build.androidupdate
1625 update_dirs = ant_subprojects(root_dir) + ['.']
1627 for d in update_dirs:
1628 subdir = os.path.join(root_dir, d)
1630 logging.debug("Updating main project")
1631 cmd = parms + ['-p', d]
1633 logging.debug("Updating subproject %s" % d)
1634 cmd = lparms + ['-p', d]
1635 p = SdkToolsPopen(cmd, cwd=root_dir)
1636 # Check to see whether an error was returned without a proper exit
1637 # code (this is the case for the 'no target set or target invalid'
1639 if p.returncode != 0 or p.output.startswith("Error: "):
1640 raise BuildException("Failed to update project at %s" % d, p.output)
1641 # Clean update dirs via ant
1643 logging.info("Cleaning subproject %s" % d)
1644 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1646 return (root_dir, srclibpaths)
1649 def getpaths_map(build_dir, globpaths):
1650 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1654 full_path = os.path.join(build_dir, p)
1655 full_path = os.path.normpath(full_path)
1656 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1658 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1662 def getpaths(build_dir, globpaths):
1663 """Extend via globbing the paths from a field and return them as a set"""
1664 paths_map = getpaths_map(build_dir, globpaths)
1666 for k, v in paths_map.items():
1673 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1677 """permanent store of existing APKs with the date they were added
1679 This is currently the only way to permanently store the "updated"
1684 '''Load filename/date info about previously seen APKs
1686 Since the appid and date strings both will never have spaces,
1687 this is parsed as a list from the end to allow the filename to
1688 have any combo of spaces.
1691 self.path = os.path.join('stats', 'known_apks.txt')
1693 if os.path.isfile(self.path):
1694 with open(self.path, 'r', encoding='utf8') as f:
1696 t = line.rstrip().split(' ')
1698 self.apks[t[0]] = (t[1], None)
1701 date = datetime.strptime(t[-1], '%Y-%m-%d')
1702 filename = line[0:line.rfind(appid) - 1]
1703 self.apks[filename] = (appid, date)
1704 self.changed = False
1706 def writeifchanged(self):
1707 if not self.changed:
1710 if not os.path.exists('stats'):
1714 for apk, app in self.apks.items():
1716 line = apk + ' ' + appid
1718 line += ' ' + added.strftime('%Y-%m-%d')
1721 with open(self.path, 'w', encoding='utf8') as f:
1722 for line in sorted(lst, key=natural_key):
1723 f.write(line + '\n')
1725 def recordapk(self, apkName, app, default_date=None):
1727 Record an apk (if it's new, otherwise does nothing)
1728 Returns the date it was added as a datetime instance
1730 if apkName not in self.apks:
1731 if default_date is None:
1732 default_date = datetime.utcnow()
1733 self.apks[apkName] = (app, default_date)
1735 _, added = self.apks[apkName]
1738 def getapp(self, apkname):
1739 """Look up information - given the 'apkname', returns (app id, date added/None).
1741 Or returns None for an unknown apk.
1743 if apkname in self.apks:
1744 return self.apks[apkname]
1747 def getlatest(self, num):
1748 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1750 for apk, app in self.apks.items():
1754 if apps[appid] > added:
1758 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1759 lst = [app for app, _ignored in sortedapps]
1764 def get_file_extension(filename):
1765 """get the normalized file extension, can be blank string but never None"""
1766 if isinstance(filename, bytes):
1767 filename = filename.decode('utf-8')
1768 return os.path.splitext(filename)[1].lower()[1:]
1771 def get_apk_debuggable_aapt(apkfile):
1772 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1774 if p.returncode != 0:
1775 raise FDroidException(_("Failed to get APK manifest information"))
1776 for line in p.output.splitlines():
1777 if 'android:debuggable' in line and not line.endswith('0x0'):
1782 def get_apk_debuggable_androguard(apkfile):
1784 from androguard.core.bytecodes.apk import APK
1786 raise FDroidException("androguard library is not installed and aapt not present")
1788 apkobject = APK(apkfile)
1789 if apkobject.is_valid_APK():
1790 debuggable = apkobject.get_element("application", "debuggable")
1791 if debuggable is not None:
1792 return bool(strtobool(debuggable))
1796 def isApkAndDebuggable(apkfile):
1797 """Returns True if the given file is an APK and is debuggable
1799 :param apkfile: full path to the apk to check"""
1801 if get_file_extension(apkfile) != 'apk':
1804 if SdkToolsPopen(['aapt', 'version'], output=False):
1805 return get_apk_debuggable_aapt(apkfile)
1807 return get_apk_debuggable_androguard(apkfile)
1810 def get_apk_id_aapt(apkfile):
1811 """Extrat identification information from APK using aapt.
1813 :param apkfile: path to an APK file.
1814 :returns: triplet (appid, version code, version name)
1816 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1817 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1818 for line in p.output.splitlines():
1821 return m.group('appid'), m.group('vercode'), m.group('vername')
1822 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1823 .format(apkfilename=apkfile))
1828 self.returncode = None
1832 def SdkToolsPopen(commands, cwd=None, output=True):
1834 if cmd not in config:
1835 config[cmd] = find_sdk_tools_cmd(commands[0])
1836 abscmd = config[cmd]
1838 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1840 test_aapt_version(config['aapt'])
1841 return FDroidPopen([abscmd] + commands[1:],
1842 cwd=cwd, output=output)
1845 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1847 Run a command and capture the possibly huge output as bytes.
1849 :param commands: command and argument list like in subprocess.Popen
1850 :param cwd: optionally specifies a working directory
1851 :param envs: a optional dictionary of environment variables and their values
1852 :returns: A PopenResult.
1857 set_FDroidPopen_env()
1859 process_env = env.copy()
1860 if envs is not None and len(envs) > 0:
1861 process_env.update(envs)
1864 cwd = os.path.normpath(cwd)
1865 logging.debug("Directory: %s" % cwd)
1866 logging.debug("> %s" % ' '.join(commands))
1868 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1869 result = PopenResult()
1872 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1873 stdout=subprocess.PIPE, stderr=stderr_param)
1874 except OSError as e:
1875 raise BuildException("OSError while trying to execute " +
1876 ' '.join(commands) + ': ' + str(e))
1878 if not stderr_to_stdout and options.verbose:
1879 stderr_queue = Queue()
1880 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1882 while not stderr_reader.eof():
1883 while not stderr_queue.empty():
1884 line = stderr_queue.get()
1885 sys.stderr.buffer.write(line)
1890 stdout_queue = Queue()
1891 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1894 # Check the queue for output (until there is no more to get)
1895 while not stdout_reader.eof():
1896 while not stdout_queue.empty():
1897 line = stdout_queue.get()
1898 if output and options.verbose:
1899 # Output directly to console
1900 sys.stderr.buffer.write(line)
1906 result.returncode = p.wait()
1907 result.output = buf.getvalue()
1909 # make sure all filestreams of the subprocess are closed
1910 for streamvar in ['stdin', 'stdout', 'stderr']:
1911 if hasattr(p, streamvar):
1912 stream = getattr(p, streamvar)
1918 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1920 Run a command and capture the possibly huge output as a str.
1922 :param commands: command and argument list like in subprocess.Popen
1923 :param cwd: optionally specifies a working directory
1924 :param envs: a optional dictionary of environment variables and their values
1925 :returns: A PopenResult.
1927 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
1928 result.output = result.output.decode('utf-8', 'ignore')
1932 gradle_comment = re.compile(r'[ ]*//')
1933 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
1934 gradle_line_matches = [
1935 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
1936 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
1937 re.compile(r'.*\.readLine\(.*'),
1941 def remove_signing_keys(build_dir):
1942 for root, dirs, files in os.walk(build_dir):
1943 if 'build.gradle' in files:
1944 path = os.path.join(root, 'build.gradle')
1946 with open(path, "r", encoding='utf8') as o:
1947 lines = o.readlines()
1953 with open(path, "w", encoding='utf8') as o:
1954 while i < len(lines):
1957 while line.endswith('\\\n'):
1958 line = line.rstrip('\\\n') + lines[i]
1961 if gradle_comment.match(line):
1966 opened += line.count('{')
1967 opened -= line.count('}')
1970 if gradle_signing_configs.match(line):
1975 if any(s.match(line) for s in gradle_line_matches):
1983 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
1986 'project.properties',
1988 'default.properties',
1989 'ant.properties', ]:
1990 if propfile in files:
1991 path = os.path.join(root, propfile)
1993 with open(path, "r", encoding='iso-8859-1') as o:
1994 lines = o.readlines()
1998 with open(path, "w", encoding='iso-8859-1') as o:
2000 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2007 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2010 def set_FDroidPopen_env(build=None):
2012 set up the environment variables for the build environment
2014 There is only a weak standard, the variables used by gradle, so also set
2015 up the most commonly used environment variables for SDK and NDK. Also, if
2016 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2018 global env, orig_path
2022 orig_path = env['PATH']
2023 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2024 env[n] = config['sdk_path']
2025 for k, v in config['java_paths'].items():
2026 env['JAVA%s_HOME' % k] = v
2028 missinglocale = True
2029 for k, v in env.items():
2030 if k == 'LANG' and v != 'C':
2031 missinglocale = False
2033 missinglocale = False
2035 env['LANG'] = 'en_US.UTF-8'
2037 if build is not None:
2038 path = build.ndk_path()
2039 paths = orig_path.split(os.pathsep)
2040 if path not in paths:
2041 paths = [path] + paths
2042 env['PATH'] = os.pathsep.join(paths)
2043 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2044 env[n] = build.ndk_path()
2047 def replace_build_vars(cmd, build):
2048 cmd = cmd.replace('$$COMMIT$$', build.commit)
2049 cmd = cmd.replace('$$VERSION$$', build.versionName)
2050 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2054 def replace_config_vars(cmd, build):
2055 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2056 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2057 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2058 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2059 if build is not None:
2060 cmd = replace_build_vars(cmd, build)
2064 def place_srclib(root_dir, number, libpath):
2067 relpath = os.path.relpath(libpath, root_dir)
2068 proppath = os.path.join(root_dir, 'project.properties')
2071 if os.path.isfile(proppath):
2072 with open(proppath, "r", encoding='iso-8859-1') as o:
2073 lines = o.readlines()
2075 with open(proppath, "w", encoding='iso-8859-1') as o:
2078 if line.startswith('android.library.reference.%d=' % number):
2079 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2084 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2087 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2090 def signer_fingerprint_short(sig):
2091 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2093 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2094 for a given pkcs7 signature.
2096 :param sig: Contents of an APK signing certificate.
2097 :returns: shortened signing-key fingerprint.
2099 return signer_fingerprint(sig)[:7]
2102 def signer_fingerprint(sig):
2103 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2105 Extracts hexadecimal sha256 signing-key fingerprint string
2106 for a given pkcs7 signature.
2108 :param: Contents of an APK signature.
2109 :returns: shortened signature fingerprint.
2111 cert_encoded = get_certificate(sig)
2112 return hashlib.sha256(cert_encoded).hexdigest()
2115 def apk_signer_fingerprint(apk_path):
2116 """Obtain sha256 signing-key fingerprint for APK.
2118 Extracts hexadecimal sha256 signing-key fingerprint string
2121 :param apkpath: path to APK
2122 :returns: signature fingerprint
2125 with zipfile.ZipFile(apk_path, 'r') as apk:
2126 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2129 logging.error("Found no signing certificates on %s" % apk_path)
2132 logging.error("Found multiple signing certificates on %s" % apk_path)
2135 cert = apk.read(certs[0])
2136 return signer_fingerprint(cert)
2139 def apk_signer_fingerprint_short(apk_path):
2140 """Obtain shortened sha256 signing-key fingerprint for APK.
2142 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2143 for a given pkcs7 APK.
2145 :param apk_path: path to APK
2146 :returns: shortened signing-key fingerprint
2148 return apk_signer_fingerprint(apk_path)[:7]
2151 def metadata_get_sigdir(appid, vercode=None):
2152 """Get signature directory for app"""
2154 return os.path.join('metadata', appid, 'signatures', vercode)
2156 return os.path.join('metadata', appid, 'signatures')
2159 def metadata_find_developer_signature(appid, vercode=None):
2160 """Tires to find the developer signature for given appid.
2162 This picks the first signature file found in metadata an returns its
2165 :returns: sha256 signing key fingerprint of the developer signing key.
2166 None in case no signature can not be found."""
2168 # fetch list of dirs for all versions of signatures
2171 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2173 appsigdir = metadata_get_sigdir(appid)
2174 if os.path.isdir(appsigdir):
2175 numre = re.compile('[0-9]+')
2176 for ver in os.listdir(appsigdir):
2177 if numre.match(ver):
2178 appversigdir = os.path.join(appsigdir, ver)
2179 appversigdirs.append(appversigdir)
2181 for sigdir in appversigdirs:
2182 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2183 glob.glob(os.path.join(sigdir, '*.EC')) + \
2184 glob.glob(os.path.join(sigdir, '*.RSA'))
2186 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2188 with open(sig, 'rb') as f:
2189 return signer_fingerprint(f.read())
2193 def metadata_find_signing_files(appid, vercode):
2194 """Gets a list of singed manifests and signatures.
2196 :param appid: app id string
2197 :param vercode: app version code
2198 :returns: a list of triplets for each signing key with following paths:
2199 (signature_file, singed_file, manifest_file)
2202 sigdir = metadata_get_sigdir(appid, vercode)
2203 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2204 glob.glob(os.path.join(sigdir, '*.EC')) + \
2205 glob.glob(os.path.join(sigdir, '*.RSA'))
2206 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2208 sf = extre.sub('.SF', sig)
2209 if os.path.isfile(sf):
2210 mf = os.path.join(sigdir, 'MANIFEST.MF')
2211 if os.path.isfile(mf):
2212 ret.append((sig, sf, mf))
2216 def metadata_find_developer_signing_files(appid, vercode):
2217 """Get developer signature files for specified app from metadata.
2219 :returns: A triplet of paths for signing files from metadata:
2220 (signature_file, singed_file, manifest_file)
2222 allsigningfiles = metadata_find_signing_files(appid, vercode)
2223 if allsigningfiles and len(allsigningfiles) == 1:
2224 return allsigningfiles[0]
2229 def apk_strip_signatures(signed_apk, strip_manifest=False):
2230 """Removes signatures from APK.
2232 :param signed_apk: path to apk file.
2233 :param strip_manifest: when set to True also the manifest file will
2234 be removed from the APK.
2236 with tempfile.TemporaryDirectory() as tmpdir:
2237 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2238 os.rename(signed_apk, tmp_apk)
2239 with ZipFile(tmp_apk, 'r') as in_apk:
2240 with ZipFile(signed_apk, 'w') as out_apk:
2241 for f in in_apk.infolist():
2242 if not apk_sigfile.match(f.filename):
2244 if f.filename != 'META-INF/MANIFEST.MF':
2245 buf = in_apk.read(f.filename)
2246 out_apk.writestr(f.filename, buf)
2248 buf = in_apk.read(f.filename)
2249 out_apk.writestr(f.filename, buf)
2252 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2253 """Implats a signature from metadata into an APK.
2255 Note: this changes there supplied APK in place. So copy it if you
2256 need the original to be preserved.
2258 :param apkpath: location of the apk
2260 # get list of available signature files in metadata
2261 with tempfile.TemporaryDirectory() as tmpdir:
2262 # orig_apk = os.path.join(tmpdir, 'orig.apk')
2263 # os.rename(apkpath, orig_apk)
2264 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2265 with ZipFile(apkpath, 'r') as in_apk:
2266 with ZipFile(apkwithnewsig, 'w') as out_apk:
2267 for sig_file in [signaturefile, signedfile, manifest]:
2268 out_apk.write(sig_file, arcname='META-INF/' +
2269 os.path.basename(sig_file))
2270 for f in in_apk.infolist():
2271 if not apk_sigfile.match(f.filename):
2272 if f.filename != 'META-INF/MANIFEST.MF':
2273 buf = in_apk.read(f.filename)
2274 out_apk.writestr(f.filename, buf)
2276 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2277 if p.returncode != 0:
2278 raise BuildException("Failed to align application")
2281 def apk_extract_signatures(apkpath, outdir, manifest=True):
2282 """Extracts a signature files from APK and puts them into target directory.
2284 :param apkpath: location of the apk
2285 :param outdir: folder where the extracted signature files will be stored
2286 :param manifest: (optionally) disable extracting manifest file
2288 with ZipFile(apkpath, 'r') as in_apk:
2289 for f in in_apk.infolist():
2290 if apk_sigfile.match(f.filename) or \
2291 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2292 newpath = os.path.join(outdir, os.path.basename(f.filename))
2293 with open(newpath, 'wb') as out_file:
2294 out_file.write(in_apk.read(f.filename))
2297 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2298 """Verify that two apks are the same
2300 One of the inputs is signed, the other is unsigned. The signature metadata
2301 is transferred from the signed to the unsigned apk, and then jarsigner is
2302 used to verify that the signature from the signed apk is also varlid for
2303 the unsigned one. If the APK given as unsigned actually does have a
2304 signature, it will be stripped out and ignored.
2306 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2307 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2308 into AndroidManifest.xml, but that makes the build not reproducible. So
2309 instead they are included as separate files in the APK's META-INF/ folder.
2310 If those files exist in the signed APK, they will be part of the signature
2311 and need to also be included in the unsigned APK for it to validate.
2313 :param signed_apk: Path to a signed apk file
2314 :param unsigned_apk: Path to an unsigned apk file expected to match it
2315 :param tmp_dir: Path to directory for temporary files
2316 :returns: None if the verification is successful, otherwise a string
2317 describing what went wrong.
2320 if not os.path.isfile(signed_apk):
2321 return 'can not verify: file does not exists: {}'.format(signed_apk)
2323 if not os.path.isfile(unsigned_apk):
2324 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2326 with ZipFile(signed_apk, 'r') as signed:
2327 meta_inf_files = ['META-INF/MANIFEST.MF']
2328 for f in signed.namelist():
2329 if apk_sigfile.match(f) \
2330 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2331 meta_inf_files.append(f)
2332 if len(meta_inf_files) < 3:
2333 return "Signature files missing from {0}".format(signed_apk)
2335 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2336 with ZipFile(unsigned_apk, 'r') as unsigned:
2337 # only read the signature from the signed APK, everything else from unsigned
2338 with ZipFile(tmp_apk, 'w') as tmp:
2339 for filename in meta_inf_files:
2340 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2341 for info in unsigned.infolist():
2342 if info.filename in meta_inf_files:
2343 logging.warning('Ignoring %s from %s',
2344 info.filename, unsigned_apk)
2346 if info.filename in tmp.namelist():
2347 return "duplicate filename found: " + info.filename
2348 tmp.writestr(info, unsigned.read(info.filename))
2350 verified = verify_apk_signature(tmp_apk)
2353 logging.info("...NOT verified - {0}".format(tmp_apk))
2354 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2355 os.path.dirname(unsigned_apk))
2357 logging.info("...successfully verified")
2361 def verify_jar_signature(jar):
2362 """Verifies the signature of a given JAR file.
2364 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2365 this has to turn on -strict then check for result 4, since this
2366 does not expect the signature to be from a CA-signed certificate.
2368 :raises: VerificationException() if the JAR's signature could not be verified
2372 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2373 raise VerificationException(_("The repository's index could not be verified."))
2376 def verify_apk_signature(apk, min_sdk_version=None):
2377 """verify the signature on an APK
2379 Try to use apksigner whenever possible since jarsigner is very
2380 shitty: unsigned APKs pass as "verified"! Warning, this does
2381 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2383 :returns: boolean whether the APK was verified
2385 if set_command_in_config('apksigner'):
2386 args = [config['apksigner'], 'verify']
2388 args += ['--min-sdk-version=' + min_sdk_version]
2389 return subprocess.call(args + [apk]) == 0
2391 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2393 verify_jar_signature(apk)
2400 def verify_old_apk_signature(apk):
2401 """verify the signature on an archived APK, supporting deprecated algorithms
2403 F-Droid aims to keep every single binary that it ever published. Therefore,
2404 it needs to be able to verify APK signatures that include deprecated/removed
2405 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2407 jarsigner passes unsigned APKs as "verified"! So this has to turn
2408 on -strict then check for result 4.
2410 :returns: boolean whether the APK was verified
2413 _java_security = os.path.join(os.getcwd(), '.java.security')
2414 with open(_java_security, 'w') as fp:
2415 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2417 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2418 '-strict', '-verify', apk]) == 4
2421 apk_badchars = re.compile('''[/ :;'"]''')
2424 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2427 Returns None if the apk content is the same (apart from the signing key),
2428 otherwise a string describing what's different, or what went wrong when
2429 trying to do the comparison.
2435 absapk1 = os.path.abspath(apk1)
2436 absapk2 = os.path.abspath(apk2)
2438 if set_command_in_config('diffoscope'):
2439 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2440 htmlfile = logfilename + '.diffoscope.html'
2441 textfile = logfilename + '.diffoscope.txt'
2442 if subprocess.call([config['diffoscope'],
2443 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2444 '--html', htmlfile, '--text', textfile,
2445 absapk1, absapk2]) != 0:
2446 return("Failed to unpack " + apk1)
2448 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2449 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2450 for d in [apk1dir, apk2dir]:
2451 if os.path.exists(d):
2454 os.mkdir(os.path.join(d, 'jar-xf'))
2456 if subprocess.call(['jar', 'xf',
2457 os.path.abspath(apk1)],
2458 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2459 return("Failed to unpack " + apk1)
2460 if subprocess.call(['jar', 'xf',
2461 os.path.abspath(apk2)],
2462 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2463 return("Failed to unpack " + apk2)
2465 if set_command_in_config('apktool'):
2466 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2468 return("Failed to unpack " + apk1)
2469 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2471 return("Failed to unpack " + apk2)
2473 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2474 lines = p.output.splitlines()
2475 if len(lines) != 1 or 'META-INF' not in lines[0]:
2476 if set_command_in_config('meld'):
2477 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2478 return("Unexpected diff output - " + p.output)
2480 # since everything verifies, delete the comparison to keep cruft down
2481 shutil.rmtree(apk1dir)
2482 shutil.rmtree(apk2dir)
2484 # If we get here, it seems like they're the same!
2488 def set_command_in_config(command):
2489 '''Try to find specified command in the path, if it hasn't been
2490 manually set in config.py. If found, it is added to the config
2491 dict. The return value says whether the command is available.
2494 if command in config:
2497 tmp = find_command(command)
2499 config[command] = tmp
2504 def find_command(command):
2505 '''find the full path of a command, or None if it can't be found in the PATH'''
2508 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2510 fpath, fname = os.path.split(command)
2515 for path in os.environ["PATH"].split(os.pathsep):
2516 path = path.strip('"')
2517 exe_file = os.path.join(path, command)
2518 if is_exe(exe_file):
2525 '''generate a random password for when generating keys'''
2526 h = hashlib.sha256()
2527 h.update(os.urandom(16)) # salt
2528 h.update(socket.getfqdn().encode('utf-8'))
2529 passwd = base64.b64encode(h.digest()).strip()
2530 return passwd.decode('utf-8')
2533 def genkeystore(localconfig):
2535 Generate a new key with password provided in :param localconfig and add it to new keystore
2536 :return: hexed public key, public key fingerprint
2538 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2539 keystoredir = os.path.dirname(localconfig['keystore'])
2540 if keystoredir is None or keystoredir == '':
2541 keystoredir = os.path.join(os.getcwd(), keystoredir)
2542 if not os.path.exists(keystoredir):
2543 os.makedirs(keystoredir, mode=0o700)
2546 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2547 'FDROID_KEY_PASS': localconfig['keypass'],
2549 p = FDroidPopen([config['keytool'], '-genkey',
2550 '-keystore', localconfig['keystore'],
2551 '-alias', localconfig['repo_keyalias'],
2552 '-keyalg', 'RSA', '-keysize', '4096',
2553 '-sigalg', 'SHA256withRSA',
2554 '-validity', '10000',
2555 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2556 '-keypass:env', 'FDROID_KEY_PASS',
2557 '-dname', localconfig['keydname']], envs=env_vars)
2558 if p.returncode != 0:
2559 raise BuildException("Failed to generate key", p.output)
2560 os.chmod(localconfig['keystore'], 0o0600)
2561 if not options.quiet:
2562 # now show the lovely key that was just generated
2563 p = FDroidPopen([config['keytool'], '-list', '-v',
2564 '-keystore', localconfig['keystore'],
2565 '-alias', localconfig['repo_keyalias'],
2566 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2567 logging.info(p.output.strip() + '\n\n')
2568 # get the public key
2569 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2570 '-keystore', localconfig['keystore'],
2571 '-alias', localconfig['repo_keyalias'],
2572 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2573 + config['smartcardoptions'],
2574 envs=env_vars, output=False, stderr_to_stdout=False)
2575 if p.returncode != 0 or len(p.output) < 20:
2576 raise BuildException("Failed to get public key", p.output)
2578 fingerprint = get_cert_fingerprint(pubkey)
2579 return hexlify(pubkey), fingerprint
2582 def get_cert_fingerprint(pubkey):
2584 Generate a certificate fingerprint the same way keytool does it
2585 (but with slightly different formatting)
2587 digest = hashlib.sha256(pubkey).digest()
2588 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2589 return " ".join(ret)
2592 def get_certificate(certificate_file):
2594 Extracts a certificate from the given file.
2595 :param certificate_file: file bytes (as string) representing the certificate
2596 :return: A binary representation of the certificate's public key, or None in case of error
2598 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2599 if content.getComponentByName('contentType') != rfc2315.signedData:
2601 content = decoder.decode(content.getComponentByName('content'),
2602 asn1Spec=rfc2315.SignedData())[0]
2604 certificates = content.getComponentByName('certificates')
2605 cert = certificates[0].getComponentByName('certificate')
2607 logging.error("Certificates not found.")
2609 return encoder.encode(cert)
2612 def load_stats_fdroid_signing_key_fingerprints():
2613 """Load list of signing-key fingerprints stored by fdroid publish from file.
2615 :returns: list of dictionanryies containing the singing-key fingerprints.
2617 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2618 if not os.path.isfile(jar_file):
2620 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2621 p = FDroidPopen(cmd, output=False)
2622 if p.returncode != 4:
2623 raise FDroidException("Signature validation of '{}' failed! "
2624 "Please run publish again to rebuild this file.".format(jar_file))
2626 jar_sigkey = apk_signer_fingerprint(jar_file)
2627 repo_key_sig = config.get('repo_key_sha256')
2629 if jar_sigkey != repo_key_sig:
2630 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2632 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2633 config['repo_key_sha256'] = jar_sigkey
2634 write_to_config(config, 'repo_key_sha256')
2636 with zipfile.ZipFile(jar_file, 'r') as f:
2637 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2640 def write_to_config(thisconfig, key, value=None, config_file=None):
2641 '''write a key/value to the local config.py
2643 NOTE: only supports writing string variables.
2645 :param thisconfig: config dictionary
2646 :param key: variable name in config.py to be overwritten/added
2647 :param value: optional value to be written, instead of fetched
2648 from 'thisconfig' dictionary.
2651 origkey = key + '_orig'
2652 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2653 cfg = config_file if config_file else 'config.py'
2655 # load config file, create one if it doesn't exist
2656 if not os.path.exists(cfg):
2657 open(cfg, 'a').close()
2658 logging.info("Creating empty " + cfg)
2659 with open(cfg, 'r', encoding="utf-8") as f:
2660 lines = f.readlines()
2662 # make sure the file ends with a carraige return
2664 if not lines[-1].endswith('\n'):
2667 # regex for finding and replacing python string variable
2668 # definitions/initializations
2669 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2670 repl = key + ' = "' + value + '"'
2671 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2672 repl2 = key + " = '" + value + "'"
2674 # If we replaced this line once, we make sure won't be a
2675 # second instance of this line for this key in the document.
2678 with open(cfg, 'w', encoding="utf-8") as f:
2680 if pattern.match(line) or pattern2.match(line):
2682 line = pattern.sub(repl, line)
2683 line = pattern2.sub(repl2, line)
2694 def parse_xml(path):
2695 return XMLElementTree.parse(path).getroot()
2698 def string_is_integer(string):
2706 def get_per_app_repos():
2707 '''per-app repos are dirs named with the packageName of a single app'''
2709 # Android packageNames are Java packages, they may contain uppercase or
2710 # lowercase letters ('A' through 'Z'), numbers, and underscores
2711 # ('_'). However, individual package name parts may only start with
2712 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2713 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2716 for root, dirs, files in os.walk(os.getcwd()):
2718 print('checking', root, 'for', d)
2719 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2720 # standard parts of an fdroid repo, so never packageNames
2723 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2729 def is_repo_file(filename):
2730 '''Whether the file in a repo is a build product to be delivered to users'''
2731 if isinstance(filename, str):
2732 filename = filename.encode('utf-8', errors="surrogateescape")
2733 return os.path.isfile(filename) \
2734 and not filename.endswith(b'.asc') \
2735 and not filename.endswith(b'.sig') \
2736 and os.path.basename(filename) not in [
2738 b'index_unsigned.jar',