3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime, timedelta
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\
57 BuildException, VerificationException
58 from .asynchronousfilereader import AsynchronousFileReader
61 # A signature block file with a .DSA, .RSA, or .EC extension
62 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
63 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
64 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
66 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
75 'sdk_path': "$ANDROID_HOME",
80 'r12b': "$ANDROID_NDK",
87 'build_tools': "25.0.2",
88 'force_build_tools': False,
93 'accepted_formats': ['txt', 'yml'],
94 'sync_from_local_copy_dir': False,
95 'allow_disabled_algorithms': False,
96 'per_app_repos': False,
97 'make_current_version_link': True,
98 'current_version_name_source': 'Name',
99 'update_stats': False,
101 'stats_server': None,
103 'stats_to_carbon': False,
105 'build_server_always': False,
106 'keystore': 'keystore.jks',
107 'smartcardoptions': [],
117 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
118 'repo_name': "My First FDroid Repo Demo",
119 'repo_icon': "fdroid-icon.png",
120 'repo_description': '''
121 This is a repository of apps to be used with FDroid. Applications in this
122 repository are either official binaries built by the original application
123 developers, or are binaries built from source by the admin of f-droid.org
124 using the tools on https://gitlab.com/u/fdroid.
130 def setup_global_opts(parser):
131 parser.add_argument("-v", "--verbose", action="store_true", default=False,
132 help=_("Spew out even more information than normal"))
133 parser.add_argument("-q", "--quiet", action="store_true", default=False,
134 help=_("Restrict output to warnings and errors"))
137 def _add_java_paths_to_config(pathlist, thisconfig):
138 def path_version_key(s):
140 for u in re.split('[^0-9]+', s):
142 versionlist.append(int(u))
147 for d in sorted(pathlist, key=path_version_key):
148 if os.path.islink(d):
150 j = os.path.basename(d)
151 # the last one found will be the canonical one, so order appropriately
153 r'^1\.([6-9])\.0\.jdk$', # OSX
154 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
155 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
156 r'^jdk([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-openjdk$', # Arch
158 r'^java-([6-9])-jdk$', # Arch (oracle)
159 r'^java-1\.([6-9])\.0-.*$', # RedHat
160 r'^java-([6-9])-oracle$', # Debian WebUpd8
161 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
162 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
164 m = re.match(regex, j)
167 for p in [d, os.path.join(d, 'Contents', 'Home')]:
168 if os.path.exists(os.path.join(p, 'bin', 'javac')):
169 thisconfig['java_paths'][m.group(1)] = p
172 def fill_config_defaults(thisconfig):
173 for k, v in default_config.items():
174 if k not in thisconfig:
177 # Expand paths (~users and $vars)
178 def expand_path(path):
182 path = os.path.expanduser(path)
183 path = os.path.expandvars(path)
188 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
193 thisconfig[k + '_orig'] = v
195 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
196 if thisconfig['java_paths'] is None:
197 thisconfig['java_paths'] = dict()
199 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
200 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
201 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
202 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
203 if os.getenv('JAVA_HOME') is not None:
204 pathlist.append(os.getenv('JAVA_HOME'))
205 if os.getenv('PROGRAMFILES') is not None:
206 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
207 _add_java_paths_to_config(pathlist, thisconfig)
209 for java_version in ('7', '8', '9'):
210 if java_version not in thisconfig['java_paths']:
212 java_home = thisconfig['java_paths'][java_version]
213 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
214 if os.path.exists(jarsigner):
215 thisconfig['jarsigner'] = jarsigner
216 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
217 break # Java7 is preferred, so quit if found
219 for k in ['ndk_paths', 'java_paths']:
225 thisconfig[k][k2] = exp
226 thisconfig[k][k2 + '_orig'] = v
229 def regsub_file(pattern, repl, path):
230 with open(path, 'rb') as f:
232 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
233 with open(path, 'wb') as f:
237 def read_config(opts, config_file='config.py'):
238 """Read the repository config
240 The config is read from config_file, which is in the current
241 directory when any of the repo management commands are used. If
242 there is a local metadata file in the git repo, then config.py is
243 not required, just use defaults.
246 global config, options
248 if config is not None:
255 if os.path.isfile(config_file):
256 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
257 with io.open(config_file, "rb") as f:
258 code = compile(f.read(), config_file, 'exec')
259 exec(code, None, config)
261 logging.warning(_("No 'config.py' found, using defaults."))
263 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
265 if not type(config[k]) in (str, list, tuple):
267 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
270 # smartcardoptions must be a list since its command line args for Popen
271 if 'smartcardoptions' in config:
272 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
273 elif 'keystore' in config and config['keystore'] == 'NONE':
274 # keystore='NONE' means use smartcard, these are required defaults
275 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
276 'SunPKCS11-OpenSC', '-providerClass',
277 'sun.security.pkcs11.SunPKCS11',
278 '-providerArg', 'opensc-fdroid.cfg']
280 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
281 st = os.stat(config_file)
282 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
283 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
284 .format(config_file=config_file))
286 fill_config_defaults(config)
288 for k in ["repo_description", "archive_description"]:
290 config[k] = clean_description(config[k])
292 if 'serverwebroot' in config:
293 if isinstance(config['serverwebroot'], str):
294 roots = [config['serverwebroot']]
295 elif all(isinstance(item, str) for item in config['serverwebroot']):
296 roots = config['serverwebroot']
298 raise TypeError(_('only accepts strings, lists, and tuples'))
300 for rootstr in roots:
301 # since this is used with rsync, where trailing slashes have
302 # meaning, ensure there is always a trailing slash
303 if rootstr[-1] != '/':
305 rootlist.append(rootstr.replace('//', '/'))
306 config['serverwebroot'] = rootlist
308 if 'servergitmirrors' in config:
309 if isinstance(config['servergitmirrors'], str):
310 roots = [config['servergitmirrors']]
311 elif all(isinstance(item, str) for item in config['servergitmirrors']):
312 roots = config['servergitmirrors']
314 raise TypeError(_('only accepts strings, lists, and tuples'))
315 config['servergitmirrors'] = roots
320 def assert_config_keystore(config):
321 """Check weather keystore is configured correctly and raise exception if not."""
324 if 'repo_keyalias' not in config:
326 logging.critical(_("'repo_keyalias' not found in config.py!"))
327 if 'keystore' not in config:
329 logging.critical(_("'keystore' not found in config.py!"))
330 elif not os.path.exists(config['keystore']):
332 logging.critical("'" + config['keystore'] + "' does not exist!")
333 if 'keystorepass' not in config:
335 logging.critical(_("'keystorepass' not found in config.py!"))
336 if 'keypass' not in config:
338 logging.critical(_("'keypass' not found in config.py!"))
340 raise FDroidException("This command requires a signing key, " +
341 "you can create one using: fdroid update --create-key")
344 def find_sdk_tools_cmd(cmd):
345 '''find a working path to a tool from the Android SDK'''
348 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
349 # try to find a working path to this command, in all the recent possible paths
350 if 'build_tools' in config:
351 build_tools = os.path.join(config['sdk_path'], 'build-tools')
352 # if 'build_tools' was manually set and exists, check only that one
353 configed_build_tools = os.path.join(build_tools, config['build_tools'])
354 if os.path.exists(configed_build_tools):
355 tooldirs.append(configed_build_tools)
357 # no configed version, so hunt known paths for it
358 for f in sorted(os.listdir(build_tools), reverse=True):
359 if os.path.isdir(os.path.join(build_tools, f)):
360 tooldirs.append(os.path.join(build_tools, f))
361 tooldirs.append(build_tools)
362 sdk_tools = os.path.join(config['sdk_path'], 'tools')
363 if os.path.exists(sdk_tools):
364 tooldirs.append(sdk_tools)
365 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
366 if os.path.exists(sdk_platform_tools):
367 tooldirs.append(sdk_platform_tools)
368 tooldirs.append('/usr/bin')
370 path = os.path.join(d, cmd)
371 if os.path.isfile(path):
373 test_aapt_version(path)
375 # did not find the command, exit with error message
376 ensure_build_tools_exists(config)
379 def test_aapt_version(aapt):
380 '''Check whether the version of aapt is new enough'''
381 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
382 if output is None or output == '':
383 logging.error(_("'{path}' failed to execute!").format(path=aapt))
385 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
390 # the Debian package has the version string like "v0.2-23.0.2"
391 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
392 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
395 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
398 def test_sdk_exists(thisconfig):
399 if 'sdk_path' not in thisconfig:
400 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
401 test_aapt_version(thisconfig['aapt'])
404 logging.error(_("'sdk_path' not set in 'config.py'!"))
406 if thisconfig['sdk_path'] == default_config['sdk_path']:
407 logging.error(_('No Android SDK found!'))
408 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
409 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
411 if not os.path.exists(thisconfig['sdk_path']):
412 logging.critical(_("Android SDK path '{path}' does not exist!")
413 .format(path=thisconfig['sdk_path']))
415 if not os.path.isdir(thisconfig['sdk_path']):
416 logging.critical(_("Android SDK path '{path}' is not a directory!")
417 .format(path=thisconfig['sdk_path']))
419 for d in ['build-tools', 'platform-tools', 'tools']:
420 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
421 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
422 .format(path=thisconfig['sdk_path'], dirname=d))
427 def ensure_build_tools_exists(thisconfig):
428 if not test_sdk_exists(thisconfig):
429 raise FDroidException(_("Android SDK not found!"))
430 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
431 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
432 if not os.path.isdir(versioned_build_tools):
433 raise FDroidException(
434 _("Android build-tools path '{path}' does not exist!")
435 .format(path=versioned_build_tools))
438 def get_local_metadata_files():
439 '''get any metadata files local to an app's source repo
441 This tries to ignore anything that does not count as app metdata,
442 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
445 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
448 def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
450 :param appids: arguments in the form of multiple appid:[vc] strings
451 :returns: a dictionary with the set of vercodes specified for each package
454 if not appid_versionCode_pairs:
457 for p in appid_versionCode_pairs:
458 if allow_vercodes and ':' in p:
459 package, vercode = p.split(':')
461 package, vercode = p, None
462 if package not in vercodes:
463 vercodes[package] = [vercode] if vercode else []
465 elif vercode and vercode not in vercodes[package]:
466 vercodes[package] += [vercode] if vercode else []
471 def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
472 """Build a list of App instances for processing
474 On top of what read_pkg_args does, this returns the whole app
475 metadata, but limiting the builds list to the builds matching the
476 appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
477 all App and Build instances are returned.
481 vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
487 for appid, app in allapps.items():
488 if appid in vercodes:
491 if len(apps) != len(vercodes):
494 logging.critical(_("No such package: %s") % p)
495 raise FDroidException(_("Found invalid appids in arguments"))
497 raise FDroidException(_("No packages specified"))
500 for appid, app in apps.items():
504 app.builds = [b for b in app.builds if b.versionCode in vc]
505 if len(app.builds) != len(vercodes[appid]):
507 allvcs = [b.versionCode for b in app.builds]
508 for v in vercodes[appid]:
510 logging.critical(_("No such versionCode {versionCode} for app {appid}")
511 .format(versionCode=v, appid=appid))
514 raise FDroidException(_("Found invalid versionCodes for some apps"))
519 def get_extension(filename):
520 base, ext = os.path.splitext(filename)
523 return base, ext.lower()[1:]
526 def has_extension(filename, ext):
527 _ignored, f_ext = get_extension(filename)
531 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
534 def clean_description(description):
535 'Remove unneeded newlines and spaces from a block of description text'
537 # this is split up by paragraph to make removing the newlines easier
538 for paragraph in re.split(r'\n\n', description):
539 paragraph = re.sub('\r', '', paragraph)
540 paragraph = re.sub('\n', ' ', paragraph)
541 paragraph = re.sub(' {2,}', ' ', paragraph)
542 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
543 returnstring += paragraph + '\n\n'
544 return returnstring.rstrip('\n')
547 def publishednameinfo(filename):
548 filename = os.path.basename(filename)
549 m = publish_name_regex.match(filename)
551 result = (m.group(1), m.group(2))
552 except AttributeError:
553 raise FDroidException(_("Invalid name for published file: %s") % filename)
557 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
558 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
561 def apk_parse_release_filename(apkname):
562 """Parses the name of an APK file according the F-Droids APK naming
563 scheme and returns the tokens.
565 WARNING: Returned values don't necessarily represent the APKs actual
566 properties, the are just paresed from the file name.
568 :returns: A triplet containing (appid, versionCode, signer), where appid
569 should be the package name, versionCode should be the integer
570 represion of the APKs version and signer should be the first 7 hex
571 digists of the sha256 signing key fingerprint which was used to sign
574 m = apk_release_filename_with_sigfp.match(apkname)
576 return m.group('appid'), m.group('vercode'), m.group('sigfp')
577 m = apk_release_filename.match(apkname)
579 return m.group('appid'), m.group('vercode'), None
580 return None, None, None
583 def get_release_filename(app, build):
585 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
587 return "%s_%s.apk" % (app.id, build.versionCode)
590 def get_toolsversion_logname(app, build):
591 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
594 def getsrcname(app, build):
595 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
607 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
610 def get_build_dir(app):
611 '''get the dir that this app will be built in'''
613 if app.RepoType == 'srclib':
614 return os.path.join('build', 'srclib', app.Repo)
616 return os.path.join('build', app.id)
620 '''checkout code from VCS and return instance of vcs and the build dir'''
621 build_dir = get_build_dir(app)
623 # Set up vcs interface and make sure we have the latest code...
624 logging.debug("Getting {0} vcs interface for {1}"
625 .format(app.RepoType, app.Repo))
626 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
630 vcs = getvcs(app.RepoType, remote, build_dir)
632 return vcs, build_dir
635 def getvcs(vcstype, remote, local):
637 return vcs_git(remote, local)
638 if vcstype == 'git-svn':
639 return vcs_gitsvn(remote, local)
641 return vcs_hg(remote, local)
643 return vcs_bzr(remote, local)
644 if vcstype == 'srclib':
645 if local != os.path.join('build', 'srclib', remote):
646 raise VCSException("Error: srclib paths are hard-coded!")
647 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
649 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
650 raise VCSException("Invalid vcs type " + vcstype)
653 def getsrclibvcs(name):
654 if name not in fdroidserver.metadata.srclibs:
655 raise VCSException("Missing srclib " + name)
656 return fdroidserver.metadata.srclibs[name]['Repo Type']
661 def __init__(self, remote, local):
663 # svn, git-svn and bzr may require auth
665 if self.repotype() in ('git-svn', 'bzr'):
667 if self.repotype == 'git-svn':
668 raise VCSException("Authentication is not supported for git-svn")
669 self.username, remote = remote.split('@')
670 if ':' not in self.username:
671 raise VCSException(_("Password required with username"))
672 self.username, self.password = self.username.split(':')
676 self.clone_failed = False
677 self.refreshed = False
683 def clientversion(self):
684 versionstr = FDroidPopen(self.clientversioncmd()).output
685 return versionstr[0:versionstr.find('\n')]
687 def clientversioncmd(self):
690 def gotorevision(self, rev, refresh=True):
691 """Take the local repository to a clean version of the given
692 revision, which is specificed in the VCS's native
693 format. Beforehand, the repository can be dirty, or even
694 non-existent. If the repository does already exist locally, it
695 will be updated from the origin, but only once in the lifetime
696 of the vcs object. None is acceptable for 'rev' if you know
697 you are cloning a clean copy of the repo - otherwise it must
698 specify a valid revision.
701 if self.clone_failed:
702 raise VCSException(_("Downloading the repository already failed once, not trying again."))
704 # The .fdroidvcs-id file for a repo tells us what VCS type
705 # and remote that directory was created from, allowing us to drop it
706 # automatically if either of those things changes.
707 fdpath = os.path.join(self.local, '..',
708 '.fdroidvcs-' + os.path.basename(self.local))
709 fdpath = os.path.normpath(fdpath)
710 cdata = self.repotype() + ' ' + self.remote
713 if os.path.exists(self.local):
714 if os.path.exists(fdpath):
715 with open(fdpath, 'r') as f:
716 fsdata = f.read().strip()
721 logging.info("Repository details for %s changed - deleting" % (
725 logging.info("Repository details for %s missing - deleting" % (
728 shutil.rmtree(self.local)
732 self.refreshed = True
735 self.gotorevisionx(rev)
736 except FDroidException as e:
739 # If necessary, write the .fdroidvcs file.
740 if writeback and not self.clone_failed:
741 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
742 with open(fdpath, 'w+') as f:
748 def gotorevisionx(self, rev): # pylint: disable=unused-argument
749 """Derived classes need to implement this.
751 It's called once basic checking has been performed.
753 raise VCSException("This VCS type doesn't define gotorevisionx")
755 # Initialise and update submodules
756 def initsubmodules(self):
757 raise VCSException('Submodules not supported for this vcs type')
759 # Get a list of all known tags
761 if not self._gettags:
762 raise VCSException('gettags not supported for this vcs type')
764 for tag in self._gettags():
765 if re.match('[-A-Za-z0-9_. /]+$', tag):
769 def latesttags(self):
770 """Get a list of all the known tags, sorted from newest to oldest"""
771 raise VCSException('latesttags not supported for this vcs type')
774 """Get current commit reference (hash, revision, etc)"""
775 raise VCSException('getref not supported for this vcs type')
778 """Returns the srclib (name, path) used in setting up the current revision, or None."""
787 def clientversioncmd(self):
788 return ['git', '--version']
790 def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
791 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
793 While fetch/pull/clone respect the command line option flags,
794 it seems that submodule commands do not. They do seem to
795 follow whatever is in env vars, if the version of git is new
796 enough. So we just throw the kitchen sink at it to see what
801 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
802 git_config.append('-c')
803 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
804 git_config.append('-c')
805 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
806 git_config.append('-c')
807 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
808 # add helpful tricks supported in git >= 2.3
809 ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
810 git_config.append('-c')
811 git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10
813 'GIT_TERMINAL_PROMPT': '0',
814 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3
816 return FDroidPopen(['git', ] + git_config + gitargs,
817 envs=envs, cwd=cwd, output=output)
820 """If the local directory exists, but is somehow not a git repository,
821 git will traverse up the directory tree until it finds one
822 that is (i.e. fdroidserver) and then we'll proceed to destroy
823 it! This is called as a safety check.
827 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
828 result = p.output.rstrip()
829 if not result.endswith(self.local):
830 raise VCSException('Repository mismatch')
832 def gotorevisionx(self, rev):
833 if not os.path.exists(self.local):
835 p = self.GitFetchFDroidPopen(['clone', self.remote, self.local])
836 if p.returncode != 0:
837 self.clone_failed = True
838 raise VCSException("Git clone failed", p.output)
842 # Discard any working tree changes
843 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
844 'git', 'reset', '--hard'], cwd=self.local, output=False)
845 if p.returncode != 0:
846 raise VCSException(_("Git reset failed"), p.output)
847 # Remove untracked files now, in case they're tracked in the target
848 # revision (it happens!)
849 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
850 'git', 'clean', '-dffx'], cwd=self.local, output=False)
851 if p.returncode != 0:
852 raise VCSException(_("Git clean failed"), p.output)
853 if not self.refreshed:
854 # Get latest commits and tags from remote
855 p = self.GitFetchFDroidPopen(['fetch', 'origin'], cwd=self.local)
856 if p.returncode != 0:
857 raise VCSException(_("Git fetch failed"), p.output)
858 p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local)
859 if p.returncode != 0:
860 raise VCSException(_("Git fetch failed"), p.output)
861 # Recreate origin/HEAD as git clone would do it, in case it disappeared
862 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
863 if p.returncode != 0:
864 lines = p.output.splitlines()
865 if 'Multiple remote HEAD branches' not in lines[0]:
866 raise VCSException(_("Git remote set-head failed"), p.output)
867 branch = lines[1].split(' ')[-1]
868 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
869 if p2.returncode != 0:
870 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
871 self.refreshed = True
872 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
873 # a github repo. Most of the time this is the same as origin/master.
874 rev = rev or 'origin/HEAD'
875 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
876 if p.returncode != 0:
877 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
878 # Get rid of any uncontrolled files left behind
879 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
880 if p.returncode != 0:
881 raise VCSException(_("Git clean failed"), p.output)
883 def initsubmodules(self):
885 submfile = os.path.join(self.local, '.gitmodules')
886 if not os.path.isfile(submfile):
887 raise NoSubmodulesException(_("No git submodules available"))
889 # fix submodules not accessible without an account and public key auth
890 with open(submfile, 'r') as f:
891 lines = f.readlines()
892 with open(submfile, 'w') as f:
894 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
895 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
898 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
899 if p.returncode != 0:
900 raise VCSException(_("Git submodule sync failed"), p.output)
901 p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
902 if p.returncode != 0:
903 raise VCSException(_("Git submodule update failed"), p.output)
907 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
908 return p.output.splitlines()
910 tag_format = re.compile(r'tag: ([^),]*)')
912 def latesttags(self):
914 p = FDroidPopen(['git', 'log', '--tags',
915 '--simplify-by-decoration', '--pretty=format:%d'],
916 cwd=self.local, output=False)
918 for line in p.output.splitlines():
919 for tag in self.tag_format.findall(line):
924 class vcs_gitsvn(vcs):
929 def clientversioncmd(self):
930 return ['git', 'svn', '--version']
933 """If the local directory exists, but is somehow not a git repository,
934 git will traverse up the directory tree until it finds one that
935 is (i.e. fdroidserver) and then we'll proceed to destory it!
936 This is called as a safety check.
939 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
940 result = p.output.rstrip()
941 if not result.endswith(self.local):
942 raise VCSException('Repository mismatch')
944 def gotorevisionx(self, rev):
945 if not os.path.exists(self.local):
947 gitsvn_args = ['git', 'svn', 'clone']
948 if ';' in self.remote:
949 remote_split = self.remote.split(';')
950 for i in remote_split[1:]:
951 if i.startswith('trunk='):
952 gitsvn_args.extend(['-T', i[6:]])
953 elif i.startswith('tags='):
954 gitsvn_args.extend(['-t', i[5:]])
955 elif i.startswith('branches='):
956 gitsvn_args.extend(['-b', i[9:]])
957 gitsvn_args.extend([remote_split[0], self.local])
958 p = FDroidPopen(gitsvn_args, output=False)
959 if p.returncode != 0:
960 self.clone_failed = True
961 raise VCSException("Git svn clone failed", p.output)
963 gitsvn_args.extend([self.remote, self.local])
964 p = FDroidPopen(gitsvn_args, output=False)
965 if p.returncode != 0:
966 self.clone_failed = True
967 raise VCSException("Git svn clone failed", p.output)
971 # Discard any working tree changes
972 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
973 if p.returncode != 0:
974 raise VCSException("Git reset failed", p.output)
975 # Remove untracked files now, in case they're tracked in the target
976 # revision (it happens!)
977 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
978 if p.returncode != 0:
979 raise VCSException("Git clean failed", p.output)
980 if not self.refreshed:
981 # Get new commits, branches and tags from repo
982 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
983 if p.returncode != 0:
984 raise VCSException("Git svn fetch failed")
985 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
986 if p.returncode != 0:
987 raise VCSException("Git svn rebase failed", p.output)
988 self.refreshed = True
990 rev = rev or 'master'
992 nospaces_rev = rev.replace(' ', '%20')
993 # Try finding a svn tag
994 for treeish in ['origin/', '']:
995 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
996 if p.returncode == 0:
998 if p.returncode != 0:
999 # No tag found, normal svn rev translation
1000 # Translate svn rev into git format
1001 rev_split = rev.split('/')
1004 for treeish in ['origin/', '']:
1005 if len(rev_split) > 1:
1006 treeish += rev_split[0]
1007 svn_rev = rev_split[1]
1010 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1014 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1016 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1017 git_rev = p.output.rstrip()
1019 if p.returncode == 0 and git_rev:
1022 if p.returncode != 0 or not git_rev:
1023 # Try a plain git checkout as a last resort
1024 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
1025 if p.returncode != 0:
1026 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1028 # Check out the git rev equivalent to the svn rev
1029 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
1030 if p.returncode != 0:
1031 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1033 # Get rid of any uncontrolled files left behind
1034 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
1035 if p.returncode != 0:
1036 raise VCSException(_("Git clean failed"), p.output)
1040 for treeish in ['origin/', '']:
1041 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1042 if os.path.isdir(d):
1043 return os.listdir(d)
1047 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1048 if p.returncode != 0:
1050 return p.output.strip()
1058 def clientversioncmd(self):
1059 return ['hg', '--version']
1061 def gotorevisionx(self, rev):
1062 if not os.path.exists(self.local):
1063 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
1064 if p.returncode != 0:
1065 self.clone_failed = True
1066 raise VCSException("Hg clone failed", p.output)
1068 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1069 if p.returncode != 0:
1070 raise VCSException("Hg status failed", p.output)
1071 for line in p.output.splitlines():
1072 if not line.startswith('? '):
1073 raise VCSException("Unexpected output from hg status -uS: " + line)
1074 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1075 if not self.refreshed:
1076 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
1077 if p.returncode != 0:
1078 raise VCSException("Hg pull failed", p.output)
1079 self.refreshed = True
1081 rev = rev or 'default'
1084 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1085 if p.returncode != 0:
1086 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1087 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1088 # Also delete untracked files, we have to enable purge extension for that:
1089 if "'purge' is provided by the following extension" in p.output:
1090 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1091 myfile.write("\n[extensions]\nhgext.purge=\n")
1092 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1093 if p.returncode != 0:
1094 raise VCSException("HG purge failed", p.output)
1095 elif p.returncode != 0:
1096 raise VCSException("HG purge failed", p.output)
1099 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1100 return p.output.splitlines()[1:]
1108 def clientversioncmd(self):
1109 return ['bzr', '--version']
1111 def gotorevisionx(self, rev):
1112 if not os.path.exists(self.local):
1113 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1114 if p.returncode != 0:
1115 self.clone_failed = True
1116 raise VCSException("Bzr branch failed", p.output)
1118 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1119 if p.returncode != 0:
1120 raise VCSException("Bzr revert failed", p.output)
1121 if not self.refreshed:
1122 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1123 if p.returncode != 0:
1124 raise VCSException("Bzr update failed", p.output)
1125 self.refreshed = True
1127 revargs = list(['-r', rev] if rev else [])
1128 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1129 if p.returncode != 0:
1130 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1133 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1134 return [tag.split(' ')[0].strip() for tag in
1135 p.output.splitlines()]
1138 def unescape_string(string):
1141 if string[0] == '"' and string[-1] == '"':
1144 return string.replace("\\'", "'")
1147 def retrieve_string(app_dir, string, xmlfiles=None):
1149 if not string.startswith('@string/'):
1150 return unescape_string(string)
1152 if xmlfiles is None:
1155 os.path.join(app_dir, 'res'),
1156 os.path.join(app_dir, 'src', 'main', 'res'),
1158 for root, dirs, files in os.walk(res_dir):
1159 if os.path.basename(root) == 'values':
1160 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1162 name = string[len('@string/'):]
1164 def element_content(element):
1165 if element.text is None:
1167 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1168 return s.decode('utf-8').strip()
1170 for path in xmlfiles:
1171 if not os.path.isfile(path):
1173 xml = parse_xml(path)
1174 element = xml.find('string[@name="' + name + '"]')
1175 if element is not None:
1176 content = element_content(element)
1177 return retrieve_string(app_dir, content, xmlfiles)
1182 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1183 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1186 def manifest_paths(app_dir, flavours):
1187 '''Return list of existing files that will be used to find the highest vercode'''
1189 possible_manifests = \
1190 [os.path.join(app_dir, 'AndroidManifest.xml'),
1191 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1192 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1193 os.path.join(app_dir, 'build.gradle')]
1195 for flavour in flavours:
1196 if flavour == 'yes':
1198 possible_manifests.append(
1199 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1201 return [path for path in possible_manifests if os.path.isfile(path)]
1204 def fetch_real_name(app_dir, flavours):
1205 '''Retrieve the package name. Returns the name, or None if not found.'''
1206 for path in manifest_paths(app_dir, flavours):
1207 if not has_extension(path, 'xml') or not os.path.isfile(path):
1209 logging.debug("fetch_real_name: Checking manifest at " + path)
1210 xml = parse_xml(path)
1211 app = xml.find('application')
1214 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1216 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1217 result = retrieve_string_singleline(app_dir, label)
1219 result = result.strip()
1224 def get_library_references(root_dir):
1226 proppath = os.path.join(root_dir, 'project.properties')
1227 if not os.path.isfile(proppath):
1229 with open(proppath, 'r', encoding='iso-8859-1') as f:
1231 if not line.startswith('android.library.reference.'):
1233 path = line.split('=')[1].strip()
1234 relpath = os.path.join(root_dir, path)
1235 if not os.path.isdir(relpath):
1237 logging.debug("Found subproject at %s" % path)
1238 libraries.append(path)
1242 def ant_subprojects(root_dir):
1243 subprojects = get_library_references(root_dir)
1244 for subpath in subprojects:
1245 subrelpath = os.path.join(root_dir, subpath)
1246 for p in get_library_references(subrelpath):
1247 relp = os.path.normpath(os.path.join(subpath, p))
1248 if relp not in subprojects:
1249 subprojects.insert(0, relp)
1253 def remove_debuggable_flags(root_dir):
1254 # Remove forced debuggable flags
1255 logging.debug("Removing debuggable flags from %s" % root_dir)
1256 for root, dirs, files in os.walk(root_dir):
1257 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1258 regsub_file(r'android:debuggable="[^"]*"',
1260 os.path.join(root, 'AndroidManifest.xml'))
1263 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1264 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1265 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1268 def app_matches_packagename(app, package):
1271 appid = app.UpdateCheckName or app.id
1272 if appid is None or appid == "Ignore":
1274 return appid == package
1277 def parse_androidmanifests(paths, app):
1279 Extract some information from the AndroidManifest.xml at the given path.
1280 Returns (version, vercode, package), any or all of which might be None.
1281 All values returned are strings.
1284 ignoreversions = app.UpdateCheckIgnore
1285 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1288 return (None, None, None)
1296 if not os.path.isfile(path):
1299 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1305 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1306 flavour = app.builds[-1].gradle[-1]
1308 if has_extension(path, 'gradle'):
1309 # first try to get version name and code from correct flavour
1310 with open(path, 'r') as f:
1311 buildfile = f.read()
1313 regex_string = r"" + flavour + ".*?}"
1314 search = re.compile(regex_string, re.DOTALL)
1315 result = search.search(buildfile)
1317 if result is not None:
1318 resultgroup = result.group()
1321 matches = psearch_g(resultgroup)
1323 s = matches.group(2)
1324 if app_matches_packagename(app, s):
1327 matches = vnsearch_g(resultgroup)
1329 version = matches.group(2)
1331 matches = vcsearch_g(resultgroup)
1333 vercode = matches.group(1)
1335 # fall back to parse file line by line
1336 with open(path, 'r') as f:
1338 if gradle_comment.match(line):
1340 # Grab first occurence of each to avoid running into
1341 # alternative flavours and builds.
1343 matches = psearch_g(line)
1345 s = matches.group(2)
1346 if app_matches_packagename(app, s):
1349 matches = vnsearch_g(line)
1351 version = matches.group(2)
1353 matches = vcsearch_g(line)
1355 vercode = matches.group(1)
1358 xml = parse_xml(path)
1359 if "package" in xml.attrib:
1360 s = xml.attrib["package"]
1361 if app_matches_packagename(app, s):
1363 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1364 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1365 base_dir = os.path.dirname(path)
1366 version = retrieve_string_singleline(base_dir, version)
1367 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1368 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1369 if string_is_integer(a):
1372 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1374 # Remember package name, may be defined separately from version+vercode
1376 package = max_package
1378 logging.debug("..got package={0}, version={1}, vercode={2}"
1379 .format(package, version, vercode))
1381 # Always grab the package name and version name in case they are not
1382 # together with the highest version code
1383 if max_package is None and package is not None:
1384 max_package = package
1385 if max_version is None and version is not None:
1386 max_version = version
1388 if vercode is not None \
1389 and (max_vercode is None or vercode > max_vercode):
1390 if not ignoresearch or not ignoresearch(version):
1391 if version is not None:
1392 max_version = version
1393 if vercode is not None:
1394 max_vercode = vercode
1395 if package is not None:
1396 max_package = package
1398 max_version = "Ignore"
1400 if max_version is None:
1401 max_version = "Unknown"
1403 if max_package and not is_valid_package_name(max_package):
1404 raise FDroidException(_("Invalid package name {0}").format(max_package))
1406 return (max_version, max_vercode, max_package)
1409 def is_valid_package_name(name):
1410 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1413 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1414 raw=False, prepare=True, preponly=False, refresh=True,
1416 """Get the specified source library.
1418 Returns the path to it. Normally this is the path to be used when
1419 referencing it, which may be a subdirectory of the actual project. If
1420 you want the base directory of the project, pass 'basepath=True'.
1429 name, ref = spec.split('@')
1431 number, name = name.split(':', 1)
1433 name, subdir = name.split('/', 1)
1435 if name not in fdroidserver.metadata.srclibs:
1436 raise VCSException('srclib ' + name + ' not found.')
1438 srclib = fdroidserver.metadata.srclibs[name]
1440 sdir = os.path.join(srclib_dir, name)
1443 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1444 vcs.srclib = (name, number, sdir)
1446 vcs.gotorevision(ref, refresh)
1453 libdir = os.path.join(sdir, subdir)
1454 elif srclib["Subdir"]:
1455 for subdir in srclib["Subdir"]:
1456 libdir_candidate = os.path.join(sdir, subdir)
1457 if os.path.exists(libdir_candidate):
1458 libdir = libdir_candidate
1464 remove_signing_keys(sdir)
1465 remove_debuggable_flags(sdir)
1469 if srclib["Prepare"]:
1470 cmd = replace_config_vars(srclib["Prepare"], build)
1472 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1473 if p.returncode != 0:
1474 raise BuildException("Error running prepare command for srclib %s"
1480 return (name, number, libdir)
1483 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1486 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1487 """ Prepare the source code for a particular build
1489 :param vcs: the appropriate vcs object for the application
1490 :param app: the application details from the metadata
1491 :param build: the build details from the metadata
1492 :param build_dir: the path to the build directory, usually 'build/app.id'
1493 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1494 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1496 Returns the (root, srclibpaths) where:
1497 :param root: is the root directory, which may be the same as 'build_dir' or may
1498 be a subdirectory of it.
1499 :param srclibpaths: is information on the srclibs being used
1502 # Optionally, the actual app source can be in a subdirectory
1504 root_dir = os.path.join(build_dir, build.subdir)
1506 root_dir = build_dir
1508 # Get a working copy of the right revision
1509 logging.info("Getting source for revision " + build.commit)
1510 vcs.gotorevision(build.commit, refresh)
1512 # Initialise submodules if required
1513 if build.submodules:
1514 logging.info(_("Initialising submodules"))
1515 vcs.initsubmodules()
1517 # Check that a subdir (if we're using one) exists. This has to happen
1518 # after the checkout, since it might not exist elsewhere
1519 if not os.path.exists(root_dir):
1520 raise BuildException('Missing subdir ' + root_dir)
1522 # Run an init command if one is required
1524 cmd = replace_config_vars(build.init, build)
1525 logging.info("Running 'init' commands in %s" % root_dir)
1527 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1528 if p.returncode != 0:
1529 raise BuildException("Error running init command for %s:%s" %
1530 (app.id, build.versionName), p.output)
1532 # Apply patches if any
1534 logging.info("Applying patches")
1535 for patch in build.patch:
1536 patch = patch.strip()
1537 logging.info("Applying " + patch)
1538 patch_path = os.path.join('metadata', app.id, patch)
1539 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1540 if p.returncode != 0:
1541 raise BuildException("Failed to apply patch %s" % patch_path)
1543 # Get required source libraries
1546 logging.info("Collecting source libraries")
1547 for lib in build.srclibs:
1548 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1549 refresh=refresh, build=build))
1551 for name, number, libpath in srclibpaths:
1552 place_srclib(root_dir, int(number) if number else None, libpath)
1554 basesrclib = vcs.getsrclib()
1555 # If one was used for the main source, add that too.
1557 srclibpaths.append(basesrclib)
1559 # Update the local.properties file
1560 localprops = [os.path.join(build_dir, 'local.properties')]
1562 parts = build.subdir.split(os.sep)
1565 cur = os.path.join(cur, d)
1566 localprops += [os.path.join(cur, 'local.properties')]
1567 for path in localprops:
1569 if os.path.isfile(path):
1570 logging.info("Updating local.properties file at %s" % path)
1571 with open(path, 'r', encoding='iso-8859-1') as f:
1575 logging.info("Creating local.properties file at %s" % path)
1576 # Fix old-fashioned 'sdk-location' by copying
1577 # from sdk.dir, if necessary
1579 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1580 re.S | re.M).group(1)
1581 props += "sdk-location=%s\n" % sdkloc
1583 props += "sdk.dir=%s\n" % config['sdk_path']
1584 props += "sdk-location=%s\n" % config['sdk_path']
1585 ndk_path = build.ndk_path()
1586 # if for any reason the path isn't valid or the directory
1587 # doesn't exist, some versions of Gradle will error with a
1588 # cryptic message (even if the NDK is not even necessary).
1589 # https://gitlab.com/fdroid/fdroidserver/issues/171
1590 if ndk_path and os.path.exists(ndk_path):
1592 props += "ndk.dir=%s\n" % ndk_path
1593 props += "ndk-location=%s\n" % ndk_path
1594 # Add java.encoding if necessary
1596 props += "java.encoding=%s\n" % build.encoding
1597 with open(path, 'w', encoding='iso-8859-1') as f:
1601 if build.build_method() == 'gradle':
1602 flavours = build.gradle
1605 n = build.target.split('-')[1]
1606 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1607 r'compileSdkVersion %s' % n,
1608 os.path.join(root_dir, 'build.gradle'))
1610 # Remove forced debuggable flags
1611 remove_debuggable_flags(root_dir)
1613 # Insert version code and number into the manifest if necessary
1614 if build.forceversion:
1615 logging.info("Changing the version name")
1616 for path in manifest_paths(root_dir, flavours):
1617 if not os.path.isfile(path):
1619 if has_extension(path, 'xml'):
1620 regsub_file(r'android:versionName="[^"]*"',
1621 r'android:versionName="%s"' % build.versionName,
1623 elif has_extension(path, 'gradle'):
1624 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1625 r"""\1versionName '%s'""" % build.versionName,
1628 if build.forcevercode:
1629 logging.info("Changing the version code")
1630 for path in manifest_paths(root_dir, flavours):
1631 if not os.path.isfile(path):
1633 if has_extension(path, 'xml'):
1634 regsub_file(r'android:versionCode="[^"]*"',
1635 r'android:versionCode="%s"' % build.versionCode,
1637 elif has_extension(path, 'gradle'):
1638 regsub_file(r'versionCode[ =]+[0-9]+',
1639 r'versionCode %s' % build.versionCode,
1642 # Delete unwanted files
1644 logging.info(_("Removing specified files"))
1645 for part in getpaths(build_dir, build.rm):
1646 dest = os.path.join(build_dir, part)
1647 logging.info("Removing {0}".format(part))
1648 if os.path.lexists(dest):
1649 # rmtree can only handle directories that are not symlinks, so catch anything else
1650 if not os.path.isdir(dest) or os.path.islink(dest):
1655 logging.info("...but it didn't exist")
1657 remove_signing_keys(build_dir)
1659 # Add required external libraries
1661 logging.info("Collecting prebuilt libraries")
1662 libsdir = os.path.join(root_dir, 'libs')
1663 if not os.path.exists(libsdir):
1665 for lib in build.extlibs:
1667 logging.info("...installing extlib {0}".format(lib))
1668 libf = os.path.basename(lib)
1669 libsrc = os.path.join(extlib_dir, lib)
1670 if not os.path.exists(libsrc):
1671 raise BuildException("Missing extlib file {0}".format(libsrc))
1672 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1674 # Run a pre-build command if one is required
1676 logging.info("Running 'prebuild' commands in %s" % root_dir)
1678 cmd = replace_config_vars(build.prebuild, build)
1680 # Substitute source library paths into prebuild commands
1681 for name, number, libpath in srclibpaths:
1682 libpath = os.path.relpath(libpath, root_dir)
1683 cmd = cmd.replace('$$' + name + '$$', libpath)
1685 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1686 if p.returncode != 0:
1687 raise BuildException("Error running prebuild command for %s:%s" %
1688 (app.id, build.versionName), p.output)
1690 # Generate (or update) the ant build file, build.xml...
1691 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1692 parms = ['android', 'update', 'lib-project']
1693 lparms = ['android', 'update', 'project']
1696 parms += ['-t', build.target]
1697 lparms += ['-t', build.target]
1698 if build.androidupdate:
1699 update_dirs = build.androidupdate
1701 update_dirs = ant_subprojects(root_dir) + ['.']
1703 for d in update_dirs:
1704 subdir = os.path.join(root_dir, d)
1706 logging.debug("Updating main project")
1707 cmd = parms + ['-p', d]
1709 logging.debug("Updating subproject %s" % d)
1710 cmd = lparms + ['-p', d]
1711 p = SdkToolsPopen(cmd, cwd=root_dir)
1712 # Check to see whether an error was returned without a proper exit
1713 # code (this is the case for the 'no target set or target invalid'
1715 if p.returncode != 0 or p.output.startswith("Error: "):
1716 raise BuildException("Failed to update project at %s" % d, p.output)
1717 # Clean update dirs via ant
1719 logging.info("Cleaning subproject %s" % d)
1720 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1722 return (root_dir, srclibpaths)
1725 def getpaths_map(build_dir, globpaths):
1726 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1730 full_path = os.path.join(build_dir, p)
1731 full_path = os.path.normpath(full_path)
1732 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1734 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1738 def getpaths(build_dir, globpaths):
1739 """Extend via globbing the paths from a field and return them as a set"""
1740 paths_map = getpaths_map(build_dir, globpaths)
1742 for k, v in paths_map.items():
1749 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1752 def check_system_clock(dt_obj, path):
1753 """Check if system clock is updated based on provided date
1755 If an APK has files newer than the system time, suggest updating
1756 the system clock. This is useful for offline systems, used for
1757 signing, which do not have another source of clock sync info. It
1758 has to be more than 24 hours newer because ZIP/APK files do not
1762 checkdt = dt_obj - timedelta(1)
1763 if datetime.today() < checkdt:
1764 logging.warning(_('System clock is older than date in {path}!').format(path=path)
1765 + '\n' + _('Set clock to that time using:') + '\n'
1766 + 'sudo date -s "' + str(dt_obj) + '"')
1770 """permanent store of existing APKs with the date they were added
1772 This is currently the only way to permanently store the "updated"
1777 '''Load filename/date info about previously seen APKs
1779 Since the appid and date strings both will never have spaces,
1780 this is parsed as a list from the end to allow the filename to
1781 have any combo of spaces.
1784 self.path = os.path.join('stats', 'known_apks.txt')
1786 if os.path.isfile(self.path):
1787 with open(self.path, 'r', encoding='utf8') as f:
1789 t = line.rstrip().split(' ')
1791 self.apks[t[0]] = (t[1], None)
1794 date = datetime.strptime(t[-1], '%Y-%m-%d')
1795 filename = line[0:line.rfind(appid) - 1]
1796 self.apks[filename] = (appid, date)
1797 check_system_clock(date, self.path)
1798 self.changed = False
1800 def writeifchanged(self):
1801 if not self.changed:
1804 if not os.path.exists('stats'):
1808 for apk, app in self.apks.items():
1810 line = apk + ' ' + appid
1812 line += ' ' + added.strftime('%Y-%m-%d')
1815 with open(self.path, 'w', encoding='utf8') as f:
1816 for line in sorted(lst, key=natural_key):
1817 f.write(line + '\n')
1819 def recordapk(self, apkName, app, default_date=None):
1821 Record an apk (if it's new, otherwise does nothing)
1822 Returns the date it was added as a datetime instance
1824 if apkName not in self.apks:
1825 if default_date is None:
1826 default_date = datetime.utcnow()
1827 self.apks[apkName] = (app, default_date)
1829 _ignored, added = self.apks[apkName]
1832 def getapp(self, apkname):
1833 """Look up information - given the 'apkname', returns (app id, date added/None).
1835 Or returns None for an unknown apk.
1837 if apkname in self.apks:
1838 return self.apks[apkname]
1841 def getlatest(self, num):
1842 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1844 for apk, app in self.apks.items():
1848 if apps[appid] > added:
1852 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1853 lst = [app for app, _ignored in sortedapps]
1858 def get_file_extension(filename):
1859 """get the normalized file extension, can be blank string but never None"""
1860 if isinstance(filename, bytes):
1861 filename = filename.decode('utf-8')
1862 return os.path.splitext(filename)[1].lower()[1:]
1865 def get_apk_debuggable_aapt(apkfile):
1866 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1868 if p.returncode != 0:
1869 raise FDroidException(_("Failed to get APK manifest information"))
1870 for line in p.output.splitlines():
1871 if 'android:debuggable' in line and not line.endswith('0x0'):
1876 def get_apk_debuggable_androguard(apkfile):
1878 from androguard.core.bytecodes.apk import APK
1880 raise FDroidException("androguard library is not installed and aapt not present")
1882 apkobject = APK(apkfile)
1883 if apkobject.is_valid_APK():
1884 debuggable = apkobject.get_element("application", "debuggable")
1885 if debuggable is not None:
1886 return bool(strtobool(debuggable))
1890 def isApkAndDebuggable(apkfile):
1891 """Returns True if the given file is an APK and is debuggable
1893 :param apkfile: full path to the apk to check"""
1895 if get_file_extension(apkfile) != 'apk':
1898 if SdkToolsPopen(['aapt', 'version'], output=False):
1899 return get_apk_debuggable_aapt(apkfile)
1901 return get_apk_debuggable_androguard(apkfile)
1904 def get_apk_id_aapt(apkfile):
1905 """Extrat identification information from APK using aapt.
1907 :param apkfile: path to an APK file.
1908 :returns: triplet (appid, version code, version name)
1910 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1911 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1912 for line in p.output.splitlines():
1915 return m.group('appid'), m.group('vercode'), m.group('vername')
1916 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1917 .format(apkfilename=apkfile))
1922 self.returncode = None
1926 def SdkToolsPopen(commands, cwd=None, output=True):
1928 if cmd not in config:
1929 config[cmd] = find_sdk_tools_cmd(commands[0])
1930 abscmd = config[cmd]
1932 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1934 test_aapt_version(config['aapt'])
1935 return FDroidPopen([abscmd] + commands[1:],
1936 cwd=cwd, output=output)
1939 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1941 Run a command and capture the possibly huge output as bytes.
1943 :param commands: command and argument list like in subprocess.Popen
1944 :param cwd: optionally specifies a working directory
1945 :param envs: a optional dictionary of environment variables and their values
1946 :returns: A PopenResult.
1951 set_FDroidPopen_env()
1953 process_env = env.copy()
1954 if envs is not None and len(envs) > 0:
1955 process_env.update(envs)
1958 cwd = os.path.normpath(cwd)
1959 logging.debug("Directory: %s" % cwd)
1960 logging.debug("> %s" % ' '.join(commands))
1962 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1963 result = PopenResult()
1966 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1967 stdout=subprocess.PIPE, stderr=stderr_param)
1968 except OSError as e:
1969 raise BuildException("OSError while trying to execute " +
1970 ' '.join(commands) + ': ' + str(e))
1972 # TODO are these AsynchronousFileReader threads always exiting?
1973 if not stderr_to_stdout and options.verbose:
1974 stderr_queue = Queue()
1975 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1977 while not stderr_reader.eof():
1978 while not stderr_queue.empty():
1979 line = stderr_queue.get()
1980 sys.stderr.buffer.write(line)
1985 stdout_queue = Queue()
1986 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1989 # Check the queue for output (until there is no more to get)
1990 while not stdout_reader.eof():
1991 while not stdout_queue.empty():
1992 line = stdout_queue.get()
1993 if output and options.verbose:
1994 # Output directly to console
1995 sys.stderr.buffer.write(line)
2001 result.returncode = p.wait()
2002 result.output = buf.getvalue()
2004 # make sure all filestreams of the subprocess are closed
2005 for streamvar in ['stdin', 'stdout', 'stderr']:
2006 if hasattr(p, streamvar):
2007 stream = getattr(p, streamvar)
2013 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
2015 Run a command and capture the possibly huge output as a str.
2017 :param commands: command and argument list like in subprocess.Popen
2018 :param cwd: optionally specifies a working directory
2019 :param envs: a optional dictionary of environment variables and their values
2020 :returns: A PopenResult.
2022 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2023 result.output = result.output.decode('utf-8', 'ignore')
2027 gradle_comment = re.compile(r'[ ]*//')
2028 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2029 gradle_line_matches = [
2030 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2031 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2032 re.compile(r'.*\.readLine\(.*'),
2036 def remove_signing_keys(build_dir):
2037 for root, dirs, files in os.walk(build_dir):
2038 if 'build.gradle' in files:
2039 path = os.path.join(root, 'build.gradle')
2041 with open(path, "r", encoding='utf8') as o:
2042 lines = o.readlines()
2048 with open(path, "w", encoding='utf8') as o:
2049 while i < len(lines):
2052 while line.endswith('\\\n'):
2053 line = line.rstrip('\\\n') + lines[i]
2056 if gradle_comment.match(line):
2061 opened += line.count('{')
2062 opened -= line.count('}')
2065 if gradle_signing_configs.match(line):
2070 if any(s.match(line) for s in gradle_line_matches):
2078 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2081 'project.properties',
2083 'default.properties',
2084 'ant.properties', ]:
2085 if propfile in files:
2086 path = os.path.join(root, propfile)
2088 with open(path, "r", encoding='iso-8859-1') as o:
2089 lines = o.readlines()
2093 with open(path, "w", encoding='iso-8859-1') as o:
2095 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2102 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2105 def set_FDroidPopen_env(build=None):
2107 set up the environment variables for the build environment
2109 There is only a weak standard, the variables used by gradle, so also set
2110 up the most commonly used environment variables for SDK and NDK. Also, if
2111 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2113 global env, orig_path
2117 orig_path = env['PATH']
2118 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2119 env[n] = config['sdk_path']
2120 for k, v in config['java_paths'].items():
2121 env['JAVA%s_HOME' % k] = v
2123 missinglocale = True
2124 for k, v in env.items():
2125 if k == 'LANG' and v != 'C':
2126 missinglocale = False
2128 missinglocale = False
2130 env['LANG'] = 'en_US.UTF-8'
2132 if build is not None:
2133 path = build.ndk_path()
2134 paths = orig_path.split(os.pathsep)
2135 if path not in paths:
2136 paths = [path] + paths
2137 env['PATH'] = os.pathsep.join(paths)
2138 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2139 env[n] = build.ndk_path()
2142 def replace_build_vars(cmd, build):
2143 cmd = cmd.replace('$$COMMIT$$', build.commit)
2144 cmd = cmd.replace('$$VERSION$$', build.versionName)
2145 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2149 def replace_config_vars(cmd, build):
2150 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2151 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2152 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2153 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2154 if build is not None:
2155 cmd = replace_build_vars(cmd, build)
2159 def place_srclib(root_dir, number, libpath):
2162 relpath = os.path.relpath(libpath, root_dir)
2163 proppath = os.path.join(root_dir, 'project.properties')
2166 if os.path.isfile(proppath):
2167 with open(proppath, "r", encoding='iso-8859-1') as o:
2168 lines = o.readlines()
2170 with open(proppath, "w", encoding='iso-8859-1') as o:
2173 if line.startswith('android.library.reference.%d=' % number):
2174 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2179 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2182 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2185 def signer_fingerprint_short(sig):
2186 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2188 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2189 for a given pkcs7 signature.
2191 :param sig: Contents of an APK signing certificate.
2192 :returns: shortened signing-key fingerprint.
2194 return signer_fingerprint(sig)[:7]
2197 def signer_fingerprint(sig):
2198 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2200 Extracts hexadecimal sha256 signing-key fingerprint string
2201 for a given pkcs7 signature.
2203 :param: Contents of an APK signature.
2204 :returns: shortened signature fingerprint.
2206 cert_encoded = get_certificate(sig)
2207 return hashlib.sha256(cert_encoded).hexdigest()
2210 def apk_signer_fingerprint(apk_path):
2211 """Obtain sha256 signing-key fingerprint for APK.
2213 Extracts hexadecimal sha256 signing-key fingerprint string
2216 :param apkpath: path to APK
2217 :returns: signature fingerprint
2220 with zipfile.ZipFile(apk_path, 'r') as apk:
2221 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2224 logging.error("Found no signing certificates on %s" % apk_path)
2227 logging.error("Found multiple signing certificates on %s" % apk_path)
2230 cert = apk.read(certs[0])
2231 return signer_fingerprint(cert)
2234 def apk_signer_fingerprint_short(apk_path):
2235 """Obtain shortened sha256 signing-key fingerprint for APK.
2237 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2238 for a given pkcs7 APK.
2240 :param apk_path: path to APK
2241 :returns: shortened signing-key fingerprint
2243 return apk_signer_fingerprint(apk_path)[:7]
2246 def metadata_get_sigdir(appid, vercode=None):
2247 """Get signature directory for app"""
2249 return os.path.join('metadata', appid, 'signatures', vercode)
2251 return os.path.join('metadata', appid, 'signatures')
2254 def metadata_find_developer_signature(appid, vercode=None):
2255 """Tires to find the developer signature for given appid.
2257 This picks the first signature file found in metadata an returns its
2260 :returns: sha256 signing key fingerprint of the developer signing key.
2261 None in case no signature can not be found."""
2263 # fetch list of dirs for all versions of signatures
2266 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2268 appsigdir = metadata_get_sigdir(appid)
2269 if os.path.isdir(appsigdir):
2270 numre = re.compile('[0-9]+')
2271 for ver in os.listdir(appsigdir):
2272 if numre.match(ver):
2273 appversigdir = os.path.join(appsigdir, ver)
2274 appversigdirs.append(appversigdir)
2276 for sigdir in appversigdirs:
2277 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2278 glob.glob(os.path.join(sigdir, '*.EC')) + \
2279 glob.glob(os.path.join(sigdir, '*.RSA'))
2281 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2283 with open(sig, 'rb') as f:
2284 return signer_fingerprint(f.read())
2288 def metadata_find_signing_files(appid, vercode):
2289 """Gets a list of singed manifests and signatures.
2291 :param appid: app id string
2292 :param vercode: app version code
2293 :returns: a list of triplets for each signing key with following paths:
2294 (signature_file, singed_file, manifest_file)
2297 sigdir = metadata_get_sigdir(appid, vercode)
2298 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2299 glob.glob(os.path.join(sigdir, '*.EC')) + \
2300 glob.glob(os.path.join(sigdir, '*.RSA'))
2301 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2303 sf = extre.sub('.SF', sig)
2304 if os.path.isfile(sf):
2305 mf = os.path.join(sigdir, 'MANIFEST.MF')
2306 if os.path.isfile(mf):
2307 ret.append((sig, sf, mf))
2311 def metadata_find_developer_signing_files(appid, vercode):
2312 """Get developer signature files for specified app from metadata.
2314 :returns: A triplet of paths for signing files from metadata:
2315 (signature_file, singed_file, manifest_file)
2317 allsigningfiles = metadata_find_signing_files(appid, vercode)
2318 if allsigningfiles and len(allsigningfiles) == 1:
2319 return allsigningfiles[0]
2324 def apk_strip_signatures(signed_apk, strip_manifest=False):
2325 """Removes signatures from APK.
2327 :param signed_apk: path to apk file.
2328 :param strip_manifest: when set to True also the manifest file will
2329 be removed from the APK.
2331 with tempfile.TemporaryDirectory() as tmpdir:
2332 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2333 os.rename(signed_apk, tmp_apk)
2334 with ZipFile(tmp_apk, 'r') as in_apk:
2335 with ZipFile(signed_apk, 'w') as out_apk:
2336 for info in in_apk.infolist():
2337 if not apk_sigfile.match(info.filename):
2339 if info.filename != 'META-INF/MANIFEST.MF':
2340 buf = in_apk.read(info.filename)
2341 out_apk.writestr(info, buf)
2343 buf = in_apk.read(info.filename)
2344 out_apk.writestr(info, buf)
2347 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2348 """Implats a signature from metadata into an APK.
2350 Note: this changes there supplied APK in place. So copy it if you
2351 need the original to be preserved.
2353 :param apkpath: location of the apk
2355 # get list of available signature files in metadata
2356 with tempfile.TemporaryDirectory() as tmpdir:
2357 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2358 with ZipFile(apkpath, 'r') as in_apk:
2359 with ZipFile(apkwithnewsig, 'w') as out_apk:
2360 for sig_file in [signaturefile, signedfile, manifest]:
2361 with open(sig_file, 'rb') as fp:
2363 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2364 info.compress_type = zipfile.ZIP_DEFLATED
2365 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2366 out_apk.writestr(info, buf)
2367 for info in in_apk.infolist():
2368 if not apk_sigfile.match(info.filename):
2369 if info.filename != 'META-INF/MANIFEST.MF':
2370 buf = in_apk.read(info.filename)
2371 out_apk.writestr(info, buf)
2373 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2374 if p.returncode != 0:
2375 raise BuildException("Failed to align application")
2378 def apk_extract_signatures(apkpath, outdir, manifest=True):
2379 """Extracts a signature files from APK and puts them into target directory.
2381 :param apkpath: location of the apk
2382 :param outdir: folder where the extracted signature files will be stored
2383 :param manifest: (optionally) disable extracting manifest file
2385 with ZipFile(apkpath, 'r') as in_apk:
2386 for f in in_apk.infolist():
2387 if apk_sigfile.match(f.filename) or \
2388 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2389 newpath = os.path.join(outdir, os.path.basename(f.filename))
2390 with open(newpath, 'wb') as out_file:
2391 out_file.write(in_apk.read(f.filename))
2394 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2395 """Verify that two apks are the same
2397 One of the inputs is signed, the other is unsigned. The signature metadata
2398 is transferred from the signed to the unsigned apk, and then jarsigner is
2399 used to verify that the signature from the signed apk is also varlid for
2400 the unsigned one. If the APK given as unsigned actually does have a
2401 signature, it will be stripped out and ignored.
2403 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2404 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2405 into AndroidManifest.xml, but that makes the build not reproducible. So
2406 instead they are included as separate files in the APK's META-INF/ folder.
2407 If those files exist in the signed APK, they will be part of the signature
2408 and need to also be included in the unsigned APK for it to validate.
2410 :param signed_apk: Path to a signed apk file
2411 :param unsigned_apk: Path to an unsigned apk file expected to match it
2412 :param tmp_dir: Path to directory for temporary files
2413 :returns: None if the verification is successful, otherwise a string
2414 describing what went wrong.
2417 if not os.path.isfile(signed_apk):
2418 return 'can not verify: file does not exists: {}'.format(signed_apk)
2420 if not os.path.isfile(unsigned_apk):
2421 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2423 with ZipFile(signed_apk, 'r') as signed:
2424 meta_inf_files = ['META-INF/MANIFEST.MF']
2425 for f in signed.namelist():
2426 if apk_sigfile.match(f) \
2427 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2428 meta_inf_files.append(f)
2429 if len(meta_inf_files) < 3:
2430 return "Signature files missing from {0}".format(signed_apk)
2432 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2433 with ZipFile(unsigned_apk, 'r') as unsigned:
2434 # only read the signature from the signed APK, everything else from unsigned
2435 with ZipFile(tmp_apk, 'w') as tmp:
2436 for filename in meta_inf_files:
2437 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2438 for info in unsigned.infolist():
2439 if info.filename in meta_inf_files:
2440 logging.warning('Ignoring %s from %s',
2441 info.filename, unsigned_apk)
2443 if info.filename in tmp.namelist():
2444 return "duplicate filename found: " + info.filename
2445 tmp.writestr(info, unsigned.read(info.filename))
2447 verified = verify_apk_signature(tmp_apk)
2450 logging.info("...NOT verified - {0}".format(tmp_apk))
2451 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2452 os.path.dirname(unsigned_apk))
2454 logging.info("...successfully verified")
2458 def verify_jar_signature(jar):
2459 """Verifies the signature of a given JAR file.
2461 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2462 this has to turn on -strict then check for result 4, since this
2463 does not expect the signature to be from a CA-signed certificate.
2465 :raises: VerificationException() if the JAR's signature could not be verified
2469 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2470 raise VerificationException(_("The repository's index could not be verified."))
2473 def verify_apk_signature(apk, min_sdk_version=None):
2474 """verify the signature on an APK
2476 Try to use apksigner whenever possible since jarsigner is very
2477 shitty: unsigned APKs pass as "verified"! Warning, this does
2478 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2480 :returns: boolean whether the APK was verified
2482 if set_command_in_config('apksigner'):
2483 args = [config['apksigner'], 'verify']
2485 args += ['--min-sdk-version=' + min_sdk_version]
2486 return subprocess.call(args + [apk]) == 0
2488 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2490 verify_jar_signature(apk)
2497 def verify_old_apk_signature(apk):
2498 """verify the signature on an archived APK, supporting deprecated algorithms
2500 F-Droid aims to keep every single binary that it ever published. Therefore,
2501 it needs to be able to verify APK signatures that include deprecated/removed
2502 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2504 jarsigner passes unsigned APKs as "verified"! So this has to turn
2505 on -strict then check for result 4.
2507 :returns: boolean whether the APK was verified
2510 _java_security = os.path.join(os.getcwd(), '.java.security')
2511 with open(_java_security, 'w') as fp:
2512 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2514 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2515 '-strict', '-verify', apk]) == 4
2518 apk_badchars = re.compile('''[/ :;'"]''')
2521 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2524 Returns None if the apk content is the same (apart from the signing key),
2525 otherwise a string describing what's different, or what went wrong when
2526 trying to do the comparison.
2532 absapk1 = os.path.abspath(apk1)
2533 absapk2 = os.path.abspath(apk2)
2535 if set_command_in_config('diffoscope'):
2536 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2537 htmlfile = logfilename + '.diffoscope.html'
2538 textfile = logfilename + '.diffoscope.txt'
2539 if subprocess.call([config['diffoscope'],
2540 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2541 '--html', htmlfile, '--text', textfile,
2542 absapk1, absapk2]) != 0:
2543 return("Failed to unpack " + apk1)
2545 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2546 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2547 for d in [apk1dir, apk2dir]:
2548 if os.path.exists(d):
2551 os.mkdir(os.path.join(d, 'jar-xf'))
2553 if subprocess.call(['jar', 'xf',
2554 os.path.abspath(apk1)],
2555 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2556 return("Failed to unpack " + apk1)
2557 if subprocess.call(['jar', 'xf',
2558 os.path.abspath(apk2)],
2559 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2560 return("Failed to unpack " + apk2)
2562 if set_command_in_config('apktool'):
2563 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2565 return("Failed to unpack " + apk1)
2566 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2568 return("Failed to unpack " + apk2)
2570 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2571 lines = p.output.splitlines()
2572 if len(lines) != 1 or 'META-INF' not in lines[0]:
2573 if set_command_in_config('meld'):
2574 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2575 return("Unexpected diff output - " + p.output)
2577 # since everything verifies, delete the comparison to keep cruft down
2578 shutil.rmtree(apk1dir)
2579 shutil.rmtree(apk2dir)
2581 # If we get here, it seems like they're the same!
2585 def set_command_in_config(command):
2586 '''Try to find specified command in the path, if it hasn't been
2587 manually set in config.py. If found, it is added to the config
2588 dict. The return value says whether the command is available.
2591 if command in config:
2594 tmp = find_command(command)
2596 config[command] = tmp
2601 def find_command(command):
2602 '''find the full path of a command, or None if it can't be found in the PATH'''
2605 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2607 fpath, fname = os.path.split(command)
2612 for path in os.environ["PATH"].split(os.pathsep):
2613 path = path.strip('"')
2614 exe_file = os.path.join(path, command)
2615 if is_exe(exe_file):
2622 '''generate a random password for when generating keys'''
2623 h = hashlib.sha256()
2624 h.update(os.urandom(16)) # salt
2625 h.update(socket.getfqdn().encode('utf-8'))
2626 passwd = base64.b64encode(h.digest()).strip()
2627 return passwd.decode('utf-8')
2630 def genkeystore(localconfig):
2632 Generate a new key with password provided in :param localconfig and add it to new keystore
2633 :return: hexed public key, public key fingerprint
2635 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2636 keystoredir = os.path.dirname(localconfig['keystore'])
2637 if keystoredir is None or keystoredir == '':
2638 keystoredir = os.path.join(os.getcwd(), keystoredir)
2639 if not os.path.exists(keystoredir):
2640 os.makedirs(keystoredir, mode=0o700)
2643 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2644 'FDROID_KEY_PASS': localconfig['keypass'],
2646 p = FDroidPopen([config['keytool'], '-genkey',
2647 '-keystore', localconfig['keystore'],
2648 '-alias', localconfig['repo_keyalias'],
2649 '-keyalg', 'RSA', '-keysize', '4096',
2650 '-sigalg', 'SHA256withRSA',
2651 '-validity', '10000',
2652 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2653 '-keypass:env', 'FDROID_KEY_PASS',
2654 '-dname', localconfig['keydname']], envs=env_vars)
2655 if p.returncode != 0:
2656 raise BuildException("Failed to generate key", p.output)
2657 os.chmod(localconfig['keystore'], 0o0600)
2658 if not options.quiet:
2659 # now show the lovely key that was just generated
2660 p = FDroidPopen([config['keytool'], '-list', '-v',
2661 '-keystore', localconfig['keystore'],
2662 '-alias', localconfig['repo_keyalias'],
2663 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2664 logging.info(p.output.strip() + '\n\n')
2665 # get the public key
2666 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2667 '-keystore', localconfig['keystore'],
2668 '-alias', localconfig['repo_keyalias'],
2669 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2670 + config['smartcardoptions'],
2671 envs=env_vars, output=False, stderr_to_stdout=False)
2672 if p.returncode != 0 or len(p.output) < 20:
2673 raise BuildException("Failed to get public key", p.output)
2675 fingerprint = get_cert_fingerprint(pubkey)
2676 return hexlify(pubkey), fingerprint
2679 def get_cert_fingerprint(pubkey):
2681 Generate a certificate fingerprint the same way keytool does it
2682 (but with slightly different formatting)
2684 digest = hashlib.sha256(pubkey).digest()
2685 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2686 return " ".join(ret)
2689 def get_certificate(certificate_file):
2691 Extracts a certificate from the given file.
2692 :param certificate_file: file bytes (as string) representing the certificate
2693 :return: A binary representation of the certificate's public key, or None in case of error
2695 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2696 if content.getComponentByName('contentType') != rfc2315.signedData:
2698 content = decoder.decode(content.getComponentByName('content'),
2699 asn1Spec=rfc2315.SignedData())[0]
2701 certificates = content.getComponentByName('certificates')
2702 cert = certificates[0].getComponentByName('certificate')
2704 logging.error("Certificates not found.")
2706 return encoder.encode(cert)
2709 def load_stats_fdroid_signing_key_fingerprints():
2710 """Load list of signing-key fingerprints stored by fdroid publish from file.
2712 :returns: list of dictionanryies containing the singing-key fingerprints.
2714 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2715 if not os.path.isfile(jar_file):
2717 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2718 p = FDroidPopen(cmd, output=False)
2719 if p.returncode != 4:
2720 raise FDroidException("Signature validation of '{}' failed! "
2721 "Please run publish again to rebuild this file.".format(jar_file))
2723 jar_sigkey = apk_signer_fingerprint(jar_file)
2724 repo_key_sig = config.get('repo_key_sha256')
2726 if jar_sigkey != repo_key_sig:
2727 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2729 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2730 config['repo_key_sha256'] = jar_sigkey
2731 write_to_config(config, 'repo_key_sha256')
2733 with zipfile.ZipFile(jar_file, 'r') as f:
2734 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2737 def write_to_config(thisconfig, key, value=None, config_file=None):
2738 '''write a key/value to the local config.py
2740 NOTE: only supports writing string variables.
2742 :param thisconfig: config dictionary
2743 :param key: variable name in config.py to be overwritten/added
2744 :param value: optional value to be written, instead of fetched
2745 from 'thisconfig' dictionary.
2748 origkey = key + '_orig'
2749 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2750 cfg = config_file if config_file else 'config.py'
2752 # load config file, create one if it doesn't exist
2753 if not os.path.exists(cfg):
2754 open(cfg, 'a').close()
2755 logging.info("Creating empty " + cfg)
2756 with open(cfg, 'r', encoding="utf-8") as f:
2757 lines = f.readlines()
2759 # make sure the file ends with a carraige return
2761 if not lines[-1].endswith('\n'):
2764 # regex for finding and replacing python string variable
2765 # definitions/initializations
2766 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2767 repl = key + ' = "' + value + '"'
2768 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2769 repl2 = key + " = '" + value + "'"
2771 # If we replaced this line once, we make sure won't be a
2772 # second instance of this line for this key in the document.
2775 with open(cfg, 'w', encoding="utf-8") as f:
2777 if pattern.match(line) or pattern2.match(line):
2779 line = pattern.sub(repl, line)
2780 line = pattern2.sub(repl2, line)
2791 def parse_xml(path):
2792 return XMLElementTree.parse(path).getroot()
2795 def string_is_integer(string):
2803 def local_rsync(options, fromdir, todir):
2804 '''Rsync method for local to local copying of things
2806 This is an rsync wrapper with all the settings for safe use within
2807 the various fdroidserver use cases. This uses stricter rsync
2808 checking on all files since people using offline mode are already
2809 prioritizing security above ease and speed.
2812 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2813 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2814 if not options.no_checksum:
2815 rsyncargs.append('--checksum')
2817 rsyncargs += ['--verbose']
2819 rsyncargs += ['--quiet']
2820 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2821 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2822 raise FDroidException()
2825 def get_per_app_repos():
2826 '''per-app repos are dirs named with the packageName of a single app'''
2828 # Android packageNames are Java packages, they may contain uppercase or
2829 # lowercase letters ('A' through 'Z'), numbers, and underscores
2830 # ('_'). However, individual package name parts may only start with
2831 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2832 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2835 for root, dirs, files in os.walk(os.getcwd()):
2837 print('checking', root, 'for', d)
2838 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2839 # standard parts of an fdroid repo, so never packageNames
2842 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2848 def is_repo_file(filename):
2849 '''Whether the file in a repo is a build product to be delivered to users'''
2850 if isinstance(filename, str):
2851 filename = filename.encode('utf-8', errors="surrogateescape")
2852 return os.path.isfile(filename) \
2853 and not filename.endswith(b'.asc') \
2854 and not filename.endswith(b'.sig') \
2855 and os.path.basename(filename) not in [
2857 b'index_unsigned.jar',
2866 def get_examples_dir():
2867 '''Return the dir where the fdroidserver example files are available'''
2869 tmp = os.path.dirname(sys.argv[0])
2870 if os.path.basename(tmp) == 'bin':
2871 egg_links = glob.glob(os.path.join(tmp, '..',
2872 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2874 # installed from local git repo
2875 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2878 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2879 if not os.path.exists(examplesdir): # use UNIX layout
2880 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2882 # we're running straight out of the git repo
2883 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2884 examplesdir = prefix + '/examples'