3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
20 # common.py is imported by all modules, so do not import third-party
21 # libraries here as they will become a requirement for all commands.
40 import xml.etree.ElementTree as XMLElementTree
42 from binascii import hexlify
43 from datetime import datetime
44 from distutils.version import LooseVersion
45 from queue import Queue
46 from zipfile import ZipFile
48 from pyasn1.codec.der import decoder, encoder
49 from pyasn1_modules import rfc2315
50 from pyasn1.error import PyAsn1Error
52 from distutils.util import strtobool
54 import fdroidserver.metadata
55 from fdroidserver import _
56 from fdroidserver.exception import FDroidException, VCSException, BuildException, VerificationException
57 from .asynchronousfilereader import AsynchronousFileReader
60 # A signature block file with a .DSA, .RSA, or .EC extension
61 CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$')
62 APK_NAME_REGEX = re.compile(r'^([a-zA-Z][\w.]*)_(-?[0-9]+)_?([0-9a-f]{7})?\.apk')
63 STANDARD_FILE_NAME_REGEX = re.compile(r'^(\w[\w.]*)_(-?[0-9]+)\.\w+')
65 XMLElementTree.register_namespace('android', 'http://schemas.android.com/apk/res/android')
74 'sdk_path': "$ANDROID_HOME",
79 'r12b': "$ANDROID_NDK",
86 'build_tools': "25.0.2",
87 'force_build_tools': False,
92 'accepted_formats': ['txt', 'yml'],
93 'sync_from_local_copy_dir': False,
94 'allow_disabled_algorithms': False,
95 'per_app_repos': False,
96 'make_current_version_link': True,
97 'current_version_name_source': 'Name',
98 'update_stats': False,
100 'stats_server': None,
102 'stats_to_carbon': False,
104 'build_server_always': False,
105 'keystore': 'keystore.jks',
106 'smartcardoptions': [],
116 'repo_url': "https://MyFirstFDroidRepo.org/fdroid/repo",
117 'repo_name': "My First FDroid Repo Demo",
118 'repo_icon': "fdroid-icon.png",
119 'repo_description': '''
120 This is a repository of apps to be used with FDroid. Applications in this
121 repository are either official binaries built by the original application
122 developers, or are binaries built from source by the admin of f-droid.org
123 using the tools on https://gitlab.com/u/fdroid.
129 def setup_global_opts(parser):
130 parser.add_argument("-v", "--verbose", action="store_true", default=False,
131 help=_("Spew out even more information than normal"))
132 parser.add_argument("-q", "--quiet", action="store_true", default=False,
133 help=_("Restrict output to warnings and errors"))
136 def _add_java_paths_to_config(pathlist, thisconfig):
137 def path_version_key(s):
139 for u in re.split('[^0-9]+', s):
141 versionlist.append(int(u))
146 for d in sorted(pathlist, key=path_version_key):
147 if os.path.islink(d):
149 j = os.path.basename(d)
150 # the last one found will be the canonical one, so order appropriately
152 r'^1\.([6-9])\.0\.jdk$', # OSX
153 r'^jdk1\.([6-9])\.0_[0-9]+.jdk$', # OSX and Oracle tarball
154 r'^jdk1\.([6-9])\.0_[0-9]+$', # Oracle Windows
155 r'^jdk([6-9])-openjdk$', # Arch
156 r'^java-([6-9])-openjdk$', # Arch
157 r'^java-([6-9])-jdk$', # Arch (oracle)
158 r'^java-1\.([6-9])\.0-.*$', # RedHat
159 r'^java-([6-9])-oracle$', # Debian WebUpd8
160 r'^jdk-([6-9])-oracle-.*$', # Debian make-jpkg
161 r'^java-([6-9])-openjdk-[^c][^o][^m].*$', # Debian
163 m = re.match(regex, j)
166 for p in [d, os.path.join(d, 'Contents', 'Home')]:
167 if os.path.exists(os.path.join(p, 'bin', 'javac')):
168 thisconfig['java_paths'][m.group(1)] = p
171 def fill_config_defaults(thisconfig):
172 for k, v in default_config.items():
173 if k not in thisconfig:
176 # Expand paths (~users and $vars)
177 def expand_path(path):
181 path = os.path.expanduser(path)
182 path = os.path.expandvars(path)
187 for k in ['sdk_path', 'ant', 'mvn3', 'gradle', 'keystore', 'repo_icon']:
192 thisconfig[k + '_orig'] = v
194 # find all installed JDKs for keytool, jarsigner, and JAVA[6-9]_HOME env vars
195 if thisconfig['java_paths'] is None:
196 thisconfig['java_paths'] = dict()
198 pathlist += glob.glob('/usr/lib/jvm/j*[6-9]*')
199 pathlist += glob.glob('/usr/java/jdk1.[6-9]*')
200 pathlist += glob.glob('/System/Library/Java/JavaVirtualMachines/1.[6-9].0.jdk')
201 pathlist += glob.glob('/Library/Java/JavaVirtualMachines/*jdk*[6-9]*')
202 if os.getenv('JAVA_HOME') is not None:
203 pathlist.append(os.getenv('JAVA_HOME'))
204 if os.getenv('PROGRAMFILES') is not None:
205 pathlist += glob.glob(os.path.join(os.getenv('PROGRAMFILES'), 'Java', 'jdk1.[6-9].*'))
206 _add_java_paths_to_config(pathlist, thisconfig)
208 for java_version in ('7', '8', '9'):
209 if java_version not in thisconfig['java_paths']:
211 java_home = thisconfig['java_paths'][java_version]
212 jarsigner = os.path.join(java_home, 'bin', 'jarsigner')
213 if os.path.exists(jarsigner):
214 thisconfig['jarsigner'] = jarsigner
215 thisconfig['keytool'] = os.path.join(java_home, 'bin', 'keytool')
216 break # Java7 is preferred, so quit if found
218 for k in ['ndk_paths', 'java_paths']:
224 thisconfig[k][k2] = exp
225 thisconfig[k][k2 + '_orig'] = v
228 def regsub_file(pattern, repl, path):
229 with open(path, 'rb') as f:
231 text = re.sub(bytes(pattern, 'utf8'), bytes(repl, 'utf8'), text)
232 with open(path, 'wb') as f:
236 def read_config(opts, config_file='config.py'):
237 """Read the repository config
239 The config is read from config_file, which is in the current
240 directory when any of the repo management commands are used. If
241 there is a local metadata file in the git repo, then config.py is
242 not required, just use defaults.
245 global config, options
247 if config is not None:
254 if os.path.isfile(config_file):
255 logging.debug(_("Reading '{config_file}'").format(config_file=config_file))
256 with io.open(config_file, "rb") as f:
257 code = compile(f.read(), config_file, 'exec')
258 exec(code, None, config)
260 logging.warning(_("No 'config.py' found, using defaults."))
262 for k in ('mirrors', 'install_list', 'uninstall_list', 'serverwebroot', 'servergitroot'):
264 if not type(config[k]) in (str, list, tuple):
266 _("'{field}' will be in random order! Use () or [] brackets if order is important!")
269 # smartcardoptions must be a list since its command line args for Popen
270 if 'smartcardoptions' in config:
271 config['smartcardoptions'] = config['smartcardoptions'].split(' ')
272 elif 'keystore' in config and config['keystore'] == 'NONE':
273 # keystore='NONE' means use smartcard, these are required defaults
274 config['smartcardoptions'] = ['-storetype', 'PKCS11', '-providerName',
275 'SunPKCS11-OpenSC', '-providerClass',
276 'sun.security.pkcs11.SunPKCS11',
277 '-providerArg', 'opensc-fdroid.cfg']
279 if any(k in config for k in ["keystore", "keystorepass", "keypass"]):
280 st = os.stat(config_file)
281 if st.st_mode & stat.S_IRWXG or st.st_mode & stat.S_IRWXO:
282 logging.warning(_("unsafe permissions on '{config_file}' (should be 0600)!")
283 .format(config_file=config_file))
285 fill_config_defaults(config)
287 for k in ["repo_description", "archive_description"]:
289 config[k] = clean_description(config[k])
291 if 'serverwebroot' in config:
292 if isinstance(config['serverwebroot'], str):
293 roots = [config['serverwebroot']]
294 elif all(isinstance(item, str) for item in config['serverwebroot']):
295 roots = config['serverwebroot']
297 raise TypeError(_('only accepts strings, lists, and tuples'))
299 for rootstr in roots:
300 # since this is used with rsync, where trailing slashes have
301 # meaning, ensure there is always a trailing slash
302 if rootstr[-1] != '/':
304 rootlist.append(rootstr.replace('//', '/'))
305 config['serverwebroot'] = rootlist
307 if 'servergitmirrors' in config:
308 if isinstance(config['servergitmirrors'], str):
309 roots = [config['servergitmirrors']]
310 elif all(isinstance(item, str) for item in config['servergitmirrors']):
311 roots = config['servergitmirrors']
313 raise TypeError(_('only accepts strings, lists, and tuples'))
314 config['servergitmirrors'] = roots
319 def assert_config_keystore(config):
320 """Check weather keystore is configured correctly and raise exception if not."""
323 if 'repo_keyalias' not in config:
325 logging.critical(_("'repo_keyalias' not found in config.py!"))
326 if 'keystore' not in config:
328 logging.critical(_("'keystore' not found in config.py!"))
329 elif not os.path.exists(config['keystore']):
331 logging.critical("'" + config['keystore'] + "' does not exist!")
332 if 'keystorepass' not in config:
334 logging.critical(_("'keystorepass' not found in config.py!"))
335 if 'keypass' not in config:
337 logging.critical(_("'keypass' not found in config.py!"))
339 raise FDroidException("This command requires a signing key, " +
340 "you can create one using: fdroid update --create-key")
343 def find_sdk_tools_cmd(cmd):
344 '''find a working path to a tool from the Android SDK'''
347 if config is not None and 'sdk_path' in config and os.path.exists(config['sdk_path']):
348 # try to find a working path to this command, in all the recent possible paths
349 if 'build_tools' in config:
350 build_tools = os.path.join(config['sdk_path'], 'build-tools')
351 # if 'build_tools' was manually set and exists, check only that one
352 configed_build_tools = os.path.join(build_tools, config['build_tools'])
353 if os.path.exists(configed_build_tools):
354 tooldirs.append(configed_build_tools)
356 # no configed version, so hunt known paths for it
357 for f in sorted(os.listdir(build_tools), reverse=True):
358 if os.path.isdir(os.path.join(build_tools, f)):
359 tooldirs.append(os.path.join(build_tools, f))
360 tooldirs.append(build_tools)
361 sdk_tools = os.path.join(config['sdk_path'], 'tools')
362 if os.path.exists(sdk_tools):
363 tooldirs.append(sdk_tools)
364 sdk_platform_tools = os.path.join(config['sdk_path'], 'platform-tools')
365 if os.path.exists(sdk_platform_tools):
366 tooldirs.append(sdk_platform_tools)
367 tooldirs.append('/usr/bin')
369 path = os.path.join(d, cmd)
370 if os.path.isfile(path):
372 test_aapt_version(path)
374 # did not find the command, exit with error message
375 ensure_build_tools_exists(config)
378 def test_aapt_version(aapt):
379 '''Check whether the version of aapt is new enough'''
380 output = subprocess.check_output([aapt, 'version'], universal_newlines=True)
381 if output is None or output == '':
382 logging.error(_("'{path}' failed to execute!").format(path=aapt))
384 m = re.match(r'.*v([0-9]+)\.([0-9]+)[.-]?([0-9.-]*)', output)
389 # the Debian package has the version string like "v0.2-23.0.2"
390 if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'):
391 logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!")
394 logging.warning(_('Unknown version of aapt, might cause problems: ') + output)
397 def test_sdk_exists(thisconfig):
398 if 'sdk_path' not in thisconfig:
399 if 'aapt' in thisconfig and os.path.isfile(thisconfig['aapt']):
400 test_aapt_version(thisconfig['aapt'])
403 logging.error(_("'sdk_path' not set in 'config.py'!"))
405 if thisconfig['sdk_path'] == default_config['sdk_path']:
406 logging.error(_('No Android SDK found!'))
407 logging.error(_('You can use ANDROID_HOME to set the path to your SDK, i.e.:'))
408 logging.error('\texport ANDROID_HOME=/opt/android-sdk')
410 if not os.path.exists(thisconfig['sdk_path']):
411 logging.critical(_("Android SDK path '{path}' does not exist!")
412 .format(path=thisconfig['sdk_path']))
414 if not os.path.isdir(thisconfig['sdk_path']):
415 logging.critical(_("Android SDK path '{path}' is not a directory!")
416 .format(path=thisconfig['sdk_path']))
418 for d in ['build-tools', 'platform-tools', 'tools']:
419 if not os.path.isdir(os.path.join(thisconfig['sdk_path'], d)):
420 logging.critical(_("Android SDK '{path}' does not have '{dirname}' installed!")
421 .format(path=thisconfig['sdk_path'], dirname=d))
426 def ensure_build_tools_exists(thisconfig):
427 if not test_sdk_exists(thisconfig):
428 raise FDroidException(_("Android SDK not found!"))
429 build_tools = os.path.join(thisconfig['sdk_path'], 'build-tools')
430 versioned_build_tools = os.path.join(build_tools, thisconfig['build_tools'])
431 if not os.path.isdir(versioned_build_tools):
432 raise FDroidException(
433 _("Android build-tools path '{path}' does not exist!")
434 .format(path=versioned_build_tools))
437 def get_local_metadata_files():
438 '''get any metadata files local to an app's source repo
440 This tries to ignore anything that does not count as app metdata,
441 including emacs cruft ending in ~ and the .fdroid.key*pass.txt files.
444 return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
447 def read_pkg_args(args, allow_vercodes=False):
449 :param args: arguments in the form of multiple appid:[vc] strings
450 :returns: a dictionary with the set of vercodes specified for each package
458 if allow_vercodes and ':' in p:
459 package, vercode = p.split(':')
461 package, vercode = p, None
462 if package not in vercodes:
463 vercodes[package] = [vercode] if vercode else []
465 elif vercode and vercode not in vercodes[package]:
466 vercodes[package] += [vercode] if vercode else []
471 def read_app_args(args, allapps, allow_vercodes=False):
473 On top of what read_pkg_args does, this returns the whole app metadata, but
474 limiting the builds list to the builds matching the vercodes specified.
477 vercodes = read_pkg_args(args, allow_vercodes)
483 for appid, app in allapps.items():
484 if appid in vercodes:
487 if len(apps) != len(vercodes):
490 logging.critical(_("No such package: %s") % p)
491 raise FDroidException(_("Found invalid appids in arguments"))
493 raise FDroidException(_("No packages specified"))
496 for appid, app in apps.items():
500 app.builds = [b for b in app.builds if b.versionCode in vc]
501 if len(app.builds) != len(vercodes[appid]):
503 allvcs = [b.versionCode for b in app.builds]
504 for v in vercodes[appid]:
506 logging.critical(_("No such versionCode {versionCode} for app {appid}")
507 .format(versionCode=v, appid=appid))
510 raise FDroidException(_("Found invalid versionCodes for some apps"))
515 def get_extension(filename):
516 base, ext = os.path.splitext(filename)
519 return base, ext.lower()[1:]
522 def has_extension(filename, ext):
523 _ignored, f_ext = get_extension(filename)
527 publish_name_regex = re.compile(r"^(.+)_([0-9]+)\.(apk|zip)$")
530 def clean_description(description):
531 'Remove unneeded newlines and spaces from a block of description text'
533 # this is split up by paragraph to make removing the newlines easier
534 for paragraph in re.split(r'\n\n', description):
535 paragraph = re.sub('\r', '', paragraph)
536 paragraph = re.sub('\n', ' ', paragraph)
537 paragraph = re.sub(' {2,}', ' ', paragraph)
538 paragraph = re.sub('^\s*(\w)', r'\1', paragraph)
539 returnstring += paragraph + '\n\n'
540 return returnstring.rstrip('\n')
543 def publishednameinfo(filename):
544 filename = os.path.basename(filename)
545 m = publish_name_regex.match(filename)
547 result = (m.group(1), m.group(2))
548 except AttributeError:
549 raise FDroidException(_("Invalid name for published file: %s") % filename)
553 apk_release_filename = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)\.apk')
554 apk_release_filename_with_sigfp = re.compile('(?P<appid>[a-zA-Z0-9_\.]+)_(?P<vercode>[0-9]+)_(?P<sigfp>[0-9a-f]{7})\.apk')
557 def apk_parse_release_filename(apkname):
558 """Parses the name of an APK file according the F-Droids APK naming
559 scheme and returns the tokens.
561 WARNING: Returned values don't necessarily represent the APKs actual
562 properties, the are just paresed from the file name.
564 :returns: A triplet containing (appid, versionCode, signer), where appid
565 should be the package name, versionCode should be the integer
566 represion of the APKs version and signer should be the first 7 hex
567 digists of the sha256 signing key fingerprint which was used to sign
570 m = apk_release_filename_with_sigfp.match(apkname)
572 return m.group('appid'), m.group('vercode'), m.group('sigfp')
573 m = apk_release_filename.match(apkname)
575 return m.group('appid'), m.group('vercode'), None
576 return None, None, None
579 def get_release_filename(app, build):
581 return "%s_%s.%s" % (app.id, build.versionCode, get_file_extension(build.output))
583 return "%s_%s.apk" % (app.id, build.versionCode)
586 def get_toolsversion_logname(app, build):
587 return "%s_%s_toolsversion.log" % (app.id, build.versionCode)
590 def getsrcname(app, build):
591 return "%s_%s_src.tar.gz" % (app.id, build.versionCode)
603 return '%s (%s)' % (app.CurrentVersion, app.CurrentVersionCode)
606 def get_build_dir(app):
607 '''get the dir that this app will be built in'''
609 if app.RepoType == 'srclib':
610 return os.path.join('build', 'srclib', app.Repo)
612 return os.path.join('build', app.id)
616 '''checkout code from VCS and return instance of vcs and the build dir'''
617 build_dir = get_build_dir(app)
619 # Set up vcs interface and make sure we have the latest code...
620 logging.debug("Getting {0} vcs interface for {1}"
621 .format(app.RepoType, app.Repo))
622 if app.RepoType == 'git' and os.path.exists('.fdroid.yml'):
626 vcs = getvcs(app.RepoType, remote, build_dir)
628 return vcs, build_dir
631 def getvcs(vcstype, remote, local):
633 return vcs_git(remote, local)
634 if vcstype == 'git-svn':
635 return vcs_gitsvn(remote, local)
637 return vcs_hg(remote, local)
639 return vcs_bzr(remote, local)
640 if vcstype == 'srclib':
641 if local != os.path.join('build', 'srclib', remote):
642 raise VCSException("Error: srclib paths are hard-coded!")
643 return getsrclib(remote, os.path.join('build', 'srclib'), raw=True)
645 raise VCSException("Deprecated vcs type 'svn' - please use 'git-svn' instead")
646 raise VCSException("Invalid vcs type " + vcstype)
649 def getsrclibvcs(name):
650 if name not in fdroidserver.metadata.srclibs:
651 raise VCSException("Missing srclib " + name)
652 return fdroidserver.metadata.srclibs[name]['Repo Type']
657 def __init__(self, remote, local):
659 # svn, git-svn and bzr may require auth
661 if self.repotype() in ('git-svn', 'bzr'):
663 if self.repotype == 'git-svn':
664 raise VCSException("Authentication is not supported for git-svn")
665 self.username, remote = remote.split('@')
666 if ':' not in self.username:
667 raise VCSException(_("Password required with username"))
668 self.username, self.password = self.username.split(':')
672 self.clone_failed = False
673 self.refreshed = False
679 def clientversion(self):
680 versionstr = FDroidPopen(self.clientversioncmd()).output
681 return versionstr[0:versionstr.find('\n')]
683 def clientversioncmd(self):
686 def gotorevision(self, rev, refresh=True):
687 """Take the local repository to a clean version of the given
688 revision, which is specificed in the VCS's native
689 format. Beforehand, the repository can be dirty, or even
690 non-existent. If the repository does already exist locally, it
691 will be updated from the origin, but only once in the lifetime
692 of the vcs object. None is acceptable for 'rev' if you know
693 you are cloning a clean copy of the repo - otherwise it must
694 specify a valid revision.
697 if self.clone_failed:
698 raise VCSException(_("Downloading the repository already failed once, not trying again."))
700 # The .fdroidvcs-id file for a repo tells us what VCS type
701 # and remote that directory was created from, allowing us to drop it
702 # automatically if either of those things changes.
703 fdpath = os.path.join(self.local, '..',
704 '.fdroidvcs-' + os.path.basename(self.local))
705 fdpath = os.path.normpath(fdpath)
706 cdata = self.repotype() + ' ' + self.remote
709 if os.path.exists(self.local):
710 if os.path.exists(fdpath):
711 with open(fdpath, 'r') as f:
712 fsdata = f.read().strip()
717 logging.info("Repository details for %s changed - deleting" % (
721 logging.info("Repository details for %s missing - deleting" % (
724 shutil.rmtree(self.local)
728 self.refreshed = True
731 self.gotorevisionx(rev)
732 except FDroidException as e:
735 # If necessary, write the .fdroidvcs file.
736 if writeback and not self.clone_failed:
737 os.makedirs(os.path.dirname(fdpath), exist_ok=True)
738 with open(fdpath, 'w+') as f:
744 def gotorevisionx(self, rev): # pylint: disable=unused-argument
745 """Derived classes need to implement this.
747 It's called once basic checking has been performed.
749 raise VCSException("This VCS type doesn't define gotorevisionx")
751 # Initialise and update submodules
752 def initsubmodules(self):
753 raise VCSException('Submodules not supported for this vcs type')
755 # Get a list of all known tags
757 if not self._gettags:
758 raise VCSException('gettags not supported for this vcs type')
760 for tag in self._gettags():
761 if re.match('[-A-Za-z0-9_. /]+$', tag):
765 def latesttags(self):
766 """Get a list of all the known tags, sorted from newest to oldest"""
767 raise VCSException('latesttags not supported for this vcs type')
770 """Get current commit reference (hash, revision, etc)"""
771 raise VCSException('getref not supported for this vcs type')
774 """Returns the srclib (name, path) used in setting up the current revision, or None."""
783 def clientversioncmd(self):
784 return ['git', '--version']
786 def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
787 '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
789 While fetch/pull/clone respect the command line option flags,
790 it seems that submodule commands do not. They do seem to
791 follow whatever is in env vars, if the version of git is new
792 enough. So we just throw the kitchen sink at it to see what
799 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
800 git_config.append('-c')
801 git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
802 git_config.append('-c')
803 git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
804 git_config.append('-c')
805 git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
806 # add helpful tricks supported in git >= 2.3
807 ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
808 git_config.append('-c')
809 git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10
811 'GIT_TERMINAL_PROMPT': '0',
812 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3
814 return FDroidPopen(['git', ] + git_config + gitargs,
815 envs=envs, cwd=cwd, output=output)
818 """If the local directory exists, but is somehow not a git repository,
819 git will traverse up the directory tree until it finds one
820 that is (i.e. fdroidserver) and then we'll proceed to destroy
821 it! This is called as a safety check.
825 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
826 result = p.output.rstrip()
827 if not result.endswith(self.local):
828 raise VCSException('Repository mismatch')
830 def gotorevisionx(self, rev):
831 if not os.path.exists(self.local):
833 p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
834 if p.returncode != 0:
835 self.clone_failed = True
836 raise VCSException("Git clone failed", p.output)
840 # Discard any working tree changes
841 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
842 'git', 'reset', '--hard'], cwd=self.local, output=False)
843 if p.returncode != 0:
844 raise VCSException(_("Git reset failed"), p.output)
845 # Remove untracked files now, in case they're tracked in the target
846 # revision (it happens!)
847 p = FDroidPopen(['git', 'submodule', 'foreach', '--recursive',
848 'git', 'clean', '-dffx'], cwd=self.local, output=False)
849 if p.returncode != 0:
850 raise VCSException(_("Git clean failed"), p.output)
851 if not self.refreshed:
852 # Get latest commits and tags from remote
853 p = self.GitFetchFDroidPopen(['fetch', 'origin'])
854 if p.returncode != 0:
855 raise VCSException(_("Git fetch failed"), p.output)
856 p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
857 if p.returncode != 0:
858 raise VCSException(_("Git fetch failed"), p.output)
859 # Recreate origin/HEAD as git clone would do it, in case it disappeared
860 p = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--auto'], cwd=self.local, output=False)
861 if p.returncode != 0:
862 lines = p.output.splitlines()
863 if 'Multiple remote HEAD branches' not in lines[0]:
864 raise VCSException(_("Git remote set-head failed"), p.output)
865 branch = lines[1].split(' ')[-1]
866 p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False)
867 if p2.returncode != 0:
868 raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output)
869 self.refreshed = True
870 # origin/HEAD is the HEAD of the remote, e.g. the "default branch" on
871 # a github repo. Most of the time this is the same as origin/master.
872 rev = rev or 'origin/HEAD'
873 p = FDroidPopen(['git', 'checkout', '-f', rev], cwd=self.local, output=False)
874 if p.returncode != 0:
875 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
876 # Get rid of any uncontrolled files left behind
877 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
878 if p.returncode != 0:
879 raise VCSException(_("Git clean failed"), p.output)
881 def initsubmodules(self):
883 submfile = os.path.join(self.local, '.gitmodules')
884 if not os.path.isfile(submfile):
885 raise VCSException(_("No git submodules available"))
887 # fix submodules not accessible without an account and public key auth
888 with open(submfile, 'r') as f:
889 lines = f.readlines()
890 with open(submfile, 'w') as f:
892 for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
893 line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
896 p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
897 if p.returncode != 0:
898 raise VCSException(_("Git submodule sync failed"), p.output)
899 p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
900 if p.returncode != 0:
901 raise VCSException(_("Git submodule update failed"), p.output)
905 p = FDroidPopen(['git', 'tag'], cwd=self.local, output=False)
906 return p.output.splitlines()
908 tag_format = re.compile(r'tag: ([^),]*)')
910 def latesttags(self):
912 p = FDroidPopen(['git', 'log', '--tags',
913 '--simplify-by-decoration', '--pretty=format:%d'],
914 cwd=self.local, output=False)
916 for line in p.output.splitlines():
917 for tag in self.tag_format.findall(line):
922 class vcs_gitsvn(vcs):
927 def clientversioncmd(self):
928 return ['git', 'svn', '--version']
931 """If the local directory exists, but is somehow not a git repository,
932 git will traverse up the directory tree until it finds one that
933 is (i.e. fdroidserver) and then we'll proceed to destory it!
934 This is called as a safety check.
937 p = FDroidPopen(['git', 'rev-parse', '--show-toplevel'], cwd=self.local, output=False)
938 result = p.output.rstrip()
939 if not result.endswith(self.local):
940 raise VCSException('Repository mismatch')
942 def gotorevisionx(self, rev):
943 if not os.path.exists(self.local):
945 gitsvn_args = ['git', 'svn', 'clone']
946 if ';' in self.remote:
947 remote_split = self.remote.split(';')
948 for i in remote_split[1:]:
949 if i.startswith('trunk='):
950 gitsvn_args.extend(['-T', i[6:]])
951 elif i.startswith('tags='):
952 gitsvn_args.extend(['-t', i[5:]])
953 elif i.startswith('branches='):
954 gitsvn_args.extend(['-b', i[9:]])
955 gitsvn_args.extend([remote_split[0], self.local])
956 p = FDroidPopen(gitsvn_args, output=False)
957 if p.returncode != 0:
958 self.clone_failed = True
959 raise VCSException("Git svn clone failed", p.output)
961 gitsvn_args.extend([self.remote, self.local])
962 p = FDroidPopen(gitsvn_args, output=False)
963 if p.returncode != 0:
964 self.clone_failed = True
965 raise VCSException("Git svn clone failed", p.output)
969 # Discard any working tree changes
970 p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False)
971 if p.returncode != 0:
972 raise VCSException("Git reset failed", p.output)
973 # Remove untracked files now, in case they're tracked in the target
974 # revision (it happens!)
975 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
976 if p.returncode != 0:
977 raise VCSException("Git clean failed", p.output)
978 if not self.refreshed:
979 # Get new commits, branches and tags from repo
980 p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False)
981 if p.returncode != 0:
982 raise VCSException("Git svn fetch failed")
983 p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False)
984 if p.returncode != 0:
985 raise VCSException("Git svn rebase failed", p.output)
986 self.refreshed = True
988 rev = rev or 'master'
990 nospaces_rev = rev.replace(' ', '%20')
991 # Try finding a svn tag
992 for treeish in ['origin/', '']:
993 p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False)
994 if p.returncode == 0:
996 if p.returncode != 0:
997 # No tag found, normal svn rev translation
998 # Translate svn rev into git format
999 rev_split = rev.split('/')
1002 for treeish in ['origin/', '']:
1003 if len(rev_split) > 1:
1004 treeish += rev_split[0]
1005 svn_rev = rev_split[1]
1008 # if no branch is specified, then assume trunk (i.e. 'master' branch):
1012 svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev
1014 p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False)
1015 git_rev = p.output.rstrip()
1017 if p.returncode == 0 and git_rev:
1020 if p.returncode != 0 or not git_rev:
1021 # Try a plain git checkout as a last resort
1022 p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False)
1023 if p.returncode != 0:
1024 raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output)
1026 # Check out the git rev equivalent to the svn rev
1027 p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False)
1028 if p.returncode != 0:
1029 raise VCSException(_("Git checkout of '%s' failed") % rev, p.output)
1031 # Get rid of any uncontrolled files left behind
1032 p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False)
1033 if p.returncode != 0:
1034 raise VCSException(_("Git clean failed"), p.output)
1038 for treeish in ['origin/', '']:
1039 d = os.path.join(self.local, '.git', 'svn', 'refs', 'remotes', treeish, 'tags')
1040 if os.path.isdir(d):
1041 return os.listdir(d)
1045 p = FDroidPopen(['git', 'svn', 'find-rev', 'HEAD'], cwd=self.local, output=False)
1046 if p.returncode != 0:
1048 return p.output.strip()
1056 def clientversioncmd(self):
1057 return ['hg', '--version']
1059 def gotorevisionx(self, rev):
1060 if not os.path.exists(self.local):
1061 p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False)
1062 if p.returncode != 0:
1063 self.clone_failed = True
1064 raise VCSException("Hg clone failed", p.output)
1066 p = FDroidPopen(['hg', 'status', '-uS'], cwd=self.local, output=False)
1067 if p.returncode != 0:
1068 raise VCSException("Hg status failed", p.output)
1069 for line in p.output.splitlines():
1070 if not line.startswith('? '):
1071 raise VCSException("Unexpected output from hg status -uS: " + line)
1072 FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False)
1073 if not self.refreshed:
1074 p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False)
1075 if p.returncode != 0:
1076 raise VCSException("Hg pull failed", p.output)
1077 self.refreshed = True
1079 rev = rev or 'default'
1082 p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False)
1083 if p.returncode != 0:
1084 raise VCSException("Hg checkout of '%s' failed" % rev, p.output)
1085 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1086 # Also delete untracked files, we have to enable purge extension for that:
1087 if "'purge' is provided by the following extension" in p.output:
1088 with open(os.path.join(self.local, '.hg', 'hgrc'), "a") as myfile:
1089 myfile.write("\n[extensions]\nhgext.purge=\n")
1090 p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False)
1091 if p.returncode != 0:
1092 raise VCSException("HG purge failed", p.output)
1093 elif p.returncode != 0:
1094 raise VCSException("HG purge failed", p.output)
1097 p = FDroidPopen(['hg', 'tags', '-q'], cwd=self.local, output=False)
1098 return p.output.splitlines()[1:]
1106 def clientversioncmd(self):
1107 return ['bzr', '--version']
1109 def gotorevisionx(self, rev):
1110 if not os.path.exists(self.local):
1111 p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False)
1112 if p.returncode != 0:
1113 self.clone_failed = True
1114 raise VCSException("Bzr branch failed", p.output)
1116 p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False)
1117 if p.returncode != 0:
1118 raise VCSException("Bzr revert failed", p.output)
1119 if not self.refreshed:
1120 p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False)
1121 if p.returncode != 0:
1122 raise VCSException("Bzr update failed", p.output)
1123 self.refreshed = True
1125 revargs = list(['-r', rev] if rev else [])
1126 p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False)
1127 if p.returncode != 0:
1128 raise VCSException("Bzr revert of '%s' failed" % rev, p.output)
1131 p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False)
1132 return [tag.split(' ')[0].strip() for tag in
1133 p.output.splitlines()]
1136 def unescape_string(string):
1139 if string[0] == '"' and string[-1] == '"':
1142 return string.replace("\\'", "'")
1145 def retrieve_string(app_dir, string, xmlfiles=None):
1147 if not string.startswith('@string/'):
1148 return unescape_string(string)
1150 if xmlfiles is None:
1153 os.path.join(app_dir, 'res'),
1154 os.path.join(app_dir, 'src', 'main', 'res'),
1156 for root, dirs, files in os.walk(res_dir):
1157 if os.path.basename(root) == 'values':
1158 xmlfiles += [os.path.join(root, x) for x in files if x.endswith('.xml')]
1160 name = string[len('@string/'):]
1162 def element_content(element):
1163 if element.text is None:
1165 s = XMLElementTree.tostring(element, encoding='utf-8', method='text')
1166 return s.decode('utf-8').strip()
1168 for path in xmlfiles:
1169 if not os.path.isfile(path):
1171 xml = parse_xml(path)
1172 element = xml.find('string[@name="' + name + '"]')
1173 if element is not None:
1174 content = element_content(element)
1175 return retrieve_string(app_dir, content, xmlfiles)
1180 def retrieve_string_singleline(app_dir, string, xmlfiles=None):
1181 return retrieve_string(app_dir, string, xmlfiles).replace('\n', ' ').strip()
1184 def manifest_paths(app_dir, flavours):
1185 '''Return list of existing files that will be used to find the highest vercode'''
1187 possible_manifests = \
1188 [os.path.join(app_dir, 'AndroidManifest.xml'),
1189 os.path.join(app_dir, 'src', 'main', 'AndroidManifest.xml'),
1190 os.path.join(app_dir, 'src', 'AndroidManifest.xml'),
1191 os.path.join(app_dir, 'build.gradle')]
1193 for flavour in flavours:
1194 if flavour == 'yes':
1196 possible_manifests.append(
1197 os.path.join(app_dir, 'src', flavour, 'AndroidManifest.xml'))
1199 return [path for path in possible_manifests if os.path.isfile(path)]
1202 def fetch_real_name(app_dir, flavours):
1203 '''Retrieve the package name. Returns the name, or None if not found.'''
1204 for path in manifest_paths(app_dir, flavours):
1205 if not has_extension(path, 'xml') or not os.path.isfile(path):
1207 logging.debug("fetch_real_name: Checking manifest at " + path)
1208 xml = parse_xml(path)
1209 app = xml.find('application')
1212 if "{http://schemas.android.com/apk/res/android}label" not in app.attrib:
1214 label = app.attrib["{http://schemas.android.com/apk/res/android}label"]
1215 result = retrieve_string_singleline(app_dir, label)
1217 result = result.strip()
1222 def get_library_references(root_dir):
1224 proppath = os.path.join(root_dir, 'project.properties')
1225 if not os.path.isfile(proppath):
1227 with open(proppath, 'r', encoding='iso-8859-1') as f:
1229 if not line.startswith('android.library.reference.'):
1231 path = line.split('=')[1].strip()
1232 relpath = os.path.join(root_dir, path)
1233 if not os.path.isdir(relpath):
1235 logging.debug("Found subproject at %s" % path)
1236 libraries.append(path)
1240 def ant_subprojects(root_dir):
1241 subprojects = get_library_references(root_dir)
1242 for subpath in subprojects:
1243 subrelpath = os.path.join(root_dir, subpath)
1244 for p in get_library_references(subrelpath):
1245 relp = os.path.normpath(os.path.join(subpath, p))
1246 if relp not in subprojects:
1247 subprojects.insert(0, relp)
1251 def remove_debuggable_flags(root_dir):
1252 # Remove forced debuggable flags
1253 logging.debug("Removing debuggable flags from %s" % root_dir)
1254 for root, dirs, files in os.walk(root_dir):
1255 if 'AndroidManifest.xml' in files and os.path.isfile(os.path.join(root, 'AndroidManifest.xml')):
1256 regsub_file(r'android:debuggable="[^"]*"',
1258 os.path.join(root, 'AndroidManifest.xml'))
1261 vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search
1262 vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search
1263 psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search
1266 def app_matches_packagename(app, package):
1269 appid = app.UpdateCheckName or app.id
1270 if appid is None or appid == "Ignore":
1272 return appid == package
1275 def parse_androidmanifests(paths, app):
1277 Extract some information from the AndroidManifest.xml at the given path.
1278 Returns (version, vercode, package), any or all of which might be None.
1279 All values returned are strings.
1282 ignoreversions = app.UpdateCheckIgnore
1283 ignoresearch = re.compile(ignoreversions).search if ignoreversions else None
1286 return (None, None, None)
1294 if not os.path.isfile(path):
1297 logging.debug(_("Parsing manifest at '{path}'").format(path=path))
1303 if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle:
1304 flavour = app.builds[-1].gradle[-1]
1306 if has_extension(path, 'gradle'):
1307 # first try to get version name and code from correct flavour
1308 with open(path, 'r') as f:
1309 buildfile = f.read()
1311 regex_string = r"" + flavour + ".*?}"
1312 search = re.compile(regex_string, re.DOTALL)
1313 result = search.search(buildfile)
1315 if result is not None:
1316 resultgroup = result.group()
1319 matches = psearch_g(resultgroup)
1321 s = matches.group(2)
1322 if app_matches_packagename(app, s):
1325 matches = vnsearch_g(resultgroup)
1327 version = matches.group(2)
1329 matches = vcsearch_g(resultgroup)
1331 vercode = matches.group(1)
1333 # fall back to parse file line by line
1334 with open(path, 'r') as f:
1336 if gradle_comment.match(line):
1338 # Grab first occurence of each to avoid running into
1339 # alternative flavours and builds.
1341 matches = psearch_g(line)
1343 s = matches.group(2)
1344 if app_matches_packagename(app, s):
1347 matches = vnsearch_g(line)
1349 version = matches.group(2)
1351 matches = vcsearch_g(line)
1353 vercode = matches.group(1)
1356 xml = parse_xml(path)
1357 if "package" in xml.attrib:
1358 s = xml.attrib["package"]
1359 if app_matches_packagename(app, s):
1361 if "{http://schemas.android.com/apk/res/android}versionName" in xml.attrib:
1362 version = xml.attrib["{http://schemas.android.com/apk/res/android}versionName"]
1363 base_dir = os.path.dirname(path)
1364 version = retrieve_string_singleline(base_dir, version)
1365 if "{http://schemas.android.com/apk/res/android}versionCode" in xml.attrib:
1366 a = xml.attrib["{http://schemas.android.com/apk/res/android}versionCode"]
1367 if string_is_integer(a):
1370 logging.warning(_("Problem with xml at '{path}'").format(path=path))
1372 # Remember package name, may be defined separately from version+vercode
1374 package = max_package
1376 logging.debug("..got package={0}, version={1}, vercode={2}"
1377 .format(package, version, vercode))
1379 # Always grab the package name and version name in case they are not
1380 # together with the highest version code
1381 if max_package is None and package is not None:
1382 max_package = package
1383 if max_version is None and version is not None:
1384 max_version = version
1386 if vercode is not None \
1387 and (max_vercode is None or vercode > max_vercode):
1388 if not ignoresearch or not ignoresearch(version):
1389 if version is not None:
1390 max_version = version
1391 if vercode is not None:
1392 max_vercode = vercode
1393 if package is not None:
1394 max_package = package
1396 max_version = "Ignore"
1398 if max_version is None:
1399 max_version = "Unknown"
1401 if max_package and not is_valid_package_name(max_package):
1402 raise FDroidException(_("Invalid package name {0}").format(max_package))
1404 return (max_version, max_vercode, max_package)
1407 def is_valid_package_name(name):
1408 return re.match("[A-Za-z_][A-Za-z_0-9.]+$", name)
1411 def getsrclib(spec, srclib_dir, subdir=None, basepath=False,
1412 raw=False, prepare=True, preponly=False, refresh=True,
1414 """Get the specified source library.
1416 Returns the path to it. Normally this is the path to be used when
1417 referencing it, which may be a subdirectory of the actual project. If
1418 you want the base directory of the project, pass 'basepath=True'.
1427 name, ref = spec.split('@')
1429 number, name = name.split(':', 1)
1431 name, subdir = name.split('/', 1)
1433 if name not in fdroidserver.metadata.srclibs:
1434 raise VCSException('srclib ' + name + ' not found.')
1436 srclib = fdroidserver.metadata.srclibs[name]
1438 sdir = os.path.join(srclib_dir, name)
1441 vcs = getvcs(srclib["Repo Type"], srclib["Repo"], sdir)
1442 vcs.srclib = (name, number, sdir)
1444 vcs.gotorevision(ref, refresh)
1451 libdir = os.path.join(sdir, subdir)
1452 elif srclib["Subdir"]:
1453 for subdir in srclib["Subdir"]:
1454 libdir_candidate = os.path.join(sdir, subdir)
1455 if os.path.exists(libdir_candidate):
1456 libdir = libdir_candidate
1462 remove_signing_keys(sdir)
1463 remove_debuggable_flags(sdir)
1467 if srclib["Prepare"]:
1468 cmd = replace_config_vars(srclib["Prepare"], build)
1470 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir)
1471 if p.returncode != 0:
1472 raise BuildException("Error running prepare command for srclib %s"
1478 return (name, number, libdir)
1481 gradle_version_regex = re.compile(r"[^/]*'com\.android\.tools\.build:gradle:([^\.]+\.[^\.]+).*'.*")
1484 def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=False, refresh=True):
1485 """ Prepare the source code for a particular build
1487 :param vcs: the appropriate vcs object for the application
1488 :param app: the application details from the metadata
1489 :param build: the build details from the metadata
1490 :param build_dir: the path to the build directory, usually 'build/app.id'
1491 :param srclib_dir: the path to the source libraries directory, usually 'build/srclib'
1492 :param extlib_dir: the path to the external libraries directory, usually 'build/extlib'
1494 Returns the (root, srclibpaths) where:
1495 :param root: is the root directory, which may be the same as 'build_dir' or may
1496 be a subdirectory of it.
1497 :param srclibpaths: is information on the srclibs being used
1500 # Optionally, the actual app source can be in a subdirectory
1502 root_dir = os.path.join(build_dir, build.subdir)
1504 root_dir = build_dir
1506 # Get a working copy of the right revision
1507 logging.info("Getting source for revision " + build.commit)
1508 vcs.gotorevision(build.commit, refresh)
1510 # Initialise submodules if required
1511 if build.submodules:
1512 logging.info(_("Initialising submodules"))
1513 vcs.initsubmodules()
1515 # Check that a subdir (if we're using one) exists. This has to happen
1516 # after the checkout, since it might not exist elsewhere
1517 if not os.path.exists(root_dir):
1518 raise BuildException('Missing subdir ' + root_dir)
1520 # Run an init command if one is required
1522 cmd = replace_config_vars(build.init, build)
1523 logging.info("Running 'init' commands in %s" % root_dir)
1525 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1526 if p.returncode != 0:
1527 raise BuildException("Error running init command for %s:%s" %
1528 (app.id, build.versionName), p.output)
1530 # Apply patches if any
1532 logging.info("Applying patches")
1533 for patch in build.patch:
1534 patch = patch.strip()
1535 logging.info("Applying " + patch)
1536 patch_path = os.path.join('metadata', app.id, patch)
1537 p = FDroidPopen(['patch', '-p1', '-i', os.path.abspath(patch_path)], cwd=build_dir)
1538 if p.returncode != 0:
1539 raise BuildException("Failed to apply patch %s" % patch_path)
1541 # Get required source libraries
1544 logging.info("Collecting source libraries")
1545 for lib in build.srclibs:
1546 srclibpaths.append(getsrclib(lib, srclib_dir, build, preponly=onserver,
1547 refresh=refresh, build=build))
1549 for name, number, libpath in srclibpaths:
1550 place_srclib(root_dir, int(number) if number else None, libpath)
1552 basesrclib = vcs.getsrclib()
1553 # If one was used for the main source, add that too.
1555 srclibpaths.append(basesrclib)
1557 # Update the local.properties file
1558 localprops = [os.path.join(build_dir, 'local.properties')]
1560 parts = build.subdir.split(os.sep)
1563 cur = os.path.join(cur, d)
1564 localprops += [os.path.join(cur, 'local.properties')]
1565 for path in localprops:
1567 if os.path.isfile(path):
1568 logging.info("Updating local.properties file at %s" % path)
1569 with open(path, 'r', encoding='iso-8859-1') as f:
1573 logging.info("Creating local.properties file at %s" % path)
1574 # Fix old-fashioned 'sdk-location' by copying
1575 # from sdk.dir, if necessary
1577 sdkloc = re.match(r".*^sdk.dir=(\S+)$.*", props,
1578 re.S | re.M).group(1)
1579 props += "sdk-location=%s\n" % sdkloc
1581 props += "sdk.dir=%s\n" % config['sdk_path']
1582 props += "sdk-location=%s\n" % config['sdk_path']
1583 ndk_path = build.ndk_path()
1584 # if for any reason the path isn't valid or the directory
1585 # doesn't exist, some versions of Gradle will error with a
1586 # cryptic message (even if the NDK is not even necessary).
1587 # https://gitlab.com/fdroid/fdroidserver/issues/171
1588 if ndk_path and os.path.exists(ndk_path):
1590 props += "ndk.dir=%s\n" % ndk_path
1591 props += "ndk-location=%s\n" % ndk_path
1592 # Add java.encoding if necessary
1594 props += "java.encoding=%s\n" % build.encoding
1595 with open(path, 'w', encoding='iso-8859-1') as f:
1599 if build.build_method() == 'gradle':
1600 flavours = build.gradle
1603 n = build.target.split('-')[1]
1604 regsub_file(r'compileSdkVersion[ =]+[0-9]+',
1605 r'compileSdkVersion %s' % n,
1606 os.path.join(root_dir, 'build.gradle'))
1608 # Remove forced debuggable flags
1609 remove_debuggable_flags(root_dir)
1611 # Insert version code and number into the manifest if necessary
1612 if build.forceversion:
1613 logging.info("Changing the version name")
1614 for path in manifest_paths(root_dir, flavours):
1615 if not os.path.isfile(path):
1617 if has_extension(path, 'xml'):
1618 regsub_file(r'android:versionName="[^"]*"',
1619 r'android:versionName="%s"' % build.versionName,
1621 elif has_extension(path, 'gradle'):
1622 regsub_file(r"""(\s*)versionName[\s'"=]+.*""",
1623 r"""\1versionName '%s'""" % build.versionName,
1626 if build.forcevercode:
1627 logging.info("Changing the version code")
1628 for path in manifest_paths(root_dir, flavours):
1629 if not os.path.isfile(path):
1631 if has_extension(path, 'xml'):
1632 regsub_file(r'android:versionCode="[^"]*"',
1633 r'android:versionCode="%s"' % build.versionCode,
1635 elif has_extension(path, 'gradle'):
1636 regsub_file(r'versionCode[ =]+[0-9]+',
1637 r'versionCode %s' % build.versionCode,
1640 # Delete unwanted files
1642 logging.info(_("Removing specified files"))
1643 for part in getpaths(build_dir, build.rm):
1644 dest = os.path.join(build_dir, part)
1645 logging.info("Removing {0}".format(part))
1646 if os.path.lexists(dest):
1647 # rmtree can only handle directories that are not symlinks, so catch anything else
1648 if not os.path.isdir(dest) or os.path.islink(dest):
1653 logging.info("...but it didn't exist")
1655 remove_signing_keys(build_dir)
1657 # Add required external libraries
1659 logging.info("Collecting prebuilt libraries")
1660 libsdir = os.path.join(root_dir, 'libs')
1661 if not os.path.exists(libsdir):
1663 for lib in build.extlibs:
1665 logging.info("...installing extlib {0}".format(lib))
1666 libf = os.path.basename(lib)
1667 libsrc = os.path.join(extlib_dir, lib)
1668 if not os.path.exists(libsrc):
1669 raise BuildException("Missing extlib file {0}".format(libsrc))
1670 shutil.copyfile(libsrc, os.path.join(libsdir, libf))
1672 # Run a pre-build command if one is required
1674 logging.info("Running 'prebuild' commands in %s" % root_dir)
1676 cmd = replace_config_vars(build.prebuild, build)
1678 # Substitute source library paths into prebuild commands
1679 for name, number, libpath in srclibpaths:
1680 libpath = os.path.relpath(libpath, root_dir)
1681 cmd = cmd.replace('$$' + name + '$$', libpath)
1683 p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir)
1684 if p.returncode != 0:
1685 raise BuildException("Error running prebuild command for %s:%s" %
1686 (app.id, build.versionName), p.output)
1688 # Generate (or update) the ant build file, build.xml...
1689 if build.build_method() == 'ant' and build.androidupdate != ['no']:
1690 parms = ['android', 'update', 'lib-project']
1691 lparms = ['android', 'update', 'project']
1694 parms += ['-t', build.target]
1695 lparms += ['-t', build.target]
1696 if build.androidupdate:
1697 update_dirs = build.androidupdate
1699 update_dirs = ant_subprojects(root_dir) + ['.']
1701 for d in update_dirs:
1702 subdir = os.path.join(root_dir, d)
1704 logging.debug("Updating main project")
1705 cmd = parms + ['-p', d]
1707 logging.debug("Updating subproject %s" % d)
1708 cmd = lparms + ['-p', d]
1709 p = SdkToolsPopen(cmd, cwd=root_dir)
1710 # Check to see whether an error was returned without a proper exit
1711 # code (this is the case for the 'no target set or target invalid'
1713 if p.returncode != 0 or p.output.startswith("Error: "):
1714 raise BuildException("Failed to update project at %s" % d, p.output)
1715 # Clean update dirs via ant
1717 logging.info("Cleaning subproject %s" % d)
1718 p = FDroidPopen(['ant', 'clean'], cwd=subdir)
1720 return (root_dir, srclibpaths)
1723 def getpaths_map(build_dir, globpaths):
1724 """Extend via globbing the paths from a field and return them as a map from original path to resulting paths"""
1728 full_path = os.path.join(build_dir, p)
1729 full_path = os.path.normpath(full_path)
1730 paths[p] = [r[len(build_dir) + 1:] for r in glob.glob(full_path)]
1732 raise FDroidException("glob path '%s' did not match any files/dirs" % p)
1736 def getpaths(build_dir, globpaths):
1737 """Extend via globbing the paths from a field and return them as a set"""
1738 paths_map = getpaths_map(build_dir, globpaths)
1740 for k, v in paths_map.items():
1747 return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
1751 """permanent store of existing APKs with the date they were added
1753 This is currently the only way to permanently store the "updated"
1758 '''Load filename/date info about previously seen APKs
1760 Since the appid and date strings both will never have spaces,
1761 this is parsed as a list from the end to allow the filename to
1762 have any combo of spaces.
1765 self.path = os.path.join('stats', 'known_apks.txt')
1767 if os.path.isfile(self.path):
1768 with open(self.path, 'r', encoding='utf8') as f:
1770 t = line.rstrip().split(' ')
1772 self.apks[t[0]] = (t[1], None)
1775 date = datetime.strptime(t[-1], '%Y-%m-%d')
1776 filename = line[0:line.rfind(appid) - 1]
1777 self.apks[filename] = (appid, date)
1778 self.changed = False
1780 def writeifchanged(self):
1781 if not self.changed:
1784 if not os.path.exists('stats'):
1788 for apk, app in self.apks.items():
1790 line = apk + ' ' + appid
1792 line += ' ' + added.strftime('%Y-%m-%d')
1795 with open(self.path, 'w', encoding='utf8') as f:
1796 for line in sorted(lst, key=natural_key):
1797 f.write(line + '\n')
1799 def recordapk(self, apkName, app, default_date=None):
1801 Record an apk (if it's new, otherwise does nothing)
1802 Returns the date it was added as a datetime instance
1804 if apkName not in self.apks:
1805 if default_date is None:
1806 default_date = datetime.utcnow()
1807 self.apks[apkName] = (app, default_date)
1809 _ignored, added = self.apks[apkName]
1812 def getapp(self, apkname):
1813 """Look up information - given the 'apkname', returns (app id, date added/None).
1815 Or returns None for an unknown apk.
1817 if apkname in self.apks:
1818 return self.apks[apkname]
1821 def getlatest(self, num):
1822 """Get the most recent 'num' apps added to the repo, as a list of package ids with the most recent first"""
1824 for apk, app in self.apks.items():
1828 if apps[appid] > added:
1832 sortedapps = sorted(apps.items(), key=operator.itemgetter(1))[-num:]
1833 lst = [app for app, _ignored in sortedapps]
1838 def get_file_extension(filename):
1839 """get the normalized file extension, can be blank string but never None"""
1840 if isinstance(filename, bytes):
1841 filename = filename.decode('utf-8')
1842 return os.path.splitext(filename)[1].lower()[1:]
1845 def get_apk_debuggable_aapt(apkfile):
1846 p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'],
1848 if p.returncode != 0:
1849 raise FDroidException(_("Failed to get APK manifest information"))
1850 for line in p.output.splitlines():
1851 if 'android:debuggable' in line and not line.endswith('0x0'):
1856 def get_apk_debuggable_androguard(apkfile):
1858 from androguard.core.bytecodes.apk import APK
1860 raise FDroidException("androguard library is not installed and aapt not present")
1862 apkobject = APK(apkfile)
1863 if apkobject.is_valid_APK():
1864 debuggable = apkobject.get_element("application", "debuggable")
1865 if debuggable is not None:
1866 return bool(strtobool(debuggable))
1870 def isApkAndDebuggable(apkfile):
1871 """Returns True if the given file is an APK and is debuggable
1873 :param apkfile: full path to the apk to check"""
1875 if get_file_extension(apkfile) != 'apk':
1878 if SdkToolsPopen(['aapt', 'version'], output=False):
1879 return get_apk_debuggable_aapt(apkfile)
1881 return get_apk_debuggable_androguard(apkfile)
1884 def get_apk_id_aapt(apkfile):
1885 """Extrat identification information from APK using aapt.
1887 :param apkfile: path to an APK file.
1888 :returns: triplet (appid, version code, version name)
1890 r = re.compile("package: name='(?P<appid>.*)' versionCode='(?P<vercode>.*)' versionName='(?P<vername>.*)' platformBuildVersionName='.*'")
1891 p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
1892 for line in p.output.splitlines():
1895 return m.group('appid'), m.group('vercode'), m.group('vername')
1896 raise FDroidException(_("Reading packageName/versionCode/versionName failed, APK invalid: '{apkfilename}'")
1897 .format(apkfilename=apkfile))
1902 self.returncode = None
1906 def SdkToolsPopen(commands, cwd=None, output=True):
1908 if cmd not in config:
1909 config[cmd] = find_sdk_tools_cmd(commands[0])
1910 abscmd = config[cmd]
1912 raise FDroidException(_("Could not find '{command}' on your system").format(command=cmd))
1914 test_aapt_version(config['aapt'])
1915 return FDroidPopen([abscmd] + commands[1:],
1916 cwd=cwd, output=output)
1919 def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1921 Run a command and capture the possibly huge output as bytes.
1923 :param commands: command and argument list like in subprocess.Popen
1924 :param cwd: optionally specifies a working directory
1925 :param envs: a optional dictionary of environment variables and their values
1926 :returns: A PopenResult.
1931 set_FDroidPopen_env()
1933 process_env = env.copy()
1934 if envs is not None and len(envs) > 0:
1935 process_env.update(envs)
1938 cwd = os.path.normpath(cwd)
1939 logging.debug("Directory: %s" % cwd)
1940 logging.debug("> %s" % ' '.join(commands))
1942 stderr_param = subprocess.STDOUT if stderr_to_stdout else subprocess.PIPE
1943 result = PopenResult()
1946 p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env,
1947 stdout=subprocess.PIPE, stderr=stderr_param)
1948 except OSError as e:
1949 raise BuildException("OSError while trying to execute " +
1950 ' '.join(commands) + ': ' + str(e))
1952 if not stderr_to_stdout and options.verbose:
1953 stderr_queue = Queue()
1954 stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue)
1956 while not stderr_reader.eof():
1957 while not stderr_queue.empty():
1958 line = stderr_queue.get()
1959 sys.stderr.buffer.write(line)
1964 stdout_queue = Queue()
1965 stdout_reader = AsynchronousFileReader(p.stdout, stdout_queue)
1968 # Check the queue for output (until there is no more to get)
1969 while not stdout_reader.eof():
1970 while not stdout_queue.empty():
1971 line = stdout_queue.get()
1972 if output and options.verbose:
1973 # Output directly to console
1974 sys.stderr.buffer.write(line)
1980 result.returncode = p.wait()
1981 result.output = buf.getvalue()
1983 # make sure all filestreams of the subprocess are closed
1984 for streamvar in ['stdin', 'stdout', 'stderr']:
1985 if hasattr(p, streamvar):
1986 stream = getattr(p, streamvar)
1992 def FDroidPopen(commands, cwd=None, envs=None, output=True, stderr_to_stdout=True):
1994 Run a command and capture the possibly huge output as a str.
1996 :param commands: command and argument list like in subprocess.Popen
1997 :param cwd: optionally specifies a working directory
1998 :param envs: a optional dictionary of environment variables and their values
1999 :returns: A PopenResult.
2001 result = FDroidPopenBytes(commands, cwd, envs, output, stderr_to_stdout)
2002 result.output = result.output.decode('utf-8', 'ignore')
2006 gradle_comment = re.compile(r'[ ]*//')
2007 gradle_signing_configs = re.compile(r'^[\t ]*signingConfigs[ \t]*{[ \t]*$')
2008 gradle_line_matches = [
2009 re.compile(r'^[\t ]*signingConfig [^ ]*$'),
2010 re.compile(r'.*android\.signingConfigs\.[^{]*$'),
2011 re.compile(r'.*\.readLine\(.*'),
2015 def remove_signing_keys(build_dir):
2016 for root, dirs, files in os.walk(build_dir):
2017 if 'build.gradle' in files:
2018 path = os.path.join(root, 'build.gradle')
2020 with open(path, "r", encoding='utf8') as o:
2021 lines = o.readlines()
2027 with open(path, "w", encoding='utf8') as o:
2028 while i < len(lines):
2031 while line.endswith('\\\n'):
2032 line = line.rstrip('\\\n') + lines[i]
2035 if gradle_comment.match(line):
2040 opened += line.count('{')
2041 opened -= line.count('}')
2044 if gradle_signing_configs.match(line):
2049 if any(s.match(line) for s in gradle_line_matches):
2057 logging.info("Cleaned build.gradle of keysigning configs at %s" % path)
2060 'project.properties',
2062 'default.properties',
2063 'ant.properties', ]:
2064 if propfile in files:
2065 path = os.path.join(root, propfile)
2067 with open(path, "r", encoding='iso-8859-1') as o:
2068 lines = o.readlines()
2072 with open(path, "w", encoding='iso-8859-1') as o:
2074 if any(line.startswith(s) for s in ('key.store', 'key.alias')):
2081 logging.info("Cleaned %s of keysigning configs at %s" % (propfile, path))
2084 def set_FDroidPopen_env(build=None):
2086 set up the environment variables for the build environment
2088 There is only a weak standard, the variables used by gradle, so also set
2089 up the most commonly used environment variables for SDK and NDK. Also, if
2090 there is no locale set, this will set the locale (e.g. LANG) to en_US.UTF-8.
2092 global env, orig_path
2096 orig_path = env['PATH']
2097 for n in ['ANDROID_HOME', 'ANDROID_SDK']:
2098 env[n] = config['sdk_path']
2099 for k, v in config['java_paths'].items():
2100 env['JAVA%s_HOME' % k] = v
2102 missinglocale = True
2103 for k, v in env.items():
2104 if k == 'LANG' and v != 'C':
2105 missinglocale = False
2107 missinglocale = False
2109 env['LANG'] = 'en_US.UTF-8'
2111 if build is not None:
2112 path = build.ndk_path()
2113 paths = orig_path.split(os.pathsep)
2114 if path not in paths:
2115 paths = [path] + paths
2116 env['PATH'] = os.pathsep.join(paths)
2117 for n in ['ANDROID_NDK', 'NDK', 'ANDROID_NDK_HOME']:
2118 env[n] = build.ndk_path()
2121 def replace_build_vars(cmd, build):
2122 cmd = cmd.replace('$$COMMIT$$', build.commit)
2123 cmd = cmd.replace('$$VERSION$$', build.versionName)
2124 cmd = cmd.replace('$$VERCODE$$', build.versionCode)
2128 def replace_config_vars(cmd, build):
2129 cmd = cmd.replace('$$SDK$$', config['sdk_path'])
2130 cmd = cmd.replace('$$NDK$$', build.ndk_path())
2131 cmd = cmd.replace('$$MVN3$$', config['mvn3'])
2132 cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '')
2133 if build is not None:
2134 cmd = replace_build_vars(cmd, build)
2138 def place_srclib(root_dir, number, libpath):
2141 relpath = os.path.relpath(libpath, root_dir)
2142 proppath = os.path.join(root_dir, 'project.properties')
2145 if os.path.isfile(proppath):
2146 with open(proppath, "r", encoding='iso-8859-1') as o:
2147 lines = o.readlines()
2149 with open(proppath, "w", encoding='iso-8859-1') as o:
2152 if line.startswith('android.library.reference.%d=' % number):
2153 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2158 o.write('android.library.reference.%d=%s\n' % (number, relpath))
2161 apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)')
2164 def signer_fingerprint_short(sig):
2165 """Obtain shortened sha256 signing-key fingerprint for pkcs7 signature.
2167 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2168 for a given pkcs7 signature.
2170 :param sig: Contents of an APK signing certificate.
2171 :returns: shortened signing-key fingerprint.
2173 return signer_fingerprint(sig)[:7]
2176 def signer_fingerprint(sig):
2177 """Obtain sha256 signing-key fingerprint for pkcs7 signature.
2179 Extracts hexadecimal sha256 signing-key fingerprint string
2180 for a given pkcs7 signature.
2182 :param: Contents of an APK signature.
2183 :returns: shortened signature fingerprint.
2185 cert_encoded = get_certificate(sig)
2186 return hashlib.sha256(cert_encoded).hexdigest()
2189 def apk_signer_fingerprint(apk_path):
2190 """Obtain sha256 signing-key fingerprint for APK.
2192 Extracts hexadecimal sha256 signing-key fingerprint string
2195 :param apkpath: path to APK
2196 :returns: signature fingerprint
2199 with zipfile.ZipFile(apk_path, 'r') as apk:
2200 certs = [n for n in apk.namelist() if CERT_PATH_REGEX.match(n)]
2203 logging.error("Found no signing certificates on %s" % apk_path)
2206 logging.error("Found multiple signing certificates on %s" % apk_path)
2209 cert = apk.read(certs[0])
2210 return signer_fingerprint(cert)
2213 def apk_signer_fingerprint_short(apk_path):
2214 """Obtain shortened sha256 signing-key fingerprint for APK.
2216 Extracts the first 7 hexadecimal digits of sha256 signing-key fingerprint
2217 for a given pkcs7 APK.
2219 :param apk_path: path to APK
2220 :returns: shortened signing-key fingerprint
2222 return apk_signer_fingerprint(apk_path)[:7]
2225 def metadata_get_sigdir(appid, vercode=None):
2226 """Get signature directory for app"""
2228 return os.path.join('metadata', appid, 'signatures', vercode)
2230 return os.path.join('metadata', appid, 'signatures')
2233 def metadata_find_developer_signature(appid, vercode=None):
2234 """Tires to find the developer signature for given appid.
2236 This picks the first signature file found in metadata an returns its
2239 :returns: sha256 signing key fingerprint of the developer signing key.
2240 None in case no signature can not be found."""
2242 # fetch list of dirs for all versions of signatures
2245 appversigdirs.append(metadata_get_sigdir(appid, vercode))
2247 appsigdir = metadata_get_sigdir(appid)
2248 if os.path.isdir(appsigdir):
2249 numre = re.compile('[0-9]+')
2250 for ver in os.listdir(appsigdir):
2251 if numre.match(ver):
2252 appversigdir = os.path.join(appsigdir, ver)
2253 appversigdirs.append(appversigdir)
2255 for sigdir in appversigdirs:
2256 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2257 glob.glob(os.path.join(sigdir, '*.EC')) + \
2258 glob.glob(os.path.join(sigdir, '*.RSA'))
2260 raise FDroidException('ambiguous signatures, please make sure there is only one signature in \'{}\'. (The signature has to be the App maintainers signature for version of the APK.)'.format(sigdir))
2262 with open(sig, 'rb') as f:
2263 return signer_fingerprint(f.read())
2267 def metadata_find_signing_files(appid, vercode):
2268 """Gets a list of singed manifests and signatures.
2270 :param appid: app id string
2271 :param vercode: app version code
2272 :returns: a list of triplets for each signing key with following paths:
2273 (signature_file, singed_file, manifest_file)
2276 sigdir = metadata_get_sigdir(appid, vercode)
2277 sigs = glob.glob(os.path.join(sigdir, '*.DSA')) + \
2278 glob.glob(os.path.join(sigdir, '*.EC')) + \
2279 glob.glob(os.path.join(sigdir, '*.RSA'))
2280 extre = re.compile('(\.DSA|\.EC|\.RSA)$')
2282 sf = extre.sub('.SF', sig)
2283 if os.path.isfile(sf):
2284 mf = os.path.join(sigdir, 'MANIFEST.MF')
2285 if os.path.isfile(mf):
2286 ret.append((sig, sf, mf))
2290 def metadata_find_developer_signing_files(appid, vercode):
2291 """Get developer signature files for specified app from metadata.
2293 :returns: A triplet of paths for signing files from metadata:
2294 (signature_file, singed_file, manifest_file)
2296 allsigningfiles = metadata_find_signing_files(appid, vercode)
2297 if allsigningfiles and len(allsigningfiles) == 1:
2298 return allsigningfiles[0]
2303 def apk_strip_signatures(signed_apk, strip_manifest=False):
2304 """Removes signatures from APK.
2306 :param signed_apk: path to apk file.
2307 :param strip_manifest: when set to True also the manifest file will
2308 be removed from the APK.
2310 with tempfile.TemporaryDirectory() as tmpdir:
2311 tmp_apk = os.path.join(tmpdir, 'tmp.apk')
2312 os.rename(signed_apk, tmp_apk)
2313 with ZipFile(tmp_apk, 'r') as in_apk:
2314 with ZipFile(signed_apk, 'w') as out_apk:
2315 for info in in_apk.infolist():
2316 if not apk_sigfile.match(info.filename):
2318 if info.filename != 'META-INF/MANIFEST.MF':
2319 buf = in_apk.read(info.filename)
2320 out_apk.writestr(info, buf)
2322 buf = in_apk.read(info.filename)
2323 out_apk.writestr(info, buf)
2326 def apk_implant_signatures(apkpath, signaturefile, signedfile, manifest):
2327 """Implats a signature from metadata into an APK.
2329 Note: this changes there supplied APK in place. So copy it if you
2330 need the original to be preserved.
2332 :param apkpath: location of the apk
2334 # get list of available signature files in metadata
2335 with tempfile.TemporaryDirectory() as tmpdir:
2336 apkwithnewsig = os.path.join(tmpdir, 'newsig.apk')
2337 with ZipFile(apkpath, 'r') as in_apk:
2338 with ZipFile(apkwithnewsig, 'w') as out_apk:
2339 for sig_file in [signaturefile, signedfile, manifest]:
2340 with open(sig_file, 'rb') as fp:
2342 info = zipfile.ZipInfo('META-INF/' + os.path.basename(sig_file))
2343 info.compress_type = zipfile.ZIP_DEFLATED
2344 info.create_system = 0 # "Windows" aka "FAT", what Android SDK uses
2345 out_apk.writestr(info, buf)
2346 for info in in_apk.infolist():
2347 if not apk_sigfile.match(info.filename):
2348 if info.filename != 'META-INF/MANIFEST.MF':
2349 buf = in_apk.read(info.filename)
2350 out_apk.writestr(info, buf)
2352 p = SdkToolsPopen(['zipalign', '-v', '4', apkwithnewsig, apkpath])
2353 if p.returncode != 0:
2354 raise BuildException("Failed to align application")
2357 def apk_extract_signatures(apkpath, outdir, manifest=True):
2358 """Extracts a signature files from APK and puts them into target directory.
2360 :param apkpath: location of the apk
2361 :param outdir: folder where the extracted signature files will be stored
2362 :param manifest: (optionally) disable extracting manifest file
2364 with ZipFile(apkpath, 'r') as in_apk:
2365 for f in in_apk.infolist():
2366 if apk_sigfile.match(f.filename) or \
2367 (manifest and f.filename == 'META-INF/MANIFEST.MF'):
2368 newpath = os.path.join(outdir, os.path.basename(f.filename))
2369 with open(newpath, 'wb') as out_file:
2370 out_file.write(in_apk.read(f.filename))
2373 def verify_apks(signed_apk, unsigned_apk, tmp_dir):
2374 """Verify that two apks are the same
2376 One of the inputs is signed, the other is unsigned. The signature metadata
2377 is transferred from the signed to the unsigned apk, and then jarsigner is
2378 used to verify that the signature from the signed apk is also varlid for
2379 the unsigned one. If the APK given as unsigned actually does have a
2380 signature, it will be stripped out and ignored.
2382 There are two SHA1 git commit IDs that fdroidserver includes in the builds
2383 it makes: fdroidserverid and buildserverid. Originally, these were inserted
2384 into AndroidManifest.xml, but that makes the build not reproducible. So
2385 instead they are included as separate files in the APK's META-INF/ folder.
2386 If those files exist in the signed APK, they will be part of the signature
2387 and need to also be included in the unsigned APK for it to validate.
2389 :param signed_apk: Path to a signed apk file
2390 :param unsigned_apk: Path to an unsigned apk file expected to match it
2391 :param tmp_dir: Path to directory for temporary files
2392 :returns: None if the verification is successful, otherwise a string
2393 describing what went wrong.
2396 if not os.path.isfile(signed_apk):
2397 return 'can not verify: file does not exists: {}'.format(signed_apk)
2399 if not os.path.isfile(unsigned_apk):
2400 return 'can not verify: file does not exists: {}'.format(unsigned_apk)
2402 with ZipFile(signed_apk, 'r') as signed:
2403 meta_inf_files = ['META-INF/MANIFEST.MF']
2404 for f in signed.namelist():
2405 if apk_sigfile.match(f) \
2406 or f in ['META-INF/fdroidserverid', 'META-INF/buildserverid']:
2407 meta_inf_files.append(f)
2408 if len(meta_inf_files) < 3:
2409 return "Signature files missing from {0}".format(signed_apk)
2411 tmp_apk = os.path.join(tmp_dir, 'sigcp_' + os.path.basename(unsigned_apk))
2412 with ZipFile(unsigned_apk, 'r') as unsigned:
2413 # only read the signature from the signed APK, everything else from unsigned
2414 with ZipFile(tmp_apk, 'w') as tmp:
2415 for filename in meta_inf_files:
2416 tmp.writestr(signed.getinfo(filename), signed.read(filename))
2417 for info in unsigned.infolist():
2418 if info.filename in meta_inf_files:
2419 logging.warning('Ignoring %s from %s',
2420 info.filename, unsigned_apk)
2422 if info.filename in tmp.namelist():
2423 return "duplicate filename found: " + info.filename
2424 tmp.writestr(info, unsigned.read(info.filename))
2426 verified = verify_apk_signature(tmp_apk)
2429 logging.info("...NOT verified - {0}".format(tmp_apk))
2430 return compare_apks(signed_apk, tmp_apk, tmp_dir,
2431 os.path.dirname(unsigned_apk))
2433 logging.info("...successfully verified")
2437 def verify_jar_signature(jar):
2438 """Verifies the signature of a given JAR file.
2440 jarsigner is very shitty: unsigned JARs pass as "verified"! So
2441 this has to turn on -strict then check for result 4, since this
2442 does not expect the signature to be from a CA-signed certificate.
2444 :raises: VerificationException() if the JAR's signature could not be verified
2448 if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4:
2449 raise VerificationException(_("The repository's index could not be verified."))
2452 def verify_apk_signature(apk, min_sdk_version=None):
2453 """verify the signature on an APK
2455 Try to use apksigner whenever possible since jarsigner is very
2456 shitty: unsigned APKs pass as "verified"! Warning, this does
2457 not work on JARs with apksigner >= 0.7 (build-tools 26.0.1)
2459 :returns: boolean whether the APK was verified
2461 if set_command_in_config('apksigner'):
2462 args = [config['apksigner'], 'verify']
2464 args += ['--min-sdk-version=' + min_sdk_version]
2465 return subprocess.call(args + [apk]) == 0
2467 logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")
2469 verify_jar_signature(apk)
2476 def verify_old_apk_signature(apk):
2477 """verify the signature on an archived APK, supporting deprecated algorithms
2479 F-Droid aims to keep every single binary that it ever published. Therefore,
2480 it needs to be able to verify APK signatures that include deprecated/removed
2481 algorithms. For example, jarsigner treats an MD5 signature as unsigned.
2483 jarsigner passes unsigned APKs as "verified"! So this has to turn
2484 on -strict then check for result 4.
2486 :returns: boolean whether the APK was verified
2489 _java_security = os.path.join(os.getcwd(), '.java.security')
2490 with open(_java_security, 'w') as fp:
2491 fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024')
2493 return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security,
2494 '-strict', '-verify', apk]) == 4
2497 apk_badchars = re.compile('''[/ :;'"]''')
2500 def compare_apks(apk1, apk2, tmp_dir, log_dir=None):
2503 Returns None if the apk content is the same (apart from the signing key),
2504 otherwise a string describing what's different, or what went wrong when
2505 trying to do the comparison.
2511 absapk1 = os.path.abspath(apk1)
2512 absapk2 = os.path.abspath(apk2)
2514 if set_command_in_config('diffoscope'):
2515 logfilename = os.path.join(log_dir, os.path.basename(absapk1))
2516 htmlfile = logfilename + '.diffoscope.html'
2517 textfile = logfilename + '.diffoscope.txt'
2518 if subprocess.call([config['diffoscope'],
2519 '--max-report-size', '12345678', '--max-diff-block-lines', '100',
2520 '--html', htmlfile, '--text', textfile,
2521 absapk1, absapk2]) != 0:
2522 return("Failed to unpack " + apk1)
2524 apk1dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk1[0:-4])) # trim .apk
2525 apk2dir = os.path.join(tmp_dir, apk_badchars.sub('_', apk2[0:-4])) # trim .apk
2526 for d in [apk1dir, apk2dir]:
2527 if os.path.exists(d):
2530 os.mkdir(os.path.join(d, 'jar-xf'))
2532 if subprocess.call(['jar', 'xf',
2533 os.path.abspath(apk1)],
2534 cwd=os.path.join(apk1dir, 'jar-xf')) != 0:
2535 return("Failed to unpack " + apk1)
2536 if subprocess.call(['jar', 'xf',
2537 os.path.abspath(apk2)],
2538 cwd=os.path.join(apk2dir, 'jar-xf')) != 0:
2539 return("Failed to unpack " + apk2)
2541 if set_command_in_config('apktool'):
2542 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk1), '--output', 'apktool'],
2544 return("Failed to unpack " + apk1)
2545 if subprocess.call([config['apktool'], 'd', os.path.abspath(apk2), '--output', 'apktool'],
2547 return("Failed to unpack " + apk2)
2549 p = FDroidPopen(['diff', '-r', apk1dir, apk2dir], output=False)
2550 lines = p.output.splitlines()
2551 if len(lines) != 1 or 'META-INF' not in lines[0]:
2552 if set_command_in_config('meld'):
2553 p = FDroidPopen([config['meld'], apk1dir, apk2dir], output=False)
2554 return("Unexpected diff output - " + p.output)
2556 # since everything verifies, delete the comparison to keep cruft down
2557 shutil.rmtree(apk1dir)
2558 shutil.rmtree(apk2dir)
2560 # If we get here, it seems like they're the same!
2564 def set_command_in_config(command):
2565 '''Try to find specified command in the path, if it hasn't been
2566 manually set in config.py. If found, it is added to the config
2567 dict. The return value says whether the command is available.
2570 if command in config:
2573 tmp = find_command(command)
2575 config[command] = tmp
2580 def find_command(command):
2581 '''find the full path of a command, or None if it can't be found in the PATH'''
2584 return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
2586 fpath, fname = os.path.split(command)
2591 for path in os.environ["PATH"].split(os.pathsep):
2592 path = path.strip('"')
2593 exe_file = os.path.join(path, command)
2594 if is_exe(exe_file):
2601 '''generate a random password for when generating keys'''
2602 h = hashlib.sha256()
2603 h.update(os.urandom(16)) # salt
2604 h.update(socket.getfqdn().encode('utf-8'))
2605 passwd = base64.b64encode(h.digest()).strip()
2606 return passwd.decode('utf-8')
2609 def genkeystore(localconfig):
2611 Generate a new key with password provided in :param localconfig and add it to new keystore
2612 :return: hexed public key, public key fingerprint
2614 logging.info('Generating a new key in "' + localconfig['keystore'] + '"...')
2615 keystoredir = os.path.dirname(localconfig['keystore'])
2616 if keystoredir is None or keystoredir == '':
2617 keystoredir = os.path.join(os.getcwd(), keystoredir)
2618 if not os.path.exists(keystoredir):
2619 os.makedirs(keystoredir, mode=0o700)
2622 'FDROID_KEY_STORE_PASS': localconfig['keystorepass'],
2623 'FDROID_KEY_PASS': localconfig['keypass'],
2625 p = FDroidPopen([config['keytool'], '-genkey',
2626 '-keystore', localconfig['keystore'],
2627 '-alias', localconfig['repo_keyalias'],
2628 '-keyalg', 'RSA', '-keysize', '4096',
2629 '-sigalg', 'SHA256withRSA',
2630 '-validity', '10000',
2631 '-storepass:env', 'FDROID_KEY_STORE_PASS',
2632 '-keypass:env', 'FDROID_KEY_PASS',
2633 '-dname', localconfig['keydname']], envs=env_vars)
2634 if p.returncode != 0:
2635 raise BuildException("Failed to generate key", p.output)
2636 os.chmod(localconfig['keystore'], 0o0600)
2637 if not options.quiet:
2638 # now show the lovely key that was just generated
2639 p = FDroidPopen([config['keytool'], '-list', '-v',
2640 '-keystore', localconfig['keystore'],
2641 '-alias', localconfig['repo_keyalias'],
2642 '-storepass:env', 'FDROID_KEY_STORE_PASS'], envs=env_vars)
2643 logging.info(p.output.strip() + '\n\n')
2644 # get the public key
2645 p = FDroidPopenBytes([config['keytool'], '-exportcert',
2646 '-keystore', localconfig['keystore'],
2647 '-alias', localconfig['repo_keyalias'],
2648 '-storepass:env', 'FDROID_KEY_STORE_PASS']
2649 + config['smartcardoptions'],
2650 envs=env_vars, output=False, stderr_to_stdout=False)
2651 if p.returncode != 0 or len(p.output) < 20:
2652 raise BuildException("Failed to get public key", p.output)
2654 fingerprint = get_cert_fingerprint(pubkey)
2655 return hexlify(pubkey), fingerprint
2658 def get_cert_fingerprint(pubkey):
2660 Generate a certificate fingerprint the same way keytool does it
2661 (but with slightly different formatting)
2663 digest = hashlib.sha256(pubkey).digest()
2664 ret = [' '.join("%02X" % b for b in bytearray(digest))]
2665 return " ".join(ret)
2668 def get_certificate(certificate_file):
2670 Extracts a certificate from the given file.
2671 :param certificate_file: file bytes (as string) representing the certificate
2672 :return: A binary representation of the certificate's public key, or None in case of error
2674 content = decoder.decode(certificate_file, asn1Spec=rfc2315.ContentInfo())[0]
2675 if content.getComponentByName('contentType') != rfc2315.signedData:
2677 content = decoder.decode(content.getComponentByName('content'),
2678 asn1Spec=rfc2315.SignedData())[0]
2680 certificates = content.getComponentByName('certificates')
2681 cert = certificates[0].getComponentByName('certificate')
2683 logging.error("Certificates not found.")
2685 return encoder.encode(cert)
2688 def load_stats_fdroid_signing_key_fingerprints():
2689 """Load list of signing-key fingerprints stored by fdroid publish from file.
2691 :returns: list of dictionanryies containing the singing-key fingerprints.
2693 jar_file = os.path.join('stats', 'publishsigkeys.jar')
2694 if not os.path.isfile(jar_file):
2696 cmd = [config['jarsigner'], '-strict', '-verify', jar_file]
2697 p = FDroidPopen(cmd, output=False)
2698 if p.returncode != 4:
2699 raise FDroidException("Signature validation of '{}' failed! "
2700 "Please run publish again to rebuild this file.".format(jar_file))
2702 jar_sigkey = apk_signer_fingerprint(jar_file)
2703 repo_key_sig = config.get('repo_key_sha256')
2705 if jar_sigkey != repo_key_sig:
2706 raise FDroidException("Signature key fingerprint of file '{}' does not match repo_key_sha256 in config.py (found fingerprint: '{}')".format(jar_file, jar_sigkey))
2708 logging.warning("repo_key_sha256 not in config.py, setting it to the signature key fingerprint of '{}'".format(jar_file))
2709 config['repo_key_sha256'] = jar_sigkey
2710 write_to_config(config, 'repo_key_sha256')
2712 with zipfile.ZipFile(jar_file, 'r') as f:
2713 return json.loads(str(f.read('publishsigkeys.json'), 'utf-8'))
2716 def write_to_config(thisconfig, key, value=None, config_file=None):
2717 '''write a key/value to the local config.py
2719 NOTE: only supports writing string variables.
2721 :param thisconfig: config dictionary
2722 :param key: variable name in config.py to be overwritten/added
2723 :param value: optional value to be written, instead of fetched
2724 from 'thisconfig' dictionary.
2727 origkey = key + '_orig'
2728 value = thisconfig[origkey] if origkey in thisconfig else thisconfig[key]
2729 cfg = config_file if config_file else 'config.py'
2731 # load config file, create one if it doesn't exist
2732 if not os.path.exists(cfg):
2733 open(cfg, 'a').close()
2734 logging.info("Creating empty " + cfg)
2735 with open(cfg, 'r', encoding="utf-8") as f:
2736 lines = f.readlines()
2738 # make sure the file ends with a carraige return
2740 if not lines[-1].endswith('\n'):
2743 # regex for finding and replacing python string variable
2744 # definitions/initializations
2745 pattern = re.compile('^[\s#]*' + key + '\s*=\s*"[^"]*"')
2746 repl = key + ' = "' + value + '"'
2747 pattern2 = re.compile('^[\s#]*' + key + "\s*=\s*'[^']*'")
2748 repl2 = key + " = '" + value + "'"
2750 # If we replaced this line once, we make sure won't be a
2751 # second instance of this line for this key in the document.
2754 with open(cfg, 'w', encoding="utf-8") as f:
2756 if pattern.match(line) or pattern2.match(line):
2758 line = pattern.sub(repl, line)
2759 line = pattern2.sub(repl2, line)
2770 def parse_xml(path):
2771 return XMLElementTree.parse(path).getroot()
2774 def string_is_integer(string):
2782 def local_rsync(options, fromdir, todir):
2783 '''Rsync method for local to local copying of things
2785 This is an rsync wrapper with all the settings for safe use within
2786 the various fdroidserver use cases. This uses stricter rsync
2787 checking on all files since people using offline mode are already
2788 prioritizing security above ease and speed.
2791 rsyncargs = ['rsync', '--recursive', '--safe-links', '--times', '--perms',
2792 '--one-file-system', '--delete', '--chmod=Da+rx,Fa-x,a+r,u+w']
2793 if not options.no_checksum:
2794 rsyncargs.append('--checksum')
2796 rsyncargs += ['--verbose']
2798 rsyncargs += ['--quiet']
2799 logging.debug(' '.join(rsyncargs + [fromdir, todir]))
2800 if subprocess.call(rsyncargs + [fromdir, todir]) != 0:
2801 raise FDroidException()
2804 def get_per_app_repos():
2805 '''per-app repos are dirs named with the packageName of a single app'''
2807 # Android packageNames are Java packages, they may contain uppercase or
2808 # lowercase letters ('A' through 'Z'), numbers, and underscores
2809 # ('_'). However, individual package name parts may only start with
2810 # letters. https://developer.android.com/guide/topics/manifest/manifest-element.html#package
2811 p = re.compile('^([a-zA-Z][a-zA-Z0-9_]*(\\.[a-zA-Z][a-zA-Z0-9_]*)*)?$')
2814 for root, dirs, files in os.walk(os.getcwd()):
2816 print('checking', root, 'for', d)
2817 if d in ('archive', 'metadata', 'repo', 'srclibs', 'tmp'):
2818 # standard parts of an fdroid repo, so never packageNames
2821 and os.path.exists(os.path.join(d, 'fdroid', 'repo', 'index.jar')):
2827 def is_repo_file(filename):
2828 '''Whether the file in a repo is a build product to be delivered to users'''
2829 if isinstance(filename, str):
2830 filename = filename.encode('utf-8', errors="surrogateescape")
2831 return os.path.isfile(filename) \
2832 and not filename.endswith(b'.asc') \
2833 and not filename.endswith(b'.sig') \
2834 and os.path.basename(filename) not in [
2836 b'index_unsigned.jar',
2845 def get_examples_dir():
2846 '''Return the dir where the fdroidserver example files are available'''
2848 tmp = os.path.dirname(sys.argv[0])
2849 if os.path.basename(tmp) == 'bin':
2850 egg_links = glob.glob(os.path.join(tmp, '..',
2851 'local/lib/python3.*/site-packages/fdroidserver.egg-link'))
2853 # installed from local git repo
2854 examplesdir = os.path.join(open(egg_links[0]).readline().rstrip(), 'examples')
2857 examplesdir = os.path.dirname(os.path.dirname(__file__)) + '/share/doc/fdroidserver/examples'
2858 if not os.path.exists(examplesdir): # use UNIX layout
2859 examplesdir = os.path.dirname(tmp) + '/share/doc/fdroidserver/examples'
2861 # we're running straight out of the git repo
2862 prefix = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
2863 examplesdir = prefix + '/examples'