X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Fcommon.py;h=76089184708cb6a39618e9468981e73faec93041;hb=70d9633555ba07b4bb83dfd7dcda9781cc80cf51;hp=6ed43d4c4e465eda49d752eb7ceea09d7eb6af5a;hpb=dcbc78d23815cd27db0414074fa5c601093ce82f;p=fdroidserver.git diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6ed43d4c..76089184 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -53,9 +53,15 @@ from distutils.util import strtobool import fdroidserver.metadata from fdroidserver import _ -from fdroidserver.exception import FDroidException, VCSException, BuildException, VerificationException +from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesException,\ + BuildException, VerificationException from .asynchronousfilereader import AsynchronousFileReader +# this is the build-tools version, aapt has a separate version that +# has to be manually set in test_aapt_version() +MINIMUM_AAPT_VERSION = '26.0.0' + +VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$') # A signature block file with a .DSA, .RSA, or .EC extension CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$') @@ -73,17 +79,15 @@ orig_path = None default_config = { 'sdk_path': "$ANDROID_HOME", 'ndk_paths': { - 'r9b': None, 'r10e': None, 'r11c': None, 'r12b': "$ANDROID_NDK", 'r13b': None, 'r14b': None, 'r15c': None, - 'r16': None, + 'r16b': None, }, - 'qt_sdk_path': None, - 'build_tools': "25.0.2", + 'build_tools': MINIMUM_AAPT_VERSION, 'force_build_tools': False, 'java_paths': None, 'ant': "ant", @@ -127,6 +131,13 @@ default_config = { def setup_global_opts(parser): + try: # the buildserver VM might not have PIL installed + from PIL import PngImagePlugin + logger = logging.getLogger(PngImagePlugin.__name__) + logger.setLevel(logging.INFO) # tame the "STREAM" debug messages + except ImportError: + pass + parser.add_argument("-v", "--verbose", action="store_true", default=False, help=_("Spew out even more information than normal")) parser.add_argument("-q", "--quiet", action="store_true", default=False, @@ -387,9 +398,15 @@ def test_aapt_version(aapt): minor = m.group(2) bugfix = m.group(3) # the Debian package has the version string like "v0.2-23.0.2" - if '.' not in bugfix and LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2166767'): - logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-23.0.0 or newer!") - .format(aapt=aapt)) + too_old = False + if '.' in bugfix: + if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION): + too_old = True + elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'): + too_old = True + if too_old: + logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!") + .format(aapt=aapt, version=MINIMUM_AAPT_VERSION)) else: logging.warning(_('Unknown version of aapt, might cause problems: ') + output) @@ -786,7 +803,7 @@ class vcs_git(vcs): def clientversioncmd(self): return ['git', '--version'] - def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True): + def git(self, args, envs=dict(), cwd=None, output=True): '''Prevent git fetch/clone/submodule from hanging at the username/password prompt While fetch/pull/clone respect the command line option flags, @@ -795,10 +812,15 @@ class vcs_git(vcs): enough. So we just throw the kitchen sink at it to see what sticks. + Also, because of CVE-2017-1000117, block all SSH URLs. ''' - if cwd is None: - cwd = self.local - git_config = [] + # + # supported in git >= 2.3 + git_config = [ + '-c', 'core.askpass=/bin/true', + '-c', 'core.sshCommand=/bin/false', + '-c', 'url.https://.insteadOf=ssh://', + ] for domain in ('bitbucket.org', 'github.com', 'gitlab.com'): git_config.append('-c') git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':') @@ -806,15 +828,13 @@ class vcs_git(vcs): git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain) git_config.append('-c') git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain) - # add helpful tricks supported in git >= 2.3 - ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes' - git_config.append('-c') - git_config.append('core.sshCommand="' + ssh_command + '"') # git >= 2.10 envs.update({ 'GIT_TERMINAL_PROMPT': '0', - 'GIT_SSH_COMMAND': ssh_command, # git >= 2.3 + 'GIT_ASKPASS': '/bin/true', + 'SSH_ASKPASS': '/bin/true', + 'GIT_SSH': '/bin/false', # for git < 2.3 }) - return FDroidPopen(['git', ] + git_config + gitargs, + return FDroidPopen(['git', ] + git_config + args, envs=envs, cwd=cwd, output=output) def checkrepo(self): @@ -833,7 +853,7 @@ class vcs_git(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None) + p = self.git(['clone', '--', self.remote, self.local]) if p.returncode != 0: self.clone_failed = True raise VCSException("Git clone failed", p.output) @@ -853,10 +873,10 @@ class vcs_git(vcs): raise VCSException(_("Git clean failed"), p.output) if not self.refreshed: # Get latest commits and tags from remote - p = self.GitFetchFDroidPopen(['fetch', 'origin']) + p = self.git(['fetch', 'origin'], cwd=self.local) if p.returncode != 0: raise VCSException(_("Git fetch failed"), p.output) - p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False) + p = self.git(['fetch', '--prune', '--tags', 'origin'], output=False, cwd=self.local) if p.returncode != 0: raise VCSException(_("Git fetch failed"), p.output) # Recreate origin/HEAD as git clone would do it, in case it disappeared @@ -866,7 +886,8 @@ class vcs_git(vcs): if 'Multiple remote HEAD branches' not in lines[0]: raise VCSException(_("Git remote set-head failed"), p.output) branch = lines[1].split(' ')[-1] - p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False) + p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch], + cwd=self.local, output=False) if p2.returncode != 0: raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output) self.refreshed = True @@ -885,7 +906,7 @@ class vcs_git(vcs): self.checkrepo() submfile = os.path.join(self.local, '.gitmodules') if not os.path.isfile(submfile): - raise VCSException(_("No git submodules available")) + raise NoSubmodulesException(_("No git submodules available")) # fix submodules not accessible without an account and public key auth with open(submfile, 'r') as f: @@ -899,7 +920,7 @@ class vcs_git(vcs): p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git submodule sync failed"), p.output) - p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive']) + p = self.git(['submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local) if p.returncode != 0: raise VCSException(_("Git submodule update failed"), p.output) @@ -942,10 +963,36 @@ class vcs_gitsvn(vcs): if not result.endswith(self.local): raise VCSException('Repository mismatch') + def git(self, args, envs=dict(), cwd=None, output=True): + '''Prevent git fetch/clone/submodule from hanging at the username/password prompt + + AskPass is set to /bin/true to let the process try to connect + without a username/password. + + The SSH command is set to /bin/false to block all SSH URLs + (supported in git >= 2.3). This protects against + CVE-2017-1000117. + + ''' + git_config = [ + '-c', 'core.askpass=/bin/true', + '-c', 'core.sshCommand=/bin/false', + ] + envs.update({ + 'GIT_TERMINAL_PROMPT': '0', + 'GIT_ASKPASS': '/bin/true', + 'SSH_ASKPASS': '/bin/true', + 'GIT_SSH': '/bin/false', # for git < 2.3 + 'SVN_SSH': '/bin/false', + }) + return FDroidPopen(['git', ] + git_config + args, + envs=envs, cwd=cwd, output=output) + def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - gitsvn_args = ['git', 'svn', 'clone'] + gitsvn_args = ['svn', 'clone'] + remote = None if ';' in self.remote: remote_split = self.remote.split(';') for i in remote_split[1:]: @@ -955,35 +1002,45 @@ class vcs_gitsvn(vcs): gitsvn_args.extend(['-t', i[5:]]) elif i.startswith('branches='): gitsvn_args.extend(['-b', i[9:]]) - gitsvn_args.extend([remote_split[0], self.local]) - p = FDroidPopen(gitsvn_args, output=False) - if p.returncode != 0: - self.clone_failed = True - raise VCSException("Git svn clone failed", p.output) + remote = remote_split[0] else: - gitsvn_args.extend([self.remote, self.local]) - p = FDroidPopen(gitsvn_args, output=False) - if p.returncode != 0: - self.clone_failed = True - raise VCSException("Git svn clone failed", p.output) + remote = self.remote + + if not remote.startswith('https://'): + raise VCSException(_('HTTPS must be used with Subversion URLs!')) + + # git-svn sucks at certificate validation, this throws useful errors: + import requests + r = requests.head(remote) + r.raise_for_status() + location = r.headers.get('location') + if location and not location.startswith('https://'): + raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ') + .format(before=remote, after=location)) + + gitsvn_args.extend(['--', remote, self.local]) + p = self.git(gitsvn_args) + if p.returncode != 0: + self.clone_failed = True + raise VCSException(_('git svn clone failed'), p.output) self.checkrepo() else: self.checkrepo() # Discard any working tree changes - p = FDroidPopen(['git', 'reset', '--hard'], cwd=self.local, output=False) + p = self.git(['reset', '--hard'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Git reset failed", p.output) # Remove untracked files now, in case they're tracked in the target # revision (it happens!) - p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False) + p = self.git(['clean', '-dffx'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Git clean failed", p.output) if not self.refreshed: # Get new commits, branches and tags from repo - p = FDroidPopen(['git', 'svn', 'fetch'], cwd=self.local, output=False) + p = self.git(['svn', 'fetch'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Git svn fetch failed") - p = FDroidPopen(['git', 'svn', 'rebase'], cwd=self.local, output=False) + p = self.git(['svn', 'rebase'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Git svn rebase failed", p.output) self.refreshed = True @@ -993,7 +1050,7 @@ class vcs_gitsvn(vcs): nospaces_rev = rev.replace(' ', '%20') # Try finding a svn tag for treeish in ['origin/', '']: - p = FDroidPopen(['git', 'checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False) + p = self.git(['checkout', treeish + 'tags/' + nospaces_rev], cwd=self.local, output=False) if p.returncode == 0: break if p.returncode != 0: @@ -1014,7 +1071,7 @@ class vcs_gitsvn(vcs): svn_rev = svn_rev if svn_rev[0] == 'r' else 'r' + svn_rev - p = FDroidPopen(['git', 'svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False) + p = self.git(['svn', 'find-rev', '--before', svn_rev, treeish], cwd=self.local, output=False) git_rev = p.output.rstrip() if p.returncode == 0 and git_rev: @@ -1022,17 +1079,17 @@ class vcs_gitsvn(vcs): if p.returncode != 0 or not git_rev: # Try a plain git checkout as a last resort - p = FDroidPopen(['git', 'checkout', rev], cwd=self.local, output=False) + p = self.git(['checkout', rev], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("No git treeish found and direct git checkout of '%s' failed" % rev, p.output) else: # Check out the git rev equivalent to the svn rev - p = FDroidPopen(['git', 'checkout', git_rev], cwd=self.local, output=False) + p = self.git(['checkout', git_rev], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git checkout of '%s' failed") % rev, p.output) # Get rid of any uncontrolled files left behind - p = FDroidPopen(['git', 'clean', '-dffx'], cwd=self.local, output=False) + p = self.git(['clean', '-dffx'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git clean failed"), p.output) @@ -1061,7 +1118,8 @@ class vcs_hg(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = FDroidPopen(['hg', 'clone', self.remote, self.local], output=False) + p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local], + output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Hg clone failed", p.output) @@ -1072,9 +1130,9 @@ class vcs_hg(vcs): for line in p.output.splitlines(): if not line.startswith('? '): raise VCSException("Unexpected output from hg status -uS: " + line) - FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False) + FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False) if not self.refreshed: - p = FDroidPopen(['hg', 'pull'], cwd=self.local, output=False) + p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg pull failed", p.output) self.refreshed = True @@ -1082,7 +1140,7 @@ class vcs_hg(vcs): rev = rev or 'default' if not rev: return - p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False) + p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg checkout of '%s' failed" % rev, p.output) p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False) @@ -1109,29 +1167,36 @@ class vcs_bzr(vcs): def clientversioncmd(self): return ['bzr', '--version'] + def bzr(self, args, envs=dict(), cwd=None, output=True): + '''Prevent bzr from ever using SSH to avoid security vulns''' + envs.update({ + 'BZR_SSH': 'false', + }) + return FDroidPopen(['bzr', ] + args, envs=envs, cwd=cwd, output=output) + def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = FDroidPopen(['bzr', 'branch', self.remote, self.local], output=False) + p = self.bzr(['branch', self.remote, self.local], output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Bzr branch failed", p.output) else: - p = FDroidPopen(['bzr', 'clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False) + p = self.bzr(['clean-tree', '--force', '--unknown', '--ignored'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Bzr revert failed", p.output) if not self.refreshed: - p = FDroidPopen(['bzr', 'pull'], cwd=self.local, output=False) + p = self.bzr(['pull'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Bzr update failed", p.output) self.refreshed = True revargs = list(['-r', rev] if rev else []) - p = FDroidPopen(['bzr', 'revert'] + revargs, cwd=self.local, output=False) + p = self.bzr(['revert'] + revargs, cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Bzr revert of '%s' failed" % rev, p.output) def _gettags(self): - p = FDroidPopen(['bzr', 'tags'], cwd=self.local, output=False) + p = self.bzr(['tags'], cwd=self.local, output=False) return [tag.split(' ')[0].strip() for tag in p.output.splitlines()] @@ -1261,9 +1326,9 @@ def remove_debuggable_flags(root_dir): os.path.join(root, 'AndroidManifest.xml')) -vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search -vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search -psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search +vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search +vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search +psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search def app_matches_packagename(app, package): @@ -1302,44 +1367,49 @@ def parse_androidmanifests(paths, app): vercode = None package = None - flavour = "" + flavour = None if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle: - flavour = app.builds[-1].gradle[-1] + flavour = app.builds[-1].gradle[-1] if has_extension(path, 'gradle'): - # first try to get version name and code from correct flavour with open(path, 'r') as f: - buildfile = f.read() - - regex_string = r"" + flavour + ".*?}" - search = re.compile(regex_string, re.DOTALL) - result = search.search(buildfile) - - if result is not None: - resultgroup = result.group() - - if not package: - matches = psearch_g(resultgroup) - if matches: - s = matches.group(2) - if app_matches_packagename(app, s): - package = s - if not version: - matches = vnsearch_g(resultgroup) - if matches: - version = matches.group(2) - if not vercode: - matches = vcsearch_g(resultgroup) - if matches: - vercode = matches.group(1) - else: - # fall back to parse file line by line - with open(path, 'r') as f: - for line in f: - if gradle_comment.match(line): - continue - # Grab first occurence of each to avoid running into - # alternative flavours and builds. + inside_flavour_group = 0 + inside_required_flavour = 0 + for line in f: + if gradle_comment.match(line): + continue + + if inside_flavour_group > 0: + if inside_required_flavour > 0: + matches = psearch_g(line) + if matches: + s = matches.group(2) + if app_matches_packagename(app, s): + package = s + + matches = vnsearch_g(line) + if matches: + version = matches.group(2) + + matches = vcsearch_g(line) + if matches: + vercode = matches.group(1) + + if '{' in line: + inside_required_flavour += 1 + if '}' in line: + inside_required_flavour -= 1 + else: + if flavour and (flavour in line): + inside_required_flavour = 1 + + if '{' in line: + inside_flavour_group += 1 + if '}' in line: + inside_flavour_group -= 1 + else: + if "productFlavors" in line: + inside_flavour_group = 1 if not package: matches = psearch_g(line) if matches: @@ -1470,7 +1540,7 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False, if srclib["Prepare"]: cmd = replace_config_vars(srclib["Prepare"], build) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir) if p.returncode != 0: raise BuildException("Error running prepare command for srclib %s" % name, p.output) @@ -1525,7 +1595,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= cmd = replace_config_vars(build.init, build) logging.info("Running 'init' commands in %s" % root_dir) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % (app.id, build.versionName), p.output) @@ -1683,7 +1753,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % (app.id, build.versionName), p.output) @@ -1863,7 +1933,25 @@ def get_file_extension(filename): return os.path.splitext(filename)[1].lower()[1:] -def get_apk_debuggable_aapt(apkfile): +def use_androguard(): + """Report if androguard is available, and config its debug logging""" + + try: + import androguard + if use_androguard.show_path: + logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__)) + use_androguard.show_path = False + if options and options.verbose: + logging.getLogger("androguard.axml").setLevel(logging.INFO) + return True + except ImportError: + return False + + +use_androguard.show_path = True + + +def is_apk_and_debuggable_aapt(apkfile): p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'], output=False) if p.returncode != 0: @@ -1874,7 +1962,7 @@ def get_apk_debuggable_aapt(apkfile): return False -def get_apk_debuggable_androguard(apkfile): +def is_apk_and_debuggable_androguard(apkfile): try: from androguard.core.bytecodes.apk import APK except ImportError: @@ -1888,7 +1976,7 @@ def get_apk_debuggable_androguard(apkfile): return False -def isApkAndDebuggable(apkfile): +def is_apk_and_debuggable(apkfile): """Returns True if the given file is an APK and is debuggable :param apkfile: full path to the apk to check""" @@ -1896,10 +1984,10 @@ def isApkAndDebuggable(apkfile): if get_file_extension(apkfile) != 'apk': return False - if SdkToolsPopen(['aapt', 'version'], output=False): - return get_apk_debuggable_aapt(apkfile) + if use_androguard(): + return is_apk_and_debuggable_androguard(apkfile) else: - return get_apk_debuggable_androguard(apkfile) + return is_apk_and_debuggable_aapt(apkfile) def get_apk_id_aapt(apkfile): @@ -1908,7 +1996,7 @@ def get_apk_id_aapt(apkfile): :param apkfile: path to an APK file. :returns: triplet (appid, version code, version name) """ - r = re.compile("package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)' platformBuildVersionName='.*'") + r = re.compile("^package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)'.*") p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) for line in p.output.splitlines(): m = r.match(line) @@ -1918,6 +2006,22 @@ def get_apk_id_aapt(apkfile): .format(apkfilename=apkfile)) +def get_minSdkVersion_aapt(apkfile): + """Extract the minimum supported Android SDK from an APK using aapt + + :param apkfile: path to an APK file. + :returns: the integer representing the SDK version + """ + r = re.compile(r"^sdkVersion:'([0-9]+)'") + p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) + for line in p.output.splitlines(): + m = r.match(line) + if m: + return int(m.group(1)) + raise FDroidException(_('Reading minSdkVersion failed: "{apkfilename}"') + .format(apkfilename=apkfile)) + + class PopenResult: def __init__(self): self.returncode = None @@ -1965,11 +2069,13 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou p = None try: p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env, - stdout=subprocess.PIPE, stderr=stderr_param) + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=stderr_param) except OSError as e: raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e)) + # TODO are these AsynchronousFileReader threads always exiting? if not stderr_to_stdout and options.verbose: stderr_queue = Queue() stderr_reader = AsynchronousFileReader(p.stderr, stderr_queue) @@ -2150,7 +2256,6 @@ def replace_config_vars(cmd, build): cmd = cmd.replace('$$SDK$$', config['sdk_path']) cmd = cmd.replace('$$NDK$$', build.ndk_path()) cmd = cmd.replace('$$MVN3$$', config['mvn3']) - cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '') if build is not None: cmd = replace_build_vars(cmd, build) return cmd @@ -2179,7 +2284,7 @@ def place_srclib(root_dir, number, libpath): o.write('android.library.reference.%d=%s\n' % (number, relpath)) -apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)') +apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)') def signer_fingerprint_short(sig): @@ -2330,7 +2435,7 @@ def apk_strip_signatures(signed_apk, strip_manifest=False): """ with tempfile.TemporaryDirectory() as tmpdir: tmp_apk = os.path.join(tmpdir, 'tmp.apk') - os.rename(signed_apk, tmp_apk) + shutil.move(signed_apk, tmp_apk) with ZipFile(tmp_apk, 'r') as in_apk: with ZipFile(signed_apk, 'w') as out_apk: for info in in_apk.infolist(): @@ -2391,6 +2496,40 @@ def apk_extract_signatures(apkpath, outdir, manifest=True): out_file.write(in_apk.read(f.filename)) +def sign_apk(unsigned_path, signed_path, keyalias): + """Sign and zipalign an unsigned APK, then save to a new file, deleting the unsigned + + android-18 (4.3) finally added support for reasonable hash + algorithms, like SHA-256, before then, the only options were MD5 + and SHA1 :-/ This aims to use SHA-256 when the APK does not target + older Android versions, and is therefore safe to do so. + + https://issuetracker.google.com/issues/36956587 + https://android-review.googlesource.com/c/platform/libcore/+/44491 + + """ + + if get_minSdkVersion_aapt(unsigned_path) < 18: + signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1'] + else: + signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256'] + + p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'], + '-storepass:env', 'FDROID_KEY_STORE_PASS', + '-keypass:env', 'FDROID_KEY_PASS'] + + signature_algorithm + [unsigned_path, keyalias], + envs={ + 'FDROID_KEY_STORE_PASS': config['keystorepass'], + 'FDROID_KEY_PASS': config['keypass'], }) + if p.returncode != 0: + raise BuildException(_("Failed to sign application"), p.output) + + p = SdkToolsPopen(['zipalign', '-v', '4', unsigned_path, signed_path]) + if p.returncode != 0: + raise BuildException(_("Failed to zipalign application")) + os.remove(unsigned_path) + + def verify_apks(signed_apk, unsigned_apk, tmp_dir): """Verify that two apks are the same @@ -2466,8 +2605,16 @@ def verify_jar_signature(jar): """ - if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4: - raise VerificationException(_("The repository's index could not be verified.")) + error = _('JAR signature failed to verify: {path}').format(path=jar) + try: + output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar], + stderr=subprocess.STDOUT) + raise VerificationException(error + '\n' + output.decode('utf-8')) + except subprocess.CalledProcessError as e: + if e.returncode == 4: + logging.debug(_('JAR signature verified: {path}').format(path=jar)) + else: + raise VerificationException(error + '\n' + e.output.decode('utf-8')) def verify_apk_signature(apk, min_sdk_version=None): @@ -2483,14 +2630,24 @@ def verify_apk_signature(apk, min_sdk_version=None): args = [config['apksigner'], 'verify'] if min_sdk_version: args += ['--min-sdk-version=' + min_sdk_version] - return subprocess.call(args + [apk]) == 0 + if options.verbose: + args += ['--verbose'] + try: + output = subprocess.check_output(args + [apk]) + if options.verbose: + logging.debug(apk + ': ' + output.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + logging.error('\n' + apk + ': ' + e.output.decode('utf-8')) else: - logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner") + if not config.get('jarsigner_warning_displayed'): + config['jarsigner_warning_displayed'] = True + logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")) try: verify_jar_signature(apk) return True - except Exception: - pass + except Exception as e: + logging.error(e) return False @@ -2511,8 +2668,23 @@ def verify_old_apk_signature(apk): with open(_java_security, 'w') as fp: fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024') - return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security, - '-strict', '-verify', apk]) == 4 + try: + cmd = [ + config['jarsigner'], + '-J-Djava.security.properties=' + _java_security, + '-strict', '-verify', apk + ] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if e.returncode != 4: + output = e.output + else: + logging.debug(_('JAR signature verified: {path}').format(path=apk)) + return True + + logging.error(_('Old APK signature failed to verify: {path}').format(path=apk) + + '\n' + output.decode('utf-8')) + return False apk_badchars = re.compile('''[/ :;'"]''') @@ -2884,3 +3056,65 @@ def get_examples_dir(): examplesdir = prefix + '/examples' return examplesdir + + +def get_wiki_timestamp(timestamp=None): + """Return current time in the standard format for posting to the wiki""" + + if timestamp is None: + timestamp = time.gmtime() + return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) + + +def get_android_tools_versions(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + + global config + sdk_path = config['sdk_path'] + if sdk_path[-1] != '/': + sdk_path += '/' + components = [] + if ndk_path: + ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') + if os.path.isfile(ndk_release_txt): + with open(ndk_release_txt, 'r') as fp: + components.append((os.path.basename(ndk_path), fp.read()[:-1])) + + pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) + for root, dirs, files in os.walk(sdk_path): + if 'source.properties' in files: + source_properties = os.path.join(root, 'source.properties') + with open(source_properties, 'r') as fp: + m = pattern.search(fp.read()) + if m: + components.append((root[len(sdk_path):], m.group(1))) + + return components + + +def get_android_tools_version_log(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + log = '== Installed Android Tools ==\n\n' + components = get_android_tools_versions(ndk_path) + for name, version in sorted(components): + log += '* ' + name + ' (' + version + ')\n' + + return log + + +def get_git_describe_link(): + """Get a link to the current fdroiddata commit, to post to the wiki + + """ + try: + output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'], + universal_newlines=True).strip() + except subprocess.CalledProcessError: + pass + if output: + commit = output.replace('-dirty', '') + return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n' + .format(commit=commit, id=output)) + else: + logging.error(_("'{path}' failed to execute!").format(path='git describe')) + return ''