X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Fcommon.py;h=76089184708cb6a39618e9468981e73faec93041;hb=70d9633555ba07b4bb83dfd7dcda9781cc80cf51;hp=6511181f0f2404b58bd0fbdc6d0e9e896e83852f;hpb=adc0c23db4eeae7476cb3256806dbd9215d1cce2;p=fdroidserver.git diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 6511181f..76089184 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -57,6 +57,11 @@ from fdroidserver.exception import FDroidException, VCSException, NoSubmodulesEx BuildException, VerificationException from .asynchronousfilereader import AsynchronousFileReader +# this is the build-tools version, aapt has a separate version that +# has to be manually set in test_aapt_version() +MINIMUM_AAPT_VERSION = '26.0.0' + +VERCODE_OPERATION_RE = re.compile(r'^([ 0-9/*+-]|%c)+$') # A signature block file with a .DSA, .RSA, or .EC extension CERT_PATH_REGEX = re.compile(r'^META-INF/.*\.(DSA|EC|RSA)$') @@ -74,17 +79,15 @@ orig_path = None default_config = { 'sdk_path': "$ANDROID_HOME", 'ndk_paths': { - 'r9b': None, 'r10e': None, 'r11c': None, 'r12b': "$ANDROID_NDK", 'r13b': None, 'r14b': None, 'r15c': None, - 'r16': None, + 'r16b': None, }, - 'qt_sdk_path': None, - 'build_tools': "25.0.2", + 'build_tools': MINIMUM_AAPT_VERSION, 'force_build_tools': False, 'java_paths': None, 'ant': "ant", @@ -128,6 +131,13 @@ default_config = { def setup_global_opts(parser): + try: # the buildserver VM might not have PIL installed + from PIL import PngImagePlugin + logger = logging.getLogger(PngImagePlugin.__name__) + logger.setLevel(logging.INFO) # tame the "STREAM" debug messages + except ImportError: + pass + parser.add_argument("-v", "--verbose", action="store_true", default=False, help=_("Spew out even more information than normal")) parser.add_argument("-q", "--quiet", action="store_true", default=False, @@ -390,13 +400,13 @@ def test_aapt_version(aapt): # the Debian package has the version string like "v0.2-23.0.2" too_old = False if '.' in bugfix: - if LooseVersion(bugfix) < LooseVersion('24.0.0'): + if LooseVersion(bugfix) < LooseVersion(MINIMUM_AAPT_VERSION): too_old = True - elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.2964546'): + elif LooseVersion('.'.join((major, minor, bugfix))) < LooseVersion('0.2.4062713'): too_old = True if too_old: - logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-24.0.0 or newer!") - .format(aapt=aapt)) + logging.warning(_("'{aapt}' is too old, fdroid requires build-tools-{version} or newer!") + .format(aapt=aapt, version=MINIMUM_AAPT_VERSION)) else: logging.warning(_('Unknown version of aapt, might cause problems: ') + output) @@ -807,7 +817,8 @@ class vcs_git(vcs): # # supported in git >= 2.3 git_config = [ - '-c', 'core.sshCommand=false', + '-c', 'core.askpass=/bin/true', + '-c', 'core.sshCommand=/bin/false', '-c', 'url.https://.insteadOf=ssh://', ] for domain in ('bitbucket.org', 'github.com', 'gitlab.com'): @@ -819,7 +830,9 @@ class vcs_git(vcs): git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain) envs.update({ 'GIT_TERMINAL_PROMPT': '0', - 'GIT_SSH': 'false', # for git < 2.3 + 'GIT_ASKPASS': '/bin/true', + 'SSH_ASKPASS': '/bin/true', + 'GIT_SSH': '/bin/false', # for git < 2.3 }) return FDroidPopen(['git', ] + git_config + args, envs=envs, cwd=cwd, output=output) @@ -840,7 +853,7 @@ class vcs_git(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - p = self.git(['clone', self.remote, self.local]) + p = self.git(['clone', '--', self.remote, self.local]) if p.returncode != 0: self.clone_failed = True raise VCSException("Git clone failed", p.output) @@ -873,7 +886,8 @@ class vcs_git(vcs): if 'Multiple remote HEAD branches' not in lines[0]: raise VCSException(_("Git remote set-head failed"), p.output) branch = lines[1].split(' ')[-1] - p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', branch], cwd=self.local, output=False) + p2 = FDroidPopen(['git', 'remote', 'set-head', 'origin', '--', branch], + cwd=self.local, output=False) if p2.returncode != 0: raise VCSException(_("Git remote set-head failed"), p.output + '\n' + p2.output) self.refreshed = True @@ -951,21 +965,34 @@ class vcs_gitsvn(vcs): def git(self, args, envs=dict(), cwd=None, output=True): '''Prevent git fetch/clone/submodule from hanging at the username/password prompt + + AskPass is set to /bin/true to let the process try to connect + without a username/password. + + The SSH command is set to /bin/false to block all SSH URLs + (supported in git >= 2.3). This protects against + CVE-2017-1000117. + ''' - # CVE-2017-1000117 block all SSH URLs (supported in git >= 2.3) - config = ['-c', 'core.sshCommand=false'] + git_config = [ + '-c', 'core.askpass=/bin/true', + '-c', 'core.sshCommand=/bin/false', + ] envs.update({ 'GIT_TERMINAL_PROMPT': '0', - 'GIT_SSH': 'false', # for git < 2.3 - 'SVN_SSH': 'false', + 'GIT_ASKPASS': '/bin/true', + 'SSH_ASKPASS': '/bin/true', + 'GIT_SSH': '/bin/false', # for git < 2.3 + 'SVN_SSH': '/bin/false', }) - return FDroidPopen(['git', ] + config + args, + return FDroidPopen(['git', ] + git_config + args, envs=envs, cwd=cwd, output=output) def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout gitsvn_args = ['svn', 'clone'] + remote = None if ';' in self.remote: remote_split = self.remote.split(';') for i in remote_split[1:]: @@ -975,17 +1002,27 @@ class vcs_gitsvn(vcs): gitsvn_args.extend(['-t', i[5:]]) elif i.startswith('branches='): gitsvn_args.extend(['-b', i[9:]]) - gitsvn_args.extend([remote_split[0], self.local]) - p = self.git(gitsvn_args, output=False) - if p.returncode != 0: - self.clone_failed = True - raise VCSException("Git svn clone failed", p.output) + remote = remote_split[0] else: - gitsvn_args.extend([self.remote, self.local]) - p = self.git(gitsvn_args, output=False) - if p.returncode != 0: - self.clone_failed = True - raise VCSException("Git svn clone failed", p.output) + remote = self.remote + + if not remote.startswith('https://'): + raise VCSException(_('HTTPS must be used with Subversion URLs!')) + + # git-svn sucks at certificate validation, this throws useful errors: + import requests + r = requests.head(remote) + r.raise_for_status() + location = r.headers.get('location') + if location and not location.startswith('https://'): + raise VCSException(_('Invalid redirect to non-HTTPS: {before} -> {after} ') + .format(before=remote, after=location)) + + gitsvn_args.extend(['--', remote, self.local]) + p = self.git(gitsvn_args) + if p.returncode != 0: + self.clone_failed = True + raise VCSException(_('git svn clone failed'), p.output) self.checkrepo() else: self.checkrepo() @@ -1081,7 +1118,8 @@ class vcs_hg(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): - p = FDroidPopen(['hg', 'clone', '--ssh', 'false', self.remote, self.local], output=False) + p = FDroidPopen(['hg', 'clone', '--ssh', '/bin/false', '--', self.remote, self.local], + output=False) if p.returncode != 0: self.clone_failed = True raise VCSException("Hg clone failed", p.output) @@ -1092,9 +1130,9 @@ class vcs_hg(vcs): for line in p.output.splitlines(): if not line.startswith('? '): raise VCSException("Unexpected output from hg status -uS: " + line) - FDroidPopen(['rm', '-rf', line[2:]], cwd=self.local, output=False) + FDroidPopen(['rm', '-rf', '--', line[2:]], cwd=self.local, output=False) if not self.refreshed: - p = FDroidPopen(['hg', 'pull', '--ssh', 'false'], cwd=self.local, output=False) + p = FDroidPopen(['hg', 'pull', '--ssh', '/bin/false'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg pull failed", p.output) self.refreshed = True @@ -1102,7 +1140,7 @@ class vcs_hg(vcs): rev = rev or 'default' if not rev: return - p = FDroidPopen(['hg', 'update', '-C', rev], cwd=self.local, output=False) + p = FDroidPopen(['hg', 'update', '-C', '--', rev], cwd=self.local, output=False) if p.returncode != 0: raise VCSException("Hg checkout of '%s' failed" % rev, p.output) p = FDroidPopen(['hg', 'purge', '--all'], cwd=self.local, output=False) @@ -1288,9 +1326,9 @@ def remove_debuggable_flags(root_dir): os.path.join(root, 'AndroidManifest.xml')) -vcsearch_g = re.compile(r'''.*[Vv]ersionCode[ =]+["']*([0-9]+)["']*''').search -vnsearch_g = re.compile(r'.*[Vv]ersionName *=* *(["\'])((?:(?=(\\?))\3.)*?)\1.*').search -psearch_g = re.compile(r'.*(packageName|applicationId) *=* *["\']([^"]+)["\'].*').search +vcsearch_g = re.compile(r'''.*[Vv]ersionCode\s*=?\s*["']*([0-9]+)["']*''').search +vnsearch_g = re.compile(r'''.*[Vv]ersionName\s*=?\s*(["'])((?:(?=(\\?))\3.)*?)\1.*''').search +psearch_g = re.compile(r'''.*(packageName|applicationId)\s*=*\s*["']([^"']+)["'].*''').search def app_matches_packagename(app, package): @@ -1329,44 +1367,49 @@ def parse_androidmanifests(paths, app): vercode = None package = None - flavour = "" + flavour = None if app.builds and 'gradle' in app.builds[-1] and app.builds[-1].gradle: - flavour = app.builds[-1].gradle[-1] + flavour = app.builds[-1].gradle[-1] if has_extension(path, 'gradle'): - # first try to get version name and code from correct flavour with open(path, 'r') as f: - buildfile = f.read() - - regex_string = r"" + flavour + ".*?}" - search = re.compile(regex_string, re.DOTALL) - result = search.search(buildfile) - - if result is not None: - resultgroup = result.group() - - if not package: - matches = psearch_g(resultgroup) - if matches: - s = matches.group(2) - if app_matches_packagename(app, s): - package = s - if not version: - matches = vnsearch_g(resultgroup) - if matches: - version = matches.group(2) - if not vercode: - matches = vcsearch_g(resultgroup) - if matches: - vercode = matches.group(1) - else: - # fall back to parse file line by line - with open(path, 'r') as f: - for line in f: - if gradle_comment.match(line): - continue - # Grab first occurence of each to avoid running into - # alternative flavours and builds. + inside_flavour_group = 0 + inside_required_flavour = 0 + for line in f: + if gradle_comment.match(line): + continue + + if inside_flavour_group > 0: + if inside_required_flavour > 0: + matches = psearch_g(line) + if matches: + s = matches.group(2) + if app_matches_packagename(app, s): + package = s + + matches = vnsearch_g(line) + if matches: + version = matches.group(2) + + matches = vcsearch_g(line) + if matches: + vercode = matches.group(1) + + if '{' in line: + inside_required_flavour += 1 + if '}' in line: + inside_required_flavour -= 1 + else: + if flavour and (flavour in line): + inside_required_flavour = 1 + + if '{' in line: + inside_flavour_group += 1 + if '}' in line: + inside_flavour_group -= 1 + else: + if "productFlavors" in line: + inside_flavour_group = 1 if not package: matches = psearch_g(line) if matches: @@ -1497,7 +1540,7 @@ def getsrclib(spec, srclib_dir, subdir=None, basepath=False, if srclib["Prepare"]: cmd = replace_config_vars(srclib["Prepare"], build) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=libdir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=libdir) if p.returncode != 0: raise BuildException("Error running prepare command for srclib %s" % name, p.output) @@ -1552,7 +1595,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= cmd = replace_config_vars(build.init, build) logging.info("Running 'init' commands in %s" % root_dir) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running init command for %s:%s" % (app.id, build.versionName), p.output) @@ -1710,7 +1753,7 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver= libpath = os.path.relpath(libpath, root_dir) cmd = cmd.replace('$$' + name + '$$', libpath) - p = FDroidPopen(['bash', '-x', '-c', cmd], cwd=root_dir) + p = FDroidPopen(['bash', '-x', '-c', '--', cmd], cwd=root_dir) if p.returncode != 0: raise BuildException("Error running prebuild command for %s:%s" % (app.id, build.versionName), p.output) @@ -1890,7 +1933,25 @@ def get_file_extension(filename): return os.path.splitext(filename)[1].lower()[1:] -def get_apk_debuggable_aapt(apkfile): +def use_androguard(): + """Report if androguard is available, and config its debug logging""" + + try: + import androguard + if use_androguard.show_path: + logging.debug(_('Using androguard from "{path}"').format(path=androguard.__file__)) + use_androguard.show_path = False + if options and options.verbose: + logging.getLogger("androguard.axml").setLevel(logging.INFO) + return True + except ImportError: + return False + + +use_androguard.show_path = True + + +def is_apk_and_debuggable_aapt(apkfile): p = SdkToolsPopen(['aapt', 'dump', 'xmltree', apkfile, 'AndroidManifest.xml'], output=False) if p.returncode != 0: @@ -1901,7 +1962,7 @@ def get_apk_debuggable_aapt(apkfile): return False -def get_apk_debuggable_androguard(apkfile): +def is_apk_and_debuggable_androguard(apkfile): try: from androguard.core.bytecodes.apk import APK except ImportError: @@ -1915,7 +1976,7 @@ def get_apk_debuggable_androguard(apkfile): return False -def isApkAndDebuggable(apkfile): +def is_apk_and_debuggable(apkfile): """Returns True if the given file is an APK and is debuggable :param apkfile: full path to the apk to check""" @@ -1923,10 +1984,10 @@ def isApkAndDebuggable(apkfile): if get_file_extension(apkfile) != 'apk': return False - if SdkToolsPopen(['aapt', 'version'], output=False): - return get_apk_debuggable_aapt(apkfile) + if use_androguard(): + return is_apk_and_debuggable_androguard(apkfile) else: - return get_apk_debuggable_androguard(apkfile) + return is_apk_and_debuggable_aapt(apkfile) def get_apk_id_aapt(apkfile): @@ -1935,7 +1996,7 @@ def get_apk_id_aapt(apkfile): :param apkfile: path to an APK file. :returns: triplet (appid, version code, version name) """ - r = re.compile("package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)' platformBuildVersionName='.*'") + r = re.compile("^package: name='(?P.*)' versionCode='(?P.*)' versionName='(?P.*)'.*") p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) for line in p.output.splitlines(): m = r.match(line) @@ -2008,7 +2069,8 @@ def FDroidPopenBytes(commands, cwd=None, envs=None, output=True, stderr_to_stdou p = None try: p = subprocess.Popen(commands, cwd=cwd, shell=False, env=process_env, - stdout=subprocess.PIPE, stderr=stderr_param) + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=stderr_param) except OSError as e: raise BuildException("OSError while trying to execute " + ' '.join(commands) + ': ' + str(e)) @@ -2194,7 +2256,6 @@ def replace_config_vars(cmd, build): cmd = cmd.replace('$$SDK$$', config['sdk_path']) cmd = cmd.replace('$$NDK$$', build.ndk_path()) cmd = cmd.replace('$$MVN3$$', config['mvn3']) - cmd = cmd.replace('$$QT$$', config['qt_sdk_path'] or '') if build is not None: cmd = replace_build_vars(cmd, build) return cmd @@ -2223,7 +2284,7 @@ def place_srclib(root_dir, number, libpath): o.write('android.library.reference.%d=%s\n' % (number, relpath)) -apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z]+\.(SF|RSA|DSA|EC)') +apk_sigfile = re.compile(r'META-INF/[0-9A-Za-z_\-]+\.(SF|RSA|DSA|EC)') def signer_fingerprint_short(sig): @@ -2451,7 +2512,7 @@ def sign_apk(unsigned_path, signed_path, keyalias): if get_minSdkVersion_aapt(unsigned_path) < 18: signature_algorithm = ['-sigalg', 'SHA1withRSA', '-digestalg', 'SHA1'] else: - signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA256'] + signature_algorithm = ['-sigalg', 'SHA256withRSA', '-digestalg', 'SHA-256'] p = FDroidPopen([config['jarsigner'], '-keystore', config['keystore'], '-storepass:env', 'FDROID_KEY_STORE_PASS', @@ -2544,8 +2605,16 @@ def verify_jar_signature(jar): """ - if subprocess.call([config['jarsigner'], '-strict', '-verify', jar]) != 4: - raise VerificationException(_("The repository's index could not be verified.")) + error = _('JAR signature failed to verify: {path}').format(path=jar) + try: + output = subprocess.check_output([config['jarsigner'], '-strict', '-verify', jar], + stderr=subprocess.STDOUT) + raise VerificationException(error + '\n' + output.decode('utf-8')) + except subprocess.CalledProcessError as e: + if e.returncode == 4: + logging.debug(_('JAR signature verified: {path}').format(path=jar)) + else: + raise VerificationException(error + '\n' + e.output.decode('utf-8')) def verify_apk_signature(apk, min_sdk_version=None): @@ -2561,14 +2630,24 @@ def verify_apk_signature(apk, min_sdk_version=None): args = [config['apksigner'], 'verify'] if min_sdk_version: args += ['--min-sdk-version=' + min_sdk_version] - return subprocess.call(args + [apk]) == 0 + if options.verbose: + args += ['--verbose'] + try: + output = subprocess.check_output(args + [apk]) + if options.verbose: + logging.debug(apk + ': ' + output.decode('utf-8')) + return True + except subprocess.CalledProcessError as e: + logging.error('\n' + apk + ': ' + e.output.decode('utf-8')) else: - logging.warning("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner") + if not config.get('jarsigner_warning_displayed'): + config['jarsigner_warning_displayed'] = True + logging.warning(_("Using Java's jarsigner, not recommended for verifying APKs! Use apksigner")) try: verify_jar_signature(apk) return True - except Exception: - pass + except Exception as e: + logging.error(e) return False @@ -2589,8 +2668,23 @@ def verify_old_apk_signature(apk): with open(_java_security, 'w') as fp: fp.write('jdk.jar.disabledAlgorithms=MD2, RSA keySize < 1024') - return subprocess.call([config['jarsigner'], '-J-Djava.security.properties=' + _java_security, - '-strict', '-verify', apk]) == 4 + try: + cmd = [ + config['jarsigner'], + '-J-Djava.security.properties=' + _java_security, + '-strict', '-verify', apk + ] + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + if e.returncode != 4: + output = e.output + else: + logging.debug(_('JAR signature verified: {path}').format(path=apk)) + return True + + logging.error(_('Old APK signature failed to verify: {path}').format(path=apk) + + '\n' + output.decode('utf-8')) + return False apk_badchars = re.compile('''[/ :;'"]''') @@ -2962,3 +3056,65 @@ def get_examples_dir(): examplesdir = prefix + '/examples' return examplesdir + + +def get_wiki_timestamp(timestamp=None): + """Return current time in the standard format for posting to the wiki""" + + if timestamp is None: + timestamp = time.gmtime() + return time.strftime("%Y-%m-%d %H:%M:%SZ", timestamp) + + +def get_android_tools_versions(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + + global config + sdk_path = config['sdk_path'] + if sdk_path[-1] != '/': + sdk_path += '/' + components = [] + if ndk_path: + ndk_release_txt = os.path.join(ndk_path, 'RELEASE.TXT') + if os.path.isfile(ndk_release_txt): + with open(ndk_release_txt, 'r') as fp: + components.append((os.path.basename(ndk_path), fp.read()[:-1])) + + pattern = re.compile('^Pkg.Revision=(.+)', re.MULTILINE) + for root, dirs, files in os.walk(sdk_path): + if 'source.properties' in files: + source_properties = os.path.join(root, 'source.properties') + with open(source_properties, 'r') as fp: + m = pattern.search(fp.read()) + if m: + components.append((root[len(sdk_path):], m.group(1))) + + return components + + +def get_android_tools_version_log(ndk_path=None): + '''get a list of the versions of all installed Android SDK/NDK components''' + log = '== Installed Android Tools ==\n\n' + components = get_android_tools_versions(ndk_path) + for name, version in sorted(components): + log += '* ' + name + ' (' + version + ')\n' + + return log + + +def get_git_describe_link(): + """Get a link to the current fdroiddata commit, to post to the wiki + + """ + try: + output = subprocess.check_output(['git', 'describe', '--always', '--dirty', '--abbrev=0'], + universal_newlines=True).strip() + except subprocess.CalledProcessError: + pass + if output: + commit = output.replace('-dirty', '') + return ('* fdroiddata: [https://gitlab.com/fdroid/fdroiddata/commit/{commit} {id}]\n' + .format(commit=commit, id=output)) + else: + logging.error(_("'{path}' failed to execute!").format(path='git describe')) + return ''