From: Hans-Christoph Steiner Date: Thu, 30 Nov 2017 12:44:47 +0000 (+0100) Subject: Merge branch 'gradleFlavor' into 'master' X-Git-Tag: 1.0.0~59 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=dcbc78d23815cd27db0414074fa5c601093ce82f;hp=c7c40cb59f232844a1d74120d8536673f655e435;p=fdroidserver.git Merge branch 'gradleFlavor' into 'master' gradle file: use flavour specific versionCode/versionName, fall back to parsing line by line See merge request fdroid/fdroidserver!389 --- diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index b79c2e4a..7cd96e0c 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -74,12 +74,25 @@ y EOH mkdir -p $ANDROID_HOME/licenses/ + cat << EOF > $ANDROID_HOME/licenses/android-sdk-license 8933bad161af4178b1185d1a37fbf41ea5269c55 + d56f5187479451eabf01fb78af6dfcb131a6481e EOF -echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license-old +79120722343a6f314e0719f863036c702b0e6b2a + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" diff --git a/completion/bash-completion b/completion/bash-completion index 8bdd333f..d18edd35 100644 --- a/completion/bash-completion +++ b/completion/bash-completion @@ -264,6 +264,12 @@ __complete_btlog() { __complete_options } +__complete_mirror() { + opts="-v" + lopts="--archive --output-dir" + __complete_options +} + __complete_nightly() { opts="-v -q" lopts="--show-secret-var" @@ -316,6 +322,7 @@ import \ init \ install \ lint \ +mirror \ nightly \ publish \ readmeta \ diff --git a/examples/config.py b/examples/config.py index 0551e1c0..f36d51d2 100644 --- a/examples/config.py +++ b/examples/config.py @@ -216,11 +216,12 @@ The repository of older versions of applications from the main demo repository. # sync_from_local_copy_dir = True -# To upload the repo to an Amazon S3 bucket using `fdroid server update`. -# Warning, this deletes and recreates the whole fdroid/ directory each -# time. This is based on apache-libcloud, which supports basically all cloud -# storage services, so it should be easy to port the fdroid server tools to -# any of them. +# To upload the repo to an Amazon S3 bucket using `fdroid server +# update`. Warning, this deletes and recreates the whole fdroid/ +# directory each time. This prefers s3cmd, but can also use +# apache-libcloud. To customize how s3cmd interacts with the cloud +# provider, create a 's3cfg' file next to this file (config.py), and +# those settings will be used instead of any 'aws' variable below. # # awsbucket = 'myawsfdroid' # awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA' diff --git a/fdroid b/fdroid index a07a4ecf..f5e6c92b 100755 --- a/fdroid +++ b/fdroid @@ -48,6 +48,7 @@ commands = OrderedDict([ ("btlog", _("Update the binary transparency log for a URL")), ("signatures", _("Extract signatures from APKs")), ("nightly", _("Set up an app build for a nightly build repo")), + ("mirror", _("Download complete mirrors of small repos")), ]) diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 740d7f2c..16901e78 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -23,6 +23,7 @@ import shutil import glob import subprocess import re +import resource import tarfile import traceback import time @@ -1120,6 +1121,19 @@ def main(): if not apps: raise FDroidException("No apps to process.") + # make sure enough open files are allowed to process everything + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + if options.latest: for app in apps.values(): for build in reversed(app.builds): diff --git a/fdroidserver/common.py b/fdroidserver/common.py index b2e918ba..6ed43d4c 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -40,7 +40,7 @@ import json import xml.etree.ElementTree as XMLElementTree from binascii import hexlify -from datetime import datetime +from datetime import datetime, timedelta from distutils.version import LooseVersion from queue import Queue from zipfile import ZipFile @@ -444,17 +444,16 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') -def read_pkg_args(args, allow_vercodes=False): +def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): """ - :param args: arguments in the form of multiple appid:[vc] strings + :param appids: arguments in the form of multiple appid:[vc] strings :returns: a dictionary with the set of vercodes specified for each package """ - vercodes = {} - if not args: + if not appid_versionCode_pairs: return vercodes - for p in args: + for p in appid_versionCode_pairs: if allow_vercodes and ':' in p: package, vercode = p.split(':') else: @@ -468,13 +467,17 @@ def read_pkg_args(args, allow_vercodes=False): return vercodes -def read_app_args(args, allapps, allow_vercodes=False): - """ - On top of what read_pkg_args does, this returns the whole app metadata, but - limiting the builds list to the builds matching the vercodes specified. +def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): + """Build a list of App instances for processing + + On top of what read_pkg_args does, this returns the whole app + metadata, but limiting the builds list to the builds matching the + appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then + all App and Build instances are returned. + """ - vercodes = read_pkg_args(args, allow_vercodes) + vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) if not vercodes: return allapps @@ -1747,6 +1750,23 @@ def natural_key(s): return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)] +def check_system_clock(dt_obj, path): + """Check if system clock is updated based on provided date + + If an APK has files newer than the system time, suggest updating + the system clock. This is useful for offline systems, used for + signing, which do not have another source of clock sync info. It + has to be more than 24 hours newer because ZIP/APK files do not + store timezone info + + """ + checkdt = dt_obj - timedelta(1) + if datetime.today() < checkdt: + logging.warning(_('System clock is older than date in {path}!').format(path=path) + + '\n' + _('Set clock to that time using:') + '\n' + + 'sudo date -s "' + str(dt_obj) + '"') + + class KnownApks: """permanent store of existing APKs with the date they were added @@ -1775,6 +1795,7 @@ class KnownApks: date = datetime.strptime(t[-1], '%Y-%m-%d') filename = line[0:line.rfind(appid) - 1] self.apks[filename] = (appid, date) + check_system_clock(date, self.path) self.changed = False def writeifchanged(self): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 9fdb5836..9d03e0b9 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -178,6 +178,7 @@ def main(): + '" does not exist, creating a new keystore there.') common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None + keydname = None if options.repo_keyalias: repo_keyalias = options.repo_keyalias common.write_to_config(test_config, 'repo_keyalias', repo_keyalias) @@ -211,7 +212,16 @@ def main(): flags=re.MULTILINE) with open('opensc-fdroid.cfg', 'w') as f: f.write(opensc_fdroid) - elif not os.path.exists(keystore): + elif os.path.exists(keystore): + to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] + if repo_keyalias: + to_set.remove('repo_keyalias') + if keydname: + to_set.remove('keydname') + logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) + + '\n' + _('Now set these in config.py:') + ' ' + + ', '.join(to_set) + '\n') + else: password = common.genpassword() c = dict(test_config) c['keystorepass'] = password diff --git a/fdroidserver/mirror.py b/fdroidserver/mirror.py new file mode 100644 index 00000000..06595a44 --- /dev/null +++ b/fdroidserver/mirror.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 + +import ipaddress +import logging +import os +import posixpath +import socket +import subprocess +import sys +from argparse import ArgumentParser +import urllib.parse + +from . import _ +from . import common +from . import index +from . import update + +options = None + + +def _run_wget(path, urls): + if options.verbose: + verbose = '--verbose' + else: + verbose = '--no-verbose' + + if not urls: + return + logging.debug(_('Running wget in {path}').format(path=path)) + os.makedirs(path, exist_ok=True) + os.chdir(path) + urls_file = '.fdroid-mirror-wget-input-file' + with open(urls_file, 'w') as fp: + for url in urls: + fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename + subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"', + '--input-file=' + urls_file]) + os.remove(urls_file) + + +def main(): + global options + + parser = ArgumentParser(usage=_("%(prog)s [options] url")) + common.setup_global_opts(parser) + parser.add_argument("url", nargs='?', + help=_('Base URL to mirror, can include the index signing key ' + + 'using the query string: ?fingerprint=')) + parser.add_argument("--archive", action='store_true', default=False, + help=_("Also mirror the full archive section")) + parser.add_argument("--output-dir", default=None, + help=_("The directory to write the mirror to")) + options = parser.parse_args() + + if options.url is None: + logging.error(_('A URL is required as an argument!') + '\n') + parser.print_help() + sys.exit(1) + + scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url) + fingerprint = urllib.parse.parse_qs(query).get('fingerprint') + + def _append_to_url_path(*args): + '''Append the list of path components to URL, keeping the rest the same''' + newpath = posixpath.join(path, *args) + return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment)) + + if fingerprint: + config = common.read_config(options) + if not ('jarsigner' in config or 'apksigner' in config): + logging.error(_('Java JDK not found! Install in standard location or set java_paths!')) + sys.exit(1) + + def _get_index(section, etag=None): + url = _append_to_url_path(section) + return index.download_repo_index(url, etag=etag) + else: + def _get_index(section, etag=None): + import io + import json + import zipfile + from . import net + url = _append_to_url_path(section, 'index-v1.jar') + content, etag = net.http_get(url) + with zipfile.ZipFile(io.BytesIO(content)) as zip: + jsoncontents = zip.open('index-v1.json').read() + data = json.loads(jsoncontents.decode('utf-8')) + return data, etag + + ip = None + try: + ip = ipaddress.ip_address(hostname) + except ValueError: + pass + if hostname == 'f-droid.org' \ + or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]): + print(_('ERROR: this command should never be used to mirror f-droid.org!\n' + 'A full mirror of f-droid.org requires more than 200GB.')) + sys.exit(1) + + path = path.rstrip('/') + if path.endswith('repo') or path.endswith('archive'): + logging.warning(_('Do not include "{path}" in URL!') + .format(path=path.split('/')[-1])) + elif not path.endswith('fdroid'): + logging.warning(_('{url} does not end with "fdroid", check the URL path!') + .format(url=options.url)) + + icondirs = ['icons', ] + for density in update.screen_densities: + icondirs.append('icons-' + density) + + if options.output_dir: + basedir = options.output_dir + else: + basedir = os.path.join(os.getcwd(), hostname, path.strip('/')) + os.makedirs(basedir, exist_ok=True) + + if options.archive: + sections = ('repo', 'archive') + else: + sections = ('repo', ) + + for section in sections: + sectiondir = os.path.join(basedir, section) + + data, etag = _get_index(section) + + os.makedirs(sectiondir, exist_ok=True) + os.chdir(sectiondir) + for icondir in icondirs: + os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True) + + urls = [] + for packageName, packageList in data['packages'].items(): + for package in packageList: + to_fetch = [] + for k in ('apkName', 'srcname'): + if k in package: + to_fetch.append(package[k]) + elif k == 'apkName': + logging.error(_('{appid} is missing {name}') + .format(appid=package['packageName'], name=k)) + for f in to_fetch: + if not os.path.exists(f) \ + or (f.endswith('.apk') and os.path.getsize(f) != package['size']): + urls.append(_append_to_url_path(section, f)) + urls.append(_append_to_url_path(section, f + '.asc')) + _run_wget(sectiondir, urls) + + for app in data['apps']: + localized = app.get('localized') + if localized: + for locale, d in localized.items(): + urls = [] + components = (section, app['packageName'], locale) + for k in update.GRAPHIC_NAMES: + f = d.get(k) + if f: + urls.append(_append_to_url_path(*components, f)) + _run_wget(os.path.join(basedir, *components), urls) + for k in update.SCREENSHOT_DIRS: + urls = [] + filelist = d.get(k) + if filelist: + components = (section, app['packageName'], locale, k) + for f in filelist: + urls.append(_append_to_url_path(*components, f)) + _run_wget(os.path.join(basedir, *components), urls) + + urls = dict() + for app in data['apps']: + if 'icon' not in app: + logging.error(_('no "icon" in {appid}').format(appid=app['packageName'])) + continue + icon = app['icon'] + for icondir in icondirs: + url = _append_to_url_path(section, icondir, icon) + if icondir not in urls: + urls[icondir] = [] + urls[icondir].append(url) + + for icondir in icondirs: + _run_wget(os.path.join(basedir, section, icondir), urls[icondir]) + + +if __name__ == "__main__": + main() diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 3834f79e..454616e4 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -155,6 +155,7 @@ def main(): repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) try: @@ -186,10 +187,10 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update README") - icon_path = os.path.join(repo_basedir, 'icon.png') + icon_path = os.path.join(git_mirror_path, 'icon.png') try: import qrcode - img = qrcode.make('Some data here') + img = qrcode.make(repo_url) with open(icon_path, 'wb') as fp: fp.write(img) except Exception: @@ -197,9 +198,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.copy(exampleicon, icon_path) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update repo/website icon") + shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) - common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_repodir): + common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_metadatadir): + common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key @@ -254,7 +259,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, except subprocess.CalledProcessError: pass - subprocess.check_call(['fdroid', 'update', '--rename-apks', '--verbose'], cwd=repo_basedir) + subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], + cwd=repo_basedir) + common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') + mirror_git_repo.git.add(all=True) + mirror_git_repo.index.commit("update app metadata") try: subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir) except subprocess.CalledProcessError: diff --git a/fdroidserver/server.py b/fdroidserver/server.py index 116c27f8..004a6068 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -38,6 +38,9 @@ options = None BINARY_TRANSPARENCY_DIR = 'binary_transparency' +AUTO_S3CFG = '.fdroid-server-update-s3cfg' +USER_S3CFG = 's3cfg' + def update_awsbucket(repo_section): ''' @@ -72,12 +75,17 @@ def update_awsbucket_s3cmd(repo_section): logging.debug(_('Using s3cmd to sync with: {url}') .format(url=config['awsbucket'])) - configfilename = '.s3cfg' - fd = os.open(configfilename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) - os.write(fd, '[default]\n'.encode('utf-8')) - os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8')) - os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8')) - os.close(fd) + if os.path.exists(USER_S3CFG): + logging.info(_('Using "{path}" for configuring s3cmd.').format(path=USER_S3CFG)) + configfilename = USER_S3CFG + else: + fd = os.open(AUTO_S3CFG, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600) + logging.debug(_('Creating "{path}" for configuring s3cmd.').format(path=AUTO_S3CFG)) + os.write(fd, '[default]\n'.encode('utf-8')) + os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8')) + os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8')) + os.close(fd) + configfilename = AUTO_S3CFG s3bucketurl = 's3://' + config['awsbucket'] s3cmd = [config['s3cmd'], '--config=' + configfilename] @@ -151,6 +159,10 @@ def update_awsbucket_libcloud(repo_section): _('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.py!')) awsbucket = config['awsbucket'] + if os.path.exists(USER_S3CFG): + raise FDroidException(_('"{path}" exists but s3cmd is not installed!') + .format(path=USER_S3CFG)) + cls = get_driver(Provider.S3) driver = cls(config['awsaccesskeyid'], config['awssecretkey']) try: @@ -501,7 +513,7 @@ def upload_to_virustotal(repo_section, vt_apikey): with open(outputfilename, 'w') as fp: json.dump(response, fp, indent=2, sort_keys=True) - if response.get('positives') > 0: + if response.get('positives', 0) > 0: logging.warning(repofilename + ' has been flagged by virustotal ' + str(response['positives']) + ' times:' + '\n\t' + response['permalink']) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 2019063c..61026cab 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -29,7 +29,7 @@ import zipfile import hashlib import pickle import time -from datetime import datetime, timedelta +from datetime import datetime from argparse import ArgumentParser import collections @@ -1297,22 +1297,11 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal apkzip = zipfile.ZipFile(apkfile, 'r') - # if an APK has files newer than the system time, suggest updating - # the system clock. This is useful for offline systems, used for - # signing, which do not have another source of clock sync info. It - # has to be more than 24 hours newer because ZIP/APK files do not - # store timezone info manifest = apkzip.getinfo('AndroidManifest.xml') if manifest.date_time[1] == 0: # month can't be zero logging.debug(_('AndroidManifest.xml has no date')) else: - dt_obj = datetime(*manifest.date_time) - checkdt = dt_obj - timedelta(1) - if datetime.today() < checkdt: - logging.warning('System clock is older than manifest in: ' - + apkfilename - + '\nSet clock to that time using:\n' - + 'sudo date -s "' + str(dt_obj) + '"') + common.check_system_clock(datetime(*manifest.date_time), apkfilename) # extract icons from APK zip file iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode']) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index aae46c75..aab7eb0d 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -29,6 +29,8 @@ import textwrap from .common import FDroidException from logging import getLogger +from fdroidserver import _ + logger = getLogger('fdroidserver-vmtools') @@ -383,7 +385,9 @@ class LibvirtBuildVm(FDroidBuildVm): vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable - _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + if not os.access(imagepath, os.R_OK): + logger.warning(_('Cannot read "{path}"!').format(path=imagepath)) + _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) diff --git a/jenkins-build-all b/jenkins-build-all index 2abac58c..d41d920f 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -31,7 +31,6 @@ else echo "No virtualization is used." fi sudo /bin/chmod -R a+rX /var/lib/libvirt/images -ulimit -n 2048 echo 'maximum allowed number of open file descriptors: ' `ulimit -n` ls -ld /var/lib/libvirt/images ls -l /var/lib/libvirt/images || echo no access diff --git a/setup.py b/setup.py index 1552a636..a382b254 100755 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ setup(name='fdroidserver', 'pyasn1-modules', 'python-vagrant', 'PyYAML', + 'qrcode', 'ruamel.yaml >= 0.13', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'docker-py >= 1.9, < 2.0',