From: Hans-Christoph Steiner Date: Thu, 30 Nov 2017 12:28:28 +0000 (+0000) Subject: Merge branch 'polish-for-1.0' into 'master' X-Git-Tag: 1.0.0~60 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=2a758886cdc659de4402153216d5e3df83b4948c;hp=1219f07d3b8ba195a8153e52cedfa9db138c0b5f;p=fdroidserver.git Merge branch 'polish-for-1.0' into 'master' Polish for 1.0 Closes #421 See merge request fdroid/fdroidserver!388 --- diff --git a/buildserver/provision-android-sdk b/buildserver/provision-android-sdk index b79c2e4a..7cd96e0c 100644 --- a/buildserver/provision-android-sdk +++ b/buildserver/provision-android-sdk @@ -74,12 +74,25 @@ y EOH mkdir -p $ANDROID_HOME/licenses/ + cat << EOF > $ANDROID_HOME/licenses/android-sdk-license 8933bad161af4178b1185d1a37fbf41ea5269c55 + d56f5187479451eabf01fb78af6dfcb131a6481e EOF -echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + +cat < $ANDROID_HOME/licenses/android-sdk-preview-license-old +79120722343a6f314e0719f863036c702b0e6b2a + +84831b9409646a918e30573bab4c9c91346d8abd +EOF + echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1" echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2" diff --git a/fdroidserver/build.py b/fdroidserver/build.py index 740d7f2c..16901e78 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -23,6 +23,7 @@ import shutil import glob import subprocess import re +import resource import tarfile import traceback import time @@ -1120,6 +1121,19 @@ def main(): if not apps: raise FDroidException("No apps to process.") + # make sure enough open files are allowed to process everything + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) + if len(apps) > soft: + try: + soft = len(apps) * 2 + if soft > hard: + soft = hard + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) + logging.debug(_('Set open file limit to {integer}') + .format(integer=soft)) + except (OSError, ValueError) as e: + logging.warning(_('Setting open file limit failed: ') + str(e)) + if options.latest: for app in apps.values(): for build in reversed(app.builds): diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 89c1a1d5..a23ae2e9 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -40,7 +40,7 @@ import json import xml.etree.ElementTree as XMLElementTree from binascii import hexlify -from datetime import datetime +from datetime import datetime, timedelta from distutils.version import LooseVersion from queue import Queue from zipfile import ZipFile @@ -444,17 +444,16 @@ def get_local_metadata_files(): return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]') -def read_pkg_args(args, allow_vercodes=False): +def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False): """ - :param args: arguments in the form of multiple appid:[vc] strings + :param appids: arguments in the form of multiple appid:[vc] strings :returns: a dictionary with the set of vercodes specified for each package """ - vercodes = {} - if not args: + if not appid_versionCode_pairs: return vercodes - for p in args: + for p in appid_versionCode_pairs: if allow_vercodes and ':' in p: package, vercode = p.split(':') else: @@ -468,13 +467,17 @@ def read_pkg_args(args, allow_vercodes=False): return vercodes -def read_app_args(args, allapps, allow_vercodes=False): - """ - On top of what read_pkg_args does, this returns the whole app metadata, but - limiting the builds list to the builds matching the vercodes specified. +def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False): + """Build a list of App instances for processing + + On top of what read_pkg_args does, this returns the whole app + metadata, but limiting the builds list to the builds matching the + appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then + all App and Build instances are returned. + """ - vercodes = read_pkg_args(args, allow_vercodes) + vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes) if not vercodes: return allapps @@ -1716,6 +1719,23 @@ def natural_key(s): return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)] +def check_system_clock(dt_obj, path): + """Check if system clock is updated based on provided date + + If an APK has files newer than the system time, suggest updating + the system clock. This is useful for offline systems, used for + signing, which do not have another source of clock sync info. It + has to be more than 24 hours newer because ZIP/APK files do not + store timezone info + + """ + checkdt = dt_obj - timedelta(1) + if datetime.today() < checkdt: + logging.warning(_('System clock is older than date in {path}!').format(path=path) + + '\n' + _('Set clock to that time using:') + '\n' + + 'sudo date -s "' + str(dt_obj) + '"') + + class KnownApks: """permanent store of existing APKs with the date they were added @@ -1744,6 +1764,7 @@ class KnownApks: date = datetime.strptime(t[-1], '%Y-%m-%d') filename = line[0:line.rfind(appid) - 1] self.apks[filename] = (appid, date) + check_system_clock(date, self.path) self.changed = False def writeifchanged(self): diff --git a/fdroidserver/init.py b/fdroidserver/init.py index 9fdb5836..9d03e0b9 100644 --- a/fdroidserver/init.py +++ b/fdroidserver/init.py @@ -178,6 +178,7 @@ def main(): + '" does not exist, creating a new keystore there.') common.write_to_config(test_config, 'keystore', keystore) repo_keyalias = None + keydname = None if options.repo_keyalias: repo_keyalias = options.repo_keyalias common.write_to_config(test_config, 'repo_keyalias', repo_keyalias) @@ -211,7 +212,16 @@ def main(): flags=re.MULTILINE) with open('opensc-fdroid.cfg', 'w') as f: f.write(opensc_fdroid) - elif not os.path.exists(keystore): + elif os.path.exists(keystore): + to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname'] + if repo_keyalias: + to_set.remove('repo_keyalias') + if keydname: + to_set.remove('keydname') + logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore) + + '\n' + _('Now set these in config.py:') + ' ' + + ', '.join(to_set) + '\n') + else: password = common.genpassword() c = dict(test_config) c['keystorepass'] = password diff --git a/fdroidserver/nightly.py b/fdroidserver/nightly.py index 3834f79e..454616e4 100644 --- a/fdroidserver/nightly.py +++ b/fdroidserver/nightly.py @@ -155,6 +155,7 @@ def main(): repo_url = repo_base + '/repo' git_mirror_path = os.path.join(repo_basedir, 'git-mirror') git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo') + git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata') if not os.path.isdir(git_mirror_repodir): logging.debug(_('cloning {url}').format(url=clone_url)) try: @@ -186,10 +187,10 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update README") - icon_path = os.path.join(repo_basedir, 'icon.png') + icon_path = os.path.join(git_mirror_path, 'icon.png') try: import qrcode - img = qrcode.make('Some data here') + img = qrcode.make(repo_url) with open(icon_path, 'wb') as fp: fp.write(img) except Exception: @@ -197,9 +198,13 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, shutil.copy(exampleicon, icon_path) mirror_git_repo.git.add(all=True) mirror_git_repo.index.commit("update repo/website icon") + shutil.copy(icon_path, repo_basedir) os.chdir(repo_basedir) - common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_repodir): + common.local_rsync(options, git_mirror_repodir + '/', 'repo/') + if os.path.isdir(git_mirror_metadatadir): + common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/') ssh_private_key_file = _ssh_key_from_debug_keystore() # this is needed for GitPython to find the SSH key @@ -254,7 +259,11 @@ Last updated: {date}'''.format(repo_git_base=repo_git_base, except subprocess.CalledProcessError: pass - subprocess.check_call(['fdroid', 'update', '--rename-apks', '--verbose'], cwd=repo_basedir) + subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'], + cwd=repo_basedir) + common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/') + mirror_git_repo.git.add(all=True) + mirror_git_repo.index.commit("update app metadata") try: subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir) except subprocess.CalledProcessError: diff --git a/fdroidserver/server.py b/fdroidserver/server.py index bd5a4426..004a6068 100644 --- a/fdroidserver/server.py +++ b/fdroidserver/server.py @@ -513,7 +513,7 @@ def upload_to_virustotal(repo_section, vt_apikey): with open(outputfilename, 'w') as fp: json.dump(response, fp, indent=2, sort_keys=True) - if response.get('positives') > 0: + if response.get('positives', 0) > 0: logging.warning(repofilename + ' has been flagged by virustotal ' + str(response['positives']) + ' times:' + '\n\t' + response['permalink']) diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 2019063c..61026cab 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -29,7 +29,7 @@ import zipfile import hashlib import pickle import time -from datetime import datetime, timedelta +from datetime import datetime from argparse import ArgumentParser import collections @@ -1297,22 +1297,11 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal apkzip = zipfile.ZipFile(apkfile, 'r') - # if an APK has files newer than the system time, suggest updating - # the system clock. This is useful for offline systems, used for - # signing, which do not have another source of clock sync info. It - # has to be more than 24 hours newer because ZIP/APK files do not - # store timezone info manifest = apkzip.getinfo('AndroidManifest.xml') if manifest.date_time[1] == 0: # month can't be zero logging.debug(_('AndroidManifest.xml has no date')) else: - dt_obj = datetime(*manifest.date_time) - checkdt = dt_obj - timedelta(1) - if datetime.today() < checkdt: - logging.warning('System clock is older than manifest in: ' - + apkfilename - + '\nSet clock to that time using:\n' - + 'sudo date -s "' + str(dt_obj) + '"') + common.check_system_clock(datetime(*manifest.date_time), apkfilename) # extract icons from APK zip file iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode']) diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py index aae46c75..aab7eb0d 100644 --- a/fdroidserver/vmtools.py +++ b/fdroidserver/vmtools.py @@ -29,6 +29,8 @@ import textwrap from .common import FDroidException from logging import getLogger +from fdroidserver import _ + logger = getLogger('fdroidserver-vmtools') @@ -383,7 +385,9 @@ class LibvirtBuildVm(FDroidBuildVm): vol = storagePool.storageVolLookupByName(self.srvname + '.img') imagepath = vol.path() # TODO use a libvirt storage pool to ensure the img file is readable - _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) + if not os.access(imagepath, os.R_OK): + logger.warning(_('Cannot read "{path}"!').format(path=imagepath)) + _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images']) shutil.copy2(imagepath, 'box.img') _check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img']) img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img']) diff --git a/jenkins-build-all b/jenkins-build-all index 2abac58c..d41d920f 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -31,7 +31,6 @@ else echo "No virtualization is used." fi sudo /bin/chmod -R a+rX /var/lib/libvirt/images -ulimit -n 2048 echo 'maximum allowed number of open file descriptors: ' `ulimit -n` ls -ld /var/lib/libvirt/images ls -l /var/lib/libvirt/images || echo no access diff --git a/setup.py b/setup.py index 1552a636..a382b254 100755 --- a/setup.py +++ b/setup.py @@ -89,6 +89,7 @@ setup(name='fdroidserver', 'pyasn1-modules', 'python-vagrant', 'PyYAML', + 'qrcode', 'ruamel.yaml >= 0.13', 'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0', 'docker-py >= 1.9, < 2.0',