gradle file: use flavour specific versionCode/versionName, fall back to parsing line by line
See merge request fdroid/fdroidserver!389
EOH
mkdir -p $ANDROID_HOME/licenses/
+
cat << EOF > $ANDROID_HOME/licenses/android-sdk-license
8933bad161af4178b1185d1a37fbf41ea5269c55
+
d56f5187479451eabf01fb78af6dfcb131a6481e
EOF
-echo -e "\n84831b9409646a918e30573bab4c9c91346d8abd" > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
+cat <<EOF > $ANDROID_HOME/licenses/android-sdk-preview-license-old
+79120722343a6f314e0719f863036c702b0e6b2a
+
+84831b9409646a918e30573bab4c9c91346d8abd
+EOF
+
echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.1"
echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout-solver;1.0.1"
echo y | $ANDROID_HOME/tools/bin/sdkmanager "extras;m2repository;com;android;support;constraint;constraint-layout;1.0.2"
__complete_options
}
+__complete_mirror() {
+ opts="-v"
+ lopts="--archive --output-dir"
+ __complete_options
+}
+
__complete_nightly() {
opts="-v -q"
lopts="--show-secret-var"
init \
install \
lint \
+mirror \
nightly \
publish \
readmeta \
# sync_from_local_copy_dir = True
-# To upload the repo to an Amazon S3 bucket using `fdroid server update`.
-# Warning, this deletes and recreates the whole fdroid/ directory each
-# time. This is based on apache-libcloud, which supports basically all cloud
-# storage services, so it should be easy to port the fdroid server tools to
-# any of them.
+# To upload the repo to an Amazon S3 bucket using `fdroid server
+# update`. Warning, this deletes and recreates the whole fdroid/
+# directory each time. This prefers s3cmd, but can also use
+# apache-libcloud. To customize how s3cmd interacts with the cloud
+# provider, create a 's3cfg' file next to this file (config.py), and
+# those settings will be used instead of any 'aws' variable below.
#
# awsbucket = 'myawsfdroid'
# awsaccesskeyid = 'SEE0CHAITHEIMAUR2USA'
("btlog", _("Update the binary transparency log for a URL")),
("signatures", _("Extract signatures from APKs")),
("nightly", _("Set up an app build for a nightly build repo")),
+ ("mirror", _("Download complete mirrors of small repos")),
])
import glob
import subprocess
import re
+import resource
import tarfile
import traceback
import time
if not apps:
raise FDroidException("No apps to process.")
+ # make sure enough open files are allowed to process everything
+ soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
+ if len(apps) > soft:
+ try:
+ soft = len(apps) * 2
+ if soft > hard:
+ soft = hard
+ resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
+ logging.debug(_('Set open file limit to {integer}')
+ .format(integer=soft))
+ except (OSError, ValueError) as e:
+ logging.warning(_('Setting open file limit failed: ') + str(e))
+
if options.latest:
for app in apps.values():
for build in reversed(app.builds):
import xml.etree.ElementTree as XMLElementTree
from binascii import hexlify
-from datetime import datetime
+from datetime import datetime, timedelta
from distutils.version import LooseVersion
from queue import Queue
from zipfile import ZipFile
return glob.glob('.fdroid.[a-jl-z]*[a-rt-z]')
-def read_pkg_args(args, allow_vercodes=False):
+def read_pkg_args(appid_versionCode_pairs, allow_vercodes=False):
"""
- :param args: arguments in the form of multiple appid:[vc] strings
+ :param appids: arguments in the form of multiple appid:[vc] strings
:returns: a dictionary with the set of vercodes specified for each package
"""
-
vercodes = {}
- if not args:
+ if not appid_versionCode_pairs:
return vercodes
- for p in args:
+ for p in appid_versionCode_pairs:
if allow_vercodes and ':' in p:
package, vercode = p.split(':')
else:
return vercodes
-def read_app_args(args, allapps, allow_vercodes=False):
- """
- On top of what read_pkg_args does, this returns the whole app metadata, but
- limiting the builds list to the builds matching the vercodes specified.
+def read_app_args(appid_versionCode_pairs, allapps, allow_vercodes=False):
+ """Build a list of App instances for processing
+
+ On top of what read_pkg_args does, this returns the whole app
+ metadata, but limiting the builds list to the builds matching the
+ appid_versionCode_pairs and vercodes specified. If no appid_versionCode_pairs are specified, then
+ all App and Build instances are returned.
+
"""
- vercodes = read_pkg_args(args, allow_vercodes)
+ vercodes = read_pkg_args(appid_versionCode_pairs, allow_vercodes)
if not vercodes:
return allapps
return [int(sp) if sp.isdigit() else sp for sp in re.split(r'(\d+)', s)]
+def check_system_clock(dt_obj, path):
+ """Check if system clock is updated based on provided date
+
+ If an APK has files newer than the system time, suggest updating
+ the system clock. This is useful for offline systems, used for
+ signing, which do not have another source of clock sync info. It
+ has to be more than 24 hours newer because ZIP/APK files do not
+ store timezone info
+
+ """
+ checkdt = dt_obj - timedelta(1)
+ if datetime.today() < checkdt:
+ logging.warning(_('System clock is older than date in {path}!').format(path=path)
+ + '\n' + _('Set clock to that time using:') + '\n'
+ + 'sudo date -s "' + str(dt_obj) + '"')
+
+
class KnownApks:
"""permanent store of existing APKs with the date they were added
date = datetime.strptime(t[-1], '%Y-%m-%d')
filename = line[0:line.rfind(appid) - 1]
self.apks[filename] = (appid, date)
+ check_system_clock(date, self.path)
self.changed = False
def writeifchanged(self):
+ '" does not exist, creating a new keystore there.')
common.write_to_config(test_config, 'keystore', keystore)
repo_keyalias = None
+ keydname = None
if options.repo_keyalias:
repo_keyalias = options.repo_keyalias
common.write_to_config(test_config, 'repo_keyalias', repo_keyalias)
flags=re.MULTILINE)
with open('opensc-fdroid.cfg', 'w') as f:
f.write(opensc_fdroid)
- elif not os.path.exists(keystore):
+ elif os.path.exists(keystore):
+ to_set = ['keystorepass', 'keypass', 'repo_keyalias', 'keydname']
+ if repo_keyalias:
+ to_set.remove('repo_keyalias')
+ if keydname:
+ to_set.remove('keydname')
+ logging.warning('\n' + _('Using existing keystore "{path}"').format(path=keystore)
+ + '\n' + _('Now set these in config.py:') + ' '
+ + ', '.join(to_set) + '\n')
+ else:
password = common.genpassword()
c = dict(test_config)
c['keystorepass'] = password
--- /dev/null
+#!/usr/bin/env python3
+
+import ipaddress
+import logging
+import os
+import posixpath
+import socket
+import subprocess
+import sys
+from argparse import ArgumentParser
+import urllib.parse
+
+from . import _
+from . import common
+from . import index
+from . import update
+
+options = None
+
+
+def _run_wget(path, urls):
+ if options.verbose:
+ verbose = '--verbose'
+ else:
+ verbose = '--no-verbose'
+
+ if not urls:
+ return
+ logging.debug(_('Running wget in {path}').format(path=path))
+ os.makedirs(path, exist_ok=True)
+ os.chdir(path)
+ urls_file = '.fdroid-mirror-wget-input-file'
+ with open(urls_file, 'w') as fp:
+ for url in urls:
+ fp.write(url.split('?')[0] + '\n') # wget puts query string in the filename
+ subprocess.call(['wget', verbose, '--continue', '--user-agent="fdroid mirror"',
+ '--input-file=' + urls_file])
+ os.remove(urls_file)
+
+
+def main():
+ global options
+
+ parser = ArgumentParser(usage=_("%(prog)s [options] url"))
+ common.setup_global_opts(parser)
+ parser.add_argument("url", nargs='?',
+ help=_('Base URL to mirror, can include the index signing key '
+ + 'using the query string: ?fingerprint='))
+ parser.add_argument("--archive", action='store_true', default=False,
+ help=_("Also mirror the full archive section"))
+ parser.add_argument("--output-dir", default=None,
+ help=_("The directory to write the mirror to"))
+ options = parser.parse_args()
+
+ if options.url is None:
+ logging.error(_('A URL is required as an argument!') + '\n')
+ parser.print_help()
+ sys.exit(1)
+
+ scheme, hostname, path, params, query, fragment = urllib.parse.urlparse(options.url)
+ fingerprint = urllib.parse.parse_qs(query).get('fingerprint')
+
+ def _append_to_url_path(*args):
+ '''Append the list of path components to URL, keeping the rest the same'''
+ newpath = posixpath.join(path, *args)
+ return urllib.parse.urlunparse((scheme, hostname, newpath, params, query, fragment))
+
+ if fingerprint:
+ config = common.read_config(options)
+ if not ('jarsigner' in config or 'apksigner' in config):
+ logging.error(_('Java JDK not found! Install in standard location or set java_paths!'))
+ sys.exit(1)
+
+ def _get_index(section, etag=None):
+ url = _append_to_url_path(section)
+ return index.download_repo_index(url, etag=etag)
+ else:
+ def _get_index(section, etag=None):
+ import io
+ import json
+ import zipfile
+ from . import net
+ url = _append_to_url_path(section, 'index-v1.jar')
+ content, etag = net.http_get(url)
+ with zipfile.ZipFile(io.BytesIO(content)) as zip:
+ jsoncontents = zip.open('index-v1.json').read()
+ data = json.loads(jsoncontents.decode('utf-8'))
+ return data, etag
+
+ ip = None
+ try:
+ ip = ipaddress.ip_address(hostname)
+ except ValueError:
+ pass
+ if hostname == 'f-droid.org' \
+ or (ip is not None and hostname in socket.gethostbyname_ex('f-droid.org')[2]):
+ print(_('ERROR: this command should never be used to mirror f-droid.org!\n'
+ 'A full mirror of f-droid.org requires more than 200GB.'))
+ sys.exit(1)
+
+ path = path.rstrip('/')
+ if path.endswith('repo') or path.endswith('archive'):
+ logging.warning(_('Do not include "{path}" in URL!')
+ .format(path=path.split('/')[-1]))
+ elif not path.endswith('fdroid'):
+ logging.warning(_('{url} does not end with "fdroid", check the URL path!')
+ .format(url=options.url))
+
+ icondirs = ['icons', ]
+ for density in update.screen_densities:
+ icondirs.append('icons-' + density)
+
+ if options.output_dir:
+ basedir = options.output_dir
+ else:
+ basedir = os.path.join(os.getcwd(), hostname, path.strip('/'))
+ os.makedirs(basedir, exist_ok=True)
+
+ if options.archive:
+ sections = ('repo', 'archive')
+ else:
+ sections = ('repo', )
+
+ for section in sections:
+ sectiondir = os.path.join(basedir, section)
+
+ data, etag = _get_index(section)
+
+ os.makedirs(sectiondir, exist_ok=True)
+ os.chdir(sectiondir)
+ for icondir in icondirs:
+ os.makedirs(os.path.join(sectiondir, icondir), exist_ok=True)
+
+ urls = []
+ for packageName, packageList in data['packages'].items():
+ for package in packageList:
+ to_fetch = []
+ for k in ('apkName', 'srcname'):
+ if k in package:
+ to_fetch.append(package[k])
+ elif k == 'apkName':
+ logging.error(_('{appid} is missing {name}')
+ .format(appid=package['packageName'], name=k))
+ for f in to_fetch:
+ if not os.path.exists(f) \
+ or (f.endswith('.apk') and os.path.getsize(f) != package['size']):
+ urls.append(_append_to_url_path(section, f))
+ urls.append(_append_to_url_path(section, f + '.asc'))
+ _run_wget(sectiondir, urls)
+
+ for app in data['apps']:
+ localized = app.get('localized')
+ if localized:
+ for locale, d in localized.items():
+ urls = []
+ components = (section, app['packageName'], locale)
+ for k in update.GRAPHIC_NAMES:
+ f = d.get(k)
+ if f:
+ urls.append(_append_to_url_path(*components, f))
+ _run_wget(os.path.join(basedir, *components), urls)
+ for k in update.SCREENSHOT_DIRS:
+ urls = []
+ filelist = d.get(k)
+ if filelist:
+ components = (section, app['packageName'], locale, k)
+ for f in filelist:
+ urls.append(_append_to_url_path(*components, f))
+ _run_wget(os.path.join(basedir, *components), urls)
+
+ urls = dict()
+ for app in data['apps']:
+ if 'icon' not in app:
+ logging.error(_('no "icon" in {appid}').format(appid=app['packageName']))
+ continue
+ icon = app['icon']
+ for icondir in icondirs:
+ url = _append_to_url_path(section, icondir, icon)
+ if icondir not in urls:
+ urls[icondir] = []
+ urls[icondir].append(url)
+
+ for icondir in icondirs:
+ _run_wget(os.path.join(basedir, section, icondir), urls[icondir])
+
+
+if __name__ == "__main__":
+ main()
repo_url = repo_base + '/repo'
git_mirror_path = os.path.join(repo_basedir, 'git-mirror')
git_mirror_repodir = os.path.join(git_mirror_path, 'fdroid', 'repo')
+ git_mirror_metadatadir = os.path.join(git_mirror_path, 'fdroid', 'metadata')
if not os.path.isdir(git_mirror_repodir):
logging.debug(_('cloning {url}').format(url=clone_url))
try:
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update README")
- icon_path = os.path.join(repo_basedir, 'icon.png')
+ icon_path = os.path.join(git_mirror_path, 'icon.png')
try:
import qrcode
- img = qrcode.make('Some data here')
+ img = qrcode.make(repo_url)
with open(icon_path, 'wb') as fp:
fp.write(img)
except Exception:
shutil.copy(exampleicon, icon_path)
mirror_git_repo.git.add(all=True)
mirror_git_repo.index.commit("update repo/website icon")
+ shutil.copy(icon_path, repo_basedir)
os.chdir(repo_basedir)
- common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+ if os.path.isdir(git_mirror_repodir):
+ common.local_rsync(options, git_mirror_repodir + '/', 'repo/')
+ if os.path.isdir(git_mirror_metadatadir):
+ common.local_rsync(options, git_mirror_metadatadir + '/', 'metadata/')
ssh_private_key_file = _ssh_key_from_debug_keystore()
# this is needed for GitPython to find the SSH key
except subprocess.CalledProcessError:
pass
- subprocess.check_call(['fdroid', 'update', '--rename-apks', '--verbose'], cwd=repo_basedir)
+ subprocess.check_call(['fdroid', 'update', '--rename-apks', '--create-metadata', '--verbose'],
+ cwd=repo_basedir)
+ common.local_rsync(options, repo_basedir + '/metadata/', git_mirror_metadatadir + '/')
+ mirror_git_repo.git.add(all=True)
+ mirror_git_repo.index.commit("update app metadata")
try:
subprocess.check_call(['fdroid', 'server', 'update', '--verbose'], cwd=repo_basedir)
except subprocess.CalledProcessError:
BINARY_TRANSPARENCY_DIR = 'binary_transparency'
+AUTO_S3CFG = '.fdroid-server-update-s3cfg'
+USER_S3CFG = 's3cfg'
+
def update_awsbucket(repo_section):
'''
logging.debug(_('Using s3cmd to sync with: {url}')
.format(url=config['awsbucket']))
- configfilename = '.s3cfg'
- fd = os.open(configfilename, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
- os.write(fd, '[default]\n'.encode('utf-8'))
- os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8'))
- os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8'))
- os.close(fd)
+ if os.path.exists(USER_S3CFG):
+ logging.info(_('Using "{path}" for configuring s3cmd.').format(path=USER_S3CFG))
+ configfilename = USER_S3CFG
+ else:
+ fd = os.open(AUTO_S3CFG, os.O_CREAT | os.O_TRUNC | os.O_WRONLY, 0o600)
+ logging.debug(_('Creating "{path}" for configuring s3cmd.').format(path=AUTO_S3CFG))
+ os.write(fd, '[default]\n'.encode('utf-8'))
+ os.write(fd, ('access_key = ' + config['awsaccesskeyid'] + '\n').encode('utf-8'))
+ os.write(fd, ('secret_key = ' + config['awssecretkey'] + '\n').encode('utf-8'))
+ os.close(fd)
+ configfilename = AUTO_S3CFG
s3bucketurl = 's3://' + config['awsbucket']
s3cmd = [config['s3cmd'], '--config=' + configfilename]
_('To use awsbucket, awssecretkey and awsaccesskeyid must also be set in config.py!'))
awsbucket = config['awsbucket']
+ if os.path.exists(USER_S3CFG):
+ raise FDroidException(_('"{path}" exists but s3cmd is not installed!')
+ .format(path=USER_S3CFG))
+
cls = get_driver(Provider.S3)
driver = cls(config['awsaccesskeyid'], config['awssecretkey'])
try:
with open(outputfilename, 'w') as fp:
json.dump(response, fp, indent=2, sort_keys=True)
- if response.get('positives') > 0:
+ if response.get('positives', 0) > 0:
logging.warning(repofilename + ' has been flagged by virustotal '
+ str(response['positives']) + ' times:'
+ '\n\t' + response['permalink'])
import hashlib
import pickle
import time
-from datetime import datetime, timedelta
+from datetime import datetime
from argparse import ArgumentParser
import collections
apkzip = zipfile.ZipFile(apkfile, 'r')
- # if an APK has files newer than the system time, suggest updating
- # the system clock. This is useful for offline systems, used for
- # signing, which do not have another source of clock sync info. It
- # has to be more than 24 hours newer because ZIP/APK files do not
- # store timezone info
manifest = apkzip.getinfo('AndroidManifest.xml')
if manifest.date_time[1] == 0: # month can't be zero
logging.debug(_('AndroidManifest.xml has no date'))
else:
- dt_obj = datetime(*manifest.date_time)
- checkdt = dt_obj - timedelta(1)
- if datetime.today() < checkdt:
- logging.warning('System clock is older than manifest in: '
- + apkfilename
- + '\nSet clock to that time using:\n'
- + 'sudo date -s "' + str(dt_obj) + '"')
+ common.check_system_clock(datetime(*manifest.date_time), apkfilename)
# extract icons from APK zip file
iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
from .common import FDroidException
from logging import getLogger
+from fdroidserver import _
+
logger = getLogger('fdroidserver-vmtools')
vol = storagePool.storageVolLookupByName(self.srvname + '.img')
imagepath = vol.path()
# TODO use a libvirt storage pool to ensure the img file is readable
- _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
+ if not os.access(imagepath, os.R_OK):
+ logger.warning(_('Cannot read "{path}"!').format(path=imagepath))
+ _check_call(['sudo', '/bin/chmod', '-R', 'a+rX', '/var/lib/libvirt/images'])
shutil.copy2(imagepath, 'box.img')
_check_call(['qemu-img', 'rebase', '-p', '-b', '', 'box.img'])
img_info_raw = _check_output(['qemu-img', 'info', '--output=json', 'box.img'])
echo "No virtualization is used."
fi
sudo /bin/chmod -R a+rX /var/lib/libvirt/images
-ulimit -n 2048
echo 'maximum allowed number of open file descriptors: ' `ulimit -n`
ls -ld /var/lib/libvirt/images
ls -l /var/lib/libvirt/images || echo no access
'pyasn1-modules',
'python-vagrant',
'PyYAML',
+ 'qrcode',
'ruamel.yaml >= 0.13',
'requests >= 2.5.2, != 2.11.0, != 2.12.2, != 2.18.0',
'docker-py >= 1.9, < 2.0',
~ianmdlvl / fdroidserver.git / commitdiff