#!/usr/bin/env python3
#
# update.py - part of the FDroid server tools
-# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
+# Copyright (C) 2016, Blue Jay Wireless
+# Copyright (C) 2014-2016, Hans-Christoph Steiner <hans@eds.org>
+# Copyright (C) 2010-2015, Ciaran Gultnieks <ciaran@ciarang.com>
+# Copyright (C) 2013-2014, Daniel Martà <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
from xml.dom.minidom import Document
from argparse import ArgumentParser
import time
+
+import collections
from pyasn1.error import PyAsn1Error
from pyasn1.codec.der import decoder, encoder
from pyasn1_modules import rfc2315
from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
from .metadata import MetaDataException
+METADATA_VERSION = 17
+
screen_densities = ['640', '480', '320', '240', '160', '120']
all_screen_densities = ['0'] + screen_densities
+UsesPermission = collections.namedtuple('UsesPermission', ['name', 'maxSdkVersion'])
+UsesPermissionSdk23 = collections.namedtuple('UsesPermissionSdk23', ['name', 'maxSdkVersion'])
+
def dpi_to_px(density):
return (int(density) * 48) / 160
apppagename = apppagename.replace('{', '')
apppagename = apppagename.replace('}', ' ')
apppagename = apppagename.replace(':', ' ')
+ apppagename = apppagename.replace('[', ' ')
+ apppagename = apppagename.replace(']', ' ')
# Drop double spaces caused mostly by replacing ':' above
apppagename = apppagename.replace(' ', ' ')
for expagename in site.allpages(prefix=apppagename,
if not os.path.isfile(iconpath):
return
+ fp = None
try:
- im = Image.open(iconpath)
+ fp = open(iconpath, 'rb')
+ im = Image.open(fp)
size = dpi_to_px(density)
if any(length > size for length in im.size):
except Exception as e:
logging.error("Failed resizing {0} - {1}".format(iconpath, e))
+ finally:
+ if fp:
+ fp.close()
+
def resize_all_icons(repodirs):
"""Resize all icons that exceed the max size
return hashlib.md5(hexlify(cert_encoded)).hexdigest()
+def get_icon_bytes(apkzip, iconsrc):
+ '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
+ try:
+ return apkzip.read(iconsrc)
+ except KeyError:
+ return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
+
+
+def sha256sum(filename):
+ '''Calculate the sha256 of the given file'''
+ sha = hashlib.sha256()
+ with open(filename, 'rb') as f:
+ while True:
+ t = f.read(16384)
+ if len(t) == 0:
+ break
+ sha.update(t)
+ return sha.hexdigest()
+
+
+def insert_obbs(repodir, apps, apks):
+ """Scans the .obb files in a given repo directory and adds them to the
+ relevant APK instances. OBB files have versionCodes like APK
+ files, and they are loosely associated. If there is an OBB file
+ present, then any APK with the same or higher versionCode will use
+ that OBB file. There are two OBB types: main and patch, each APK
+ can only have only have one of each.
+
+ https://developer.android.com/google/play/expansion-files.html
+
+ :param repodir: repo directory to scan
+ :param apps: list of current, valid apps
+ :param apks: current information on all APKs
+
+ """
+
+ def obbWarnDelete(f, msg):
+ logging.warning(msg + f)
+ if options.delete_unknown:
+ logging.error("Deleting unknown file: " + f)
+ os.remove(f)
+
+ obbs = []
+ java_Integer_MIN_VALUE = -pow(2, 31)
+ for f in glob.glob(os.path.join(repodir, '*.obb')):
+ obbfile = os.path.basename(f)
+ # obbfile looks like: [main|patch].<expansion-version>.<package-name>.obb
+ chunks = obbfile.split('.')
+ if chunks[0] != 'main' and chunks[0] != 'patch':
+ obbWarnDelete(f, 'OBB filename must start with "main." or "patch.": ')
+ continue
+ if not re.match(r'^-?[0-9]+$', chunks[1]):
+ obbWarnDelete('The OBB version code must come after "' + chunks[0] + '.": ')
+ continue
+ versioncode = int(chunks[1])
+ packagename = ".".join(chunks[2:-1])
+
+ highestVersionCode = java_Integer_MIN_VALUE
+ if packagename not in apps.keys():
+ obbWarnDelete(f, "OBB's packagename does not match a supported APK: ")
+ continue
+ for apk in apks:
+ if packagename == apk['id'] and apk['versioncode'] > highestVersionCode:
+ highestVersionCode = apk['versioncode']
+ if versioncode > highestVersionCode:
+ obbWarnDelete(f, 'OBB file has newer versioncode(' + str(versioncode)
+ + ') than any APK: ')
+ continue
+ obbsha256 = sha256sum(f)
+ obbs.append((packagename, versioncode, obbfile, obbsha256))
+
+ for apk in apks:
+ for (packagename, versioncode, obbfile, obbsha256) in sorted(obbs, reverse=True):
+ if versioncode <= apk['versioncode'] and packagename == apk['id']:
+ if obbfile.startswith('main.') and 'obbMainFile' not in apk:
+ apk['obbMainFile'] = obbfile
+ apk['obbMainFileSha256'] = obbsha256
+ elif obbfile.startswith('patch.') and 'obbPatchFile' not in apk:
+ apk['obbPatchFile'] = obbfile
+ apk['obbPatchFileSha256'] = obbsha256
+ if 'obbMainFile' in apk and 'obbPatchFile' in apk:
+ break
+
+
def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
"""Scan the apks in the given repo directory.
icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
sdkversion_pat = re.compile(".*'([0-9]*)'.*")
- string_pat = re.compile(".*'([^']*)'.*")
+ permission_pat = re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
+ feature_pat = re.compile(".*name='([^']*)'.*")
for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
apkfilename = apkfile[len(repodir) + 1:]
logging.critical("Spaces in filenames are not allowed.")
sys.exit(1)
- # Calculate the sha256...
- sha = hashlib.sha256()
- with open(apkfile, 'rb') as f:
- while True:
- t = f.read(16384)
- if len(t) == 0:
- break
- sha.update(t)
- shasum = sha.hexdigest()
+ shasum = sha256sum(apkfile)
usecache = False
if apkfilename in apkcache:
if os.path.exists(os.path.join(repodir, srcfilename)):
apk['srcname'] = srcfilename
apk['size'] = os.path.getsize(apkfile)
- apk['permissions'] = set()
+ apk['uses-permission'] = set()
+ apk['uses-permission-sdk-23'] = set()
apk['features'] = set()
apk['icons_src'] = {}
apk['icons'] = {}
+ ' is not a valid minSdkVersion!')
else:
apk['minSdkVersion'] = m.group(1)
+ # if target not set, default to min
+ if 'targetSdkVersion' not in apk:
+ apk['targetSdkVersion'] = m.group(1)
+ elif line.startswith("targetSdkVersion:"):
+ m = re.match(sdkversion_pat, line)
+ if m is None:
+ logging.error(line.replace('targetSdkVersion:', '')
+ + ' is not a valid targetSdkVersion!')
+ else:
+ apk['targetSdkVersion'] = m.group(1)
elif line.startswith("maxSdkVersion:"):
apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
elif line.startswith("native-code:"):
apk['nativecode'] = []
for arch in line[13:].split(' '):
apk['nativecode'].append(arch[1:-1])
- elif line.startswith("uses-permission:"):
- perm = re.match(string_pat, line).group(1)
- if perm.startswith("android.permission."):
- perm = perm[19:]
- apk['permissions'].add(perm)
- elif line.startswith("uses-feature:"):
- perm = re.match(string_pat, line).group(1)
+ elif line.startswith('uses-permission:'):
+ perm_match = re.match(permission_pat, line).groupdict()
+
+ permission = UsesPermission(
+ perm_match['name'],
+ perm_match['maxSdkVersion']
+ )
+
+ apk['uses-permission'].add(permission)
+ elif line.startswith('uses-permission-sdk-23:'):
+ perm_match = re.match(permission_pat, line).groupdict()
+
+ permission_sdk_23 = UsesPermissionSdk23(
+ perm_match['name'],
+ perm_match['maxSdkVersion']
+ )
+
+ apk['uses-permission-sdk-23'].add(permission_sdk_23)
+
+ elif line.startswith('uses-feature:'):
+ feature = re.match(feature_pat, line).group(1)
# Filter out this, it's only added with the latest SDK tools and
# causes problems for lots of apps.
- if perm != "android.hardware.screen.portrait" \
- and perm != "android.hardware.screen.landscape":
- if perm.startswith("android.feature."):
- perm = perm[16:]
- apk['features'].add(perm)
+ if feature != "android.hardware.screen.portrait" \
+ and feature != "android.hardware.screen.landscape":
+ if feature.startswith("android.feature."):
+ feature = feature[16:]
+ apk['features'].add(feature)
if 'minSdkVersion' not in apk:
logging.warn("No SDK version information found in {0}".format(apkfile))
# Check for debuggable apks...
if common.isApkDebuggable(apkfile, config):
- logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
+ logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
# Get the signature (or md5 of, to be precise)...
logging.debug('Getting signature of {0}'.format(apkfile))
try:
with open(icondest, 'wb') as f:
- f.write(apkzip.read(iconsrc))
+ f.write(get_icon_bytes(apkzip, iconsrc))
apk['icons'][density] = iconfilename
except:
iconpath = os.path.join(
get_icon_dir(repodir, '0'), iconfilename)
with open(iconpath, 'wb') as f:
- f.write(apkzip.read(iconsrc))
+ f.write(get_icon_bytes(apkzip, iconsrc))
try:
im = Image.open(iconpath)
dpi = px_to_dpi(im.size[0])
get_icon_dir(repodir, last_density), iconfilename)
iconpath = os.path.join(
get_icon_dir(repodir, density), iconfilename)
+ fp = None
try:
- im = Image.open(last_iconpath)
- except:
- logging.warn("Invalid image file at %s" % last_iconpath)
- continue
+ fp = open(last_iconpath, 'rb')
+ im = Image.open(fp)
- size = dpi_to_px(density)
+ size = dpi_to_px(density)
- im.thumbnail((size, size), Image.ANTIALIAS)
- im.save(iconpath, "PNG")
- empty_densities.remove(density)
+ im.thumbnail((size, size), Image.ANTIALIAS)
+ im.save(iconpath, "PNG")
+ empty_densities.remove(density)
+ except:
+ logging.warning("Invalid image file at %s" % last_iconpath)
+ finally:
+ if fp:
+ fp.close()
# Then just copy from the highest resolution available
last_density = None
shutil.copyfile(baseline,
os.path.join(get_icon_dir(repodir, '0'), iconfilename))
+ if use_date_from_apk and manifest.date_time[1] != 0:
+ default_date_param = datetime(*manifest.date_time).utctimetuple()
+ else:
+ default_date_param = None
+
# Record in known apks, getting the added date at the same time..
- added = knownapks.recordapk(apk['apkname'], apk['id'])
+ added = knownapks.recordapk(apk['apkname'], apk['id'], default_date=default_date_param)
if added:
- if use_date_from_apk and manifest.date_time[1] != 0:
- added = datetime(*manifest.date_time).timetuple()
- logging.debug("Using date from APK")
-
apk['added'] = added
apkcache[apkfilename] = apk
for mirror in config.get('mirrors', []):
addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
- repoel.setAttribute("version", "15")
+ repoel.setAttribute("version", str(METADATA_VERSION))
repoel.setAttribute("timestamp", str(int(time.time())))
nosigningkey = False
repoel.setAttribute("pubkey", extract_pubkey().decode('utf-8'))
root.appendChild(repoel)
+ for command in ('install', 'delete'):
+ packageNames = []
+ key = command + '_list'
+ if key in config:
+ if isinstance(config[key], str):
+ packageNames = [config[key]]
+ elif all(isinstance(item, str) for item in config[key]):
+ packageNames = config[key]
+ else:
+ raise TypeError('only accepts strings, lists, and tuples')
+ for packageName in packageNames:
+ element = doc.createElement(command)
+ root.appendChild(element)
+ element.setAttribute('packageName', packageName)
+
for appid in sortedids:
app = apps[appid]
addElement('sig', apk['sig'], doc, apkel)
addElement('size', str(apk['size']), doc, apkel)
addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
+ if 'targetSdkVersion' in apk:
+ addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
if 'maxSdkVersion' in apk:
addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
+ addElementNonEmpty('obbMainFile', apk.get('obbMainFile'), doc, apkel)
+ addElementNonEmpty('obbMainFileSha256', apk.get('obbMainFileSha256'), doc, apkel)
+ addElementNonEmpty('obbPatchFile', apk.get('obbPatchFile'), doc, apkel)
+ addElementNonEmpty('obbPatchFileSha256', apk.get('obbPatchFileSha256'), doc, apkel)
if 'added' in apk:
addElement('added', time.strftime('%Y-%m-%d', apk['added']), doc, apkel)
- addElementNonEmpty('permissions', ','.join(apk['permissions']), doc, apkel)
+
+ # TODO: remove old permission format
+ old_permissions = set()
+ for perm in apk['uses-permission']:
+ perm_name = perm.name
+ if perm_name.startswith("android.permission."):
+ perm_name = perm_name[19:]
+ old_permissions.add(perm_name)
+ addElementNonEmpty('permissions', ','.join(old_permissions), doc, apkel)
+
+ for permission in apk['uses-permission']:
+ permel = doc.createElement('uses-permission')
+ permel.setAttribute('name', permission.name)
+ if permission.maxSdkVersion is not None:
+ permel.setAttribute('maxSdkVersion', permission.maxSdkVersion)
+ apkel.appendChild(permel)
+ for permission_sdk_23 in apk['uses-permission-sdk-23']:
+ permel = doc.createElement('uses-permission-sdk-23')
+ permel.setAttribute('name', permission_sdk_23.name)
+ if permission_sdk_23.maxSdkVersion is not None:
+ permel.setAttribute('maxSdkVersion', permission_sdk_23.maxSdkVersion)
+ apkel.appendChild(permel)
if 'nativecode' in apk:
addElement('nativecode', ','.join(apk['nativecode']), doc, apkel)
addElementNonEmpty('features', ','.join(apk['features']), doc, apkel)
parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
help="Create skeleton metadata files that are missing")
parser.add_argument("--delete-unknown", action="store_true", default=False,
- help="Delete APKs without metadata from the repo")
+ help="Delete APKs and/or OBBs without metadata from the repo")
parser.add_argument("-b", "--buildreport", action="store_true", default=False,
help="Report on build data status")
parser.add_argument("-i", "--interactive", default=False, action="store_true",
if not options.clean and os.path.exists(apkcachefile):
with open(apkcachefile, 'rb') as cf:
apkcache = pickle.load(cf, encoding='utf-8')
+ if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+ apkcache = {}
else:
apkcache = {}
if newmetadata:
apps = metadata.read_metadata()
+ insert_obbs(repodirs[0], apps, apks)
+
# Scan the archive repo for apks as well
if len(repodirs) > 1:
archapks, cc = scan_apks(apps, apkcache, repodirs[1], knownapks, options.use_date_from_apk)
f.write(data)
if cachechanged:
+ apkcache["METADATA_VERSION"] = METADATA_VERSION
with open(apkcachefile, 'wb') as cf:
pickle.dump(apkcache, cf)
~ianmdlvl / fdroidserver.git / blobdiff