X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Fupdate.py;h=c22ac0f32b70d6add7f64a2367a24ef0e6091761;hb=70d9633555ba07b4bb83dfd7dcda9781cc80cf51;hp=6b31f1622360e8155208803c4ec805a23f9da57c;hpb=95c5b0840ce796a07a1163ad563af54dcf412658;p=fdroidserver.git diff --git a/fdroidserver/update.py b/fdroidserver/update.py index 6b31f162..c22ac0f3 100644 --- a/fdroidserver/update.py +++ b/fdroidserver/update.py @@ -29,13 +29,13 @@ import zipfile import hashlib import pickle import time -from datetime import datetime, timedelta +from datetime import datetime from argparse import ArgumentParser import collections from binascii import hexlify -from PIL import Image +from PIL import Image, PngImagePlugin import logging from . import _ @@ -53,15 +53,13 @@ UNSET_VERSION_CODE = -0x100000000 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*") APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*") APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*") -APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*") -APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*") -APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*") +APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'") APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*") APK_PERMISSION_PAT = \ re.compile(".*(name='(?P.*?)')(.*maxSdkVersion='(?P.*?)')?.*") APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*") -screen_densities = ['640', '480', '320', '240', '160', '120'] +screen_densities = ['65534', '640', '480', '320', '240', '160', '120'] screen_resolutions = { "xxxhdpi": '640', "xxhdpi": '480', @@ -84,6 +82,8 @@ GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner') SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots', 'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots') +BLANK_PNG_INFO = PngImagePlugin.PngInfo() + def dpi_to_px(density): return (int(density) * 48) / 160 @@ -94,9 +94,10 @@ def px_to_dpi(px): def get_icon_dir(repodir, density): - if density == '0': + if density == '0' or density == '65534': return os.path.join(repodir, "icons") - return os.path.join(repodir, "icons-%s" % density) + else: + return os.path.join(repodir, "icons-%s" % density) def get_icon_dirs(repodir): @@ -132,13 +133,13 @@ def update_wiki(apps, sortedids, apks): if app.Disabled: wikidata += '{{Disabled|' + app.Disabled + '}}\n' if app.AntiFeatures: - for af in app.AntiFeatures: + for af in sorted(app.AntiFeatures): wikidata += '{{AntiFeature|' + af + '}}\n' if app.RequiresRoot: requiresroot = 'Yes' else: requiresroot = 'No' - wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % ( + wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % ( appid, app.Name, app.added.strftime('%Y-%m-%d') if app.added else '', @@ -149,6 +150,7 @@ def update_wiki(apps, sortedids, apks): app.Changelog, app.Donate, app.FlattrID, + app.LiberapayID, app.Bitcoin, app.Litecoin, app.License, @@ -318,7 +320,21 @@ def update_wiki(apps, sortedids, apks): logging.error("...FAILED to create page '{0}': {1}".format(pagename, e)) # Purge server cache to ensure counts are up to date - site.pages['Repository Maintenance'].purge() + site.Pages['Repository Maintenance'].purge() + + # Write a page with the last build log for this version code + wiki_page_path = 'update_' + time.strftime('%s', start_timestamp) + newpage = site.Pages[wiki_page_path] + txt = '' + txt += "* command line: " + ' '.join(sys.argv) + "\n" + txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n' + txt += "* completed at " + common.get_wiki_timestamp() + '\n' + txt += common.get_git_describe_link() + txt += "\n\n" + txt += common.get_android_tools_version_log() + newpage.save(txt, summary='Run log') + newpage = site.Pages['update'] + newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect') def delete_disabled_builds(apps, apkcache, repodirs): @@ -370,7 +386,8 @@ def resize_icon(iconpath, density): im.thumbnail((size, size), Image.ANTIALIAS) logging.debug("%s was too large at %s - new size is %s" % ( iconpath, oldsize, im.size)) - im.save(iconpath, "PNG") + im.save(iconpath, "PNG", optimize=True, + pnginfo=BLANK_PNG_INFO, icc_profile=None) except Exception as e: logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e))) @@ -495,14 +512,25 @@ def has_known_vulnerability(filename): Checks whether there are more than one classes.dex or AndroidManifest.xml files, which is invalid and an essential part of the "Master Key" attack. - http://www.saurik.com/id/17 + + Janus is similar to Master Key but is perhaps easier to scan for. + https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures """ + found_vuln = False + # statically load this pattern if not hasattr(has_known_vulnerability, "pattern"): has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)') + with open(filename.encode(), 'rb') as fp: + first4 = fp.read(4) + if first4 != b'\x50\x4b\x03\x04': + raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!') + .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n' + + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures') + files_in_apk = set() with zipfile.ZipFile(filename) as zf: for name in zf.namelist(): @@ -523,14 +551,15 @@ def has_known_vulnerability(filename): else: logging.warning(_('"{path}" contains outdated {name} ({version})') .format(path=filename, name=name, version=version)) - return True + found_vuln = True break elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'): if name in files_in_apk: - return True + logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!') + .format(apkfilename=filename, name=name)) + found_vuln = True files_in_apk.add(name) - - return False + return found_vuln def insert_obbs(repodir, apps, apks): @@ -659,6 +688,35 @@ def _set_author_entry(app, key, f): app[key] = text +def _strip_and_copy_image(inpath, outpath): + """Remove any metadata from image and copy it to new path + + Sadly, image metadata like EXIF can be used to exploit devices. + It is not used at all in the F-Droid ecosystem, so its much safer + just to remove it entirely. + + """ + + extension = common.get_extension(inpath)[1] + if os.path.isdir(outpath): + outpath = os.path.join(outpath, os.path.basename(inpath)) + if extension == 'png': + with open(inpath, 'rb') as fp: + in_image = Image.open(fp) + in_image.save(outpath, "PNG", optimize=True, + pnginfo=BLANK_PNG_INFO, icc_profile=None) + elif extension == 'jpg' or extension == 'jpeg': + with open(inpath, 'rb') as fp: + in_image = Image.open(fp) + data = list(in_image.getdata()) + out_image = Image.new(in_image.mode, in_image.size) + out_image.putdata(data) + out_image.save(outpath, "JPEG", optimize=True) + else: + raise FDroidException(_('Unsupported file type "{extension}" for repo graphic') + .format(extension=extension)) + + def copy_triple_t_store_metadata(apps): """Include store metadata from the app's source repo @@ -731,7 +789,7 @@ def copy_triple_t_store_metadata(apps): sourcefile = os.path.join(root, f) destfile = os.path.join(destdir, os.path.basename(f)) logging.debug('copying ' + sourcefile + ' ' + destfile) - shutil.copy(sourcefile, destfile) + _strip_and_copy_image(sourcefile, destfile) def insert_localized_app_metadata(apps): @@ -772,7 +830,8 @@ def insert_localized_app_metadata(apps): """ - sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*')) + sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*')) + sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*')) sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*')) sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*')) @@ -787,6 +846,17 @@ def insert_localized_app_metadata(apps): continue locale = segments[-1] destdir = os.path.join('repo', packageName, locale) + + # flavours specified in build receipt + build_flavours = "" + if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\ + and 'gradle' in apps[packageName].builds[-1]: + build_flavours = apps[packageName].builds[-1].gradle + + if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours: + logging.debug("ignoring due to wrong flavour") + continue + for f in files: if f in ('description.txt', 'full_description.txt'): _set_localized_text_entry(apps[packageName], locale, 'description', @@ -817,19 +887,19 @@ def insert_localized_app_metadata(apps): if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS: os.makedirs(destdir, mode=0o755, exist_ok=True) logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir) - shutil.copy(os.path.join(root, f), destdir) + _strip_and_copy_image(os.path.join(root, f), destdir) for d in dirs: if d in SCREENSHOT_DIRS: if locale == 'images': locale = segments[-2] destdir = os.path.join('repo', packageName, locale) for f in glob.glob(os.path.join(root, d, '*.*')): - _, extension = common.get_extension(f) + _ignored, extension = common.get_extension(f) if extension in ALLOWED_EXTENSIONS: screenshotdestdir = os.path.join(destdir, d) os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True) logging.debug('copying ' + f + ' ' + screenshotdestdir) - shutil.copy(f, screenshotdestdir) + _strip_and_copy_image(f, screenshotdestdir) repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*'))) for d in repofiles: @@ -846,24 +916,24 @@ def insert_localized_app_metadata(apps): base, extension = common.get_extension(filename) if packageName not in apps: - logging.warning('Found "%s" graphic without metadata for app "%s"!' - % (filename, packageName)) + logging.warning(_('Found "{path}" graphic without metadata for app "{name}"!') + .format(path=filename, name=packageName)) continue graphics = _get_localized_dict(apps[packageName], locale) if extension not in ALLOWED_EXTENSIONS: - logging.warning('Only PNG and JPEG are supported for graphics, found: ' + f) + logging.warning(_('Only PNG and JPEG are supported for graphics, found: {path}').format(path=f)) elif base in GRAPHIC_NAMES: # there can only be zero or one of these per locale graphics[base] = filename elif screenshotdir in SCREENSHOT_DIRS: # there can any number of these per locale - logging.debug('adding to ' + screenshotdir + ': ' + f) + logging.debug(_('adding to {name}: {path}').format(name=screenshotdir, path=f)) if screenshotdir not in graphics: graphics[screenshotdir] = [] graphics[screenshotdir].append(filename) else: - logging.warning('Unsupported graphics file found: ' + f) + logging.warning(_('Unsupported graphics file found: {path}').format(path=f)) def scan_repo_files(apkcache, repodir, knownapks, use_date_from_file=False): @@ -978,10 +1048,10 @@ def scan_apk(apk_file): 'antiFeatures': set(), } - if SdkToolsPopen(['aapt', 'version'], output=False): - scan_apk_aapt(apk, apk_file) - else: + if common.use_androguard(): scan_apk_androguard(apk, apk_file) + else: + scan_apk_aapt(apk, apk_file) # Get the signature, or rather the signing key fingerprints logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file))) @@ -998,7 +1068,9 @@ def scan_apk(apk_file): if 'minSdkVersion' not in apk: logging.warning("No SDK version information found in {0}".format(apk_file)) - apk['minSdkVersion'] = 1 + apk['minSdkVersion'] = 3 # aapt defaults to 3 as the min + if 'targetSdkVersion' not in apk: + apk['targetSdkVersion'] = apk['minSdkVersion'] # Check for known vulnerabilities if has_known_vulnerability(apk_file): @@ -1007,6 +1079,27 @@ def scan_apk(apk_file): return apk +def _get_apk_icons_src(apkfile, icon_name): + """Extract the paths to the app icon in all available densities + + """ + icons_src = dict() + density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$') + with zipfile.ZipFile(apkfile) as zf: + for filename in zf.namelist(): + m = density_re.match(filename) + if m: + folder = m.group(1).split('-') + if len(folder) > 1: + density = screen_resolutions[folder[1]] + else: + density = '160' + icons_src[density] = m.group(0) + if icons_src.get('-1') is None: + icons_src['-1'] = icons_src['160'] + return icons_src + + def scan_apk_aapt(apk, apkfile): p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False) if p.returncode != 0: @@ -1019,6 +1112,7 @@ def scan_apk_aapt(apk, apkfile): else: logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile)) raise BuildException(_("Invalid APK")) + icon_name = None for line in p.output.splitlines(): if line.startswith("package:"): try: @@ -1028,25 +1122,13 @@ def scan_apk_aapt(apk, apkfile): except Exception as e: raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line) elif line.startswith("application:"): - apk['name'] = re.match(APK_LABEL_PAT, line).group(1) - # Keep path to non-dpi icon in case we need it - match = re.match(APK_ICON_PAT_NODPI, line) - if match: - apk['icons_src']['-1'] = match.group(1) - elif line.startswith("launchable-activity:"): + m = re.match(APK_LABEL_ICON_PAT, line) + if m: + apk['name'] = m.group(1) + icon_name = os.path.splitext(os.path.basename(m.group(2)))[0] + elif not apk.get('name') and line.startswith("launchable-activity:"): # Only use launchable-activity as fallback to application - if not apk['name']: - apk['name'] = re.match(APK_LABEL_PAT, line).group(1) - if '-1' not in apk['icons_src']: - match = re.match(APK_ICON_PAT_NODPI, line) - if match: - apk['icons_src']['-1'] = match.group(1) - elif line.startswith("application-icon-"): - match = re.match(APK_ICON_PAT, line) - if match: - density = match.group(1) - path = match.group(2) - apk['icons_src'][density] = path + apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1) elif line.startswith("sdkVersion:"): m = re.match(APK_SDK_VERSION_PAT, line) if m is None: @@ -1054,9 +1136,6 @@ def scan_apk_aapt(apk, apkfile): + ' is not a valid minSdkVersion!') else: apk['minSdkVersion'] = m.group(1) - # if target not set, default to min - if 'targetSdkVersion' not in apk: - apk['targetSdkVersion'] = m.group(1) elif line.startswith("targetSdkVersion:"): m = re.match(APK_SDK_VERSION_PAT, line) if m is None: @@ -1100,6 +1179,7 @@ def scan_apk_aapt(apk, apkfile): if feature.startswith("android.feature."): feature = feature[16:] apk['features'].add(feature) + apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name) def scan_apk_androguard(apk, apkfile): @@ -1138,27 +1218,14 @@ def scan_apk_androguard(apk, apkfile): if apkobject.get_max_sdk_version() is not None: apk['maxSdkVersion'] = apkobject.get_max_sdk_version() - apk['minSdkVersion'] = apkobject.get_min_sdk_version() - apk['targetSdkVersion'] = apkobject.get_target_sdk_version() + if apkobject.get_min_sdk_version() is not None: + apk['minSdkVersion'] = apkobject.get_min_sdk_version() + if apkobject.get_target_sdk_version() is not None: + apk['targetSdkVersion'] = apkobject.get_target_sdk_version() icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16) icon_name = arsc.get_id(apk['packageName'], icon_id)[1] - - density_re = re.compile("^res/(.*)/" + icon_name + ".*$") - - for file in apkobject.get_files(): - d_re = density_re.match(file) - if d_re: - folder = d_re.group(1).split('-') - if len(folder) > 1: - resolution = folder[1] - else: - resolution = 'mdpi' - density = screen_resolutions[resolution] - apk['icons_src'][density] = d_re.group(0) - - if apk['icons_src'].get('-1') is None: - apk['icons_src']['-1'] = apk['icons_src']['160'] + apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name) arch_re = re.compile("^lib/(.*)/.*$") arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)]) @@ -1168,33 +1235,41 @@ def scan_apk_androguard(apk, apkfile): xml = apkobject.get_android_manifest_xml() - for item in xml.getElementsByTagName('uses-permission'): - name = str(item.getAttribute("android:name")) - maxSdkVersion = item.getAttribute("android:maxSdkVersion") - maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion) + for item in xml.findall('uses-permission'): + name = str(item.attrib['{' + xml.nsmap['android'] + '}name']) + maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion') + maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None + permission = UsesPermission( + name, + maxSdkVersion + ) + apk['uses-permission'].append(permission) + for name, maxSdkVersion in apkobject.get_uses_implied_permission_list(): permission = UsesPermission( name, maxSdkVersion ) apk['uses-permission'].append(permission) - for item in xml.getElementsByTagName('uses-permission-sdk-23'): - name = str(item.getAttribute("android:name")) - maxSdkVersion = item.getAttribute("android:maxSdkVersion") - maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion) + for item in xml.findall('uses-permission-sdk-23'): + name = str(item.attrib['{' + xml.nsmap['android'] + '}name']) + maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion') + maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None permission_sdk_23 = UsesPermissionSdk23( name, maxSdkVersion ) apk['uses-permission-sdk-23'].append(permission_sdk_23) - for item in xml.getElementsByTagName('uses-feature'): - feature = str(item.getAttribute("android:name")) + for item in xml.findall('uses-feature'): + feature = str(item.attrib['{' + xml.nsmap['android'] + '}name']) if feature != "android.hardware.screen.portrait" \ and feature != "android.hardware.screen.landscape": if feature.startswith("android.feature."): feature = feature[16:] - apk['features'].append(feature) + required = item.attrib.get('{' + xml.nsmap['android'] + '}required') + if required is None or required == 'true': + apk['features'].append(feature) def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False, @@ -1242,7 +1317,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal return True, None, False # Check for debuggable apks... - if common.isApkAndDebuggable(apkfile): + if common.is_apk_and_debuggable(apkfile): logging.warning('{0} is set to android:debuggable="true"'.format(apkfile)) if options.rename_apks: @@ -1297,25 +1372,17 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal apkzip = zipfile.ZipFile(apkfile, 'r') - # if an APK has files newer than the system time, suggest updating - # the system clock. This is useful for offline systems, used for - # signing, which do not have another source of clock sync info. It - # has to be more than 24 hours newer because ZIP/APK files do not - # store timezone info manifest = apkzip.getinfo('AndroidManifest.xml') - if manifest.date_time[1] == 0: # month can't be zero - logging.debug(_('AndroidManifest.xml has no date')) - else: - dt_obj = datetime(*manifest.date_time) - checkdt = dt_obj - timedelta(1) - if datetime.today() < checkdt: - logging.warning('System clock is older than manifest in: ' - + apkfilename - + '\nSet clock to that time using:\n' - + 'sudo date -s "' + str(dt_obj) + '"') + # 1980-0-0 means zeroed out, any other invalid date should trigger a warning + if (1980, 0, 0) != manifest.date_time[0:3]: + try: + common.check_system_clock(datetime(*manifest.date_time), apkfilename) + except ValueError as e: + logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ") + .format(apkfilename=apkfile) + str(e)) # extract icons from APK zip file - iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode']) + iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode']) try: empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir) finally: @@ -1380,10 +1447,12 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False): def extract_apk_icons(icon_filename, apk, apkzip, repo_dir): - """ - Extracts icons from the given APK zip in various densities, - saves them into given repo directory - and stores their names in the APK metadata dictionary. + """Extracts PNG icons from an APK with the supported pixel densities + + Extracts icons from the given APK zip in various densities, saves + them into given repo directory and stores their names in the APK + metadata dictionary. If the icon is an XML icon, then this tries + to find PNG icon that can replace it. :param icon_filename: A string representing the icon's file name :param apk: A populated dictionary containing APK metadata. @@ -1391,7 +1460,17 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir): :param apkzip: An opened zipfile.ZipFile of the APK file :param repo_dir: The directory of the APK's repository :return: A list of icon densities that are missing + """ + res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)') + pngs = dict() + for f in apkzip.namelist(): + m = res_name_re.match(f) + if m and m.group(4) == 'png': + density = screen_resolutions[m.group(2)] + pngs[m.group(3) + '/' + density] = m.group(0) + + icon_type = None empty_densities = [] for density in screen_densities: if density not in apk['icons_src']: @@ -1399,67 +1478,79 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir): continue icon_src = apk['icons_src'][density] icon_dir = get_icon_dir(repo_dir, density) - icon_dest = os.path.join(icon_dir, icon_filename) + icon_type = '.png' # Extract the icon files per density if icon_src.endswith('.xml'): - png = os.path.basename(icon_src)[:-4] + '.png' - for f in apkzip.namelist(): - if f.endswith(png): - m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f) - if m and screen_resolutions[m.group(2)] == density: - icon_src = f + m = res_name_re.match(icon_src) + if m: + name = pngs.get(m.group(3) + '/' + str(density)) + if name: + icon_src = name if icon_src.endswith('.xml'): empty_densities.append(density) - continue + icon_type = '.xml' + icon_dest = os.path.join(icon_dir, icon_filename + icon_type) + try: with open(icon_dest, 'wb') as f: f.write(get_icon_bytes(apkzip, icon_src)) - apk['icons'][density] = icon_filename + apk['icons'][density] = icon_filename + icon_type except (zipfile.BadZipFile, ValueError, KeyError) as e: logging.warning("Error retrieving icon file: %s %s", icon_dest, e) del apk['icons_src'][density] empty_densities.append(density) + # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified" if '-1' in apk['icons_src']: icon_src = apk['icons_src']['-1'] - icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename) + icon_type = icon_src[-4:] + icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type) with open(icon_path, 'wb') as f: f.write(get_icon_bytes(apkzip, icon_src)) - try: - im = Image.open(icon_path) - dpi = px_to_dpi(im.size[0]) - for density in screen_densities: - if density in apk['icons']: - break - if density == screen_densities[-1] or dpi >= int(density): - apk['icons'][density] = icon_filename - shutil.move(icon_path, - os.path.join(get_icon_dir(repo_dir, density), icon_filename)) - empty_densities.remove(density) - break - except Exception as e: - logging.warning(_("Failed reading {path}: {error}") - .format(path=icon_path, error=e)) + if icon_type == '.png': + im = None + try: + im = Image.open(icon_path) + dpi = px_to_dpi(im.size[0]) + for density in screen_densities: + if density in apk['icons']: + break + if density == screen_densities[-1] or dpi >= int(density): + apk['icons'][density] = icon_filename + icon_type + shutil.move(icon_path, + os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type)) + empty_densities.remove(density) + break + except Exception as e: + logging.warning(_("Failed reading {path}: {error}") + .format(path=icon_path, error=e)) + finally: + if im and hasattr(im, 'close'): + im.close() if apk['icons']: - apk['icon'] = icon_filename + apk['icon'] = icon_filename + icon_type return empty_densities def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir): """ - Resize existing icons for densities missing in the APK to ensure all densities are available + Resize existing PNG icons for densities missing in the APK to ensure all densities are available :param empty_densities: A list of icon densities that are missing :param icon_filename: A string representing the icon's file name :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key :param repo_dir: The directory of the APK's repository + """ + icon_filename += '.png' # First try resizing down to not lose quality last_density = None for density in screen_densities: + if density == '65534': # not possible to generate 'anydpi' from other densities + continue if density not in empty_densities: last_density = density continue @@ -1477,7 +1568,8 @@ def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir): size = dpi_to_px(density) im.thumbnail((size, size), Image.ANTIALIAS) - im.save(icon_path, "PNG") + im.save(icon_path, "PNG", optimize=True, + pnginfo=BLANK_PNG_INFO, icc_profile=None) empty_densities.remove(density) except Exception as e: logging.warning("Invalid image file at %s: %s", last_icon_path, e) @@ -1625,7 +1717,7 @@ def move_apk_between_sections(from_dir, to_dir, apk): for density in all_screen_densities: from_icon_dir = get_icon_dir(from_dir, density) to_icon_dir = get_icon_dir(to_dir, density) - if density not in apk['icons']: + if density not in apk.get('icons', []): continue _move_file(from_icon_dir, to_icon_dir, apk['icons'][density], True) if 'srcname' in apk: @@ -1669,7 +1761,7 @@ def create_metadata_from_template(apk): with open('template.yml') as f: metatxt = f.read() if 'name' in apk and apk['name'] != '': - metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$', + metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''', r'\1 ' + apk['name'], metatxt, flags=re.IGNORECASE | re.MULTILINE) @@ -1705,6 +1797,7 @@ def create_metadata_from_template(apk): config = None options = None +start_timestamp = time.gmtime() def main(): @@ -1715,9 +1808,9 @@ def main(): parser = ArgumentParser() common.setup_global_opts(parser) parser.add_argument("--create-key", action="store_true", default=False, - help=_("Create a repo signing key in a keystore")) + help=_("Add a repo signing key to an unsigned repo")) parser.add_argument("-c", "--create-metadata", action="store_true", default=False, - help=_("Create skeleton metadata files that are missing")) + help=_("Add skeleton metadata files for APKs that are missing them")) parser.add_argument("--delete-unknown", action="store_true", default=False, help=_("Delete APKs and/or OBBs without metadata from the repo")) parser.add_argument("-b", "--buildreport", action="store_true", default=False, @@ -1732,7 +1825,7 @@ def main(): parser.add_argument("-w", "--wiki", default=False, action="store_true", help=_("Update the wiki")) parser.add_argument("--pretty", action="store_true", default=False, - help=_("Produce human-readable index.xml")) + help=_("Produce human-readable XML/JSON for index files")) parser.add_argument("--clean", action="store_true", default=False, help=_("Clean update - don't uses caches, reprocess all APKs")) parser.add_argument("--nosign", action="store_true", default=False,