build/checkupdates/update: log current fdroiddata commit to wiki

[fdroidserver.git] / fdroidserver / update.py

diff --git a/fdroidserver/update.py b/fdroidserver/update.py
index 574f1c243e6464f00d4051119b136394da13dd44..c22ac0f32b70d6add7f64a2367a24ef0e6091761 100644 (file)
--- a/fdroidserver/update.py
+++ b/fdroidserver/update.py
@@ -29,13 +29,13 @@ import zipfile
 import hashlib
 import pickle
 import time
 import hashlib
 import pickle
 import time

-from datetime import datetime, timedelta
+from datetime import datetime

 from argparse import ArgumentParser
 
 import collections
 from binascii import hexlify
 
 from argparse import ArgumentParser
 
 import collections
 from binascii import hexlify
 

-from PIL import Image
+from PIL import Image, PngImagePlugin

 import logging
 
 from . import _
 import logging
 
 from . import _


@@ -53,15 +53,13 @@ UNSET_VERSION_CODE = -0x100000000
 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")
 APK_NAME_PAT = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
 APK_VERCODE_PAT = re.compile(".*versionCode='([0-9]*)'.*")
 APK_VERNAME_PAT = re.compile(".*versionName='([^']*)'.*")

-APK_LABEL_PAT = re.compile(".*label='(.*?)'(\n| [a-z]*?=).*")
-APK_ICON_PAT = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
-APK_ICON_PAT_NODPI = re.compile(".*icon='([^']+?)'.*")
+APK_LABEL_ICON_PAT = re.compile(".*\s+label='(.*)'\s+icon='(.*)'")

 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
 APK_PERMISSION_PAT = \
     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
 
 APK_SDK_VERSION_PAT = re.compile(".*'([0-9]*)'.*")
 APK_PERMISSION_PAT = \
     re.compile(".*(name='(?P<name>.*?)')(.*maxSdkVersion='(?P<maxSdkVersion>.*?)')?.*")
 APK_FEATURE_PAT = re.compile(".*name='([^']*)'.*")
 

-screen_densities = ['640', '480', '320', '240', '160', '120']
+screen_densities = ['65534', '640', '480', '320', '240', '160', '120']

 screen_resolutions = {
     "xxxhdpi": '640',
     "xxhdpi": '480',
 screen_resolutions = {
     "xxxhdpi": '640',
     "xxhdpi": '480',


@@ -84,6 +82,8 @@ GRAPHIC_NAMES = ('featureGraphic', 'icon', 'promoGraphic', 'tvBanner')
 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
 
 SCREENSHOT_DIRS = ('phoneScreenshots', 'sevenInchScreenshots',
                    'tenInchScreenshots', 'tvScreenshots', 'wearScreenshots')
 

+BLANK_PNG_INFO = PngImagePlugin.PngInfo()
+

 
 def dpi_to_px(density):
     return (int(density) * 48) / 160
 
 def dpi_to_px(density):
     return (int(density) * 48) / 160


@@ -94,9 +94,10 @@ def px_to_dpi(px):
 
 
 def get_icon_dir(repodir, density):
 
 
 def get_icon_dir(repodir, density):

-    if density == '0':
+    if density == '0' or density == '65534':

         return os.path.join(repodir, "icons")
         return os.path.join(repodir, "icons")

-    return os.path.join(repodir, "icons-%s" % density)
+    else:
+        return os.path.join(repodir, "icons-%s" % density)

 
 
 def get_icon_dirs(repodir):
 
 
 def get_icon_dirs(repodir):


@@ -132,13 +133,13 @@ def update_wiki(apps, sortedids, apks):
         if app.Disabled:
             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
         if app.AntiFeatures:
         if app.Disabled:
             wikidata += '{{Disabled|' + app.Disabled + '}}\n'
         if app.AntiFeatures:

-            for af in app.AntiFeatures:
+            for af in sorted(app.AntiFeatures):

                 wikidata += '{{AntiFeature|' + af + '}}\n'
         if app.RequiresRoot:
             requiresroot = 'Yes'
         else:
             requiresroot = 'No'
                 wikidata += '{{AntiFeature|' + af + '}}\n'
         if app.RequiresRoot:
             requiresroot = 'Yes'
         else:
             requiresroot = 'No'

-        wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (
+        wikidata += '{{App|id=%s|name=%s|added=%s|lastupdated=%s|source=%s|tracker=%s|web=%s|changelog=%s|donate=%s|flattr=%s|liberapay=%s|bitcoin=%s|litecoin=%s|license=%s|root=%s|author=%s|email=%s}}\n' % (

             appid,
             app.Name,
             app.added.strftime('%Y-%m-%d') if app.added else '',
             appid,
             app.Name,
             app.added.strftime('%Y-%m-%d') if app.added else '',


@@ -149,6 +150,7 @@ def update_wiki(apps, sortedids, apks):
             app.Changelog,
             app.Donate,
             app.FlattrID,
             app.Changelog,
             app.Donate,
             app.FlattrID,

+            app.LiberapayID,

             app.Bitcoin,
             app.Litecoin,
             app.License,
             app.Bitcoin,
             app.Litecoin,
             app.License,


@@ -318,7 +320,21 @@ def update_wiki(apps, sortedids, apks):
                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 
     # Purge server cache to ensure counts are up to date
                     logging.error("...FAILED to create page '{0}': {1}".format(pagename, e))
 
     # Purge server cache to ensure counts are up to date

-    site.pages['Repository Maintenance'].purge()
+    site.Pages['Repository Maintenance'].purge()
+
+    # Write a page with the last build log for this version code
+    wiki_page_path = 'update_' + time.strftime('%s', start_timestamp)
+    newpage = site.Pages[wiki_page_path]
+    txt = ''
+    txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+    txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+    txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+    txt += common.get_git_describe_link()
+    txt += "\n\n"
+    txt += common.get_android_tools_version_log()
+    newpage.save(txt, summary='Run log')
+    newpage = site.Pages['update']
+    newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')

 
 
 def delete_disabled_builds(apps, apkcache, repodirs):
 
 
 def delete_disabled_builds(apps, apkcache, repodirs):


@@ -370,7 +386,8 @@ def resize_icon(iconpath, density):
             im.thumbnail((size, size), Image.ANTIALIAS)
             logging.debug("%s was too large at %s - new size is %s" % (
                 iconpath, oldsize, im.size))
             im.thumbnail((size, size), Image.ANTIALIAS)
             logging.debug("%s was too large at %s - new size is %s" % (
                 iconpath, oldsize, im.size))

-            im.save(iconpath, "PNG")
+            im.save(iconpath, "PNG", optimize=True,
+                    pnginfo=BLANK_PNG_INFO, icc_profile=None)

 
     except Exception as e:
         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))
 
     except Exception as e:
         logging.error(_("Failed resizing {path}: {error}".format(path=iconpath, error=e)))


@@ -495,14 +512,25 @@ def has_known_vulnerability(filename):
 
     Checks whether there are more than one classes.dex or AndroidManifest.xml
     files, which is invalid and an essential part of the "Master Key" attack.
 
     Checks whether there are more than one classes.dex or AndroidManifest.xml
     files, which is invalid and an essential part of the "Master Key" attack.

-

     http://www.saurik.com/id/17
     http://www.saurik.com/id/17

+
+    Janus is similar to Master Key but is perhaps easier to scan for.
+    https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures

     """
 
     """
 

+    found_vuln = False
+

     # statically load this pattern
     if not hasattr(has_known_vulnerability, "pattern"):
         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 
     # statically load this pattern
     if not hasattr(has_known_vulnerability, "pattern"):
         has_known_vulnerability.pattern = re.compile(b'.*OpenSSL ([01][0-9a-z.-]+)')
 

+    with open(filename.encode(), 'rb') as fp:
+        first4 = fp.read(4)
+    if first4 != b'\x50\x4b\x03\x04':
+        raise FDroidException(_('{path} has bad file signature "{pattern}", possible Janus exploit!')
+                              .format(path=filename, pattern=first4.decode().replace('\n', ' ')) + '\n'
+                              + 'https://www.guardsquare.com/en/blog/new-android-vulnerability-allows-attackers-modify-apps-without-affecting-their-signatures')
+

     files_in_apk = set()
     with zipfile.ZipFile(filename) as zf:
         for name in zf.namelist():
     files_in_apk = set()
     with zipfile.ZipFile(filename) as zf:
         for name in zf.namelist():


@@ -523,14 +551,15 @@ def has_known_vulnerability(filename):
                         else:
                             logging.warning(_('"{path}" contains outdated {name} ({version})')
                                             .format(path=filename, name=name, version=version))
                         else:
                             logging.warning(_('"{path}" contains outdated {name} ({version})')
                                             .format(path=filename, name=name, version=version))

-                            return True
+                            found_vuln = True

                         break
             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
                 if name in files_in_apk:
                         break
             elif name == 'AndroidManifest.xml' or name == 'classes.dex' or name.endswith('.so'):
                 if name in files_in_apk:

-                    return True
+                    logging.warning(_('{apkfilename} has multiple {name} files, looks like Master Key exploit!')
+                                    .format(apkfilename=filename, name=name))
+                    found_vuln = True

                 files_in_apk.add(name)
                 files_in_apk.add(name)

-
-    return False
+    return found_vuln

 
 
 def insert_obbs(repodir, apps, apks):
 
 
 def insert_obbs(repodir, apps, apks):


@@ -659,6 +688,35 @@ def _set_author_entry(app, key, f):
             app[key] = text
 
 
             app[key] = text
 
 

+def _strip_and_copy_image(inpath, outpath):
+    """Remove any metadata from image and copy it to new path
+
+    Sadly, image metadata like EXIF can be used to exploit devices.
+    It is not used at all in the F-Droid ecosystem, so its much safer
+    just to remove it entirely.
+
+    """
+
+    extension = common.get_extension(inpath)[1]
+    if os.path.isdir(outpath):
+        outpath = os.path.join(outpath, os.path.basename(inpath))
+    if extension == 'png':
+        with open(inpath, 'rb') as fp:
+            in_image = Image.open(fp)
+            in_image.save(outpath, "PNG", optimize=True,
+                          pnginfo=BLANK_PNG_INFO, icc_profile=None)
+    elif extension == 'jpg' or extension == 'jpeg':
+        with open(inpath, 'rb') as fp:
+            in_image = Image.open(fp)
+            data = list(in_image.getdata())
+            out_image = Image.new(in_image.mode, in_image.size)
+        out_image.putdata(data)
+        out_image.save(outpath, "JPEG", optimize=True)
+    else:
+        raise FDroidException(_('Unsupported file type "{extension}" for repo graphic')
+                              .format(extension=extension))
+
+

 def copy_triple_t_store_metadata(apps):
     """Include store metadata from the app's source repo
 
 def copy_triple_t_store_metadata(apps):
     """Include store metadata from the app's source repo
 


@@ -731,7 +789,7 @@ def copy_triple_t_store_metadata(apps):
                         sourcefile = os.path.join(root, f)
                         destfile = os.path.join(destdir, os.path.basename(f))
                         logging.debug('copying ' + sourcefile + ' ' + destfile)
                         sourcefile = os.path.join(root, f)
                         destfile = os.path.join(destdir, os.path.basename(f))
                         logging.debug('copying ' + sourcefile + ' ' + destfile)

-                        shutil.copy(sourcefile, destfile)
+                        _strip_and_copy_image(sourcefile, destfile)

 
 
 def insert_localized_app_metadata(apps):
 
 
 def insert_localized_app_metadata(apps):


@@ -772,7 +830,8 @@ def insert_localized_app_metadata(apps):
 
     """
 
 
     """
 

-    sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
+    sourcedirs = glob.glob(os.path.join('build', '[A-Za-z]*', 'src', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))
+    sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'fastlane', 'metadata', 'android', '[a-z][a-z]*'))

     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 
     sourcedirs += glob.glob(os.path.join('build', '[A-Za-z]*', 'metadata', '[a-z][a-z]*'))
     sourcedirs += glob.glob(os.path.join('metadata', '[A-Za-z]*', '[a-z][a-z]*'))
 


@@ -787,6 +846,17 @@ def insert_localized_app_metadata(apps):
                 continue
             locale = segments[-1]
             destdir = os.path.join('repo', packageName, locale)
                 continue
             locale = segments[-1]
             destdir = os.path.join('repo', packageName, locale)

+
+            # flavours specified in build receipt
+            build_flavours = ""
+            if apps[packageName] and 'builds' in apps[packageName] and len(apps[packageName].builds) > 0\
+                    and 'gradle' in apps[packageName].builds[-1]:
+                build_flavours = apps[packageName].builds[-1].gradle
+
+            if len(segments) >= 5 and segments[4] == "fastlane" and segments[3] not in build_flavours:
+                logging.debug("ignoring due to wrong flavour")
+                continue
+

             for f in files:
                 if f in ('description.txt', 'full_description.txt'):
                     _set_localized_text_entry(apps[packageName], locale, 'description',
             for f in files:
                 if f in ('description.txt', 'full_description.txt'):
                     _set_localized_text_entry(apps[packageName], locale, 'description',


@@ -817,7 +887,7 @@ def insert_localized_app_metadata(apps):
                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
                     os.makedirs(destdir, mode=0o755, exist_ok=True)
                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)
                 if base in GRAPHIC_NAMES and extension in ALLOWED_EXTENSIONS:
                     os.makedirs(destdir, mode=0o755, exist_ok=True)
                     logging.debug('copying ' + os.path.join(root, f) + ' ' + destdir)

-                    shutil.copy(os.path.join(root, f), destdir)
+                    _strip_and_copy_image(os.path.join(root, f), destdir)

             for d in dirs:
                 if d in SCREENSHOT_DIRS:
                     if locale == 'images':
             for d in dirs:
                 if d in SCREENSHOT_DIRS:
                     if locale == 'images':


@@ -829,7 +899,7 @@ def insert_localized_app_metadata(apps):
                             screenshotdestdir = os.path.join(destdir, d)
                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
                             logging.debug('copying ' + f + ' ' + screenshotdestdir)
                             screenshotdestdir = os.path.join(destdir, d)
                             os.makedirs(screenshotdestdir, mode=0o755, exist_ok=True)
                             logging.debug('copying ' + f + ' ' + screenshotdestdir)

-                            shutil.copy(f, screenshotdestdir)
+                            _strip_and_copy_image(f, screenshotdestdir)

 
     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
     for d in repofiles:
 
     repofiles = sorted(glob.glob(os.path.join('repo', '[A-Za-z]*', '[a-z][a-z][A-Z-.@]*')))
     for d in repofiles:


@@ -978,10 +1048,10 @@ def scan_apk(apk_file):
         'antiFeatures': set(),
     }
 
         'antiFeatures': set(),
     }
 

-    if SdkToolsPopen(['aapt', 'version'], output=False):
-        scan_apk_aapt(apk, apk_file)
-    else:
+    if common.use_androguard():

         scan_apk_androguard(apk, apk_file)
         scan_apk_androguard(apk, apk_file)

+    else:
+        scan_apk_aapt(apk, apk_file)

 
     # Get the signature, or rather the signing key fingerprints
     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))
 
     # Get the signature, or rather the signing key fingerprints
     logging.debug('Getting signature of {0}'.format(os.path.basename(apk_file)))


@@ -998,7 +1068,9 @@ def scan_apk(apk_file):
 
     if 'minSdkVersion' not in apk:
         logging.warning("No SDK version information found in {0}".format(apk_file))
 
     if 'minSdkVersion' not in apk:
         logging.warning("No SDK version information found in {0}".format(apk_file))

-        apk['minSdkVersion'] = 1
+        apk['minSdkVersion'] = 3  # aapt defaults to 3 as the min
+    if 'targetSdkVersion' not in apk:
+        apk['targetSdkVersion'] = apk['minSdkVersion']

 
     # Check for known vulnerabilities
     if has_known_vulnerability(apk_file):
 
     # Check for known vulnerabilities
     if has_known_vulnerability(apk_file):


@@ -1007,6 +1079,27 @@ def scan_apk(apk_file):
     return apk
 
 
     return apk
 
 

+def _get_apk_icons_src(apkfile, icon_name):
+    """Extract the paths to the app icon in all available densities
+
+    """
+    icons_src = dict()
+    density_re = re.compile('^res/(.*)/' + icon_name + '\.(png|xml)$')
+    with zipfile.ZipFile(apkfile) as zf:
+        for filename in zf.namelist():
+            m = density_re.match(filename)
+            if m:
+                folder = m.group(1).split('-')
+                if len(folder) > 1:
+                    density = screen_resolutions[folder[1]]
+                else:
+                    density = '160'
+                icons_src[density] = m.group(0)
+    if icons_src.get('-1') is None:
+        icons_src['-1'] = icons_src['160']
+    return icons_src
+
+

 def scan_apk_aapt(apk, apkfile):
     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
     if p.returncode != 0:
 def scan_apk_aapt(apk, apkfile):
     p = SdkToolsPopen(['aapt', 'dump', 'badging', apkfile], output=False)
     if p.returncode != 0:


@@ -1019,6 +1112,7 @@ def scan_apk_aapt(apk, apkfile):
         else:
             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
         raise BuildException(_("Invalid APK"))
         else:
             logging.error(_("Failed to get apk information, skipping {path}").format(path=apkfile))
         raise BuildException(_("Invalid APK"))

+    icon_name = None

     for line in p.output.splitlines():
         if line.startswith("package:"):
             try:
     for line in p.output.splitlines():
         if line.startswith("package:"):
             try:


@@ -1028,25 +1122,13 @@ def scan_apk_aapt(apk, apkfile):
             except Exception as e:
                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
         elif line.startswith("application:"):
             except Exception as e:
                 raise FDroidException("Package matching failed: " + str(e) + "\nLine was: " + line)
         elif line.startswith("application:"):

-            apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
-            # Keep path to non-dpi icon in case we need it
-            match = re.match(APK_ICON_PAT_NODPI, line)
-            if match:
-                apk['icons_src']['-1'] = match.group(1)
-        elif line.startswith("launchable-activity:"):
+            m = re.match(APK_LABEL_ICON_PAT, line)
+            if m:
+                apk['name'] = m.group(1)
+                icon_name = os.path.splitext(os.path.basename(m.group(2)))[0]
+        elif not apk.get('name') and line.startswith("launchable-activity:"):

             # Only use launchable-activity as fallback to application
             # Only use launchable-activity as fallback to application

-            if not apk['name']:
-                apk['name'] = re.match(APK_LABEL_PAT, line).group(1)
-            if '-1' not in apk['icons_src']:
-                match = re.match(APK_ICON_PAT_NODPI, line)
-                if match:
-                    apk['icons_src']['-1'] = match.group(1)
-        elif line.startswith("application-icon-"):
-            match = re.match(APK_ICON_PAT, line)
-            if match:
-                density = match.group(1)
-                path = match.group(2)
-                apk['icons_src'][density] = path
+            apk['name'] = re.match(APK_LABEL_ICON_PAT, line).group(1)

         elif line.startswith("sdkVersion:"):
             m = re.match(APK_SDK_VERSION_PAT, line)
             if m is None:
         elif line.startswith("sdkVersion:"):
             m = re.match(APK_SDK_VERSION_PAT, line)
             if m is None:


@@ -1054,9 +1136,6 @@ def scan_apk_aapt(apk, apkfile):
                               + ' is not a valid minSdkVersion!')
             else:
                 apk['minSdkVersion'] = m.group(1)
                               + ' is not a valid minSdkVersion!')
             else:
                 apk['minSdkVersion'] = m.group(1)

-                # if target not set, default to min
-                if 'targetSdkVersion' not in apk:
-                    apk['targetSdkVersion'] = m.group(1)

         elif line.startswith("targetSdkVersion:"):
             m = re.match(APK_SDK_VERSION_PAT, line)
             if m is None:
         elif line.startswith("targetSdkVersion:"):
             m = re.match(APK_SDK_VERSION_PAT, line)
             if m is None:


@@ -1100,6 +1179,7 @@ def scan_apk_aapt(apk, apkfile):
                 if feature.startswith("android.feature."):
                     feature = feature[16:]
                 apk['features'].add(feature)
                 if feature.startswith("android.feature."):
                     feature = feature[16:]
                 apk['features'].add(feature)

+    apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)

 
 
 def scan_apk_androguard(apk, apkfile):
 
 
 def scan_apk_androguard(apk, apkfile):


@@ -1138,27 +1218,14 @@ def scan_apk_androguard(apk, apkfile):
 
     if apkobject.get_max_sdk_version() is not None:
         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()
 
     if apkobject.get_max_sdk_version() is not None:
         apk['maxSdkVersion'] = apkobject.get_max_sdk_version()

-    apk['minSdkVersion'] = apkobject.get_min_sdk_version()
-    apk['targetSdkVersion'] = apkobject.get_target_sdk_version()
+    if apkobject.get_min_sdk_version() is not None:
+        apk['minSdkVersion'] = apkobject.get_min_sdk_version()
+    if apkobject.get_target_sdk_version() is not None:
+        apk['targetSdkVersion'] = apkobject.get_target_sdk_version()

 
     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]
 
     icon_id = int(apkobject.get_element("application", "icon").replace("@", "0x"), 16)
     icon_name = arsc.get_id(apk['packageName'], icon_id)[1]

-
-    density_re = re.compile("^res/(.*)/" + icon_name + ".*$")
-
-    for file in apkobject.get_files():
-        d_re = density_re.match(file)
-        if d_re:
-            folder = d_re.group(1).split('-')
-            if len(folder) > 1:
-                resolution = folder[1]
-            else:
-                resolution = 'mdpi'
-            density = screen_resolutions[resolution]
-            apk['icons_src'][density] = d_re.group(0)
-
-    if apk['icons_src'].get('-1') is None:
-        apk['icons_src']['-1'] = apk['icons_src']['160']
+    apk['icons_src'] = _get_apk_icons_src(apkfile, icon_name)

 
     arch_re = re.compile("^lib/(.*)/.*$")
     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])
 
     arch_re = re.compile("^lib/(.*)/.*$")
     arch = set([arch_re.match(file).group(1) for file in apkobject.get_files() if arch_re.match(file)])


@@ -1168,33 +1235,41 @@ def scan_apk_androguard(apk, apkfile):
 
     xml = apkobject.get_android_manifest_xml()
 
 
     xml = apkobject.get_android_manifest_xml()
 

-    for item in xml.getElementsByTagName('uses-permission'):
-        name = str(item.getAttribute("android:name"))
-        maxSdkVersion = item.getAttribute("android:maxSdkVersion")
-        maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
+    for item in xml.findall('uses-permission'):
+        name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
+        maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
+        maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None
+        permission = UsesPermission(
+            name,
+            maxSdkVersion
+        )
+        apk['uses-permission'].append(permission)
+    for name, maxSdkVersion in apkobject.get_uses_implied_permission_list():

         permission = UsesPermission(
             name,
             maxSdkVersion
         )
         apk['uses-permission'].append(permission)
 
         permission = UsesPermission(
             name,
             maxSdkVersion
         )
         apk['uses-permission'].append(permission)
 

-    for item in xml.getElementsByTagName('uses-permission-sdk-23'):
-        name = str(item.getAttribute("android:name"))
-        maxSdkVersion = item.getAttribute("android:maxSdkVersion")
-        maxSdkVersion = None if maxSdkVersion is '' else int(maxSdkVersion)
+    for item in xml.findall('uses-permission-sdk-23'):
+        name = str(item.attrib['{' + xml.nsmap['android'] + '}name'])
+        maxSdkVersion = item.attrib.get('{' + xml.nsmap['android'] + '}maxSdkVersion')
+        maxSdkVersion = int(maxSdkVersion) if maxSdkVersion else None

         permission_sdk_23 = UsesPermissionSdk23(
             name,
             maxSdkVersion
         )
         apk['uses-permission-sdk-23'].append(permission_sdk_23)
 
         permission_sdk_23 = UsesPermissionSdk23(
             name,
             maxSdkVersion
         )
         apk['uses-permission-sdk-23'].append(permission_sdk_23)
 

-    for item in xml.getElementsByTagName('uses-feature'):
-        feature = str(item.getAttribute("android:name"))
+    for item in xml.findall('uses-feature'):
+        feature = str(item.attrib['{' + xml.nsmap['android'] + '}name'])

         if feature != "android.hardware.screen.portrait" \
                 and feature != "android.hardware.screen.landscape":
             if feature.startswith("android.feature."):
                 feature = feature[16:]
         if feature != "android.hardware.screen.portrait" \
                 and feature != "android.hardware.screen.landscape":
             if feature.startswith("android.feature."):
                 feature = feature[16:]

-        apk['features'].append(feature)
+        required = item.attrib.get('{' + xml.nsmap['android'] + '}required')
+        if required is None or required == 'true':
+            apk['features'].append(feature)

 
 
 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,
 
 
 def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=False,


@@ -1242,7 +1317,7 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
             return True, None, False
 
         # Check for debuggable apks...
             return True, None, False
 
         # Check for debuggable apks...

-        if common.isApkAndDebuggable(apkfile):
+        if common.is_apk_and_debuggable(apkfile):

             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 
         if options.rename_apks:
             logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 
         if options.rename_apks:


@@ -1297,25 +1372,17 @@ def process_apk(apkcache, apkfilename, repodir, knownapks, use_date_from_apk=Fal
 
         apkzip = zipfile.ZipFile(apkfile, 'r')
 
 
         apkzip = zipfile.ZipFile(apkfile, 'r')
 

-        # if an APK has files newer than the system time, suggest updating
-        # the system clock.  This is useful for offline systems, used for
-        # signing, which do not have another source of clock sync info. It
-        # has to be more than 24 hours newer because ZIP/APK files do not
-        # store timezone info

         manifest = apkzip.getinfo('AndroidManifest.xml')
         manifest = apkzip.getinfo('AndroidManifest.xml')

-        if manifest.date_time[1] == 0:  # month can't be zero
-            logging.debug(_('AndroidManifest.xml has no date'))
-        else:
-            dt_obj = datetime(*manifest.date_time)
-            checkdt = dt_obj - timedelta(1)
-            if datetime.today() < checkdt:
-                logging.warning('System clock is older than manifest in: '
-                                + apkfilename
-                                + '\nSet clock to that time using:\n'
-                                + 'sudo date -s "' + str(dt_obj) + '"')
+        # 1980-0-0 means zeroed out, any other invalid date should trigger a warning
+        if (1980, 0, 0) != manifest.date_time[0:3]:
+            try:
+                common.check_system_clock(datetime(*manifest.date_time), apkfilename)
+            except ValueError as e:
+                logging.warning(_("{apkfilename}'s AndroidManifest.xml has a bad date: ")
+                                .format(apkfilename=apkfile) + str(e))

 
         # extract icons from APK zip file
 
         # extract icons from APK zip file

-        iconfilename = "%s.%s.png" % (apk['packageName'], apk['versionCode'])
+        iconfilename = "%s.%s" % (apk['packageName'], apk['versionCode'])

         try:
             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
         finally:
         try:
             empty_densities = extract_apk_icons(iconfilename, apk, apkzip, repodir)
         finally:


@@ -1380,10 +1447,12 @@ def process_apks(apkcache, repodir, knownapks, use_date_from_apk=False):
 
 
 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
 
 
 def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):

-    """
-    Extracts icons from the given APK zip in various densities,
-    saves them into given repo directory
-    and stores their names in the APK metadata dictionary.
+    """Extracts PNG icons from an APK with the supported pixel densities
+
+    Extracts icons from the given APK zip in various densities, saves
+    them into given repo directory and stores their names in the APK
+    metadata dictionary.  If the icon is an XML icon, then this tries
+    to find PNG icon that can replace it.

 
     :param icon_filename: A string representing the icon's file name
     :param apk: A populated dictionary containing APK metadata.
 
     :param icon_filename: A string representing the icon's file name
     :param apk: A populated dictionary containing APK metadata.


@@ -1391,7 +1460,17 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
     :param apkzip: An opened zipfile.ZipFile of the APK file
     :param repo_dir: The directory of the APK's repository
     :return: A list of icon densities that are missing
     :param apkzip: An opened zipfile.ZipFile of the APK file
     :param repo_dir: The directory of the APK's repository
     :return: A list of icon densities that are missing

+

     """
     """

+    res_name_re = re.compile(r'res/(drawable|mipmap)-(x*[hlm]dpi|anydpi).*/(.*)_[0-9]+dp.(png|xml)')
+    pngs = dict()
+    for f in apkzip.namelist():
+        m = res_name_re.match(f)
+        if m and m.group(4) == 'png':
+            density = screen_resolutions[m.group(2)]
+            pngs[m.group(3) + '/' + density] = m.group(0)
+
+    icon_type = None

     empty_densities = []
     for density in screen_densities:
         if density not in apk['icons_src']:
     empty_densities = []
     for density in screen_densities:
         if density not in apk['icons_src']:


@@ -1399,67 +1478,79 @@ def extract_apk_icons(icon_filename, apk, apkzip, repo_dir):
             continue
         icon_src = apk['icons_src'][density]
         icon_dir = get_icon_dir(repo_dir, density)
             continue
         icon_src = apk['icons_src'][density]
         icon_dir = get_icon_dir(repo_dir, density)

-        icon_dest = os.path.join(icon_dir, icon_filename)
+        icon_type = '.png'

 
         # Extract the icon files per density
         if icon_src.endswith('.xml'):
 
         # Extract the icon files per density
         if icon_src.endswith('.xml'):

-            png = os.path.basename(icon_src)[:-4] + '.png'
-            for f in apkzip.namelist():
-                if f.endswith(png):
-                    m = re.match(r'res/(drawable|mipmap)-(x*[hlm]dpi).*/', f)
-                    if m and screen_resolutions[m.group(2)] == density:
-                        icon_src = f
+            m = res_name_re.match(icon_src)
+            if m:
+                name = pngs.get(m.group(3) + '/' + str(density))
+                if name:
+                    icon_src = name

             if icon_src.endswith('.xml'):
                 empty_densities.append(density)
             if icon_src.endswith('.xml'):
                 empty_densities.append(density)

-                continue
+                icon_type = '.xml'
+        icon_dest = os.path.join(icon_dir, icon_filename + icon_type)
+

         try:
             with open(icon_dest, 'wb') as f:
                 f.write(get_icon_bytes(apkzip, icon_src))
         try:
             with open(icon_dest, 'wb') as f:
                 f.write(get_icon_bytes(apkzip, icon_src))

-            apk['icons'][density] = icon_filename
+            apk['icons'][density] = icon_filename + icon_type

         except (zipfile.BadZipFile, ValueError, KeyError) as e:
             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
             del apk['icons_src'][density]
             empty_densities.append(density)
 
         except (zipfile.BadZipFile, ValueError, KeyError) as e:
             logging.warning("Error retrieving icon file: %s %s", icon_dest, e)
             del apk['icons_src'][density]
             empty_densities.append(density)
 

+    # '-1' here is a remnant of the parsing of aapt output, meaning "no DPI specified"

     if '-1' in apk['icons_src']:
         icon_src = apk['icons_src']['-1']
     if '-1' in apk['icons_src']:
         icon_src = apk['icons_src']['-1']

-        icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename)
+        icon_type = icon_src[-4:]
+        icon_path = os.path.join(get_icon_dir(repo_dir, '0'), icon_filename + icon_type)

         with open(icon_path, 'wb') as f:
             f.write(get_icon_bytes(apkzip, icon_src))
         with open(icon_path, 'wb') as f:
             f.write(get_icon_bytes(apkzip, icon_src))

-        try:
-            im = Image.open(icon_path)
-            dpi = px_to_dpi(im.size[0])
-            for density in screen_densities:
-                if density in apk['icons']:
-                    break
-                if density == screen_densities[-1] or dpi >= int(density):
-                    apk['icons'][density] = icon_filename
-                    shutil.move(icon_path,
-                                os.path.join(get_icon_dir(repo_dir, density), icon_filename))
-                    empty_densities.remove(density)
-                    break
-        except Exception as e:
-            logging.warning(_("Failed reading {path}: {error}")
-                            .format(path=icon_path, error=e))
+        if icon_type == '.png':
+            im = None
+            try:
+                im = Image.open(icon_path)
+                dpi = px_to_dpi(im.size[0])
+                for density in screen_densities:
+                    if density in apk['icons']:
+                        break
+                    if density == screen_densities[-1] or dpi >= int(density):
+                        apk['icons'][density] = icon_filename + icon_type
+                        shutil.move(icon_path,
+                                    os.path.join(get_icon_dir(repo_dir, density), icon_filename + icon_type))
+                        empty_densities.remove(density)
+                        break
+            except Exception as e:
+                logging.warning(_("Failed reading {path}: {error}")
+                                .format(path=icon_path, error=e))
+            finally:
+                if im and hasattr(im, 'close'):
+                    im.close()

 
     if apk['icons']:
 
     if apk['icons']:

-        apk['icon'] = icon_filename
+        apk['icon'] = icon_filename + icon_type

 
     return empty_densities
 
 
 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
     """
 
     return empty_densities
 
 
 def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
     """

-    Resize existing icons for densities missing in the APK to ensure all densities are available
+    Resize existing PNG icons for densities missing in the APK to ensure all densities are available

 
     :param empty_densities: A list of icon densities that are missing
     :param icon_filename: A string representing the icon's file name
     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
     :param repo_dir: The directory of the APK's repository
 
     :param empty_densities: A list of icon densities that are missing
     :param icon_filename: A string representing the icon's file name
     :param apk: A populated dictionary containing APK metadata. Needs to have 'icons' key
     :param repo_dir: The directory of the APK's repository

+

     """
     """

+    icon_filename += '.png'

     # First try resizing down to not lose quality
     last_density = None
     for density in screen_densities:
     # First try resizing down to not lose quality
     last_density = None
     for density in screen_densities:

+        if density == '65534':  # not possible to generate 'anydpi' from other densities
+            continue

         if density not in empty_densities:
             last_density = density
             continue
         if density not in empty_densities:
             last_density = density
             continue


@@ -1477,7 +1568,8 @@ def fill_missing_icon_densities(empty_densities, icon_filename, apk, repo_dir):
             size = dpi_to_px(density)
 
             im.thumbnail((size, size), Image.ANTIALIAS)
             size = dpi_to_px(density)
 
             im.thumbnail((size, size), Image.ANTIALIAS)

-            im.save(icon_path, "PNG")
+            im.save(icon_path, "PNG", optimize=True,
+                    pnginfo=BLANK_PNG_INFO, icc_profile=None)

             empty_densities.remove(density)
         except Exception as e:
             logging.warning("Invalid image file at %s: %s", last_icon_path, e)
             empty_densities.remove(density)
         except Exception as e:
             logging.warning("Invalid image file at %s: %s", last_icon_path, e)


@@ -1669,7 +1761,7 @@ def create_metadata_from_template(apk):
         with open('template.yml') as f:
             metatxt = f.read()
         if 'name' in apk and apk['name'] != '':
         with open('template.yml') as f:
             metatxt = f.read()
         if 'name' in apk and apk['name'] != '':

-            metatxt = re.sub(r'^(((Auto)?Name|Summary):).*$',
+            metatxt = re.sub(r'''^(((Auto)?Name|Summary):)[ '"\.]*$''',

                              r'\1 ' + apk['name'],
                              metatxt,
                              flags=re.IGNORECASE | re.MULTILINE)
                              r'\1 ' + apk['name'],
                              metatxt,
                              flags=re.IGNORECASE | re.MULTILINE)


@@ -1705,6 +1797,7 @@ def create_metadata_from_template(apk):
 
 config = None
 options = None
 
 config = None
 options = None

+start_timestamp = time.gmtime()

 
 
 def main():
 
 
 def main():


@@ -1715,9 +1808,9 @@ def main():
     parser = ArgumentParser()
     common.setup_global_opts(parser)
     parser.add_argument("--create-key", action="store_true", default=False,
     parser = ArgumentParser()
     common.setup_global_opts(parser)
     parser.add_argument("--create-key", action="store_true", default=False,

-                        help=_("Create a repo signing key in a keystore"))
+                        help=_("Add a repo signing key to an unsigned repo"))

     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,
     parser.add_argument("-c", "--create-metadata", action="store_true", default=False,

-                        help=_("Create skeleton metadata files that are missing"))
+                        help=_("Add skeleton metadata files for APKs that are missing them"))

     parser.add_argument("--delete-unknown", action="store_true", default=False,
                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
     parser.add_argument("-b", "--buildreport", action="store_true", default=False,
     parser.add_argument("--delete-unknown", action="store_true", default=False,
                         help=_("Delete APKs and/or OBBs without metadata from the repo"))
     parser.add_argument("-b", "--buildreport", action="store_true", default=False,


@@ -1732,7 +1825,7 @@ def main():
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
                         help=_("Update the wiki"))
     parser.add_argument("--pretty", action="store_true", default=False,
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
                         help=_("Update the wiki"))
     parser.add_argument("--pretty", action="store_true", default=False,

-                        help=_("Produce human-readable index.xml"))
+                        help=_("Produce human-readable XML/JSON for index files"))

     parser.add_argument("--clean", action="store_true", default=False,
                         help=_("Clean update - don't uses caches, reprocess all APKs"))
     parser.add_argument("--nosign", action="store_true", default=False,
     parser.add_argument("--clean", action="store_true", default=False,
                         help=_("Clean update - don't uses caches, reprocess all APKs"))
     parser.add_argument("--nosign", action="store_true", default=False,