handle APKs with filenames encoded with CP437

[fdroidserver.git] / fdroidserver / update.py

diff --git a/fdroidserver/update.py b/fdroidserver/update.py
index 570d249b2cfb36e650bd502ad83b16572109859e..abff8b6516281bfed6e6c452cc499814d5fc8fc8 100644 (file)
--- a/fdroidserver/update.py
+++ b/fdroidserver/update.py
@@ -44,6 +44,8 @@ from . import metadata
 from .common import FDroidPopen, FDroidPopenBytes, SdkToolsPopen
 from .metadata import MetaDataException
 
+METADATA_VERSION = 16
+
 screen_densities = ['640', '480', '320', '240', '160', '120']
 
 all_screen_densities = ['0'] + screen_densities
@@ -321,8 +323,10 @@ def resize_icon(iconpath, density):
     if not os.path.isfile(iconpath):
         return
 
+    fp = None
     try:
-        im = Image.open(iconpath)
+        fp = open(iconpath, 'rb')
+        im = Image.open(fp)
         size = dpi_to_px(density)
 
         if any(length > size for length in im.size):
@@ -335,6 +339,10 @@ def resize_icon(iconpath, density):
     except Exception as e:
         logging.error("Failed resizing {0} - {1}".format(iconpath, e))
 
+    finally:
+        if fp:
+            fp.close()
+
 
 def resize_all_icons(repodirs):
     """Resize all icons that exceed the max size
@@ -403,6 +411,14 @@ def getsig(apkpath):
     return hashlib.md5(hexlify(cert_encoded)).hexdigest()
 
 
+def get_icon_bytes(apkzip, iconsrc):
+    '''ZIP has no official encoding, UTF-* and CP437 are defacto'''
+    try:
+        return apkzip.read(iconsrc)
+    except KeyError:
+        return apkzip.read(iconsrc.encode('utf-8').decode('cp437'))
+
+
 def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
     """Scan the apks in the given repo directory.
 
@@ -436,7 +452,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
     icon_pat = re.compile(".*application-icon-([0-9]+):'([^']+?)'.*")
     icon_pat_nodpi = re.compile(".*icon='([^']+?)'.*")
     sdkversion_pat = re.compile(".*'([0-9]*)'.*")
-    string_pat = re.compile(".*'([^']*)'.*")
+    string_pat = re.compile(".* name='([^']*)'.*")
     for apkfile in glob.glob(os.path.join(repodir, '*.apk')):
 
         apkfilename = apkfile[len(repodir) + 1:]
@@ -524,6 +540,16 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
                                       + ' is not a valid minSdkVersion!')
                     else:
                         apk['minSdkVersion'] = m.group(1)
+                        # if target not set, default to min
+                        if 'targetSdkVersion' not in apk:
+                            apk['targetSdkVersion'] = m.group(1)
+                elif line.startswith("targetSdkVersion:"):
+                    m = re.match(sdkversion_pat, line)
+                    if m is None:
+                        logging.error(line.replace('targetSdkVersion:', '')
+                                      + ' is not a valid targetSdkVersion!')
+                    else:
+                        apk['targetSdkVersion'] = m.group(1)
                 elif line.startswith("maxSdkVersion:"):
                     apk['maxSdkVersion'] = re.match(sdkversion_pat, line).group(1)
                 elif line.startswith("native-code:"):
@@ -551,7 +577,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 
             # Check for debuggable apks...
             if common.isApkDebuggable(apkfile, config):
-                logging.warn('{0} is set to android:debuggable="true"'.format(apkfile))
+                logging.warning('{0} is set to android:debuggable="true"'.format(apkfile))
 
             # Get the signature (or md5 of, to be precise)...
             logging.debug('Getting signature of {0}'.format(apkfile))
@@ -595,7 +621,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
 
                 try:
                     with open(icondest, 'wb') as f:
-                        f.write(apkzip.read(iconsrc))
+                        f.write(get_icon_bytes(apkzip, iconsrc))
                     apk['icons'][density] = iconfilename
 
                 except:
@@ -609,7 +635,7 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
                 iconpath = os.path.join(
                     get_icon_dir(repodir, '0'), iconfilename)
                 with open(iconpath, 'wb') as f:
-                    f.write(apkzip.read(iconsrc))
+                    f.write(get_icon_bytes(apkzip, iconsrc))
                 try:
                     im = Image.open(iconpath)
                     dpi = px_to_dpi(im.size[0])
@@ -645,17 +671,21 @@ def scan_apks(apps, apkcache, repodir, knownapks, use_date_from_apk=False):
                     get_icon_dir(repodir, last_density), iconfilename)
                 iconpath = os.path.join(
                     get_icon_dir(repodir, density), iconfilename)
+                fp = None
                 try:
-                    im = Image.open(last_iconpath)
-                except:
-                    logging.warn("Invalid image file at %s" % last_iconpath)
-                    continue
+                    fp = open(last_iconpath, 'rb')
+                    im = Image.open(fp)
 
-                size = dpi_to_px(density)
+                    size = dpi_to_px(density)
 
-                im.thumbnail((size, size), Image.ANTIALIAS)
-                im.save(iconpath, "PNG")
-                empty_densities.remove(density)
+                    im.thumbnail((size, size), Image.ANTIALIAS)
+                    im.save(iconpath, "PNG")
+                    empty_densities.remove(density)
+                except:
+                    logging.warning("Invalid image file at %s" % last_iconpath)
+                finally:
+                    if fp:
+                        fp.close()
 
             # Then just copy from the highest resolution available
             last_density = None
@@ -801,7 +831,7 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
         for mirror in config.get('mirrors', []):
             addElement('mirror', urllib.parse.urljoin(mirror, urlbasepath), doc, repoel)
 
-    repoel.setAttribute("version", "15")
+    repoel.setAttribute("version", str(METADATA_VERSION))
     repoel.setAttribute("timestamp", str(int(time.time())))
 
     nosigningkey = False
@@ -937,6 +967,8 @@ def make_index(apps, sortedids, apks, repodir, archive, categories):
             addElement('sig', apk['sig'], doc, apkel)
             addElement('size', str(apk['size']), doc, apkel)
             addElement('sdkver', str(apk['minSdkVersion']), doc, apkel)
+            if 'targetSdkVersion' in apk:
+                addElement('targetSdkVersion', str(apk['targetSdkVersion']), doc, apkel)
             if 'maxSdkVersion' in apk:
                 addElement('maxsdkver', str(apk['maxSdkVersion']), doc, apkel)
             if 'added' in apk:
@@ -1212,6 +1244,8 @@ def main():
     if not options.clean and os.path.exists(apkcachefile):
         with open(apkcachefile, 'rb') as cf:
             apkcache = pickle.load(cf, encoding='utf-8')
+        if apkcache.get("METADATA_VERSION") != METADATA_VERSION:
+            apkcache = {}
     else:
         apkcache = {}
 
@@ -1352,6 +1386,7 @@ def main():
                 f.write(data)
 
     if cachechanged:
+        apkcache["METADATA_VERSION"] = METADATA_VERSION
         with open(apkcachefile, 'wb') as cf:
             pickle.dump(apkcache, cf)