Improved metadata handling

Two main points: Firstly, there is no longer a random mapping between
app['key'] and a corresponding field in the metadata file - the key and
field are now the same. Secondly, more information (including comments)
is retrieved from the metadata, to facilitate being able to re-write it
which is necessary for various support utilities.

diff --git a/build.py b/build.py
index fadb7a54502560bd08ea7b377551fc7526ed961d..b69020b5253b2f0fb68f16c5f08d6b7c1ee6b2ce 100755 (executable)
--- a/build.py
+++ b/build.py
@@ -74,13 +74,13 @@ if not os.path.isdir(build_dir):
 
 for app in apps:
 
-    if app['disabled']:
+    if app['Disabled']:
         print "Skipping %s: disabled" % app['id']
     elif not app['builds']:
         print "Skipping %s: no builds specified" % app['id']
 
-    if (app['disabled'] is None and app['repo'] != '' 
-            and app['repotype'] != '' and (options.package is None or
+    if (app['Disabled'] is None and app['Repo'] != '' 
+            and app['Repo Type'] != '' and (options.package is None or
             options.package == app['id']) and len(app['builds']) > 0):
 
         print "Processing " + app['id']
@@ -88,7 +88,7 @@ for app in apps:
         build_dir = 'build/' + app['id']
 
         # Set up vcs interface and make sure we have the latest code...
-        vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
+        vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
 
         refreshed_source = False
 

diff --git a/checkmarket2.py b/checkmarket2.py
index 193a63fa8670f043ccae443b8730521b646a72b1..f1a7d805888e34cfd1762ad71ff06bf71f52662c 100755 (executable)
--- a/checkmarket2.py
+++ b/checkmarket2.py
@@ -63,7 +63,7 @@ for app in apps:
         print "...couldn't find version code"
     elif not version:
         print "...couldn't find version"
-    elif vercode == app['marketvercode'] and version == app['marketversion']:
+    elif vercode == app['Market Version Code'] and version == app['Market Version']:
         print "...up to date"
     else:
         print '...updating to version:' + version + ' vercode:' + vercode

diff --git a/common.py b/common.py
index d54ed432e0009327a18798419c2beb9a827fa223..d1bd617025fb058d309496ca7850f9de53082be5 100644 (file)
--- a/common.py
+++ b/common.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # common.py - part of the FDroid server tools
-# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -245,15 +245,53 @@ class vcs_bzr(vcs):
             raise VCSException("Bzr update failed")
 
 
+# Get the type expected for a given metadata field.
+def metafieldtype(name):
+    if name == 'Description':
+        return 'multiline'
+    if name == 'Requires Root':
+        return 'flag'
+    if name == 'Build Version':
+        return 'build'
+    if name == 'Use Built':
+        return 'obsolete'
+    return 'string'
 
+
+# Parse metadata for a single application.
+#
+#  'metafile' - the filename to read. The package id for the application comes
+#               from this filename.
+#
+# Returns a dictionary containing all the details of the application. There are
+# two major kinds of information in the dictionary. Keys beginning with capital
+# letters correspond directory to identically named keys in the metadata file.
+# Keys beginning with lower case letters are generated in one way or another,
+# and are not found verbatim in the metadata.
+#
+# Known keys not originating from the metadata are:
+#
+#  'id'               - the application's package ID
+#  'builds'           - a list of dictionaries containing build information
+#                       for each defined build
+#  'comments'         - a list of comments from the metadata file. Each is
+#                       a tuple of the form (field, comment) where field is
+#                       the name of the field it preceded in the metadata
+#                       file
+#  'name'             - the application's name, as displayed. This will
+#                       always be None on return from this parser. The name
+#                       is discovered from built APK files.
+#
 def parse_metadata(metafile, **kw):
 
-    def parse_buildline(value):
+    def parse_buildline(lines):
+        value = "".join(lines)
         parts = [p.replace("\\,", ",")
                  for p in re.split(r"(?<!\\),", value)]
         if len(parts) < 3:
             raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
         thisbuild = {}
+        thisbuild['origlines'] = lines
         thisbuild['version'] = parts[0]
         thisbuild['vercode'] = parts[1]
         thisbuild['commit'] = parts[2]
@@ -268,29 +306,37 @@ def parse_metadata(metafile, **kw):
     thisinfo['id'] = metafile.name[9:-4]
     if kw.get("verbose", False):
         print "Reading metadata for " + thisinfo['id']
-    thisinfo['description'] = ''
-    thisinfo['name'] = None
-    thisinfo['summary'] = ''
-    thisinfo['license'] = 'Unknown'
-    thisinfo['web'] = ''
-    thisinfo['source'] = ''
-    thisinfo['tracker'] = ''
-    thisinfo['donate'] = None
-    thisinfo['disabled'] = None
-    thisinfo['antifeatures'] = None
-    thisinfo['marketversion'] = ''
-    thisinfo['marketvercode'] = '0'
-    thisinfo['repotype'] = ''
-    thisinfo['repo'] = ''
+
+    # Defaults for fields that come from metadata...
+    thisinfo['Description'] = ''
+    thisinfo['Summary'] = ''
+    thisinfo['License'] = 'Unknown'
+    thisinfo['Web Site'] = ''
+    thisinfo['Source Code'] = ''
+    thisinfo['Issue Tracker'] = ''
+    thisinfo['Donate'] = None
+    thisinfo['Disabled'] = None
+    thisinfo['AntiFeatures'] = None
+    thisinfo['Market Version'] = ''
+    thisinfo['Market Version Code'] = '0'
+    thisinfo['Repo Type'] = ''
+    thisinfo['Repo'] = ''
+    thisinfo['Requires Root'] = False
+
+    # General defaults...
     thisinfo['builds'] = []
-    thisinfo['requiresroot'] = False
+    thisinfo['name'] = None
+    thisinfo['comments'] = []
+
     mode = 0
-    buildline = []
+    buildlines = []
+    curcomments = []
+
     for line in metafile:
         line = line.rstrip('\r\n')
         if line.startswith("#"):
-            continue
-        if mode == 0:
+            curcomments.append(line)
+        elif mode == 0:
             if len(line) == 0:
                 continue
             index = line.find(':')
@@ -298,83 +344,77 @@ def parse_metadata(metafile, **kw):
                 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
             field = line[:index]
             value = line[index+1:]
-            if field == 'Description':
+
+            for comment in curcomments:
+                thisinfo['comments'].append((field,comment))
+
+            fieldtype = metafieldtype(field)
+            if fieldtype == 'multiline':
                 mode = 1
-            elif field == 'Name':
-                thisinfo['name'] = value
-            elif field == 'Summary':
-                thisinfo['summary'] = value
-            elif field == 'Source Code':
-                thisinfo['source'] = value
-            elif field == 'License':
-                thisinfo['license'] = value
-            elif field == 'Category':
-                thisinfo['category'] = value
-            elif field == 'Web Site':
-                thisinfo['web'] = value
-            elif field == 'Issue Tracker':
-                thisinfo['tracker'] = value
-            elif field == 'Donate':
-                thisinfo['donate'] = value
-            elif field == 'Disabled':
-                thisinfo['disabled'] = value
-            elif field == 'Use Built':
-                pass  #Ignoring this - will be removed
-            elif field == 'AntiFeatures':
-                parts = value.split(",")
-                for part in parts:
-                    if (part != "Ads" and
-                        part != "Tracking" and
-                        part != "NonFreeNet" and
-                        part != "NonFreeDep" and
-                        part != "NonFreeAdd"):
-                        raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
-                            + metafile.name)
-                thisinfo['antifeatures'] = value
-            elif field == 'Market Version':
-                thisinfo['marketversion'] = value
-            elif field == 'Market Version Code':
-                thisinfo['marketvercode'] = value
-            elif field == 'Repo Type':
-                thisinfo['repotype'] = value
-            elif field == 'Repo':
-                thisinfo['repo'] = value
-            elif field == 'Build Version':
+                if len(value) > 0:
+                    raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
+            elif fieldtype == 'string':
+                thisinfo[field] = value
+            elif fieldtype == 'flag':
+                if value == 'Yes':
+                    thisinfo[field] = True
+                elif value == 'No':
+                    thisinfo[field] = False
+                else:
+                    raise MetaDataException("Expected Yes or No for " + field + " in " + metafile.name)
+            elif fieldtype == 'build':
                 if value.endswith("\\"):
                     mode = 2
-                    buildline = [value[:-1]]
+                    buildlines = [value[:-1]]
                 else:
-                    thisinfo['builds'].append(parse_buildline(value))
-            elif field == "Requires Root":
-                if value == "Yes":
-                    thisinfo['requiresroot'] = True
+                    thisinfo['builds'].append(parse_buildline([value]))
+            elif fieldtype == 'obsolete':
+                pass        # Just throw it away!
             else:
-                raise MetaDataException("Unrecognised field " + field + " in " + metafile.name)
-        elif mode == 1:       # multi-line description
+                raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
+        elif mode == 1:     # Multiline field
             if line == '.':
                 mode = 0
             else:
                 if len(line) == 0:
-                    thisinfo['description'] += '\n\n'
+                    thisinfo[field] += '\n\n'
                 else:
-                    if (not thisinfo['description'].endswith('\n') and
-                        len(thisinfo['description']) > 0):
-                        thisinfo['description'] += ' '
-                    thisinfo['description'] += line
-        elif mode == 2:       # line continuation
+                    if (not thisinfo[field].endswith('\n') and
+                        len(thisinfo[field]) > 0):
+                        thisinfo[field] += ' '
+                    thisinfo[field] += line
+        elif mode == 2:     # Line continuation mode in Build Version
             if line.endswith("\\"):
-                buildline.append(line[:-1])
+                buildlines.append(line[:-1])
             else:
-                buildline.append(line)
+                buildlines.append(line)
                 thisinfo['builds'].append(
-                    parse_buildline("".join(buildline)))
+                    parse_buildline(buildlines))
                 mode = 0
+
     if mode == 1:
-        raise MetaDataException("Description not terminated in " + metafile.name)
-    if len(thisinfo['description']) == 0:
-        thisinfo['description'] = 'No description available'
+        raise MetaDataException(field + " not terminated in " + metafile.name)
+    elif mode == 2:
+        raise MetaDataException("Unterminated continuation in " + metafile.name)
+
+    if len(thisinfo['Description']) == 0:
+        thisinfo['Description'] = 'No description available'
+
+    # Ensure all AntiFeatures are recognised...
+    if thisinfo['AntiFeatures']:
+        parts = thisinfo['AntiFeatures'].split(",")
+        for part in parts:
+            if (part != "Ads" and
+                part != "Tracking" and
+                part != "NonFreeNet" and
+                part != "NonFreeDep" and
+                part != "NonFreeAdd"):
+                raise MetaDataException("Unrecognised antifeature '" + part + "' in " \
+                            + metafile.name)
+
     return thisinfo
 
+
 def read_metadata(verbose=False):
     apps = []
     for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):

diff --git a/scanner.py b/scanner.py
index 3291a2dcc3c0e3d4273b82dab3889080b08a3f5e..f4d63e350bea75de8d5655312aa69777208dd4f7 100755 (executable)
--- a/scanner.py
+++ b/scanner.py
@@ -55,7 +55,7 @@ for app in apps:
     skip = False
     if options.package and app['id'] != options.package:
         skip = True
-    elif app['disabled']:
+    elif app['Disabled']:
         print "Skipping %s: disabled" % app['id']
         skip = True
     elif not app['builds']:
@@ -71,7 +71,7 @@ for app in apps:
             build_dir = 'build/' + app['id']
 
             # Set up vcs interface and make sure we have the latest code...
-            vcs = common.getvcs(app['repotype'], app['repo'], build_dir)
+            vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
 
             refreshed_source = False
 

diff --git a/update.py b/update.py
index a943bb7e8ee6dace79cd35ac8390bd9a0b0b217f..b974099245abb1888f245cd998c76fa06d2299a5 100755 (executable)
--- a/update.py
+++ b/update.py
@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # update.py - part of the FDroid server tools
-# Copyright (C) 2010-11, Ciaran Gultnieks, ciaran@ciarang.com
+# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -196,7 +196,7 @@ for app in apps:
         if app['name'] is None:
             app['name'] = app['id']
         app['icon'] = ''
-        if app['disabled'] is None:
+        if app['Disabled'] is None:
             print "WARNING: Application " + app['id'] + " has no packages"
     else:
         if app['name'] is None:
@@ -283,14 +283,14 @@ apps_nopkg = 0
 
 for app in apps:
 
-    if app['disabled'] is None:
+    if app['Disabled'] is None:
 
         # Get a list of the apks for this app...
         gotmarketver = False
         apklist = []
         for apk in apks:
             if apk['id'] == app['id']:
-                if str(apk['versioncode']) == app['marketvercode']:
+                if str(apk['versioncode']) == app['Market Version Code']:
                     gotmarketver = True
                 apklist.append(apk)
 
@@ -304,22 +304,22 @@ for app in apps:
 
             addElement('id', app['id'], doc, apel)
             addElement('name', app['name'], doc, apel)
-            addElement('summary', app['summary'], doc, apel)
+            addElement('summary', app['Summary'], doc, apel)
             addElement('icon', app['icon'], doc, apel)
-            addElement('description', app['description'], doc, apel)
-            addElement('license', app['license'], doc, apel)
-            if 'category' in app:
-                addElement('category', app['category'], doc, apel)
-            addElement('web', app['web'], doc, apel)
-            addElement('source', app['source'], doc, apel)
-            addElement('tracker', app['tracker'], doc, apel)
-            if app['donate'] != None:
-                addElement('donate', app['donate'], doc, apel)
-            addElement('marketversion', app['marketversion'], doc, apel)
-            addElement('marketvercode', app['marketvercode'], doc, apel)
-            if not (app['antifeatures'] is None):
-                addElement('antifeatures', app['antifeatures'], doc, apel)
-            if app['requiresroot']:
+            addElement('description', app['Description'], doc, apel)
+            addElement('license', app['License'], doc, apel)
+            if 'Category' in app:
+                addElement('category', app['Category'], doc, apel)
+            addElement('web', app['Web Site'], doc, apel)
+            addElement('source', app['Source Code'], doc, apel)
+            addElement('tracker', app['Issue Tracker'], doc, apel)
+            if app['Donate'] != None:
+                addElement('donate', app['Donate'], doc, apel)
+            addElement('marketversion', app['Market Version'], doc, apel)
+            addElement('marketvercode', app['Market Version Code'], doc, apel)
+            if not (app['AntiFeatures'] is None):
+                addElement('antifeatures', app['AntiFeatures'], doc, apel)
+            if app['Requires Root']:
                 addElement('requirements', 'root', doc, apel)
 
             # Sort the apk list into version order, just so the web site
@@ -370,38 +370,38 @@ for app in apps:
         if options.buildreport:
             if len(app['builds']) == 0:
                 print ("WARNING: No builds defined for " + app['id'] +
-                        " Source: " + app['source'])
+                        " Source: " + app['Source Code'])
                 warnings += 1
             else:
-                if app['marketvercode'] != '0':
+                if app['Market Version Code'] != '0':
                     gotbuild = False
                     for build in app['builds']:
-                        if build['vercode'] == app['marketvercode']:
+                        if build['vercode'] == app['Market Version Code']:
                             gotbuild = True
                     if not gotbuild:
                         print ("WARNING: No build data for market version of "
-                                + app['id'] + " (" + app['marketversion']
-                                + ") " + app['source'])
+                                + app['id'] + " (" + app['Market Version']
+                                + ") " + app['Source Code'])
                         warnings += 1
 
         # If we don't have the market version, check if there is a build
         # with a commit ID starting with '!' - this means we can't build it
         # for some reason, and don't want hassling about it...
-        if not gotmarketver and app['marketvercode'] != '0':
+        if not gotmarketver and app['Market Version Code'] != '0':
             for build in app['builds']:
-                if build['vercode'] == app['marketvercode']:
+                if build['vercode'] == app['Market Version Code']:
                     gotmarketver = True
 
         # Output a message of harassment if we don't have the market version:
-        if not gotmarketver and app['marketvercode'] != '0':
-            addr = app['source']
-            print "WARNING: Don't have market version (" + app['marketversion'] + ") of " + app['name']
+        if not gotmarketver and app['Market Version Code'] != '0':
+            addr = app['Source Code']
+            print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['name']
             print "         (" + app['id'] + ") " + addr
             warnings += 1
             if options.verbose:
                 # A bit of extra debug info, basically for diagnosing
                 # app developer mistakes:
-                print "         Market vercode:" + app['marketvercode']
+                print "         Market vercode:" + app['Market Version Code']
                 print "         Got:"
                 for apk in apks:
                     if apk['id'] == app['id']: