Metadata parsing changes

A few small changes to the format of metadata files:

"Build Version" lines can now contain include commas by escaping them
with a backslash. They can also be split over multiple lines by ending
each line (except the last) with a backslash.

Additionally, empty lines in descriptions should now work correctly.

diff --git a/README b/README
index 0f3737e8ec1f84c832f89996b1ab1988f129c938..4b45fe7cc1051a1a3603a60a920cc8a6172ac0bd 100644 (file)
--- a/README
+++ b/README
@@ -169,9 +169,14 @@ configuration to the build. These are:
                    the SDK's ant rules, and forces the Java compiler to interpret
                    source files with this encoding. If you receive warnings during
                    the compile about character encodings, you probably need this.
- prebuild=xxxx     Specifies a shell command (or commands - chain with &&) to run
-                   before the build takes place - the only proviso being that you
-                   can't use commas in the command.
+
+ prebuild=xxxx     Specifies a shell command (or commands - chain with &&) to
+                   run before the build takes place. Backslash can be used
+                   as an escape character to insert literal commas, or as the
+                   last character on a line to join that line with the next.
+                   They have no special meaning in other contexts; in
+                   particular, literal backslashes should not be escaped.
+
  novcheck=yes      Don't check that the version name and code in the resulting apk
                    are correct by looking at the build output - assume the metadata
                    is correct. This takes away a useful level of sanity checking, and

diff --git a/common.py b/common.py
index 0d3a06ca693e697e85b8f10fa9e2449ae95b26bf..b66e04a8bfd4f2af6ba9854b80dc04c8fd3dbb55 100644 (file)
--- a/common.py
+++ b/common.py
@@ -16,123 +16,139 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import glob, os, sys
+import glob, os, sys, re
 
-def read_metadata(verbose=False):
-
-    apps = []
-
-    for metafile in glob.glob(os.path.join('metadata','*.txt')):
+def parse_metadata(metafile, **kw):
 
-        thisinfo = {}
-
-        # Get metadata...
-        thisinfo['id'] = metafile[9:-4]
-        if verbose:
-            print "Reading metadata for " + thisinfo['id']
-        thisinfo['description'] = ''
-        thisinfo['name'] = None
-        thisinfo['summary'] = ''
-        thisinfo['license'] = 'Unknown'
-        thisinfo['web'] = ''
-        thisinfo['source'] = ''
-        thisinfo['tracker'] = ''
-        thisinfo['donate'] = None
-        thisinfo['disabled'] = None
-        thisinfo['antifeatures'] = None
-        thisinfo['marketversion'] = ''
-        thisinfo['marketvercode'] = '0'
-        thisinfo['repotype'] = ''
-        thisinfo['repo'] = ''
-        thisinfo['builds'] = []
-        thisinfo['usebuilt'] = False
-        f = open(metafile, 'r')
-        mode = 0
-        for line in f.readlines():
-            if not line.startswith("#"):
-                line = line.rstrip('\r\n')
-                if len(line) == 0:
-                    pass
-                elif mode == 0:
-                    index = line.find(':')
-                    if index == -1:
-                        print "Invalid metadata in " + metafile + " at:" + line
-                        sys.exit(1)
-                    field = line[:index]
-                    value = line[index+1:]
-                    if field == 'Description':
-                        mode = 1
-                    elif field == 'Name':
-                        thisinfo['name'] = value
-                    elif field == 'Summary':
-                        thisinfo['summary'] = value
-                    elif field == 'Source Code':
-                        thisinfo['source'] = value
-                    elif field == 'License':
-                        thisinfo['license'] = value
-                    elif field == 'Web Site':
-                        thisinfo['web'] = value
-                    elif field == 'Issue Tracker':
-                        thisinfo['tracker'] = value
-                    elif field == 'Donate':
-                        thisinfo['donate'] = value
-                    elif field == 'Disabled':
-                        thisinfo['disabled'] = value
-                    elif field == 'AntiFeatures':
-                        parts = value.split(",")
-                        for part in parts:
-                            if (part != "Ads" and
-                                part != "Tracking" and
-                                part != "NonFreeNet" and
-                                part != "NonFreeAdd"):
-                                print "Unrecognised antifeature '" + part + "' in "+ metafile
-                                sys.exit(1)
-                        thisinfo['antifeatures'] = value
-                    elif field == 'Market Version':
-                        thisinfo['marketversion'] = value
-                    elif field == 'Market Version Code':
-                        thisinfo['marketvercode'] = value
-                    elif field == 'Repo Type':
-                        thisinfo['repotype'] = value
-                    elif field == 'Repo':
-                        thisinfo['repo'] = value
-                    elif field == 'Build Version':
-                        parts = value.split(",")
-                        if len(parts) < 3:
-                            print "Invalid build format: " + value + " in " + metafile
-                            sys.exit(1)
-                        thisbuild = {}
-                        thisbuild['version'] = parts[0]
-                        thisbuild['vercode'] = parts[1]
-                        thisbuild['commit'] = parts[2]
-                        for p in parts[3:]:
-                            pk, pv = p.split('=', 1)
-                            thisbuild[pk] = pv
-                        thisinfo['builds'].append(thisbuild)
-                    elif field == "Use Built":
-                        if value == "Yes":
-                            thisinfo['usebuilt'] = True
-                    else:
-                        print "Unrecognised field " + field + " in " + metafile
-                        sys.exit(1)
-                elif mode == 1:
-                    if line == '.':
-                        mode = 0
-                    else:
-                        if len(line) == 0:
-                            thisinfo['description'] += '\n\n'
-                        else:
-                            if (not thisinfo['description'].endswith('\n') and
-                                len(thisinfo['description']) > 0):
-                                thisinfo['description'] += ' '
-                            thisinfo['description'] += line
-
-        if mode == 1:
-            print "Description not terminated in " + metafile
+    def parse_buildline(value):
+        parts = [p.replace("\\,", ",")
+                 for p in re.split(r"(?<!\\),", value)]
+        if len(parts) < 3:
+            print "Invalid build format: " + value + " in " + metafile.name
             sys.exit(1)
-        if len(thisinfo['description']) == 0:
-            thisinfo['description'] = 'No description available'
+        thisbuild = {}
+        thisbuild['version'] = parts[0]
+        thisbuild['vercode'] = parts[1]
+        thisbuild['commit'] = parts[2]
+        for p in parts[3:]:
+            pk, pv = p.split('=', 1)
+            thisbuild[pk] = pv
+        return thisbuild
 
-        apps.append(thisinfo)
+    if not isinstance(metafile, file):
+        metafile = open(metafile, "r")
+    thisinfo = {}
+    thisinfo['id'] = metafile.name[9:-4]
+    if kw.get("verbose", False):
+        print "Reading metadata for " + thisinfo['id']
+    thisinfo['description'] = ''
+    thisinfo['name'] = None
+    thisinfo['summary'] = ''
+    thisinfo['license'] = 'Unknown'
+    thisinfo['web'] = ''
+    thisinfo['source'] = ''
+    thisinfo['tracker'] = ''
+    thisinfo['donate'] = None
+    thisinfo['disabled'] = None
+    thisinfo['antifeatures'] = None
+    thisinfo['marketversion'] = ''
+    thisinfo['marketvercode'] = '0'
+    thisinfo['repotype'] = ''
+    thisinfo['repo'] = ''
+    thisinfo['builds'] = []
+    thisinfo['usebuilt'] = False
+    mode = 0
+    buildline = []
+    for line in metafile:
+        line = line.rstrip('\r\n')
+        if line.startswith("#"):
+            continue
+        if mode == 0:
+            if len(line) == 0:
+                continue
+            index = line.find(':')
+            if index == -1:
+                print "Invalid metadata in " + metafile.name + " at: " + line
+                sys.exit(1)
+            field = line[:index]
+            value = line[index+1:]
+            if field == 'Description':
+                mode = 1
+            elif field == 'Name':
+                thisinfo['name'] = value
+            elif field == 'Summary':
+                thisinfo['summary'] = value
+            elif field == 'Source Code':
+                thisinfo['source'] = value
+            elif field == 'License':
+                thisinfo['license'] = value
+            elif field == 'Web Site':
+                thisinfo['web'] = value
+            elif field == 'Issue Tracker':
+                thisinfo['tracker'] = value
+            elif field == 'Donate':
+                thisinfo['donate'] = value
+            elif field == 'Disabled':
+                thisinfo['disabled'] = value
+            elif field == 'AntiFeatures':
+                parts = value.split(",")
+                for part in parts:
+                    if (part != "Ads" and
+                        part != "Tracking" and
+                        part != "NonFreeNet" and
+                        part != "NonFreeAdd"):
+                        print "Unrecognised antifeature '" + part + "' in " \
+                            + metafile.name
+                        sys.exit(1)
+                thisinfo['antifeatures'] = value
+            elif field == 'Market Version':
+                thisinfo['marketversion'] = value
+            elif field == 'Market Version Code':
+                thisinfo['marketvercode'] = value
+            elif field == 'Repo Type':
+                thisinfo['repotype'] = value
+            elif field == 'Repo':
+                thisinfo['repo'] = value
+            elif field == 'Build Version':
+                if value.endswith("\\"):
+                    mode = 2
+                    buildline = [value[:-1]]
+                else:
+                    thisinfo['builds'].append(parse_buildline(value))
+            elif field == "Use Built":
+                if value == "Yes":
+                    thisinfo['usebuilt'] = True
+            else:
+                print "Unrecognised field " + field + " in " + metafile.name
+                sys.exit(1)
+        elif mode == 1:       # multi-line description
+            if line == '.':
+                mode = 0
+            else:
+                if len(line) == 0:
+                    thisinfo['description'] += '\n\n'
+                else:
+                    if (not thisinfo['description'].endswith('\n') and
+                        len(thisinfo['description']) > 0):
+                        thisinfo['description'] += ' '
+                    thisinfo['description'] += line
+        elif mode == 2:       # line continuation
+            if line.endswith("\\"):
+                buildline.append(line[:-1])
+            else:
+                buildline.append(line)
+                thisinfo['builds'].append(
+                    parse_buildline("".join(buildline)))
+                mode = 0
+    if mode == 1:
+        print "Description not terminated in " + metafile.name
+        sys.exit(1)
+    if len(thisinfo['description']) == 0:
+        thisinfo['description'] = 'No description available'
+    return thisinfo
 
+def read_metadata(verbose=False):
+    apps = []
+    for metafile in glob.glob(os.path.join('metadata', '*.txt')):
+        apps.append(parse_metadata(metafile, verbose=verbose))
     return apps