support app metadata in XML format

While the current text metadata format is good for human readability and
editability, it is difficult to produce and parse using code.  XML is a
widespread standard format for easy automatic parsing and creating, while
having decent human readability.

The .pickle for testing is a lightly edited version of the real metadata
for net.osmand.plus:

* comments were removed
* "NonFreeNet" was added as an AntiFeature

diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index 2b75389fde502150169c02e8c1e72ce06fcab262..3a389ec2d2f0a2505b126ee341e8cfac2d31af9a 100644 (file)
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -20,10 +20,14 @@
 import json
 import os
 import re
+import sys
 import glob
 import cgi
 import logging
 
+# use the C implementation when available
+import xml.etree.cElementTree as ElementTree
+
 from collections import OrderedDict
 
 import common
@@ -79,6 +83,8 @@ app_defaults = OrderedDict([
 
 # In the order in which they are laid out on files
 # Sorted by their action and their place in the build timeline
+# These variables can have varying datatypes. For example, anything with
+# flagtype(v) == 'list' is inited as False, then set as a list of strings.
 flag_defaults = OrderedDict([
     ('disable', False),
     ('commit', None),
@@ -494,6 +500,11 @@ def read_metadata(xref=True):
         check_metadata(appinfo)
         apps[appid] = appinfo
 
+    for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))):
+        appid, appinfo = parse_xml_metadata(metafile)
+        check_metadata(appinfo)
+        apps[appid] = appinfo
+
     if xref:
         # Parse all descriptions at load time, just to ensure cross-referencing
         # errors are caught early rather than when they hit the build server.
@@ -579,6 +590,20 @@ def get_default_app_info_list():
 
 def post_metadata_parse(thisinfo):
 
+    for build in thisinfo['builds']:
+        for k, v in build.iteritems():
+            if k == 'versionCode':
+                build['vercode'] = str(v)
+                del build['versionCode']
+            elif k == 'versionName':
+                build['version'] = str(v)
+                del build['versionName']
+            elif flagtype(k) == 'bool':
+                if v == 'no':
+                    build[k] = False
+                else:
+                    build[k] = True
+
     if not thisinfo['Description']:
         thisinfo['Description'].append('No description available')
 
@@ -682,12 +707,6 @@ def parse_json_metadata(metafile):
                         build[k] = ['yes']
                     else:
                         build[k] = ['no']
-            elif k == 'versionCode':
-                build['vercode'] = v
-                del build['versionCode']
-            elif k == 'versionName':
-                build['version'] = v
-                del build['versionName']
 
     # TODO create schema using https://pypi.python.org/pypi/jsonschema
     post_metadata_parse(thisinfo)
@@ -695,6 +714,75 @@ def parse_json_metadata(metafile):
     return (appid, thisinfo)
 
 
+def parse_xml_metadata(metafile):
+
+    appid = os.path.basename(metafile)[0:-4]  # strip path and .xml
+    thisinfo = get_default_app_info_list()
+    thisinfo['id'] = appid
+
+    tree = ElementTree.ElementTree(file=metafile)
+    root = tree.getroot()
+
+    if root.tag != 'resources':
+        logging.critical(metafile + ' does not have root as <resources></resources>!')
+        sys.exit(1)
+
+    supported_metadata = app_defaults.keys()
+    for child in root:
+        if child.tag != 'builds':
+            # builds does not have name="" attrib
+            name = child.attrib['name']
+            if name not in supported_metadata:
+                raise MetaDataException("Unrecognised metadata: <"
+                                        + child.tag + ' name="' + name + '">'
+                                        + child.text
+                                        + "</" + child.tag + '>')
+
+        if child.tag == 'string':
+            thisinfo[name] = child.text
+        elif child.tag == 'string-array':
+            items = []
+            for item in child:
+                items.append(item.text)
+            thisinfo[name] = items
+        elif child.tag == 'builds':
+            builds = []
+            for build in child:
+                builddict = dict()
+                for key in build:
+                    builddict[key.tag] = key.text
+                builds.append(builddict)
+            thisinfo['builds'] = builds
+
+    # convert to the odd internal format
+    for k in ('Description', 'Maintainer Notes'):
+        if isinstance(thisinfo[k], basestring):
+            text = thisinfo[k].rstrip().lstrip()
+            thisinfo[k] = text.split('\n')
+
+    supported_flags = flag_defaults.keys() + ['versionCode', 'versionName']
+    for build in thisinfo['builds']:
+        for k, v in build.iteritems():
+            if k not in supported_flags:
+                raise MetaDataException("Unrecognised build flag: {0}={1}"
+                                        .format(k, v))
+            keyflagtype = flagtype(k)
+            if keyflagtype == 'bool':
+                # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+                if isinstance(v, basestring):
+                    if v == 'true':
+                        build[k] = True
+                    else:
+                        build[k] = False
+            elif keyflagtype == 'list':
+                if isinstance(v, basestring):
+                    build[k] = [v]
+
+    post_metadata_parse(thisinfo)
+
+    return (appid, thisinfo)
+
+
 def parse_txt_metadata(metafile):
 
     appid = None

diff --git a/tests/metadata.TestCase b/tests/metadata.TestCase
index 6778606eb2dd96fae4029c3acf1f295a64159c82..d5c873e79b0de21ab1057c306037c8d6a9ceb24a 100755 (executable)
--- a/tests/metadata.TestCase
+++ b/tests/metadata.TestCase
@@ -37,7 +37,7 @@ class MetadataTest(unittest.TestCase):
         fdroidserver.common.config = config
 
         apps = fdroidserver.metadata.read_metadata(xref=True)
-        for appid in ('org.smssecure.smssecure', 'org.adaway'):
+        for appid in ('org.smssecure.smssecure', 'org.adaway', 'net.osmand.plus'):
             frompickle = pickle.load(open(os.path.join('metadata', appid + '.pickle')))
             self.assertTrue(appid in apps.keys())
             self.assertEquals(apps[appid], frompickle)

diff --git a/tests/metadata/net.osmand.plus.pickle b/tests/metadata/net.osmand.plus.pickle
new file mode 100644 (file)
index 0000000..6a2fa2b
--- /dev/null
+++ b/tests/metadata/net.osmand.plus.pickle
@@ -0,0 +1,491 @@
+(dp0
+S'Update Check Data'
+p1
+NsS'Bitcoin'
+p2
+NsS'AntiFeatures'
+p3
+(lp4
+S'Tracking'
+p5
+aS'NonFreeNet'
+p6
+asS'Web Site'
+p7
+S'http://osmand.net'
+p8
+sS'Auto Update Mode'
+p9
+S'None'
+p10
+sS'Provides'
+p11
+NsS'Issue Tracker'
+p12
+S'https://github.com/osmandapp/Osmand/issues'
+p13
+sS'Donate'
+p14
+S'https://code.google.com/p/osmand/#Please_support_the_project'
+p15
+sS'Repo Type'
+p16
+S'git'
+p17
+sS'Description'
+p18
+(lp19
+S"Osmand~'s features can be extended by enabling the plugins via the settings,"
+p20
+aS'which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and'
+p21
+aS'accessibility enhancements.'
+p22
+aS''
+p23
+aS'Map data of both vector and raster types can be stored on the phone memory'
+p24
+aS'card for offline usage, and navigation by default uses offline methods. Map'
+p25
+aS'data packages for many territories can be downloaded from within the app and'
+p26
+aS'there is a desktop program available on the website as well for creating your'
+p27
+aS'own.'
+p28
+ag23
+aS'Anti-Features: Tracking - It will send your device and application specs to an'
+p29
+aS'Analytics server upon downloading the list of maps you can download.'
+p30
+ag23
+aS'[https://osmandapp.github.io/changes.html Changelog]'
+p31
+asS'Requires Root'
+p32
+I00
+sS'comments'
+p33
+(lp34
+sS'id'
+p35
+S'net.osmand.plus'
+p36
+sS'Repo'
+p37
+S'https://github.com/mvdan/OsmAnd-submodules'
+p38
+sS'No Source Since'
+p39
+g23
+sS'Auto Name'
+p40
+g23
+sS'Categories'
+p41
+(lp42
+S'Navigation'
+p43
+asS'Source Code'
+p44
+S'https://github.com/osmandapp/Osmand'
+p45
+sS'Litecoin'
+p46
+NsS'Update Check Ignore'
+p47
+NsS'Name'
+p48
+S'OsmAnd~'
+p49
+sS'License'
+p50
+S'GPLv3'
+p51
+sS'Changelog'
+p52
+g23
+sS'Update Check Mode'
+p53
+S'None'
+p54
+sS'Summary'
+p55
+S'Offline/online maps and navigation'
+p56
+sS'Dogecoin'
+p57
+NsS'Maintainer Notes'
+p58
+(lp59
+S'No UCMs apply because git never contains actual releases, only pre-releses.'
+p60
+ag23
+aS'The build instructions have been moved to a script in the root of the repo,'
+p61
+aS"'build'. This way it can be updated along with the submodules."
+p62
+asS'Current Version Code'
+p63
+S'197'
+p64
+sS'Binaries'
+p65
+NsS'Archive Policy'
+p66
+NsS'builds'
+p67
+(lp68
+(dp69
+S'submodules'
+p70
+I01
+sS'vercode'
+p71
+S'182'
+p72
+sS'forceversion'
+p73
+I00
+sS'oldsdkloc'
+p74
+I00
+sS'kivy'
+p75
+I00
+sS'gradle'
+p76
+I00
+sS'scanignore'
+p77
+(lp78
+sS'srclibs'
+p79
+(lp80
+sS'encoding'
+p81
+NsS'extlibs'
+p82
+(lp83
+sS'init'
+p84
+g23
+sS'version'
+p85
+S'1.8.2'
+p86
+sS'build'
+p87
+S'./old-ndk-build.sh && ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release'
+p88
+sS'rm'
+p89
+(lp90
+sS'type'
+p91
+S'raw'
+p92
+sS'subdir'
+p93
+S'android/OsmAnd'
+p94
+sS'ndk_path'
+p95
+g23
+sS'forcevercode'
+p96
+I00
+sS'preassemble'
+p97
+(lp98
+sS'update'
+p99
+(lp100
+S'auto'
+p101
+asS'maven'
+p102
+I00
+sS'disable'
+p103
+I00
+sS'commit'
+p104
+S'76ada6c8a08afe69acb755503373ac36328ef665'
+p105
+sS'scandelete'
+p106
+(lp107
+sS'buildjni'
+p108
+(lp109
+S'no'
+p110
+asS'ndk'
+p111
+S'r10e'
+p112
+sS'target'
+p113
+NsS'antcommands'
+p114
+NsS'patch'
+p115
+(lp116
+sS'prebuild'
+p117
+S'sed -i \'s/"OsmAnd+"/"OsmAnd~"/g\' build.xml'
+p118
+sS'novcheck'
+p119
+I00
+sS'output'
+p120
+S'bin/OsmAnd-release-unsigned.apk'
+p121
+sa(dp122
+S'submodules'
+p123
+I01
+sg71
+S'183'
+p124
+sg73
+I00
+sg74
+I00
+sg75
+I00
+sg76
+I00
+sg77
+g78
+sg79
+g80
+sg81
+Nsg82
+g83
+sg84
+g23
+sg85
+S'1.8.3'
+p125
+sS'subdir'
+p126
+S'android/OsmAnd'
+p127
+sg89
+g90
+sg91
+g92
+sS'build'
+p128
+S'../../build'
+p129
+sg95
+g23
+sg96
+I00
+sg97
+g98
+sg99
+g100
+sg102
+I00
+sg103
+I00
+sS'commit'
+p130
+S'1.8.3'
+p131
+sg106
+g107
+sS'buildjni'
+p132
+(lp133
+S'no'
+p134
+asg111
+g112
+sg113
+Nsg114
+Nsg115
+g116
+sS'prebuild'
+p135
+g23
+sg119
+I00
+sS'output'
+p136
+S'bin/OsmAnd-release-unsigned.apk'
+p137
+sa(dp138
+S'submodules'
+p139
+I01
+sg71
+S'196'
+p140
+sg73
+I00
+sg74
+I00
+sg75
+I00
+sg76
+I00
+sg77
+g78
+sg79
+g80
+sg81
+Nsg82
+g83
+sg84
+g23
+sg85
+S'1.9.4'
+p141
+sS'subdir'
+p142
+S'android/OsmAnd'
+p143
+sg89
+g90
+sg91
+g92
+sS'build'
+p144
+S'../../build'
+p145
+sg95
+g23
+sg96
+I00
+sg97
+g98
+sg99
+g100
+sg102
+I00
+sg103
+I00
+sS'commit'
+p146
+S'1.9.4'
+p147
+sg106
+g107
+sS'buildjni'
+p148
+(lp149
+S'no'
+p150
+asS'ndk'
+p151
+S'r10d'
+p152
+sg113
+Nsg114
+Nsg115
+g116
+sg135
+g23
+sg119
+I00
+sS'output'
+p153
+S'bin/OsmAnd-release-unsigned.apk'
+p154
+sa(dp155
+S'submodules'
+p156
+I01
+sg71
+S'197'
+p157
+sg73
+I00
+sg74
+I00
+sg75
+I00
+sg76
+I00
+sg77
+g78
+sg79
+g80
+sg81
+Nsg82
+g83
+sg84
+g23
+sg85
+S'1.9.5'
+p158
+sS'subdir'
+p159
+S'android/OsmAnd'
+p160
+sg89
+g90
+sg91
+g92
+sS'build'
+p161
+S'../../build'
+p162
+sg95
+g23
+sg96
+I00
+sg97
+g98
+sg99
+g100
+sg102
+I00
+sg103
+I00
+sS'commit'
+p163
+S'1.9.5'
+p164
+sg106
+g107
+sS'buildjni'
+p165
+(lp166
+S'no'
+p167
+asS'ndk'
+p168
+S'r10d'
+p169
+sg113
+Nsg114
+Nsg115
+g116
+sg135
+g23
+sg119
+I00
+sS'output'
+p170
+S'bin/OsmAnd-release-unsigned.apk'
+p171
+sasS'FlattrID'
+p172
+NsS'Disabled'
+p173
+NsS'Update Check Name'
+p174
+NsS'Vercode Operation'
+p175
+NsS'Current Version'
+p176
+S'1.9.5'
+p177
+s.

diff --git a/tests/metadata/net.osmand.plus.xml b/tests/metadata/net.osmand.plus.xml
new file mode 100644 (file)
index 0000000..4b7ac8a
--- /dev/null
+++ b/tests/metadata/net.osmand.plus.xml
@@ -0,0 +1,180 @@
+<?xml version="1.0" encoding="utf-8"?>
+
+<resources>
+  <string-array name="AntiFeatures">
+    <item>Tracking</item>
+    <item>NonFreeNet</item>
+  </string-array>
+
+  <string-array name="Categories">
+    <item>Navigation</item>
+  </string-array>
+
+  <string name="License">GPLv3</string>
+  <string name="Web Site">http://osmand.net</string>
+  <string name="Source Code">https://github.com/osmandapp/Osmand</string>
+  <string name="Issue Tracker">https://github.com/osmandapp/Osmand/issues</string>
+  <string name="Donate">https://code.google.com/p/osmand/#Please_support_the_project</string>
+
+  <string name="Name">OsmAnd~</string>
+  <string name="Summary">Offline/online maps and navigation</string>
+  <string name="Description">Osmand~'s features can be extended by enabling the plugins via the settings,
+which include online maps from many sources, tracking, OpenStreetMap (OSM) editing and
+accessibility enhancements.
+
+Map data of both vector and raster types can be stored on the phone memory
+card for offline usage, and navigation by default uses offline methods. Map
+data packages for many territories can be downloaded from within the app and
+there is a desktop program available on the website as well for creating your
+own.
+
+Anti-Features: Tracking - It will send your device and application specs to an
+Analytics server upon downloading the list of maps you can download.
+
+[https://osmandapp.github.io/changes.html Changelog]
+</string>
+
+  <string name="Repo Type">git</string>
+  <string name="Repo">https://github.com/mvdan/OsmAnd-submodules</string>
+<!--  <string name="Repo">https://github.com/osmandapp/Osmand</string/ -->
+
+<!--
+# Old builds with the old repo
+#Build:0.6.5,34
+#    commit=v0.6.5
+#    subdir=OsmAnd
+#    encoding=utf-8
+#    prebuild=mkdir assets && \
+#        mkdir raw
+#
+#Build:0.6.6,36
+#    commit=v0.6.6_2
+#    subdir=OsmAnd
+#    encoding=utf-8
+#    prebuild=mkdir raw
+#
+#Build:0.6.7,37
+#    commit=v0.6.7
+#    subdir=OsmAnd
+#    encoding=utf-8
+#    patch=code37.patch
+#    prebuild=mkdir raw
+#
+#Build:0.6.8,39
+#    commit=v0.6.8
+#    subdir=OsmAnd
+#    encoding=utf-8
+#    prebuild=mkdir raw
+#
+#Build:0.6.8',41
+#    disable=No corresponding source for whatever this is
+#    commit=unknown - see disabled
+#
+#Build:0.6.9,42
+#    commit=v0.6.9
+#    subdir=OsmAnd
+#    encoding=utf-8
+#    prebuild=mkdir raw
+#
+#Build:0.6.9',43
+#    disable=No corresponding source for whatever this is
+#    commit=unknown - see disabled
+#
+#Build:0.8.1,65
+#    commit=d62472532d8
+#    subdir=OsmAnd
+#    target=android-8
+#    init=rm -f build.xml
+#    encoding=utf-8
+#    forceversion=yes
+#    prebuild=cd ../DataExtractionOSM && \
+#        ant compile build && \
+#        cd ../OsmAnd/ && \
+#        cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
+#        zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
+#        cp -r ../DataExtractionOSM/build/lib/ libs/
+#    buildjni=no
+#
+#Build:0.8.2,71
+#    commit=50a4733475cd
+#    subdir=OsmAnd
+#    submodules=yes
+#    target=android-8
+#    init=rm -f build.xml
+#    encoding=utf-8
+#    forceversion=yes
+#    forcevercode=yes
+#    prebuild=cd ../DataExtractionOSM && \
+#        ant compile build && \
+#        cd ../OsmAnd/ && \
+#        sed -i 's/app_version">[^<]*/app_version">0.8.2-fdroid/' res/values/no_translate.xml && \
+#        cp ../DataExtractionOSM/build/OsmAndMapCreator.jar libs/ && \
+#        zip -d libs/OsmAndMapCreator.jar net/osmand/LogUtil.class && \
+#        cp -r ../DataExtractionOSM/build/lib/ libs/
+#    buildjni=yes
+-->
+
+<builds>
+
+  <build>
+    <versionCode>182</versionCode>
+    <versionName>1.8.2</versionName>
+    <commit>76ada6c8a08afe69acb755503373ac36328ef665</commit>
+    <subdir>android/OsmAnd</subdir>
+    <submodules>true</submodules>
+    <output>bin/OsmAnd-release-unsigned.apk</output>
+    <prebuild>sed -i 's/"OsmAnd+"/"OsmAnd~"/g' build.xml</prebuild>
+    <build>./old-ndk-build.sh &amp;&amp; ant -Dsdk.dir="$ANDROID_SDK" -Dndk.dir="$ANDROID_NDK" -DBLACKBERRY_BUILD=false -DBUILD_SUFFIX= -DAPK_NUMBER_VERSION=182 "-DFEATURES=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" -DCLEAN_CPP=false -DPACKAGE_TO_BUILT=net.osmand.plus -DAPK_VERSION=1.8.2 -Dnet.osmand.plus= -Dbuild.version=1.8.2 -Dbuild.version.code=182 -Dnativeoff=false "-DversionFeatures=+play_market +gps_status -parking_plugin -blackberry -amazon -route_nav" clean release</build>
+    <buildjni>no</buildjni>
+  </build>
+
+  <build>
+    <versionName>1.8.3</versionName>
+    <versionCode>183</versionCode>
+    <commit>1.8.3</commit>
+    <subdir>android/OsmAnd</subdir>
+    <submodules>true</submodules>
+    <output>bin/OsmAnd-release-unsigned.apk</output>
+    <build>../../build</build>
+    <buildjni>no</buildjni>
+  </build>
+
+  <build>
+    <versionName>1.9.4</versionName>
+    <versionCode>196</versionCode>
+    <commit>1.9.4</commit>
+    <subdir>android/OsmAnd</subdir>
+    <submodules>true</submodules>
+    <output>bin/OsmAnd-release-unsigned.apk</output>
+    <build>../../build</build>
+    <buildjni>no</buildjni>
+    <ndk>r10d</ndk>
+  </build>
+
+  <build>
+    <versionName>1.9.5</versionName>
+    <versionCode>197</versionCode>
+    <commit>1.9.5</commit>
+    <subdir>android/OsmAnd</subdir>
+    <submodules>true</submodules>
+    <output>bin/OsmAnd-release-unsigned.apk</output>
+    <build>../../build</build>
+    <buildjni>no</buildjni>
+    <ndk>r10d</ndk>
+  </build>
+
+</builds>
+
+  <string name="Maintainer Notes">
+No UCMs apply because git never contains actual releases, only pre-releses.
+
+The build instructions have been moved to a script in the root of the repo,
+'build'. This way it can be updated along with the submodules.
+  </string>
+
+  <string name="Auto Update Mode">None</string>
+  <string name="Update Check Mode">None</string>
+  <string name="Current Version">1.9.5</string>
+  <string name="Current Version Code">197</string>
+
+</resources>
\ No newline at end of file