import json
import os
import re
+import sys
import glob
import cgi
import logging
+import yaml
+# use libyaml if it is available
+try:
+ from yaml import CLoader
+ YamlLoader = CLoader
+except ImportError:
+ from yaml import Loader
+ YamlLoader = Loader
+
+# use the C implementation when available
+import xml.etree.cElementTree as ElementTree
+
from collections import OrderedDict
import common
# In the order in which they are laid out on files
app_defaults = OrderedDict([
('Disabled', None),
- ('AntiFeatures', None),
+ ('AntiFeatures', []),
('Provides', None),
('Categories', ['None']),
('License', 'Unknown'),
# In the order in which they are laid out on files
# Sorted by their action and their place in the build timeline
+# These variables can have varying datatypes. For example, anything with
+# flagtype(v) == 'list' is inited as False, then set as a list of strings.
flag_defaults = OrderedDict([
('disable', False),
('commit', None),
["Dogecoin"],
[]),
- FieldValidator("Boolean",
- ['Yes', 'No'], None,
- ["Requires Root"],
- []),
-
FieldValidator("bool",
- ['yes', 'no'], None,
- [],
+ r'([Yy]es|[Nn]o|[Tt]rue|[Ff]alse)', None,
+ ["Requires Root"],
['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']),
check_metadata(appinfo)
apps[appid] = appinfo
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))):
+ appid, appinfo = parse_xml_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.yaml'))):
+ appid, appinfo = parse_yaml_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
if xref:
# Parse all descriptions at load time, just to ensure cross-referencing
# errors are caught early rather than when they hit the build server.
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
- if name in ['Categories']:
+ if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
return [v for v in l if v]
-def get_default_app_info_list():
+def get_default_app_info_list(appid=None):
thisinfo = {}
thisinfo.update(app_defaults)
+ if appid is not None:
+ thisinfo['id'] = appid
# General defaults...
thisinfo['builds'] = []
def post_metadata_parse(thisinfo):
+ supported_metadata = app_defaults.keys() + ['comments', 'builds', 'id']
+ for k, v in thisinfo.iteritems():
+ if k not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: {0}: {1}"
+ .format(k, v))
+ if type(v) in (float, int):
+ thisinfo[k] = str(v)
+
+ # convert to the odd internal format
+ for k in ('Description', 'Maintainer Notes'):
+ if isinstance(thisinfo[k], basestring):
+ text = thisinfo[k].rstrip().lstrip()
+ thisinfo[k] = text.split('\n')
+
+ supported_flags = (flag_defaults.keys()
+ + ['vercode', 'version', 'versionCode', 'versionName'])
+ esc_newlines = re.compile('\\\\( |\\n)')
+
+ for build in thisinfo['builds']:
+ for k, v in build.items():
+ if k not in supported_flags:
+ raise MetaDataException("Unrecognised build flag: {0}={1}"
+ .format(k, v))
+
+ if k == 'versionCode':
+ build['vercode'] = str(v)
+ del build['versionCode']
+ elif k == 'versionName':
+ build['version'] = str(v)
+ del build['versionName']
+ elif type(v) in (float, int):
+ build[k] = str(v)
+ else:
+ keyflagtype = flagtype(k)
+ if keyflagtype == 'list':
+ # these can be bools, strings or lists, but ultimately are lists
+ if isinstance(v, basestring):
+ build[k] = [v]
+ elif isinstance(v, bool):
+ if v:
+ build[k] = ['yes']
+ else:
+ build[k] = ['no']
+ elif keyflagtype == 'script':
+ build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
+ elif keyflagtype == 'bool':
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if isinstance(v, basestring):
+ if v == 'true':
+ build[k] = True
+ else:
+ build[k] = False
+
if not thisinfo['Description']:
thisinfo['Description'].append('No description available')
# 'builds' - a list of dictionaries containing build information
# for each defined build
# 'comments' - a list of comments from the metadata file. Each is
-# a tuple of the form (field, comment) where field is
+# a list of the form [field, comment] where field is
# the name of the field it preceded in the metadata
# file. Where field is None, the comment goes at the
# end of the file. Alternatively, 'build:version' is
#
+def _decode_list(data):
+ '''convert items in a list from unicode to basestring'''
+ rv = []
+ for item in data:
+ if isinstance(item, unicode):
+ item = item.encode('utf-8')
+ elif isinstance(item, list):
+ item = _decode_list(item)
+ elif isinstance(item, dict):
+ item = _decode_dict(item)
+ rv.append(item)
+ return rv
+
+
+def _decode_dict(data):
+ '''convert items in a dict from unicode to basestring'''
+ rv = {}
+ for key, value in data.iteritems():
+ if isinstance(key, unicode):
+ key = key.encode('utf-8')
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+ elif isinstance(value, list):
+ value = _decode_list(value)
+ elif isinstance(value, dict):
+ value = _decode_dict(value)
+ rv[key] = value
+ return rv
+
+
def parse_json_metadata(metafile):
appid = os.path.basename(metafile)[0:-5] # strip path and .json
- thisinfo = get_default_app_info_list()
- thisinfo['id'] = appid
+ thisinfo = get_default_app_info_list(appid)
# fdroid metadata is only strings and booleans, no floats or ints. And
# json returns unicode, and fdroidserver still uses plain python strings
+ # TODO create schema using https://pypi.python.org/pypi/jsonschema
jsoninfo = json.load(open(metafile, 'r'),
+ object_hook=_decode_dict,
parse_int=lambda s: s,
parse_float=lambda s: s)
- supported_metadata = app_defaults.keys() + ['builds', 'comments']
- for k, v in jsoninfo.iteritems():
- if k == 'Requires Root':
- if isinstance(v, basestring):
- if re.match('^\s*(yes|true).*', v, flags=re.IGNORECASE):
- jsoninfo[k] = 'Yes'
- elif re.match('^\s*(no|false).*', v, flags=re.IGNORECASE):
- jsoninfo[k] = 'No'
- if isinstance(v, bool):
- if v:
- jsoninfo[k] = 'Yes'
- else:
- jsoninfo[k] = 'No'
- if k not in supported_metadata:
- logging.warn(metafile + ' contains unknown metadata key, ignoring: ' + k)
thisinfo.update(jsoninfo)
+ post_metadata_parse(thisinfo)
- for build in thisinfo['builds']:
- for k, v in build.iteritems():
- if k in ('buildjni', 'gradle', 'maven', 'kivy'):
- # convert standard types to mixed datatype legacy format
- if isinstance(v, bool):
- if v:
- build[k] = ['yes']
- else:
- build[k] = ['no']
- elif k == 'versionCode':
- build['vercode'] = v
- del build['versionCode']
- elif k == 'versionName':
- build['version'] = v
- del build['versionName']
+ return (appid, thisinfo)
- # TODO create schema using https://pypi.python.org/pypi/jsonschema
+
+def parse_xml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-4] # strip path and .xml
+ thisinfo = get_default_app_info_list(appid)
+
+ tree = ElementTree.ElementTree(file=metafile)
+ root = tree.getroot()
+
+ if root.tag != 'resources':
+ logging.critical(metafile + ' does not have root as <resources></resources>!')
+ sys.exit(1)
+
+ supported_metadata = app_defaults.keys()
+ for child in root:
+ if child.tag != 'builds':
+ # builds does not have name="" attrib
+ name = child.attrib['name']
+ if name not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: <"
+ + child.tag + ' name="' + name + '">'
+ + child.text
+ + "</" + child.tag + '>')
+
+ if child.tag == 'string':
+ thisinfo[name] = child.text
+ elif child.tag == 'string-array':
+ items = []
+ for item in child:
+ items.append(item.text)
+ thisinfo[name] = items
+ elif child.tag == 'builds':
+ builds = []
+ for build in child:
+ builddict = dict()
+ for key in build:
+ builddict[key.tag] = key.text
+ builds.append(builddict)
+ thisinfo['builds'] = builds
+
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if not isinstance(thisinfo['Requires Root'], bool):
+ if thisinfo['Requires Root'] == 'true':
+ thisinfo['Requires Root'] = True
+ else:
+ thisinfo['Requires Root'] = False
+
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_yaml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-5] # strip path and .yaml
+ thisinfo = get_default_app_info_list(appid)
+
+ yamlinfo = yaml.load(open(metafile, 'r'), Loader=YamlLoader)
+ thisinfo.update(yamlinfo)
post_metadata_parse(thisinfo)
return (appid, thisinfo)
if not curcomments:
return
for comment in curcomments:
- thisinfo['comments'].append((key, comment))
+ thisinfo['comments'].append([key, comment])
del curcomments[:]
thisinfo = get_default_app_info_list()
mf = open(dest, 'w')
writefield_nonempty('Disabled')
- writefield_nonempty('AntiFeatures')
+ writefield('AntiFeatures')
writefield_nonempty('Provides')
writefield('Categories')
writefield('License')
mf.write('.\n')
mf.write('\n')
if app['Requires Root']:
- writefield('Requires Root', 'Yes')
+ writefield('Requires Root', 'yes')
mf.write('\n')
if app['Repo Type']:
writefield('Repo Type')
~ianmdlvl / fdroidserver.git / blobdiff