# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
+import json
import os
import re
+import sys
import glob
import cgi
import logging
+import yaml
+# use libyaml if it is available
+try:
+ from yaml import CLoader
+ YamlLoader = CLoader
+except ImportError:
+ from yaml import Loader
+ YamlLoader = Loader
+
+# use the C implementation when available
+import xml.etree.cElementTree as ElementTree
+
from collections import OrderedDict
+import common
+
srclibs = None
class MetaDataException(Exception):
+
def __init__(self, value):
self.value = value
# In the order in which they are laid out on files
app_defaults = OrderedDict([
('Disabled', None),
- ('AntiFeatures', None),
+ ('AntiFeatures', []),
('Provides', None),
('Categories', ['None']),
('License', 'Unknown'),
('Web Site', ''),
('Source Code', ''),
('Issue Tracker', ''),
+ ('Changelog', ''),
('Donate', None),
('FlattrID', None),
('Bitcoin', None),
('Requires Root', False),
('Repo Type', ''),
('Repo', ''),
+ ('Binaries', None),
('Maintainer Notes', []),
('Archive Policy', None),
('Auto Update Mode', 'None'),
('Current Version', ''),
('Current Version Code', '0'),
('No Source Since', ''),
- ])
+])
# In the order in which they are laid out on files
# Sorted by their action and their place in the build timeline
+# These variables can have varying datatypes. For example, anything with
+# flagtype(v) == 'list' is inited as False, then set as a list of strings.
flag_defaults = OrderedDict([
('disable', False),
('commit', None),
('scandelete', []),
('build', ''),
('buildjni', []),
+ ('ndk', 'r10e'), # defaults to latest
('preassemble', []),
- ('antcommand', None),
+ ('gradleprops', []),
+ ('antcommands', None),
('novcheck', False),
- ])
+])
# Designates a metadata field type and checks that it matches
FieldValidator("HTTP link",
r'^http[s]?://', None,
- ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
+ ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
FieldValidator("Bitcoin address",
r'^[a-zA-Z0-9]{27,34}$', None,
["Dogecoin"],
[]),
- FieldValidator("Boolean",
- ['Yes', 'No'], None,
- ["Requires Root"],
- []),
-
FieldValidator("bool",
- ['yes', 'no'], None,
- [],
+ r'([Yy]es|[Nn]o|[Tt]rue|[Ff]alse)', None,
+ ["Requires Root"],
['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
'novcheck']),
["Repo Type"],
[]),
+ FieldValidator("Binaries",
+ r'^http[s]?://', None,
+ ["Binaries"],
+ []),
+
FieldValidator("Archive Policy",
r'^[0-9]+ versions$', None,
["Archive Policy"],
r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
["Update Check Mode"],
[])
- }
+}
# Check an app's metadata information for integrity errors
# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
-# end() and then text_plain, text_wiki and text_html will contain the result.
+# end() and then text_wiki and text_html will contain the result.
class DescriptionFormatter:
stNONE = 0
stPARA = 1
bold = False
ital = False
state = stNONE
- text_plain = ''
text_wiki = ''
text_html = ''
linkResolver = None
self.endol()
def endpara(self):
- self.text_plain += '\n'
self.text_html += '</p>'
self.state = self.stNONE
else:
urltxt = url[index2 + 1:]
url = url[:index2]
+ if url == urltxt:
+ raise MetaDataException("Url title is just the URL - use [url]")
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
linkified_plain += urltxt
if urltxt != url:
def addtext(self, txt):
p, h = self.linkify(txt)
- self.text_plain += p
self.text_html += h
def parseline(self, line):
self.text_html += '<ul>'
self.state = self.stUL
self.text_html += '<li>'
- self.text_plain += '* '
self.addtext(line[1:])
self.text_html += '</li>'
elif line.startswith('# '):
self.text_html += '<ol>'
self.state = self.stOL
self.text_html += '<li>'
- self.text_plain += '* ' # TODO: lazy - put the numbers in!
self.addtext(line[1:])
self.text_html += '</li>'
else:
self.state = self.stPARA
elif self.state == self.stPARA:
self.text_html += ' '
- self.text_plain += ' '
self.addtext(line)
def end(self):
self.endcur()
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in plain text format.
-def description_plain(lines, linkres):
- ps = DescriptionFormatter(linkres)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_plain
-
-
# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format. Used for the Maintainer Notes field as well,
# because it's the same format.
thisinfo['Repo'] = ''
thisinfo['Subdir'] = None
thisinfo['Prepare'] = None
- thisinfo['Srclibs'] = None
if metafile is None:
return thisinfo
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
-# returned by the parse_metadata function.
+# returned by the parse_txt_metadata function.
def read_metadata(xref=True):
# Always read the srclibs before the apps, since they can use a srlib as
os.makedirs(basedir)
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
- appid, appinfo = parse_metadata(metafile)
+ appid, appinfo = parse_txt_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.json'))):
+ appid, appinfo = parse_json_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))):
+ appid, appinfo = parse_xml_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.yaml'))):
+ appid, appinfo = parse_yaml_metadata(metafile)
check_metadata(appinfo)
apps[appid] = appinfo
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
- if name in ['Categories']:
+ if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
def flagtype(name):
- if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
- 'update', 'scanignore', 'scandelete']:
+ if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
+ 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
+ 'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
continue
build[flag] = value
build['type'] = get_build_type()
+ build['ndk_path'] = common.get_ndk_path(build['ndk'])
+
+
+def split_list_values(s):
+ # Port legacy ';' separators
+ l = [v.strip() for v in s.replace(';', ',').split(',')]
+ return [v for v in l if v]
+
+
+def get_default_app_info_list(appid=None):
+ thisinfo = {}
+ thisinfo.update(app_defaults)
+ if appid is not None:
+ thisinfo['id'] = appid
+
+ # General defaults...
+ thisinfo['builds'] = []
+ thisinfo['comments'] = []
+
+ return thisinfo
+
+
+def post_metadata_parse(thisinfo):
+
+ supported_metadata = app_defaults.keys() + ['comments', 'builds', 'id']
+ for k, v in thisinfo.iteritems():
+ if k not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: {0}: {1}"
+ .format(k, v))
+ if type(v) in (float, int):
+ thisinfo[k] = str(v)
+
+ # convert to the odd internal format
+ for k in ('Description', 'Maintainer Notes'):
+ if isinstance(thisinfo[k], basestring):
+ text = thisinfo[k].rstrip().lstrip()
+ thisinfo[k] = text.split('\n')
+
+ supported_flags = (flag_defaults.keys()
+ + ['vercode', 'version', 'versionCode', 'versionName'])
+ esc_newlines = re.compile('\\\\( |\\n)')
+
+ for build in thisinfo['builds']:
+ for k, v in build.items():
+ if k not in supported_flags:
+ raise MetaDataException("Unrecognised build flag: {0}={1}"
+ .format(k, v))
+
+ if k == 'versionCode':
+ build['vercode'] = str(v)
+ del build['versionCode']
+ elif k == 'versionName':
+ build['version'] = str(v)
+ del build['versionName']
+ elif type(v) in (float, int):
+ build[k] = str(v)
+ else:
+ keyflagtype = flagtype(k)
+ if keyflagtype == 'list':
+ # these can be bools, strings or lists, but ultimately are lists
+ if isinstance(v, basestring):
+ build[k] = [v]
+ elif isinstance(v, bool):
+ if v:
+ build[k] = ['yes']
+ else:
+ build[k] = ['no']
+ elif keyflagtype == 'script':
+ build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
+ elif keyflagtype == 'bool':
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if isinstance(v, basestring):
+ if v == 'true':
+ build[k] = True
+ else:
+ build[k] = False
+
+ if not thisinfo['Description']:
+ thisinfo['Description'].append('No description available')
+
+ for build in thisinfo['builds']:
+ fill_build_defaults(build)
+
+ thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
# Parse metadata for a single application.
# 'builds' - a list of dictionaries containing build information
# for each defined build
# 'comments' - a list of comments from the metadata file. Each is
-# a tuple of the form (field, comment) where field is
+# a list of the form [field, comment] where field is
# the name of the field it preceded in the metadata
# file. Where field is None, the comment goes at the
# end of the file. Alternatively, 'build:version' is
# 'descriptionlines' - original lines of description as formatted in the
# metadata file.
#
-def parse_metadata(metafile):
+
+
+def _decode_list(data):
+ '''convert items in a list from unicode to basestring'''
+ rv = []
+ for item in data:
+ if isinstance(item, unicode):
+ item = item.encode('utf-8')
+ elif isinstance(item, list):
+ item = _decode_list(item)
+ elif isinstance(item, dict):
+ item = _decode_dict(item)
+ rv.append(item)
+ return rv
+
+
+def _decode_dict(data):
+ '''convert items in a dict from unicode to basestring'''
+ rv = {}
+ for key, value in data.iteritems():
+ if isinstance(key, unicode):
+ key = key.encode('utf-8')
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+ elif isinstance(value, list):
+ value = _decode_list(value)
+ elif isinstance(value, dict):
+ value = _decode_dict(value)
+ rv[key] = value
+ return rv
+
+
+def parse_json_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-5] # strip path and .json
+ thisinfo = get_default_app_info_list(appid)
+
+ # fdroid metadata is only strings and booleans, no floats or ints. And
+ # json returns unicode, and fdroidserver still uses plain python strings
+ # TODO create schema using https://pypi.python.org/pypi/jsonschema
+ jsoninfo = json.load(open(metafile, 'r'),
+ object_hook=_decode_dict,
+ parse_int=lambda s: s,
+ parse_float=lambda s: s)
+ thisinfo.update(jsoninfo)
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_xml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-4] # strip path and .xml
+ thisinfo = get_default_app_info_list(appid)
+
+ tree = ElementTree.ElementTree(file=metafile)
+ root = tree.getroot()
+
+ if root.tag != 'resources':
+ logging.critical(metafile + ' does not have root as <resources></resources>!')
+ sys.exit(1)
+
+ supported_metadata = app_defaults.keys()
+ for child in root:
+ if child.tag != 'builds':
+ # builds does not have name="" attrib
+ name = child.attrib['name']
+ if name not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: <"
+ + child.tag + ' name="' + name + '">'
+ + child.text
+ + "</" + child.tag + '>')
+
+ if child.tag == 'string':
+ thisinfo[name] = child.text
+ elif child.tag == 'string-array':
+ items = []
+ for item in child:
+ items.append(item.text)
+ thisinfo[name] = items
+ elif child.tag == 'builds':
+ builds = []
+ for build in child:
+ builddict = dict()
+ for key in build:
+ builddict[key.tag] = key.text
+ builds.append(builddict)
+ thisinfo['builds'] = builds
+
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if not isinstance(thisinfo['Requires Root'], bool):
+ if thisinfo['Requires Root'] == 'true':
+ thisinfo['Requires Root'] = True
+ else:
+ thisinfo['Requires Root'] = False
+
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_yaml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-5] # strip path and .yaml
+ thisinfo = get_default_app_info_list(appid)
+
+ yamlinfo = yaml.load(open(metafile, 'r'), Loader=YamlLoader)
+ thisinfo.update(yamlinfo)
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_txt_metadata(metafile):
appid = None
linedesc = None
def add_buildflag(p, thisbuild):
+ if not p.strip():
+ raise MetaDataException("Empty build flag at {1}"
+ .format(buildlines[0], linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
raise MetaDataException("Invalid build flag at {0} in {1}"
.format(p, linedesc))
t = flagtype(pk)
if t == 'list':
- # Port legacy ';' separators
- thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
+ pv = split_list_values(pv)
+ if pk == 'gradle':
+ if len(pv) == 1 and pv[0] in ['main', 'yes']:
+ pv = ['yes']
+ thisbuild[pk] = pv
elif t == 'string' or t == 'script':
thisbuild[pk] = pv
elif t == 'bool':
if not curcomments:
return
for comment in curcomments:
- thisinfo['comments'].append((key, comment))
+ thisinfo['comments'].append([key, comment])
del curcomments[:]
- thisinfo = {}
+ thisinfo = get_default_app_info_list()
if metafile:
if not isinstance(metafile, file):
metafile = open(metafile, "r")
appid = metafile.name[9:-4]
-
- thisinfo.update(app_defaults)
- thisinfo['id'] = appid
-
- # General defaults...
- thisinfo['builds'] = []
- thisinfo['comments'] = []
-
- if metafile is None:
+ thisinfo['id'] = appid
+ else:
return appid, thisinfo
mode = 0
elif fieldtype == 'string':
thisinfo[field] = value
elif fieldtype == 'list':
- thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
+ thisinfo[field] = split_list_values(value)
elif fieldtype == 'build':
if value.endswith("\\"):
mode = 2
elif mode == 3:
raise MetaDataException("Unterminated build in " + metafile.name)
- if not thisinfo['Description']:
- thisinfo['Description'].append('No description available')
-
- for build in thisinfo['builds']:
- fill_build_defaults(build)
-
- thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
+ post_metadata_parse(thisinfo)
return (appid, thisinfo)
mf = open(dest, 'w')
writefield_nonempty('Disabled')
- writefield_nonempty('AntiFeatures')
+ writefield('AntiFeatures')
writefield_nonempty('Provides')
writefield('Categories')
writefield('License')
writefield('Web Site')
writefield('Source Code')
writefield('Issue Tracker')
+ writefield_nonempty('Changelog')
writefield_nonempty('Donate')
writefield_nonempty('FlattrID')
writefield_nonempty('Bitcoin')
mf.write('.\n')
mf.write('\n')
if app['Requires Root']:
- writefield('Requires Root', 'Yes')
+ writefield('Requires Root', 'yes')
mf.write('\n')
if app['Repo Type']:
writefield('Repo Type')
writefield('Repo')
+ if app['Binaries']:
+ writefield('Binaries')
mf.write('\n')
for build in app['builds']:
if t == 'string':
outline += value
- if t == 'bool':
+ elif t == 'bool':
outline += 'yes'
elif t == 'script':
outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
~ianmdlvl / fdroidserver.git / blobdiff