# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
-import os, re, glob
+import json
+import os
+import re
+import sys
+import glob
import cgi
import logging
+import yaml
+# use libyaml if it is available
+try:
+ from yaml import CLoader
+ YamlLoader = CLoader
+except ImportError:
+ from yaml import Loader
+ YamlLoader = Loader
+
+# use the C implementation when available
+import xml.etree.cElementTree as ElementTree
+
+from collections import OrderedDict
+
+import common
+
+srclibs = None
+
+
class MetaDataException(Exception):
+
def __init__(self, value):
self.value = value
def __str__(self):
- return repr(self.value)
-
-app_defaults = {
- 'Name': None,
- 'Provides': None,
- 'Auto Name': '',
- 'Categories': ['None'],
- 'Description': [],
- 'Summary': '',
- 'License': 'Unknown',
- 'Web Site': '',
- 'Source Code': '',
- 'Issue Tracker': '',
- 'Donate': None,
- 'FlattrID': None,
- 'Bitcoin': None,
- 'Litecoin': None,
- 'Dogecoin': None,
- 'Disabled': None,
- 'AntiFeatures': None,
- 'Archive Policy': None,
- 'Update Check Mode': 'None',
- 'Update Check Name': None,
- 'Update Check Data': None,
- 'Vercode Operation': None,
- 'Auto Update Mode': 'None',
- 'Current Version': '',
- 'Current Version Code': '0',
- 'Repo Type': '',
- 'Repo': '',
- 'Requires Root': False,
- 'No Source Since': ''
-}
-
-
-# This defines the preferred order for the build items - as in the
-# manual, they're roughly in order of application.
-ordered_flags = [
- 'disable', 'commit', 'subdir', 'submodules', 'init',
- 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
- 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
- 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
- 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
- 'antcommand', 'novcheck'
-]
+ return self.value
+
+# In the order in which they are laid out on files
+app_defaults = OrderedDict([
+ ('Disabled', None),
+ ('AntiFeatures', []),
+ ('Provides', None),
+ ('Categories', ['None']),
+ ('License', 'Unknown'),
+ ('Web Site', ''),
+ ('Source Code', ''),
+ ('Issue Tracker', ''),
+ ('Changelog', ''),
+ ('Donate', None),
+ ('FlattrID', None),
+ ('Bitcoin', None),
+ ('Litecoin', None),
+ ('Dogecoin', None),
+ ('Name', None),
+ ('Auto Name', ''),
+ ('Summary', ''),
+ ('Description', []),
+ ('Requires Root', False),
+ ('Repo Type', ''),
+ ('Repo', ''),
+ ('Binaries', None),
+ ('Maintainer Notes', []),
+ ('Archive Policy', None),
+ ('Auto Update Mode', 'None'),
+ ('Update Check Mode', 'None'),
+ ('Update Check Ignore', None),
+ ('Vercode Operation', None),
+ ('Update Check Name', None),
+ ('Update Check Data', None),
+ ('Current Version', ''),
+ ('Current Version Code', '0'),
+ ('No Source Since', ''),
+])
+
+
+# In the order in which they are laid out on files
+# Sorted by their action and their place in the build timeline
+# These variables can have varying datatypes. For example, anything with
+# flagtype(v) == 'list' is inited as False, then set as a list of strings.
+flag_defaults = OrderedDict([
+ ('disable', False),
+ ('commit', None),
+ ('subdir', None),
+ ('submodules', False),
+ ('init', ''),
+ ('patch', []),
+ ('gradle', False),
+ ('maven', False),
+ ('kivy', False),
+ ('output', None),
+ ('srclibs', []),
+ ('oldsdkloc', False),
+ ('encoding', None),
+ ('forceversion', False),
+ ('forcevercode', False),
+ ('rm', []),
+ ('extlibs', []),
+ ('prebuild', ''),
+ ('update', ['auto']),
+ ('target', None),
+ ('scanignore', []),
+ ('scandelete', []),
+ ('build', ''),
+ ('buildjni', []),
+ ('ndk', 'r10e'), # defaults to latest
+ ('preassemble', []),
+ ('gradleprops', []),
+ ('antcommands', None),
+ ('novcheck', False),
+])
# Designates a metadata field type and checks that it matches
# 'fields' - Metadata fields (Field:Value) of this type
# 'attrs' - Build attributes (attr=value) of this type
#
-class FieldType():
+class FieldValidator():
+
def __init__(self, name, matching, sep, fields, attrs):
self.name = name
self.matching = matching
for v in values:
if not self.compiled.match(v):
raise MetaDataException("'%s' is not a valid %s in %s. "
- % (v, self.name, appid) +
- "Regex pattern: %s" % (self.matching))
+ % (v, self.name, appid) +
+ "Regex pattern: %s" % (self.matching))
def _assert_list(self, values, appid):
for v in values:
if v not in self.matching:
raise MetaDataException("'%s' is not a valid %s in %s. "
- % (v, self.name, appid) +
- "Possible values: %s" % (", ".join(self.matching)))
+ % (v, self.name, appid) +
+ "Possible values: %s" % (", ".join(self.matching)))
def check(self, value, appid):
if type(value) is not str or not value:
# Generic value types
valuetypes = {
- 'int': FieldType("Integer",
- r'^[1-9][0-9]*$', None,
- ['FlattrID' ],
- ['vercode' ]),
-
- 'http': FieldType("HTTP link",
- r'^http[s]?://', None,
- ["Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
-
- 'bitcoin': FieldType("Bitcoin address",
- r'^[a-zA-Z0-9]{27,34}$', None,
- ["Bitcoin" ],
- []),
-
- 'litecoin': FieldType("Litecoin address",
- r'^L[a-zA-Z0-9]{33}$', None,
- ["Litecoin" ],
- []),
-
- 'dogecoin': FieldType("Dogecoin address",
- r'^D[a-zA-Z0-9]{33}$', None,
- ["Dogecoin" ],
- []),
-
- 'Bool': FieldType("Boolean",
- ['Yes', 'No'], None,
- ["Requires Root" ],
- []),
-
- 'bool': FieldType("Boolean",
- ['yes', 'no'], None,
- [],
- ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
- 'novcheck' ]),
-
- 'Repo Type': FieldType("Repo Type",
- ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
- ["Repo Type" ],
- []),
-
- 'archive': FieldType("Archive Policy",
- r'^[0-9]+ versions$', None,
- ["Archive Policy" ],
- []),
-
- 'antifeatures': FieldType("Anti-Feature",
- ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
- ["AntiFeatures" ],
- []),
-
- 'autoupdatemodes': FieldType("Auto Update Mode",
- r"^(Version .+|None)$", None,
- ["Auto Update Mode" ],
- []),
-
- 'updatecheckmodes': FieldType("Update Check Mode",
- r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
- ["Update Check Mode" ],
- [])
+ FieldValidator("Integer",
+ r'^[1-9][0-9]*$', None,
+ [],
+ ['vercode']),
+
+ FieldValidator("Hexadecimal",
+ r'^[0-9a-f]+$', None,
+ ['FlattrID'],
+ []),
+
+ FieldValidator("HTTP link",
+ r'^http[s]?://', None,
+ ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
+
+ FieldValidator("Bitcoin address",
+ r'^[a-zA-Z0-9]{27,34}$', None,
+ ["Bitcoin"],
+ []),
+
+ FieldValidator("Litecoin address",
+ r'^L[a-zA-Z0-9]{33}$', None,
+ ["Litecoin"],
+ []),
+
+ FieldValidator("Dogecoin address",
+ r'^D[a-zA-Z0-9]{33}$', None,
+ ["Dogecoin"],
+ []),
+
+ FieldValidator("bool",
+ r'([Yy]es|[Nn]o|[Tt]rue|[Ff]alse)', None,
+ ["Requires Root"],
+ ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
+ 'novcheck']),
+
+ FieldValidator("Repo Type",
+ ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
+ ["Repo Type"],
+ []),
+
+ FieldValidator("Binaries",
+ r'^http[s]?://', None,
+ ["Binaries"],
+ []),
+
+ FieldValidator("Archive Policy",
+ r'^[0-9]+ versions$', None,
+ ["Archive Policy"],
+ []),
+
+ FieldValidator("Anti-Feature",
+ ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
+ ["AntiFeatures"],
+ []),
+
+ FieldValidator("Auto Update Mode",
+ r"^(Version .+|None)$", None,
+ ["Auto Update Mode"],
+ []),
+
+ FieldValidator("Update Check Mode",
+ r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
+ ["Update Check Mode"],
+ [])
}
+
# Check an app's metadata information for integrity errors
def check_metadata(info):
- for k, t in valuetypes.iteritems():
- for field in t.fields:
- if field in info:
- t.check(info[field], info['id'])
- if k == 'Bool':
- info[field] = info[field] == "Yes"
+ for v in valuetypes:
+ for field in v.fields:
+ v.check(info[field], info['id'])
for build in info['builds']:
- for attr in t.attrs:
- if attr in build:
- t.check(build[attr], info['id'])
- if k == 'bool':
- build[attr] = build[attr] == "yes"
- elif k == 'bool':
- build[attr] = False
+ for attr in v.attrs:
+ v.check(build[attr], info['id'])
+
# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
-# end() and then text_plain, text_wiki and text_html will contain the result.
+# end() and then text_wiki and text_html will contain the result.
class DescriptionFormatter:
stNONE = 0
stPARA = 1
bold = False
ital = False
state = stNONE
- text_plain = ''
text_wiki = ''
text_html = ''
linkResolver = None
+
def __init__(self, linkres):
self.linkResolver = linkres
+
def endcur(self, notstates=None):
if notstates and self.state in notstates:
return
self.endul()
elif self.state == self.stOL:
self.endol()
+
def endpara(self):
- self.text_plain += '\n'
self.text_html += '</p>'
self.state = self.stNONE
+
def endul(self):
self.text_html += '</ul>'
self.state = self.stNONE
+
def endol(self):
self.text_html += '</ol>'
self.state = self.stNONE
self.ital = not self.ital
txt = txt[2:]
-
def linkify(self, txt):
linkified_plain = ''
linkified_html = ''
urltext = url
linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
linkified_plain += urltext
- txt = txt[index+2:]
+ txt = txt[index + 2:]
else:
index = txt.find("]")
if index == -1:
else:
urltxt = url[index2 + 1:]
url = url[:index2]
+ if url == urltxt:
+ raise MetaDataException("Url title is just the URL - use [url]")
linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
linkified_plain += urltxt
if urltxt != url:
linkified_plain += ' (' + url + ')'
- txt = txt[index+1:]
+ txt = txt[index + 1:]
def addtext(self, txt):
p, h = self.linkify(txt)
- self.text_plain += p
self.text_html += h
def parseline(self, line):
self.text_html += '<ul>'
self.state = self.stUL
self.text_html += '<li>'
- self.text_plain += '* '
self.addtext(line[1:])
self.text_html += '</li>'
elif line.startswith('# '):
self.text_html += '<ol>'
self.state = self.stOL
self.text_html += '<li>'
- self.text_plain += '* ' #TODO: lazy - put the numbers in!
self.addtext(line[1:])
self.text_html += '</li>'
else:
self.state = self.stPARA
elif self.state == self.stPARA:
self.text_html += ' '
- self.text_plain += ' '
self.addtext(line)
def end(self):
self.endcur()
-# Parse multiple lines of description as written in a metadata file, returning
-# a single string in plain text format.
-def description_plain(lines, linkres):
- ps = DescriptionFormatter(linkres)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_plain
# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format. Used for the Maintainer Notes field as well,
ps.end()
return ps.text_wiki
+
# Parse multiple lines of description as written in a metadata file, returning
# a single string in HTML format.
def description_html(lines, linkres):
ps.end()
return ps.text_html
-def parse_srclib(metafile, **kw):
+
+def parse_srclib(metafile):
thisinfo = {}
if metafile and not isinstance(metafile, file):
thisinfo['Repo'] = ''
thisinfo['Subdir'] = None
thisinfo['Prepare'] = None
- thisinfo['Srclibs'] = None
if metafile is None:
return thisinfo
return thisinfo
+
+def read_srclibs():
+ """Read all srclib metadata.
+
+ The information read will be accessible as metadata.srclibs, which is a
+ dictionary, keyed on srclib name, with the values each being a dictionary
+ in the same format as that returned by the parse_srclib function.
+
+ A MetaDataException is raised if there are any problems with the srclib
+ metadata.
+ """
+ global srclibs
+
+ # They were already loaded
+ if srclibs is not None:
+ return
+
+ srclibs = {}
+
+ srcdir = 'srclibs'
+ if not os.path.exists(srcdir):
+ os.makedirs(srcdir)
+
+ for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
+ srclibname = os.path.basename(metafile[:-4])
+ srclibs[srclibname] = parse_srclib(metafile)
+
+
# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
-# returned by the parse_metadata function.
-def read_metadata(xref=True, package=None, store=True):
- apps = []
+# returned by the parse_txt_metadata function.
+def read_metadata(xref=True):
+
+ # Always read the srclibs before the apps, since they can use a srlib as
+ # their source repository.
+ read_srclibs()
+
+ apps = {}
for basedir in ('metadata', 'tmp'):
if not os.path.exists(basedir):
os.makedirs(basedir)
for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
- if package is None or metafile == os.path.join('metadata', package + '.txt'):
- appinfo = parse_metadata(metafile)
- check_metadata(appinfo)
- apps.append(appinfo)
+ appid, appinfo = parse_txt_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.json'))):
+ appid, appinfo = parse_json_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.xml'))):
+ appid, appinfo = parse_xml_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
+
+ for metafile in sorted(glob.glob(os.path.join('metadata', '*.yaml'))):
+ appid, appinfo = parse_yaml_metadata(metafile)
+ check_metadata(appinfo)
+ apps[appid] = appinfo
if xref:
# Parse all descriptions at load time, just to ensure cross-referencing
# errors are caught early rather than when they hit the build server.
- def linkres(link):
- for app in apps:
- if app['id'] == link:
- return ("fdroid.app:" + link, "Dummy name - don't know yet")
- raise MetaDataException("Cannot resolve app id " + link)
- for app in apps:
+ def linkres(appid):
+ if appid in apps:
+ return ("fdroid.app:" + appid, "Dummy name - don't know yet")
+ raise MetaDataException("Cannot resolve app id " + appid)
+
+ for appid, app in apps.iteritems():
try:
description_html(app['Description'], linkres)
- except Exception, e:
- raise MetaDataException("Problem with description of " + app['id'] +
- " - " + str(e))
+ except MetaDataException, e:
+ raise MetaDataException("Problem with description of " + appid +
+ " - " + str(e))
return apps
+
# Get the type expected for a given metadata field.
def metafieldtype(name):
if name in ['Description', 'Maintainer Notes']:
return 'multiline'
- if name in ['Categories']:
+ if name in ['Categories', 'AntiFeatures']:
return 'list'
if name == 'Build Version':
return 'build'
return 'unknown'
return 'string'
+
def flagtype(name):
- if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
- 'update', 'scanignore', 'scandelete']:
+ if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
+ 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
+ 'gradleprops']:
return 'list'
if name in ['init', 'prebuild', 'build']:
return 'script'
+ if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
+ 'novcheck']:
+ return 'bool'
return 'string'
+
+def fill_build_defaults(build):
+
+ def get_build_type():
+ for t in ['maven', 'gradle', 'kivy']:
+ if build[t]:
+ return t
+ if build['output']:
+ return 'raw'
+ return 'ant'
+
+ for flag, value in flag_defaults.iteritems():
+ if flag in build:
+ continue
+ build[flag] = value
+ build['type'] = get_build_type()
+ build['ndk_path'] = common.get_ndk_path(build['ndk'])
+
+
+def split_list_values(s):
+ # Port legacy ';' separators
+ l = [v.strip() for v in s.replace(';', ',').split(',')]
+ return [v for v in l if v]
+
+
+def get_default_app_info_list(appid=None):
+ thisinfo = {}
+ thisinfo.update(app_defaults)
+ if appid is not None:
+ thisinfo['id'] = appid
+
+ # General defaults...
+ thisinfo['builds'] = []
+ thisinfo['comments'] = []
+
+ return thisinfo
+
+
+def post_metadata_parse(thisinfo):
+
+ supported_metadata = app_defaults.keys() + ['comments', 'builds', 'id']
+ for k, v in thisinfo.iteritems():
+ if k not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: {0}: {1}"
+ .format(k, v))
+ if type(v) in (float, int):
+ thisinfo[k] = str(v)
+
+ # convert to the odd internal format
+ for k in ('Description', 'Maintainer Notes'):
+ if isinstance(thisinfo[k], basestring):
+ text = thisinfo[k].rstrip().lstrip()
+ thisinfo[k] = text.split('\n')
+
+ supported_flags = (flag_defaults.keys()
+ + ['vercode', 'version', 'versionCode', 'versionName'])
+ esc_newlines = re.compile('\\\\( |\\n)')
+
+ for build in thisinfo['builds']:
+ for k, v in build.items():
+ if k not in supported_flags:
+ raise MetaDataException("Unrecognised build flag: {0}={1}"
+ .format(k, v))
+
+ if k == 'versionCode':
+ build['vercode'] = str(v)
+ del build['versionCode']
+ elif k == 'versionName':
+ build['version'] = str(v)
+ del build['versionName']
+ elif type(v) in (float, int):
+ build[k] = str(v)
+ else:
+ keyflagtype = flagtype(k)
+ if keyflagtype == 'list':
+ # these can be bools, strings or lists, but ultimately are lists
+ if isinstance(v, basestring):
+ build[k] = [v]
+ elif isinstance(v, bool):
+ if v:
+ build[k] = ['yes']
+ else:
+ build[k] = ['no']
+ elif keyflagtype == 'script':
+ build[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
+ elif keyflagtype == 'bool':
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if isinstance(v, basestring):
+ if v == 'true':
+ build[k] = True
+ else:
+ build[k] = False
+
+ if not thisinfo['Description']:
+ thisinfo['Description'].append('No description available')
+
+ for build in thisinfo['builds']:
+ fill_build_defaults(build)
+
+ thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
+
+
# Parse metadata for a single application.
#
# 'metafile' - the filename to read. The package id for the application comes
#
# Known keys not originating from the metadata are:
#
-# 'id' - the application's package ID
# 'builds' - a list of dictionaries containing build information
# for each defined build
# 'comments' - a list of comments from the metadata file. Each is
-# a tuple of the form (field, comment) where field is
+# a list of the form [field, comment] where field is
# the name of the field it preceded in the metadata
# file. Where field is None, the comment goes at the
# end of the file. Alternatively, 'build:version' is
# 'descriptionlines' - original lines of description as formatted in the
# metadata file.
#
-def parse_metadata(metafile):
+
+def _decode_list(data):
+ '''convert items in a list from unicode to basestring'''
+ rv = []
+ for item in data:
+ if isinstance(item, unicode):
+ item = item.encode('utf-8')
+ elif isinstance(item, list):
+ item = _decode_list(item)
+ elif isinstance(item, dict):
+ item = _decode_dict(item)
+ rv.append(item)
+ return rv
+
+
+def _decode_dict(data):
+ '''convert items in a dict from unicode to basestring'''
+ rv = {}
+ for key, value in data.iteritems():
+ if isinstance(key, unicode):
+ key = key.encode('utf-8')
+ if isinstance(value, unicode):
+ value = value.encode('utf-8')
+ elif isinstance(value, list):
+ value = _decode_list(value)
+ elif isinstance(value, dict):
+ value = _decode_dict(value)
+ rv[key] = value
+ return rv
+
+
+def parse_json_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-5] # strip path and .json
+ thisinfo = get_default_app_info_list(appid)
+
+ # fdroid metadata is only strings and booleans, no floats or ints. And
+ # json returns unicode, and fdroidserver still uses plain python strings
+ # TODO create schema using https://pypi.python.org/pypi/jsonschema
+ jsoninfo = json.load(open(metafile, 'r'),
+ object_hook=_decode_dict,
+ parse_int=lambda s: s,
+ parse_float=lambda s: s)
+ thisinfo.update(jsoninfo)
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_xml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-4] # strip path and .xml
+ thisinfo = get_default_app_info_list(appid)
+
+ tree = ElementTree.ElementTree(file=metafile)
+ root = tree.getroot()
+
+ if root.tag != 'resources':
+ logging.critical(metafile + ' does not have root as <resources></resources>!')
+ sys.exit(1)
+
+ supported_metadata = app_defaults.keys()
+ for child in root:
+ if child.tag != 'builds':
+ # builds does not have name="" attrib
+ name = child.attrib['name']
+ if name not in supported_metadata:
+ raise MetaDataException("Unrecognised metadata: <"
+ + child.tag + ' name="' + name + '">'
+ + child.text
+ + "</" + child.tag + '>')
+
+ if child.tag == 'string':
+ thisinfo[name] = child.text
+ elif child.tag == 'string-array':
+ items = []
+ for item in child:
+ items.append(item.text)
+ thisinfo[name] = items
+ elif child.tag == 'builds':
+ builds = []
+ for build in child:
+ builddict = dict()
+ for key in build:
+ builddict[key.tag] = key.text
+ builds.append(builddict)
+ thisinfo['builds'] = builds
+
+ # TODO handle this using <xsd:element type="xsd:boolean> in a schema
+ if not isinstance(thisinfo['Requires Root'], bool):
+ if thisinfo['Requires Root'] == 'true':
+ thisinfo['Requires Root'] = True
+ else:
+ thisinfo['Requires Root'] = False
+
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_yaml_metadata(metafile):
+
+ appid = os.path.basename(metafile)[0:-5] # strip path and .yaml
+ thisinfo = get_default_app_info_list(appid)
+
+ yamlinfo = yaml.load(open(metafile, 'r'), Loader=YamlLoader)
+ thisinfo.update(yamlinfo)
+ post_metadata_parse(thisinfo)
+
+ return (appid, thisinfo)
+
+
+def parse_txt_metadata(metafile):
+
+ appid = None
linedesc = None
def add_buildflag(p, thisbuild):
+ if not p.strip():
+ raise MetaDataException("Empty build flag at {1}"
+ .format(buildlines[0], linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
- raise MetaDataException("Invalid build flag at {0} in {1}".
- format(buildlines[0], linedesc))
+ raise MetaDataException("Invalid build flag at {0} in {1}"
+ .format(buildlines[0], linedesc))
pk, pv = bv
if pk in thisbuild:
- raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
- format(pk, thisbuild['version'], linedesc))
+ raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
+ .format(pk, thisbuild['version'], linedesc))
pk = pk.lstrip()
- if pk not in ordered_flags:
- raise MetaDataException("Unrecognised build flag at {0} in {1}".
- format(p, linedesc))
+ if pk not in flag_defaults:
+ raise MetaDataException("Unrecognised build flag at {0} in {1}"
+ .format(p, linedesc))
t = flagtype(pk)
if t == 'list':
- # Port legacy ';' separators
- thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
- elif t == 'string':
+ pv = split_list_values(pv)
+ if pk == 'gradle':
+ if len(pv) == 1 and pv[0] in ['main', 'yes']:
+ pv = ['yes']
thisbuild[pk] = pv
- elif t == 'script':
+ elif t == 'string' or t == 'script':
thisbuild[pk] = pv
+ elif t == 'bool':
+ value = pv == 'yes'
+ if value:
+ thisbuild[pk] = True
+ else:
+ logging.debug("...ignoring bool flag %s" % p)
+
else:
- raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % (
- t, p, linedesc))
+ raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
+ % (t, p, linedesc))
def parse_buildline(lines):
value = "".join(lines)
commit = 'unknown - see disabled'
index = parts[2].rfind('at ')
if index != -1:
- commit = parts[2][index+3:]
+ commit = parts[2][index + 3:]
if commit.endswith(')'):
commit = commit[:-1]
thisbuild['commit'] = commit
if not curcomments:
return
for comment in curcomments:
- thisinfo['comments'].append((key, comment))
+ thisinfo['comments'].append([key, comment])
del curcomments[:]
- def get_build_type(build):
- for t in ['maven', 'gradle', 'kivy']:
- if build.get(t, 'no') != 'no':
- return t
- if 'output' in build:
- return 'raw'
- return 'ant'
-
- thisinfo = {}
+ thisinfo = get_default_app_info_list()
if metafile:
if not isinstance(metafile, file):
metafile = open(metafile, "r")
- thisinfo['id'] = metafile.name[9:-4]
+ appid = metafile.name[9:-4]
+ thisinfo['id'] = appid
else:
- thisinfo['id'] = None
-
- thisinfo.update(app_defaults)
-
- # General defaults...
- thisinfo['builds'] = []
- thisinfo['comments'] = []
-
- if metafile is None:
- return thisinfo
+ return appid, thisinfo
mode = 0
buildlines = []
curcomments = []
curbuild = None
+ vc_seen = {}
c = 0
for line in metafile:
line = line.rstrip('\r\n')
if mode == 3:
if not any(line.startswith(s) for s in (' ', '\t')):
- if 'commit' not in curbuild and 'disable' not in curbuild:
- raise MetaDataException("No commit specified for {0} in {1}".format(
- curbuild['version'], linedesc))
+ commit = curbuild['commit'] if 'commit' in curbuild else None
+ if not commit and 'disable' not in curbuild:
+ raise MetaDataException("No commit specified for {0} in {1}"
+ .format(curbuild['version'], linedesc))
+
thisinfo['builds'].append(curbuild)
- add_comments('build:' + curbuild['version'])
+ add_comments('build:' + curbuild['vercode'])
mode = 0
else:
if line.endswith('\\'):
try:
field, value = line.split(':', 1)
except ValueError:
- raise MetaDataException("Invalid metadata in "+linedesc)
+ raise MetaDataException("Invalid metadata in " + linedesc)
if field != field.strip() or value != value.strip():
- raise MetaDataException("Extra spacing found in "+linedesc)
+ raise MetaDataException("Extra spacing found in " + linedesc)
# Translate obsolete fields...
if field == 'Market Version':
elif fieldtype == 'string':
thisinfo[field] = value
elif fieldtype == 'list':
- thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
+ thisinfo[field] = split_list_values(value)
elif fieldtype == 'build':
if value.endswith("\\"):
mode = 2
buildlines = [value[:-1]]
else:
- thisinfo['builds'].append(parse_buildline([value]))
- add_comments('build:' + thisinfo['builds'][-1]['version'])
+ curbuild = parse_buildline([value])
+ thisinfo['builds'].append(curbuild)
+ add_comments('build:' + thisinfo['builds'][-1]['vercode'])
elif fieldtype == 'buildv2':
curbuild = {}
vv = value.split(',')
if len(vv) != 2:
- raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
- format(value, linedesc))
+ raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
+ .format(value, linedesc))
curbuild['version'] = vv[0]
curbuild['vercode'] = vv[1]
+ if curbuild['vercode'] in vc_seen:
+ raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
+ curbuild['vercode'], linedesc))
+ vc_seen[curbuild['vercode']] = True
buildlines = []
mode = 3
elif fieldtype == 'obsolete':
buildlines.append(line[:-1])
else:
buildlines.append(line)
- thisinfo['builds'].append(
- parse_buildline(buildlines))
- add_comments('build:' + thisinfo['builds'][-1]['version'])
+ curbuild = parse_buildline(buildlines)
+ thisinfo['builds'].append(curbuild)
+ add_comments('build:' + thisinfo['builds'][-1]['vercode'])
mode = 0
add_comments(None)
elif mode == 3:
raise MetaDataException("Unterminated build in " + metafile.name)
- if not thisinfo['Description']:
- thisinfo['Description'].append('No description available')
+ post_metadata_parse(thisinfo)
- for build in thisinfo['builds']:
- build['type'] = get_build_type(build)
+ return (appid, thisinfo)
- return thisinfo
# Write a metadata file.
#
mf.write("%s\n" % comment)
written += 1
if written > 0:
- logging.debug("...writing comments for " + (key if key else 'EOF'))
+ logging.debug("...writing comments for " + (key or 'EOF'))
def writefield(field, value=None):
writecomments(field)
value = ','.join(value)
mf.write("%s:%s\n" % (field, value))
+ def writefield_nonempty(field, value=None):
+ if value is None:
+ value = app[field]
+ if value:
+ writefield(field, value)
+
mf = open(dest, 'w')
- if app['Disabled']:
- writefield('Disabled')
- if app['AntiFeatures']:
- writefield('AntiFeatures')
- if app['Provides']:
- writefield('Provides')
+ writefield_nonempty('Disabled')
+ writefield('AntiFeatures')
+ writefield_nonempty('Provides')
writefield('Categories')
writefield('License')
writefield('Web Site')
writefield('Source Code')
writefield('Issue Tracker')
- if app['Donate']:
- writefield('Donate')
- if app['FlattrID']:
- writefield('FlattrID')
- if app['Bitcoin']:
- writefield('Bitcoin')
- if app['Litecoin']:
- writefield('Litecoin')
- if app['Dogecoin']:
- writefield('Dogecoin')
+ writefield_nonempty('Changelog')
+ writefield_nonempty('Donate')
+ writefield_nonempty('FlattrID')
+ writefield_nonempty('Bitcoin')
+ writefield_nonempty('Litecoin')
+ writefield_nonempty('Dogecoin')
mf.write('\n')
- if app['Name']:
- writefield('Name')
- if app['Auto Name']:
- writefield('Auto Name')
+ writefield_nonempty('Name')
+ writefield_nonempty('Auto Name')
writefield('Summary')
writefield('Description', '')
for line in app['Description']:
mf.write('.\n')
mf.write('\n')
if app['Requires Root']:
- writefield('Requires Root', 'Yes')
+ writefield('Requires Root', 'yes')
mf.write('\n')
if app['Repo Type']:
writefield('Repo Type')
writefield('Repo')
+ if app['Binaries']:
+ writefield('Binaries')
mf.write('\n')
for build in app['builds']:
- writecomments('build:' + build['version'])
+
+ if build['version'] == "Ignore":
+ continue
+
+ writecomments('build:' + build['vercode'])
mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
def write_builditem(key, value):
- if key in ['version', 'vercode', 'origlines', 'type']:
+
+ if key in ['version', 'vercode']:
+ return
+
+ if value == flag_defaults[key]:
return
- if key in valuetypes['bool'].attrs:
- if not value:
- return
- value = 'yes'
+
t = flagtype(key)
+
logging.debug("...writing {0} : {1}".format(key, value))
outline = ' %s=' % key
+
if t == 'string':
outline += value
+ elif t == 'bool':
+ outline += 'yes'
elif t == 'script':
outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
elif t == 'list':
outline += ','.join(value) if type(value) == list else value
+
outline += '\n'
mf.write(outline)
- for key in ordered_flags:
- if key in build:
- write_builditem(key, build[key])
+ for flag in flag_defaults:
+ value = build[flag]
+ if value:
+ write_builditem(flag, value)
mf.write('\n')
- if 'Maintainer Notes' in app:
+ if app['Maintainer Notes']:
writefield('Maintainer Notes', '')
for line in app['Maintainer Notes']:
mf.write("%s\n" % line)
mf.write('.\n')
mf.write('\n')
-
- if app['Archive Policy']:
- writefield('Archive Policy')
+ writefield_nonempty('Archive Policy')
writefield('Auto Update Mode')
writefield('Update Check Mode')
- if app['Vercode Operation']:
- writefield('Vercode Operation')
- if app['Update Check Data']:
- writefield('Update Check Data')
+ writefield_nonempty('Update Check Ignore')
+ writefield_nonempty('Vercode Operation')
+ writefield_nonempty('Update Check Name')
+ writefield_nonempty('Update Check Data')
if app['Current Version']:
writefield('Current Version')
writefield('Current Version Code')
mf.write('\n')
writecomments(None)
mf.close()
-
-
~ianmdlvl / fdroidserver.git / blobdiff