-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# metadata.py - part of the FDroid server tools
# Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
import re
import glob
import cgi
+import logging
import textwrap
+import io
import yaml
# use libyaml if it is available
# use the C implementation when available
import xml.etree.cElementTree as ElementTree
-import common
+import fdroidserver.common
srclibs = None
'Provides',
'Categories',
'License',
+ 'Author Name',
+ 'Author Email',
'Web Site',
'Source Code',
'Issue Tracker',
self.Provides = None
self.Categories = ['None']
self.License = 'Unknown'
+ self.AuthorName = None
+ self.AuthorEmail = None
self.WebSite = ''
self.SourceCode = ''
self.IssueTracker = ''
self.Name = None
self.AutoName = ''
self.Summary = ''
- self.Description = []
+ self.Description = ''
self.RequiresRoot = False
self.RepoType = ''
self.Repo = ''
self.Binaries = None
- self.MaintainerNotes = []
+ self.MaintainerNotes = ''
self.ArchivePolicy = None
self.AutoUpdateMode = 'None'
self.UpdateCheckMode = 'None'
self.UpdateCheckName = None
self.UpdateCheckData = None
self.CurrentVersion = ''
- self.CurrentVersionCode = '0'
+ self.CurrentVersionCode = None
self.NoSourceSince = ''
self.id = None
self.comments = {}
self.added = None
self.lastupdated = None
+ self._modified = set()
# Translates human-readable field names to attribute names, e.g.
# 'Auto Name' to 'AutoName'
# names. Should only be used for tests.
def field_dict(self):
d = {}
- for k, v in self.__dict__.iteritems():
+ for k, v in self.__dict__.items():
if k == 'builds':
d['builds'] = []
for build in v:
- d['builds'].append(build.__dict__)
- else:
- k = App.attr_to_field(k)
- d[k] = v
+ b = {k: v for k, v in build.__dict__.items() if not k.startswith('_')}
+ d['builds'].append(b)
+ elif not k.startswith('_'):
+ f = App.attr_to_field(k)
+ d[f] = v
return d
# Gets the value associated to a field name, e.g. 'Auto Name'
raise MetaDataException('Unrecognised app field: ' + f)
k = App.field_to_attr(f)
self.__dict__[k] = v
+ self._modified.add(k)
# Appends to the value associated to a field name, e.g. 'Auto Name'
def append_field(self, f, v):
# Like dict.update(), but using human-readable field names
def update_fields(self, d):
- for f, v in d.iteritems():
+ for f, v in d.items():
if f == 'builds':
for b in v:
build = Build()
else:
self.set_field(f, v)
+TYPE_UNKNOWN = 0
+TYPE_OBSOLETE = 1
+TYPE_STRING = 2
+TYPE_BOOL = 3
+TYPE_LIST = 4
+TYPE_SCRIPT = 5
+TYPE_MULTILINE = 6
+TYPE_BUILD = 7
+TYPE_BUILD_V2 = 8
+
+fieldtypes = {
+ 'Description': TYPE_MULTILINE,
+ 'Maintainer Notes': TYPE_MULTILINE,
+ 'Categories': TYPE_LIST,
+ 'AntiFeatures': TYPE_LIST,
+ 'Build Version': TYPE_BUILD,
+ 'Build': TYPE_BUILD_V2,
+ 'Use Built': TYPE_OBSOLETE,
+}
+
-def metafieldtype(name):
- if name in ['Description', 'Maintainer Notes']:
- return 'multiline'
- if name in ['Categories', 'AntiFeatures']:
- return 'list'
- if name == 'Build Version':
- return 'build'
- if name == 'Build':
- return 'buildv2'
- if name == 'Use Built':
- return 'obsolete'
- if name not in app_fields:
- return 'unknown'
- return 'string'
+def fieldtype(name):
+ if name in fieldtypes:
+ return fieldtypes[name]
+ return TYPE_STRING
# In the order in which they are laid out on files
self.submodules = False
self.init = ''
self.patch = []
- self.gradle = False
+ self.gradle = []
self.maven = False
self.kivy = False
self.output = None
self.rm = []
self.extlibs = []
self.prebuild = ''
- self.update = None
+ self.update = []
self.target = None
self.scanignore = []
self.scandelete = []
self.ndk = None
self.preassemble = []
self.gradleprops = []
- self.antcommands = None
+ self.antcommands = []
self.novcheck = False
+ self._modified = set()
+
def get_flag(self, f):
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
if f not in build_flags:
raise MetaDataException('Unrecognised build flag: ' + f)
self.__dict__[f] = v
+ self._modified.add(f)
def append_flag(self, f, v):
if f not in build_flags:
else:
self.__dict__[f].append(v)
- def method(self):
+ def build_method(self):
for f in ['maven', 'gradle', 'kivy']:
if self.get_flag(f):
return f
return 'raw'
return 'ant'
+ # like build_method, but prioritize output=
+ def output_method(self):
+ if self.output:
+ return 'raw'
+ for f in ['maven', 'gradle', 'kivy']:
+ if self.get_flag(f):
+ return f
+ return 'ant'
+
def ndk_path(self):
version = self.ndk
if not version:
version = 'r10e' # falls back to latest
- paths = common.config['ndk_paths']
+ paths = fdroidserver.common.config['ndk_paths']
if version not in paths:
return ''
return paths[version]
def update_flags(self, d):
- for f, v in d.iteritems():
+ for f, v in d.items():
self.set_flag(f, v)
-list_flags = set(['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
- 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
- 'gradleprops'])
-script_flags = set(['init', 'prebuild', 'build'])
-bool_flags = set(['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
- 'novcheck'])
+flagtypes = {
+ 'extlibs': TYPE_LIST,
+ 'srclibs': TYPE_LIST,
+ 'patch': TYPE_LIST,
+ 'rm': TYPE_LIST,
+ 'buildjni': TYPE_LIST,
+ 'preassemble': TYPE_LIST,
+ 'update': TYPE_LIST,
+ 'scanignore': TYPE_LIST,
+ 'scandelete': TYPE_LIST,
+ 'gradle': TYPE_LIST,
+ 'antcommands': TYPE_LIST,
+ 'gradleprops': TYPE_LIST,
+ 'init': TYPE_SCRIPT,
+ 'prebuild': TYPE_SCRIPT,
+ 'build': TYPE_SCRIPT,
+ 'submodules': TYPE_BOOL,
+ 'oldsdkloc': TYPE_BOOL,
+ 'forceversion': TYPE_BOOL,
+ 'forcevercode': TYPE_BOOL,
+ 'novcheck': TYPE_BOOL,
+}
def flagtype(name):
- if name in list_flags:
- return 'list'
- if name in script_flags:
- return 'script'
- if name in bool_flags:
- return 'bool'
- return 'string'
+ if name in flagtypes:
+ return flagtypes[name]
+ return TYPE_STRING
# Designates a metadata field type and checks that it matches
#
class FieldValidator():
- def __init__(self, name, matching, sep, fields, flags):
+ def __init__(self, name, matching, fields, flags):
self.name = name
self.matching = matching
- if type(matching) is str:
- self.compiled = re.compile(matching)
- self.sep = sep
+ self.compiled = re.compile(matching)
self.fields = fields
self.flags = flags
- def _assert_regex(self, values, appid):
- for v in values:
- if not self.compiled.match(v):
- raise MetaDataException("'%s' is not a valid %s in %s. "
- % (v, self.name, appid) +
- "Regex pattern: %s" % (self.matching))
-
- def _assert_list(self, values, appid):
- for v in values:
- if v not in self.matching:
- raise MetaDataException("'%s' is not a valid %s in %s. "
- % (v, self.name, appid) +
- "Possible values: %s" % (", ".join(self.matching)))
-
def check(self, v, appid):
- if type(v) is not str or not v:
+ if not v:
return
- if self.sep is not None:
- values = v.split(self.sep)
+ if type(v) == list:
+ values = v
else:
values = [v]
- if type(self.matching) is list:
- self._assert_list(values, appid)
- else:
- self._assert_regex(values, appid)
-
+ for v in values:
+ if not self.compiled.match(v):
+ raise MetaDataException("'%s' is not a valid %s in %s. Regex pattern: %s"
+ % (v, self.name, appid, self.matching))
# Generic value types
valuetypes = {
FieldValidator("Integer",
- r'^[1-9][0-9]*$', None,
+ r'^[1-9][0-9]*$',
[],
['vercode']),
FieldValidator("Hexadecimal",
- r'^[0-9a-f]+$', None,
+ r'^[0-9a-f]+$',
['FlattrID'],
[]),
FieldValidator("HTTP link",
- r'^http[s]?://', None,
- ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
+ r'^http[s]?://',
+ ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"], []),
+
+ FieldValidator("Email",
+ r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
+ ["AuthorEmail"], []),
FieldValidator("Bitcoin address",
- r'^[a-zA-Z0-9]{27,34}$', None,
+ r'^[a-zA-Z0-9]{27,34}$',
["Bitcoin"],
[]),
FieldValidator("Litecoin address",
- r'^L[a-zA-Z0-9]{33}$', None,
+ r'^L[a-zA-Z0-9]{33}$',
["Litecoin"],
[]),
- FieldValidator("bool",
- r'([Yy]es|[Nn]o|[Tt]rue|[Ff]alse)', None,
- ["Requires Root"],
- ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
- 'novcheck']),
-
FieldValidator("Repo Type",
- ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
- ["Repo Type"],
+ r'^(git|git-svn|svn|hg|bzr|srclib)$',
+ ["RepoType"],
[]),
FieldValidator("Binaries",
- r'^http[s]?://', None,
+ r'^http[s]?://',
["Binaries"],
[]),
FieldValidator("Archive Policy",
- r'^[0-9]+ versions$', None,
- ["Archive Policy"],
+ r'^[0-9]+ versions$',
+ ["ArchivePolicy"],
[]),
FieldValidator("Anti-Feature",
- ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
+ r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets)$',
["AntiFeatures"],
[]),
FieldValidator("Auto Update Mode",
- r"^(Version .+|None)$", None,
- ["Auto Update Mode"],
+ r"^(Version .+|None)$",
+ ["AutoUpdateMode"],
[]),
FieldValidator("Update Check Mode",
- r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
- ["Update Check Mode"],
+ r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
+ ["UpdateCheckMode"],
[])
}
# Check an app's metadata information for integrity errors
def check_metadata(app):
for v in valuetypes:
- for f in v.fields:
- v.check(app.get_field(f), app.id)
+ for k in v.fields:
+ if k not in app._modified:
+ continue
+ v.check(app.__dict__[k], app.id)
for build in app.builds:
- for f in v.flags:
- v.check(build.get_flag(f), app.id)
+ for k in v.flags:
+ if k not in build._modified:
+ continue
+ v.check(build.__dict__[k], app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
# each line of the description source from the metadata. At the end, call
-# end() and then text_wiki and text_html will contain the result.
+# end() and then text_txt and text_html will contain the result.
class DescriptionFormatter:
+
stNONE = 0
stPARA = 1
stUL = 2
stOL = 3
- bold = False
- ital = False
- state = stNONE
- text_wiki = ''
- text_html = ''
- text_txt = ''
- para_lines = []
- linkResolver = None
def __init__(self, linkres):
+ self.bold = False
+ self.ital = False
+ self.state = self.stNONE
+ self.laststate = self.stNONE
+ self.text_html = ''
+ self.text_txt = ''
+ self.html = io.StringIO()
+ self.text = io.StringIO()
+ self.para_lines = []
+ self.linkResolver = None
self.linkResolver = linkres
def endcur(self, notstates=None):
self.endol()
def endpara(self):
+ self.laststate = self.state
self.state = self.stNONE
whole_para = ' '.join(self.para_lines)
self.addtext(whole_para)
- self.text_txt += textwrap.fill(whole_para, 80,
- break_long_words=False,
- break_on_hyphens=False) + '\n\n'
- self.text_html += '</p>'
+ wrapped = textwrap.fill(whole_para, 80,
+ break_long_words=False,
+ break_on_hyphens=False)
+ self.text.write(wrapped)
+ self.html.write('</p>')
del self.para_lines[:]
def endul(self):
- self.text_html += '</ul>'
- self.text_txt += '\n'
+ self.html.write('</ul>')
+ self.laststate = self.state
self.state = self.stNONE
def endol(self):
- self.text_html += '</ol>'
- self.text_txt += '\n'
+ self.html.write('</ol>')
+ self.laststate = self.state
self.state = self.stNONE
def formatted(self, txt, html):
- formatted = ''
+ res = ''
if html:
txt = cgi.escape(txt)
while True:
index = txt.find("''")
if index == -1:
- return formatted + txt
- formatted += txt[:index]
+ return res + txt
+ res += txt[:index]
txt = txt[index:]
if txt.startswith("'''"):
if html:
if self.bold:
- formatted += '</b>'
+ res += '</b>'
else:
- formatted += '<b>'
+ res += '<b>'
self.bold = not self.bold
txt = txt[3:]
else:
if html:
if self.ital:
- formatted += '</i>'
+ res += '</i>'
else:
- formatted += '<i>'
+ res += '<i>'
self.ital = not self.ital
txt = txt[2:]
def linkify(self, txt):
- linkified_plain = ''
- linkified_html = ''
+ res_plain = ''
+ res_html = ''
while True:
index = txt.find("[")
if index == -1:
- return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
- linkified_plain += self.formatted(txt[:index], False)
- linkified_html += self.formatted(txt[:index], True)
+ return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
+ res_plain += self.formatted(txt[:index], False)
+ res_html += self.formatted(txt[:index], True)
txt = txt[index:]
if txt.startswith("[["):
index = txt.find("]]")
url, urltext = self.linkResolver(url)
else:
urltext = url
- linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
- linkified_plain += urltext
+ res_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
+ res_plain += urltext
txt = txt[index + 2:]
else:
index = txt.find("]")
url = url[:index2]
if url == urltxt:
raise MetaDataException("Url title is just the URL - use [url]")
- linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
- linkified_plain += urltxt
+ res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
+ res_plain += urltxt
if urltxt != url:
- linkified_plain += ' (' + url + ')'
+ res_plain += ' (' + url + ')'
txt = txt[index + 1:]
def addtext(self, txt):
p, h = self.linkify(txt)
- self.text_html += h
+ self.html.write(h)
def parseline(self, line):
- self.text_wiki += "%s\n" % line
if not line:
self.endcur()
elif line.startswith('* '):
self.endcur([self.stUL])
- self.text_txt += "%s\n" % line
if self.state != self.stUL:
- self.text_html += '<ul>'
+ self.html.write('<ul>')
self.state = self.stUL
- self.text_html += '<li>'
+ if self.laststate != self.stNONE:
+ self.text.write('\n\n')
+ else:
+ self.text.write('\n')
+ self.text.write(line)
+ self.html.write('<li>')
self.addtext(line[1:])
- self.text_html += '</li>'
+ self.html.write('</li>')
elif line.startswith('# '):
self.endcur([self.stOL])
- self.text_txt += "%s\n" % line
if self.state != self.stOL:
- self.text_html += '<ol>'
+ self.html.write('<ol>')
self.state = self.stOL
- self.text_html += '<li>'
+ if self.laststate != self.stNONE:
+ self.text.write('\n\n')
+ else:
+ self.text.write('\n')
+ self.text.write(line)
+ self.html.write('<li>')
self.addtext(line[1:])
- self.text_html += '</li>'
+ self.html.write('</li>')
else:
self.para_lines.append(line)
self.endcur([self.stPARA])
if self.state == self.stNONE:
- self.text_html += '<p>'
self.state = self.stPARA
+ if self.laststate != self.stNONE:
+ self.text.write('\n\n')
+ self.html.write('<p>')
def end(self):
self.endcur()
- self.text_txt = self.text_txt.strip()
+ self.text_txt = self.text.getvalue()
+ self.text_html = self.html.getvalue()
+ self.text.close()
+ self.html.close()
# Parse multiple lines of description as written in a metadata file, returning
# a single string in text format and wrapped to 80 columns.
-def description_txt(lines):
+def description_txt(s):
ps = DescriptionFormatter(None)
- for line in lines:
+ for line in s.splitlines():
ps.parseline(line)
ps.end()
return ps.text_txt
# Parse multiple lines of description as written in a metadata file, returning
# a single string in wiki format. Used for the Maintainer Notes field as well,
# because it's the same format.
-def description_wiki(lines):
- ps = DescriptionFormatter(None)
- for line in lines:
- ps.parseline(line)
- ps.end()
- return ps.text_wiki
+def description_wiki(s):
+ return s
# Parse multiple lines of description as written in a metadata file, returning
# a single string in HTML format.
-def description_html(lines, linkres):
+def description_html(s, linkres):
ps = DescriptionFormatter(linkres)
- for line in lines:
+ for line in s.splitlines():
ps.parseline(line)
ps.end()
return ps.text_html
if not os.path.exists(metadatapath):
return thisinfo
- metafile = open(metadatapath, "r")
+ metafile = open(metadatapath, "r", encoding='utf-8')
n = 0
for line in metafile:
else:
thisinfo[f] = v
+ metafile.close()
+
return thisinfo
for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
+ glob.glob(os.path.join('metadata', '*.json'))
+ glob.glob(os.path.join('metadata', '*.xml'))
- + glob.glob(os.path.join('metadata', '*.yaml'))):
+ + glob.glob(os.path.join('metadata', '*.yml'))
+ + glob.glob('.fdroid.json')
+ + glob.glob('.fdroid.xml')
+ + glob.glob('.fdroid.yml')):
+ packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+ if packageName in apps:
+ raise MetaDataException("Found multiple metadata files for " + packageName)
app = parse_metadata(metadatapath)
- if app.id in apps:
- raise MetaDataException("Found multiple metadata files for " + app.id)
check_metadata(app)
apps[app.id] = app
return ("fdroid.app:" + appid, "Dummy name - don't know yet")
raise MetaDataException("Cannot resolve app id " + appid)
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
try:
description_html(app.Description, linkres)
- except MetaDataException, e:
+ except MetaDataException as e:
raise MetaDataException("Problem with description of " + appid +
" - " + str(e))
return apps
+# Port legacy ';' separators
+list_sep = re.compile(r'[,;]')
+
def split_list_values(s):
- # Port legacy ';' separators
- l = [v.strip() for v in s.replace(';', ',').split(',')]
- return [v for v in l if v]
+ res = []
+ for v in re.split(list_sep, s):
+ if not v:
+ continue
+ v = v.strip()
+ if not v:
+ continue
+ res.append(v)
+ return res
def get_default_app_info(metadatapath=None):
if metadatapath is None:
appid = None
else:
- appid, _ = common.get_extension(os.path.basename(metadatapath))
+ appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+
+ if appid == '.fdroid': # we have local metadata in the app's source
+ if os.path.exists('AndroidManifest.xml'):
+ manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
+ else:
+ pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
+ for root, dirs, files in os.walk(os.getcwd()):
+ if 'build.gradle' in files:
+ p = os.path.join(root, 'build.gradle')
+ with open(p, 'rb') as f:
+ data = f.read()
+ m = pattern.search(data)
+ if m:
+ logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
+ manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
+ break
+ if manifestroot is None:
+ raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath))
+ appid = manifestroot.attrib['package']
app = App()
app.metadatapath = metadatapath
return sorted(builds, key=lambda build: int(build.vercode))
-esc_newlines = re.compile('\\\\( |\\n)')
+esc_newlines = re.compile(r'\\( |\n)')
# This function uses __dict__ to be faster
def post_metadata_parse(app):
- for k, v in app.__dict__.iteritems():
+ for k in app._modified:
+ v = app.__dict__[k]
if type(v) in (float, int):
- app.__dict__[f] = str(v)
+ app.__dict__[k] = str(v)
for build in app.builds:
- for k, v in app.__dict__.iteritems():
-
+ for k in build._modified:
+ v = build.__dict__[k]
if type(v) in (float, int):
build.__dict__[k] = str(v)
continue
-
ftype = flagtype(k)
- if ftype == 'script':
+ if ftype == TYPE_SCRIPT:
build.__dict__[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
- elif ftype == 'bool':
+ elif ftype == TYPE_BOOL:
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
- if isinstance(v, basestring) and v == 'true':
- build.__dict__[k] = True
- elif ftype == 'string':
+ if isinstance(v, str):
+ build.__dict__[k] = _decode_bool(v)
+ elif ftype == TYPE_STRING:
if isinstance(v, bool) and v:
build.__dict__[k] = 'yes'
- # convert to the odd internal format
- for f in ('Description', 'Maintainer Notes'):
- v = app.get_field(f)
- if isinstance(v, basestring):
- text = v.rstrip().lstrip()
- app.set_field(f, text.split('\n'))
-
if not app.Description:
- app.Description = ['No description available']
+ app.Description = 'No description available'
app.builds = sorted_builds(app.builds)
#
-def _decode_list(data):
- '''convert items in a list from unicode to basestring'''
- rv = []
- for item in data:
- if isinstance(item, unicode):
- item = item.encode('utf-8')
- elif isinstance(item, list):
- item = _decode_list(item)
- elif isinstance(item, dict):
- item = _decode_dict(item)
- rv.append(item)
- return rv
-
-
-def _decode_dict(data):
- '''convert items in a dict from unicode to basestring'''
- rv = {}
- for k, v in data.iteritems():
- if isinstance(k, unicode):
- k = k.encode('utf-8')
- if isinstance(v, unicode):
- v = v.encode('utf-8')
- elif isinstance(v, list):
- v = _decode_list(v)
- elif isinstance(v, dict):
- v = _decode_dict(v)
- rv[k] = v
- return rv
+bool_true = re.compile(r'([Yy]es|[Tt]rue)')
+bool_false = re.compile(r'([Nn]o|[Ff]alse)')
+
+
+def _decode_bool(s):
+ if bool_true.match(s):
+ return True
+ if bool_false.match(s):
+ return False
+ raise MetaDataException("Invalid bool '%s'" % s)
def parse_metadata(metadatapath):
- _, ext = common.get_extension(metadatapath)
- accepted = common.config['accepted_formats']
+ _, ext = fdroidserver.common.get_extension(metadatapath)
+ accepted = fdroidserver.common.config['accepted_formats']
if ext not in accepted:
raise MetaDataException('"%s" is not an accepted format, convert to: %s' % (
metadatapath, ', '.join(accepted)))
- app = None
- if ext == 'txt':
- app = parse_txt_metadata(metadatapath)
- elif ext == 'json':
- app = parse_json_metadata(metadatapath)
- elif ext == 'xml':
- app = parse_xml_metadata(metadatapath)
- elif ext == 'yaml':
- app = parse_yaml_metadata(metadatapath)
- else:
- raise MetaDataException('Unknown metadata format: %s' % metadatapath)
+ app = App()
+ app.metadatapath = metadatapath
+ app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+
+ with open(metadatapath, 'r', encoding='utf-8') as mf:
+ if ext == 'txt':
+ parse_txt_metadata(mf, app)
+ elif ext == 'json':
+ parse_json_metadata(mf, app)
+ elif ext == 'xml':
+ parse_xml_metadata(mf, app)
+ elif ext == 'yml':
+ parse_yaml_metadata(mf, app)
+ else:
+ raise MetaDataException('Unknown metadata format: %s' % metadatapath)
post_metadata_parse(app)
return app
-def parse_json_metadata(metadatapath):
+def parse_json_metadata(mf, app):
- app = get_default_app_info(metadatapath)
-
- # fdroid metadata is only strings and booleans, no floats or ints. And
- # json returns unicode, and fdroidserver still uses plain python strings
+ # fdroid metadata is only strings and booleans, no floats or ints.
# TODO create schema using https://pypi.python.org/pypi/jsonschema
- jsoninfo = json.load(open(metadatapath, 'r'),
- object_hook=_decode_dict,
- parse_int=lambda s: s,
+ jsoninfo = json.load(mf, parse_int=lambda s: s,
parse_float=lambda s: s)
app.update_fields(jsoninfo)
+ for f in ['Description', 'Maintainer Notes']:
+ v = app.get_field(f)
+ app.set_field(f, '\n'.join(v))
return app
-def parse_xml_metadata(metadatapath):
+def parse_xml_metadata(mf, app):
- app = get_default_app_info(metadatapath)
-
- tree = ElementTree.ElementTree(file=metadatapath)
+ tree = ElementTree.ElementTree(file=mf)
root = tree.getroot()
if root.tag != 'resources':
- raise MetaDataException('%s does not have root as <resources></resources>!' % metadatapath)
+ raise MetaDataException('resources file does not have root element <resources/>')
for child in root:
if child.tag != 'builds':
# TODO handle this using <xsd:element type="xsd:boolean> in a schema
if not isinstance(app.RequiresRoot, bool):
- if app.RequiresRoot == 'true':
- app.RequiresRoot = True
- else:
- app.RequiresRoot = False
+ app.RequiresRoot = app.RequiresRoot == 'true'
return app
-def parse_yaml_metadata(metadatapath):
-
- app = get_default_app_info(metadatapath)
+def parse_yaml_metadata(mf, app):
- yamlinfo = yaml.load(open(metadatapath, 'r'), Loader=YamlLoader)
+ yamlinfo = yaml.load(mf, Loader=YamlLoader)
app.update_fields(yamlinfo)
return app
-build_line_sep = re.compile(r"(?<!\\),")
+build_line_sep = re.compile(r'(?<!\\),')
+build_cont = re.compile(r'^[ \t]')
-def parse_txt_metadata(metadatapath):
+def parse_txt_metadata(mf, app):
linedesc = None
pk, pv = bv
pk = pk.lstrip()
- if pk not in build_flags:
- raise MetaDataException("Unrecognised build flag at {0} in {1}"
- .format(p, linedesc))
t = flagtype(pk)
- if t == 'list':
+ if t == TYPE_LIST:
pv = split_list_values(pv)
- if pk == 'gradle':
- if len(pv) == 1 and pv[0] in ['main', 'yes']:
- pv = ['yes']
build.set_flag(pk, pv)
- elif t == 'string' or t == 'script':
+ elif t == TYPE_STRING or t == TYPE_SCRIPT:
build.set_flag(pk, pv)
- elif t == 'bool':
- v = pv == 'yes'
- if v:
- build.set_flag(pk, True)
-
- else:
- raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
- % (t, p, linedesc))
+ elif t == TYPE_BOOL:
+ build.set_flag(pk, _decode_bool(pv))
def parse_buildline(lines):
v = "".join(lines)
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
if len(parts) < 3:
- raise MetaDataException("Invalid build format: " + v + " in " + metafile.name)
+ raise MetaDataException("Invalid build format: " + v + " in " + mf.name)
build = Build()
- build.origlines = lines
build.version = parts[0]
build.vercode = parts[1]
if parts[2].startswith('!'):
app.comments[key] = list(curcomments)
del curcomments[:]
- app = get_default_app_info(metadatapath)
- metafile = open(metadatapath, "r")
-
mode = 0
buildlines = []
+ multiline_lines = []
curcomments = []
build = None
- vc_seen = {}
+ vc_seen = set()
c = 0
- for line in metafile:
+ for line in mf:
c += 1
- linedesc = "%s:%d" % (metafile.name, c)
+ linedesc = "%s:%d" % (mf.name, c)
line = line.rstrip('\r\n')
if mode == 3:
- if not any(line.startswith(s) for s in (' ', '\t')):
+ if build_cont.match(line):
+ if line.endswith('\\'):
+ buildlines.append(line[:-1].lstrip())
+ else:
+ buildlines.append(line.lstrip())
+ bl = ''.join(buildlines)
+ add_buildflag(bl, build)
+ del buildlines[:]
+ else:
if not build.commit and not build.disable:
raise MetaDataException("No commit specified for {0} in {1}"
.format(build.version, linedesc))
app.builds.append(build)
add_comments('build:' + build.vercode)
mode = 0
- else:
- if line.endswith('\\'):
- buildlines.append(line[:-1].lstrip())
- else:
- buildlines.append(line.lstrip())
- bl = ''.join(buildlines)
- add_buildflag(bl, build)
- buildlines = []
if mode == 0:
if not line:
f, v = line.split(':', 1)
except ValueError:
raise MetaDataException("Invalid metadata in " + linedesc)
- if f != f.strip() or v != v.strip():
- raise MetaDataException("Extra spacing found in " + linedesc)
# Translate obsolete fields...
if f == 'Market Version':
if f == 'Market Version Code':
f = 'Current Version Code'
- fieldtype = metafieldtype(f)
- if fieldtype not in ['build', 'buildv2']:
+ ftype = fieldtype(f)
+ if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
add_comments(f)
- if fieldtype == 'multiline':
+ if ftype == TYPE_MULTILINE:
mode = 1
if v:
raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
- elif fieldtype == 'string':
+ elif ftype == TYPE_STRING:
app.set_field(f, v)
- elif fieldtype == 'list':
+ elif ftype == TYPE_LIST:
app.set_field(f, split_list_values(v))
- elif fieldtype == 'build':
+ elif ftype == TYPE_BUILD:
if v.endswith("\\"):
mode = 2
- buildlines = [v[:-1]]
+ del buildlines[:]
+ buildlines.append(v[:-1])
else:
build = parse_buildline([v])
app.builds.append(build)
add_comments('build:' + app.builds[-1].vercode)
- elif fieldtype == 'buildv2':
- build = Build()
+ elif ftype == TYPE_BUILD_V2:
vv = v.split(',')
if len(vv) != 2:
raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
.format(v, linedesc))
+ build = Build()
build.version = vv[0]
build.vercode = vv[1]
if build.vercode in vc_seen:
raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
build.vercode, linedesc))
- vc_seen[build.vercode] = True
- buildlines = []
+ vc_seen.add(build.vercode)
+ del buildlines[:]
mode = 3
- elif fieldtype == 'obsolete':
+ elif ftype == TYPE_OBSOLETE:
pass # Just throw it away!
else:
- raise MetaDataException("Unrecognised field type for " + f + " in " + linedesc)
+ raise MetaDataException("Unrecognised field '" + f + "' in " + linedesc)
elif mode == 1: # Multiline field
if line == '.':
mode = 0
+ app.set_field(f, '\n'.join(multiline_lines))
+ del multiline_lines[:]
else:
- app.append_field(f, line)
+ multiline_lines.append(line)
elif mode == 2: # Line continuation mode in Build Version
if line.endswith("\\"):
buildlines.append(line[:-1])
mode = 0
add_comments(None)
- # Mode at end of file should always be 0...
+ # Mode at end of file should always be 0
if mode == 1:
- raise MetaDataException(f + " not terminated in " + metafile.name)
- elif mode == 2:
- raise MetaDataException("Unterminated continuation in " + metafile.name)
- elif mode == 3:
- raise MetaDataException("Unterminated build in " + metafile.name)
+ raise MetaDataException(f + " not terminated in " + mf.name)
+ if mode == 2:
+ raise MetaDataException("Unterminated continuation in " + mf.name)
+ if mode == 3:
+ raise MetaDataException("Unterminated build in " + mf.name)
return app
w_field(f, v)
w_field_nonempty('Disabled')
- if app.AntiFeatures:
- w_field_always('AntiFeatures')
+ w_field_nonempty('AntiFeatures')
w_field_nonempty('Provides')
w_field_always('Categories')
w_field_always('License')
+ w_field_nonempty('Author Name')
+ w_field_nonempty('Author Email')
w_field_always('Web Site')
w_field_always('Source Code')
w_field_always('Issue Tracker')
w_field_always('Binaries')
mf.write('\n')
- for build in sorted_builds(app.builds):
+ for build in app.builds:
if build.version == "Ignore":
continue
#
# 'mf' - Writer interface (file, StringIO, ...)
# 'app' - The app data
-def write_txt_metadata(mf, app):
+def write_txt(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def w_field(f, v):
- t = metafieldtype(f)
- if t == 'list':
+ t = fieldtype(f)
+ if t == TYPE_LIST:
v = ','.join(v)
- elif t == 'multiline':
- if type(v) == list:
- v = '\n' + '\n'.join(v) + '\n.'
- else:
- v = '\n' + v + '\n.'
+ elif t == TYPE_MULTILINE:
+ v = '\n' + v + '\n.'
mf.write("%s:%s\n" % (f, v))
def w_build(build):
continue
t = flagtype(f)
- out = ' %s=' % f
- if t == 'string':
- out += v
- elif t == 'bool':
- out += 'yes'
- elif t == 'script':
- out += '&& \\\n '.join([s.lstrip() for s in v.split('&& ')])
- elif t == 'list':
- out += ','.join(v) if type(v) == list else v
-
- mf.write(out)
+ mf.write(' %s=' % f)
+ if t == TYPE_STRING:
+ mf.write(v)
+ elif t == TYPE_BOOL:
+ mf.write('yes')
+ elif t == TYPE_SCRIPT:
+ first = True
+ for s in v.split(' && '):
+ if first:
+ first = False
+ else:
+ mf.write(' && \\\n ')
+ mf.write(s)
+ elif t == TYPE_LIST:
+ mf.write(','.join(v))
+
mf.write('\n')
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-def write_yaml_metadata(mf, app):
+def write_yaml(mf, app):
def w_comment(line):
mf.write("# %s\n" % line)
def w_field(f, v, prefix='', t=None):
if t is None:
- t = metafieldtype(f)
+ t = fieldtype(f)
v = ''
- if t == 'list':
+ if t == TYPE_LIST:
v = '\n'
for e in v:
v += prefix + ' - ' + escape(e) + '\n'
- elif t == 'multiline':
+ elif t == TYPE_MULTILINE:
v = ' |\n'
- lines = v
- if type(v) == str:
- lines = v.splitlines()
- for l in lines:
+ for l in v.splitlines():
if l:
v += prefix + ' ' + l + '\n'
else:
v += '\n'
- elif t == 'bool':
+ elif t == TYPE_BOOL:
v = ' yes\n'
- elif t == 'script':
+ elif t == TYPE_SCRIPT:
cmds = [s + '&& \\' for s in v.split('&& ')]
if len(cmds) > 0:
cmds[-1] = cmds[-1][:-len('&& \\')]
mf.write("builds:\n")
first_build = False
- w_field('versionName', build.version, ' - ', 'string')
- w_field('versionCode', build.vercode, ' ', 'strsng')
+ w_field('versionName', build.version, ' - ', TYPE_STRING)
+ w_field('versionCode', build.vercode, ' ', TYPE_STRING)
for f in build_flags_order:
v = build.get_flag(f)
if not v:
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-def write_metadata(fmt, mf, app):
- if fmt == 'txt':
- return write_txt_metadata(mf, app)
- if fmt == 'yaml':
- return write_yaml_metadata(mf, app)
- raise MetaDataException("Unknown metadata format given")
+def write_metadata(metadatapath, app):
+ _, ext = fdroidserver.common.get_extension(metadatapath)
+ accepted = fdroidserver.common.config['accepted_formats']
+ if ext not in accepted:
+ raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
+ metadatapath, ', '.join(accepted)))
+
+ with open(metadatapath, 'w', encoding='utf8') as mf:
+ if ext == 'txt':
+ return write_txt(mf, app)
+ elif ext == 'yml':
+ return write_yaml(mf, app)
+ raise MetaDataException('Unknown metadata format: %s' % metadatapath)