import os
import re
import glob
-import cgi
+import html
import logging
import textwrap
import io
-
import yaml
# use libyaml if it is available
try:
from yaml import Loader
YamlLoader = Loader
-# use the C implementation when available
-import xml.etree.cElementTree as ElementTree
-
import fdroidserver.common
+from fdroidserver.exception import MetaDataException, FDroidException
srclibs = None
+warnings_action = None
-class MetaDataException(Exception):
-
- def __init__(self, value):
- self.value = value
+def warn_or_exception(value):
+ '''output warning or Exception depending on -W'''
+ if warnings_action == 'ignore':
+ pass
+ elif warnings_action == 'error':
+ raise MetaDataException(value)
+ else:
+ logging.warning(value)
- def __str__(self):
- return self.value
# To filter which ones should be written to the metadata files if
# present
'License',
'Author Name',
'Author Email',
+ 'Author Web Site',
'Web Site',
'Source Code',
'Issue Tracker',
'Current Version',
'Current Version Code',
'No Source Since',
+ 'Build',
'comments', # For formats that don't do inline comments
'builds', # For formats that do builds as a list
])
-class App():
+class App(dict):
+
+ def __init__(self, copydict=None):
+ if copydict:
+ super().__init__(copydict)
+ return
+ super().__init__()
- def __init__(self):
self.Disabled = None
self.AntiFeatures = []
self.Provides = None
self.License = 'Unknown'
self.AuthorName = None
self.AuthorEmail = None
+ self.AuthorWebSite = None
self.WebSite = ''
self.SourceCode = ''
self.IssueTracker = ''
self.builds = []
self.comments = {}
self.added = None
- self.lastupdated = None
- self._modified = set()
+ self.lastUpdated = None
- # Translates human-readable field names to attribute names, e.g.
- # 'Auto Name' to 'AutoName'
- @classmethod
- def field_to_attr(cls, f):
- return f.replace(' ', '')
+ def __getattr__(self, name):
+ if name in self:
+ return self[name]
+ else:
+ raise AttributeError("No such attribute: " + name)
- # Translates attribute names to human-readable field names, e.g.
- # 'AutoName' to 'Auto Name'
- @classmethod
- def attr_to_field(cls, k):
- if k in app_fields:
- return k
- f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
- return f
+ def __setattr__(self, name, value):
+ self[name] = value
- # Constructs an old-fashioned dict with the human-readable field
- # names. Should only be used for tests.
- def field_dict(self):
- d = {}
- for k, v in self.__dict__.items():
- if k == 'builds':
- d['builds'] = []
- for build in v:
- b = {k: v for k, v in build.__dict__.items() if not k.startswith('_')}
- d['builds'].append(b)
- elif not k.startswith('_'):
- f = App.attr_to_field(k)
- d[f] = v
- return d
-
- # Gets the value associated to a field name, e.g. 'Auto Name'
- def get_field(self, f):
- if f not in app_fields:
- raise MetaDataException('Unrecognised app field: ' + f)
- k = App.field_to_attr(f)
- return getattr(self, k)
-
- # Sets the value associated to a field name, e.g. 'Auto Name'
- def set_field(self, f, v):
- if f not in app_fields:
- raise MetaDataException('Unrecognised app field: ' + f)
- k = App.field_to_attr(f)
- self.__dict__[k] = v
- self._modified.add(k)
-
- # Appends to the value associated to a field name, e.g. 'Auto Name'
- def append_field(self, f, v):
- if f not in app_fields:
- raise MetaDataException('Unrecognised app field: ' + f)
- k = App.field_to_attr(f)
- if k not in self.__dict__:
- self.__dict__[k] = [v]
+ def __delattr__(self, name):
+ if name in self:
+ del self[name]
else:
- self.__dict__[k].append(v)
-
- # Like dict.update(), but using human-readable field names
- def update_fields(self, d):
- for f, v in d.items():
- if f == 'builds':
- for b in v:
- build = Build()
- build.update_flags(b)
- self.builds.append(build)
- else:
- self.set_field(f, v)
+ raise AttributeError("No such attribute: " + name)
+
+ def get_last_build(self):
+ if len(self.builds) > 0:
+ return self.builds[-1]
+ else:
+ return Build()
+
TYPE_UNKNOWN = 0
TYPE_OBSOLETE = 1
TYPE_MULTILINE = 6
TYPE_BUILD = 7
TYPE_BUILD_V2 = 8
+TYPE_INT = 9
fieldtypes = {
'Description': TYPE_MULTILINE,
- 'Maintainer Notes': TYPE_MULTILINE,
+ 'MaintainerNotes': TYPE_MULTILINE,
'Categories': TYPE_LIST,
'AntiFeatures': TYPE_LIST,
- 'Build Version': TYPE_BUILD,
+ 'BuildVersion': TYPE_BUILD,
'Build': TYPE_BUILD_V2,
- 'Use Built': TYPE_OBSOLETE,
+ 'UseBuilt': TYPE_OBSOLETE,
}
def fieldtype(name):
+ name = name.replace(' ', '')
if name in fieldtypes:
return fieldtypes[name]
return TYPE_STRING
'commit',
'subdir',
'submodules',
+ 'sudo',
'init',
'patch',
'gradle',
'maven',
'kivy',
+ 'buildozer',
'output',
'srclibs',
'oldsdkloc',
'rm',
'extlibs',
'prebuild',
- 'update',
+ 'androidupdate',
'target',
'scanignore',
'scandelete',
'gradleprops',
'antcommands',
'novcheck',
+ 'antifeatures',
]
+# old .txt format has version name/code inline in the 'Build:' line
+# but YAML and JSON have a explicit key for them
+build_flags = ['versionName', 'versionCode'] + build_flags_order
-build_flags = set(build_flags_order + ['version', 'vercode'])
+class Build(dict):
-class Build():
-
- def __init__(self):
+ def __init__(self, copydict=None):
+ super().__init__()
self.disable = False
self.commit = None
self.subdir = None
self.submodules = False
+ self.sudo = ''
self.init = ''
self.patch = []
self.gradle = []
self.maven = False
self.kivy = False
+ self.buildozer = False
self.output = None
self.srclibs = []
self.oldsdkloc = False
self.rm = []
self.extlibs = []
self.prebuild = ''
- self.update = []
+ self.androidupdate = []
self.target = None
self.scanignore = []
self.scandelete = []
self.gradleprops = []
self.antcommands = []
self.novcheck = False
+ self.antifeatures = []
+ if copydict:
+ super().__init__(copydict)
+ return
+
+ def __getattr__(self, name):
+ if name in self:
+ return self[name]
+ else:
+ raise AttributeError("No such attribute: " + name)
+
+ def __setattr__(self, name, value):
+ self[name] = value
- self._modified = set()
-
- def get_flag(self, f):
- if f not in build_flags:
- raise MetaDataException('Unrecognised build flag: ' + f)
- return getattr(self, f)
-
- def set_flag(self, f, v):
- if f == 'versionName':
- f = 'version'
- if f == 'versionCode':
- f = 'vercode'
- if f not in build_flags:
- raise MetaDataException('Unrecognised build flag: ' + f)
- self.__dict__[f] = v
- self._modified.add(f)
-
- def append_flag(self, f, v):
- if f not in build_flags:
- raise MetaDataException('Unrecognised build flag: ' + f)
- if f not in self.__dict__:
- self.__dict__[f] = [v]
+ def __delattr__(self, name):
+ if name in self:
+ del self[name]
else:
- self.__dict__[f].append(v)
+ raise AttributeError("No such attribute: " + name)
def build_method(self):
- for f in ['maven', 'gradle', 'kivy']:
- if self.get_flag(f):
+ for f in ['maven', 'gradle', 'kivy', 'buildozer']:
+ if self.get(f):
return f
if self.output:
return 'raw'
def output_method(self):
if self.output:
return 'raw'
- for f in ['maven', 'gradle', 'kivy']:
- if self.get_flag(f):
+ for f in ['maven', 'gradle', 'kivy', 'buildozer']:
+ if self.get(f):
return f
return 'ant'
def ndk_path(self):
version = self.ndk
if not version:
- version = 'r10e' # falls back to latest
+ version = 'r12b' # falls back to latest
paths = fdroidserver.common.config['ndk_paths']
if version not in paths:
return ''
return paths[version]
- def update_flags(self, d):
- for f, v in d.items():
- self.set_flag(f, v)
flagtypes = {
+ 'versionCode': TYPE_INT,
'extlibs': TYPE_LIST,
'srclibs': TYPE_LIST,
'patch': TYPE_LIST,
'rm': TYPE_LIST,
'buildjni': TYPE_LIST,
'preassemble': TYPE_LIST,
- 'update': TYPE_LIST,
+ 'androidupdate': TYPE_LIST,
'scanignore': TYPE_LIST,
'scandelete': TYPE_LIST,
'gradle': TYPE_LIST,
'antcommands': TYPE_LIST,
'gradleprops': TYPE_LIST,
+ 'sudo': TYPE_SCRIPT,
'init': TYPE_SCRIPT,
'prebuild': TYPE_SCRIPT,
'build': TYPE_SCRIPT,
'forceversion': TYPE_BOOL,
'forcevercode': TYPE_BOOL,
'novcheck': TYPE_BOOL,
+ 'antifeatures': TYPE_LIST,
}
return TYPE_STRING
-# Designates a metadata field type and checks that it matches
-#
-# 'name' - The long name of the field type
-# 'matching' - List of possible values or regex expression
-# 'sep' - Separator to use if value may be a list
-# 'fields' - Metadata fields (Field:Value) of this type
-# 'flags' - Build flags (flag=value) of this type
-#
class FieldValidator():
+ """
+ Designates App metadata field types and checks that it matches
- def __init__(self, name, matching, fields, flags):
+ 'name' - The long name of the field type
+ 'matching' - List of possible values or regex expression
+ 'sep' - Separator to use if value may be a list
+ 'fields' - Metadata fields (Field:Value) of this type
+ """
+
+ def __init__(self, name, matching, fields):
self.name = name
self.matching = matching
self.compiled = re.compile(matching)
self.fields = fields
- self.flags = flags
def check(self, v, appid):
if not v:
values = [v]
for v in values:
if not self.compiled.match(v):
- raise MetaDataException("'%s' is not a valid %s in %s. Regex pattern: %s"
- % (v, self.name, appid, self.matching))
+ warn_or_exception("'%s' is not a valid %s in %s. Regex pattern: %s"
+ % (v, self.name, appid, self.matching))
+
# Generic value types
valuetypes = {
- FieldValidator("Integer",
- r'^[1-9][0-9]*$',
- [],
- ['vercode']),
-
- FieldValidator("Hexadecimal",
- r'^[0-9a-f]+$',
- ['FlattrID'],
- []),
+ FieldValidator("Flattr ID",
+ r'^[0-9a-z]+$',
+ ['FlattrID']),
FieldValidator("HTTP link",
r'^http[s]?://',
- ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"], []),
+ ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
FieldValidator("Email",
r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
- ["AuthorEmail"], []),
+ ["AuthorEmail"]),
FieldValidator("Bitcoin address",
r'^[a-zA-Z0-9]{27,34}$',
- ["Bitcoin"],
- []),
+ ["Bitcoin"]),
FieldValidator("Litecoin address",
r'^L[a-zA-Z0-9]{33}$',
- ["Litecoin"],
- []),
+ ["Litecoin"]),
FieldValidator("Repo Type",
r'^(git|git-svn|svn|hg|bzr|srclib)$',
- ["RepoType"],
- []),
+ ["RepoType"]),
FieldValidator("Binaries",
r'^http[s]?://',
- ["Binaries"],
- []),
+ ["Binaries"]),
FieldValidator("Archive Policy",
r'^[0-9]+ versions$',
- ["ArchivePolicy"],
- []),
+ ["ArchivePolicy"]),
FieldValidator("Anti-Feature",
- r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets)$',
- ["AntiFeatures"],
- []),
+ r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
+ ["AntiFeatures"]),
FieldValidator("Auto Update Mode",
r"^(Version .+|None)$",
- ["AutoUpdateMode"],
- []),
+ ["AutoUpdateMode"]),
FieldValidator("Update Check Mode",
r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
- ["UpdateCheckMode"],
- [])
+ ["UpdateCheckMode"])
}
def check_metadata(app):
for v in valuetypes:
for k in v.fields:
- if k not in app._modified:
- continue
- v.check(app.__dict__[k], app.id)
- for build in app.builds:
- for k in v.flags:
- if k not in build._modified:
- continue
- v.check(build.__dict__[k], app.id)
+ v.check(app[k], app.id)
# Formatter for descriptions. Create an instance, and call parseline() with
self.laststate = self.state
self.state = self.stNONE
- def formatted(self, txt, html):
+ def formatted(self, txt, htmlbody):
res = ''
- if html:
- txt = cgi.escape(txt)
+ if htmlbody:
+ txt = html.escape(txt, quote=False)
while True:
index = txt.find("''")
if index == -1:
res += txt[:index]
txt = txt[index:]
if txt.startswith("'''"):
- if html:
+ if htmlbody:
if self.bold:
res += '</b>'
else:
self.bold = not self.bold
txt = txt[3:]
else:
- if html:
+ if htmlbody:
if self.ital:
res += '</i>'
else:
if txt.startswith("[["):
index = txt.find("]]")
if index == -1:
- raise MetaDataException("Unterminated ]]")
+ warn_or_exception("Unterminated ]]")
url = txt[2:index]
if self.linkResolver:
url, urltext = self.linkResolver(url)
else:
urltext = url
- res_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
+ res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
res_plain += urltext
txt = txt[index + 2:]
else:
index = txt.find("]")
if index == -1:
- raise MetaDataException("Unterminated ]")
+ warn_or_exception("Unterminated ]")
url = txt[1:index]
index2 = url.find(' ')
if index2 == -1:
urltxt = url[index2 + 1:]
url = url[:index2]
if url == urltxt:
- raise MetaDataException("Url title is just the URL - use [url]")
- res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
+ warn_or_exception("Url title is just the URL - use [url]")
+ res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
res_plain += urltxt
if urltxt != url:
res_plain += ' (' + url + ')'
try:
f, v = line.split(':', 1)
except ValueError:
- raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
+ warn_or_exception("Invalid metadata in %s:%d" % (line, n))
if f == "Subdir":
thisinfo[f] = v.split(',')
srclibs[srclibname] = parse_srclib(metadatapath)
-# Read all metadata. Returns a list of 'app' objects (which are dictionaries as
-# returned by the parse_txt_metadata function.
-def read_metadata(xref=True):
+def read_metadata(xref=True, check_vcs=[]):
+ """
+ Read all metadata. Returns a list of 'app' objects (which are dictionaries as
+ returned by the parse_txt_metadata function.
+
+ check_vcs is the list of packageNames to check for .fdroid.yml in source
+ """
# Always read the srclibs before the apps, since they can use a srlib as
# their source repository.
for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
+ glob.glob(os.path.join('metadata', '*.json'))
- + glob.glob(os.path.join('metadata', '*.xml'))
+ glob.glob(os.path.join('metadata', '*.yml'))
+ + glob.glob('.fdroid.txt')
+ glob.glob('.fdroid.json')
- + glob.glob('.fdroid.xml')
+ glob.glob('.fdroid.yml')):
packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
if packageName in apps:
- raise MetaDataException("Found multiple metadata files for " + packageName)
- app = parse_metadata(metadatapath)
+ warn_or_exception("Found multiple metadata files for " + packageName)
+ app = parse_metadata(metadatapath, packageName in check_vcs)
check_metadata(app)
apps[app.id] = app
def linkres(appid):
if appid in apps:
return ("fdroid.app:" + appid, "Dummy name - don't know yet")
- raise MetaDataException("Cannot resolve app id " + appid)
+ warn_or_exception("Cannot resolve app id " + appid)
for appid, app in apps.items():
try:
description_html(app.Description, linkres)
except MetaDataException as e:
- raise MetaDataException("Problem with description of " + appid +
- " - " + str(e))
+ warn_or_exception("Problem with description of " + appid +
+ " - " + str(e))
return apps
+
# Port legacy ';' separators
list_sep = re.compile(r'[,;]')
manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
break
if manifestroot is None:
- raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath))
+ warn_or_exception("Cannot find a packageName for {0}!".format(metadatapath))
appid = manifestroot.attrib['package']
app = App()
def sorted_builds(builds):
- return sorted(builds, key=lambda build: int(build.vercode))
+ return sorted(builds, key=lambda build: int(build.versionCode))
esc_newlines = re.compile(r'\\( |\n)')
-# This function uses __dict__ to be faster
def post_metadata_parse(app):
-
- for k in app._modified:
- v = app.__dict__[k]
+ # TODO keep native types, convert only for .txt metadata
+ for k, v in app.items():
if type(v) in (float, int):
- app.__dict__[k] = str(v)
+ app[k] = str(v)
- for build in app.builds:
- for k in build._modified:
- v = build.__dict__[k]
- if type(v) in (float, int):
- build.__dict__[k] = str(v)
- continue
- ftype = flagtype(k)
-
- if ftype == TYPE_SCRIPT:
- build.__dict__[k] = re.sub(esc_newlines, '', v).lstrip().rstrip()
- elif ftype == TYPE_BOOL:
- # TODO handle this using <xsd:element type="xsd:boolean> in a schema
- if isinstance(v, str):
- build.__dict__[k] = _decode_bool(v)
- elif ftype == TYPE_STRING:
- if isinstance(v, bool) and v:
- build.__dict__[k] = 'yes'
+ if 'Builds' in app:
+ app['builds'] = app.pop('Builds')
- if not app.Description:
- app.Description = 'No description available'
+ if 'flavours' in app and app['flavours'] == [True]:
+ app['flavours'] = 'yes'
- app.builds = sorted_builds(app.builds)
+ if isinstance(app.Categories, str):
+ app.Categories = [app.Categories]
+ elif app.Categories is None:
+ app.Categories = ['None']
+ else:
+ app.Categories = [str(i) for i in app.Categories]
+
+ def _yaml_bool_unmapable(v):
+ return v in (True, False, [True], [False])
+
+ def _yaml_bool_unmap(v):
+ if v is True:
+ return 'yes'
+ elif v is False:
+ return 'no'
+ elif v == [True]:
+ return ['yes']
+ elif v == [False]:
+ return ['no']
+
+ _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
+
+ builds = []
+ if 'builds' in app:
+ for build in app['builds']:
+ if not isinstance(build, Build):
+ build = Build(build)
+ for k, v in build.items():
+ if not (v is None):
+ if flagtype(k) == TYPE_LIST:
+ if _yaml_bool_unmapable(v):
+ build[k] = _yaml_bool_unmap(v)
+
+ if isinstance(v, str):
+ build[k] = [v]
+ elif isinstance(v, bool):
+ if v:
+ build[k] = ['yes']
+ else:
+ build[k] = []
+ elif flagtype(k) is TYPE_INT:
+ build[k] = str(v)
+ elif flagtype(k) is TYPE_STRING:
+ if isinstance(v, bool) and k in _bool_allowed:
+ build[k] = v
+ else:
+ if _yaml_bool_unmapable(v):
+ build[k] = _yaml_bool_unmap(v)
+ else:
+ build[k] = str(v)
+ builds.append(build)
+
+ app.builds = sorted_builds(builds)
# Parse metadata for a single application.
return True
if bool_false.match(s):
return False
- raise MetaDataException("Invalid bool '%s'" % s)
+ warn_or_exception("Invalid bool '%s'" % s)
+
+def parse_metadata(metadatapath, check_vcs=False):
+ '''parse metadata file, optionally checking the git repo for metadata first'''
-def parse_metadata(metadatapath):
_, ext = fdroidserver.common.get_extension(metadatapath)
accepted = fdroidserver.common.config['accepted_formats']
if ext not in accepted:
- raise MetaDataException('"%s" is not an accepted format, convert to: %s' % (
+ warn_or_exception('"%s" is not an accepted format, convert to: %s' % (
metadatapath, ', '.join(accepted)))
app = App()
app.metadatapath = metadatapath
- app.id, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+ name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
+ if name == '.fdroid':
+ check_vcs = False
+ else:
+ app.id = name
with open(metadatapath, 'r', encoding='utf-8') as mf:
if ext == 'txt':
parse_txt_metadata(mf, app)
elif ext == 'json':
parse_json_metadata(mf, app)
- elif ext == 'xml':
- parse_xml_metadata(mf, app)
elif ext == 'yml':
parse_yaml_metadata(mf, app)
else:
- raise MetaDataException('Unknown metadata format: %s' % metadatapath)
+ warn_or_exception('Unknown metadata format: %s' % metadatapath)
+
+ if check_vcs and app.Repo:
+ build_dir = fdroidserver.common.get_build_dir(app)
+ metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
+ if not os.path.isfile(metadata_in_repo):
+ vcs, build_dir = fdroidserver.common.setup_vcs(app)
+ if isinstance(vcs, fdroidserver.common.vcs_git):
+ vcs.gotorevision('HEAD') # HEAD since we can't know where else to go
+ if os.path.isfile(metadata_in_repo):
+ logging.debug('Including metadata from ' + metadata_in_repo)
+ # do not include fields already provided by main metadata file
+ app_in_repo = parse_metadata(metadata_in_repo)
+ for k, v in app_in_repo.items():
+ if k not in app:
+ app[k] = v
post_metadata_parse(app)
+
+ if not app.id:
+ if app.builds:
+ build = app.builds[-1]
+ if build.subdir:
+ root_dir = build.subdir
+ else:
+ root_dir = '.'
+ paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
+ _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
+
return app
# TODO create schema using https://pypi.python.org/pypi/jsonschema
jsoninfo = json.load(mf, parse_int=lambda s: s,
parse_float=lambda s: s)
- app.update_fields(jsoninfo)
+ app.update(jsoninfo)
for f in ['Description', 'Maintainer Notes']:
- v = app.get_field(f)
- app.set_field(f, '\n'.join(v))
+ v = app.get(f)
+ if v:
+ app[f] = '\n'.join(v)
return app
-def parse_xml_metadata(mf, app):
-
- tree = ElementTree.ElementTree(file=mf)
- root = tree.getroot()
-
- if root.tag != 'resources':
- raise MetaDataException('resources file does not have root element <resources/>')
-
- for child in root:
- if child.tag != 'builds':
- # builds does not have name="" attrib
- name = child.attrib['name']
-
- if child.tag == 'string':
- app.set_field(name, child.text)
- elif child.tag == 'string-array':
- for item in child:
- app.append_field(name, item.text)
- elif child.tag == 'builds':
- for b in child:
- build = Build()
- for key in b:
- build.set_flag(key.tag, key.text)
- app.builds.append(build)
-
- # TODO handle this using <xsd:element type="xsd:boolean> in a schema
- if not isinstance(app.RequiresRoot, bool):
- app.RequiresRoot = app.RequiresRoot == 'true'
-
+def parse_yaml_metadata(mf, app):
+ yamldata = yaml.load(mf, Loader=YamlLoader)
+ if yamldata:
+ app.update(yamldata)
return app
-def parse_yaml_metadata(mf, app):
+def write_yaml(mf, app):
- yamlinfo = yaml.load(mf, Loader=YamlLoader)
- app.update_fields(yamlinfo)
- return app
+ # import rumael.yaml and check version
+ try:
+ import ruamel.yaml
+ except ImportError as e:
+ raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
+ if not ruamel.yaml.__version__:
+ raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
+ m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
+ ruamel.yaml.__version__)
+ if not m:
+ raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
+ if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
+ raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
+ # suiteable version ruamel.yaml imported successfully
+
+ _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
+ 'true', 'True', 'TRUE',
+ 'on', 'On', 'ON')
+ _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
+ 'false', 'False', 'FALSE',
+ 'off', 'Off', 'OFF')
+ _yaml_bools_plus_lists = []
+ _yaml_bools_plus_lists.extend(_yaml_bools_true)
+ _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
+ _yaml_bools_plus_lists.extend(_yaml_bools_false)
+ _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
+
+ def _class_as_dict_representer(dumper, data):
+ '''Creates a YAML representation of a App/Build instance'''
+ return dumper.represent_dict(data)
+
+ def _field_to_yaml(typ, value):
+ if typ is TYPE_STRING:
+ if value in _yaml_bools_plus_lists:
+ return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
+ return str(value)
+ elif typ is TYPE_INT:
+ return int(value)
+ elif typ is TYPE_MULTILINE:
+ if '\n' in value:
+ return ruamel.yaml.scalarstring.preserve_literal(str(value))
+ else:
+ return str(value)
+ elif typ is TYPE_SCRIPT:
+ if len(value) > 50:
+ return ruamel.yaml.scalarstring.preserve_literal(value)
+ else:
+ return value
+ else:
+ return value
+
+ def _app_to_yaml(app):
+ cm = ruamel.yaml.comments.CommentedMap()
+ insert_newline = False
+ for field in yaml_app_field_order:
+ if field is '\n':
+ # next iteration will need to insert a newline
+ insert_newline = True
+ else:
+ if app.get(field) or field is 'Builds':
+ # .txt calls it 'builds' internally, everywhere else its 'Builds'
+ if field is 'Builds':
+ if app.get('builds'):
+ cm.update({field: _builds_to_yaml(app)})
+ elif field is 'CurrentVersionCode':
+ cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
+ else:
+ cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
+
+ if insert_newline:
+ # we need to prepend a newline in front of this field
+ insert_newline = False
+ # inserting empty lines is not supported so we add a
+ # bogus comment and over-write its value
+ cm.yaml_set_comment_before_after_key(field, 'bogus')
+ cm.ca.items[field][1][-1].value = '\n'
+ return cm
+
+ def _builds_to_yaml(app):
+ fields = ['versionName', 'versionCode']
+ fields.extend(build_flags_order)
+ builds = ruamel.yaml.comments.CommentedSeq()
+ for build in app.builds:
+ b = ruamel.yaml.comments.CommentedMap()
+ for field in fields:
+ if hasattr(build, field) and getattr(build, field):
+ value = getattr(build, field)
+ if field == 'gradle' and value == ['off']:
+ value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
+ if field in ('disable', 'kivy', 'maven', 'buildozer'):
+ if value == 'no':
+ continue
+ elif value == 'yes':
+ value = 'yes'
+ b.update({field: _field_to_yaml(flagtype(field), value)})
+ builds.append(b)
+
+ # insert extra empty lines between build entries
+ for i in range(1, len(builds)):
+ builds.yaml_set_comment_before_after_key(i, 'bogus')
+ builds.ca.items[i][1][-1].value = '\n'
+
+ return builds
+
+ yaml_app_field_order = [
+ 'Disabled',
+ 'AntiFeatures',
+ 'Provides',
+ 'Categories',
+ 'License',
+ 'AuthorName',
+ 'AuthorEmail',
+ 'AuthorWebSite',
+ 'WebSite',
+ 'SourceCode',
+ 'IssueTracker',
+ 'Changelog',
+ 'Donate',
+ 'FlattrID',
+ 'Bitcoin',
+ 'Litecoin',
+ '\n',
+ 'Name',
+ 'AutoName',
+ 'Summary',
+ 'Description',
+ '\n',
+ 'RequiresRoot',
+ '\n',
+ 'RepoType',
+ 'Repo',
+ 'Binaries',
+ '\n',
+ 'Builds',
+ '\n',
+ 'MaintainerNotes',
+ '\n',
+ 'ArchivePolicy',
+ 'AutoUpdateMode',
+ 'UpdateCheckMode',
+ 'UpdateCheckIgnore',
+ 'VercodeOperation',
+ 'UpdateCheckName',
+ 'UpdateCheckData',
+ 'CurrentVersion',
+ 'CurrentVersionCode',
+ '\n',
+ 'NoSourceSince',
+ ]
+
+ yaml_app = _app_to_yaml(app)
+ ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
build_line_sep = re.compile(r'(?<!\\),')
def add_buildflag(p, build):
if not p.strip():
- raise MetaDataException("Empty build flag at {1}"
- .format(buildlines[0], linedesc))
+ warn_or_exception("Empty build flag at {1}"
+ .format(buildlines[0], linedesc))
bv = p.split('=', 1)
if len(bv) != 2:
- raise MetaDataException("Invalid build flag at {0} in {1}"
- .format(buildlines[0], linedesc))
+ warn_or_exception("Invalid build flag at {0} in {1}"
+ .format(buildlines[0], linedesc))
pk, pv = bv
pk = pk.lstrip()
+ if pk == 'update':
+ pk = 'androidupdate' # avoid conflicting with Build(dict).update()
t = flagtype(pk)
if t == TYPE_LIST:
pv = split_list_values(pv)
- build.set_flag(pk, pv)
+ build[pk] = pv
elif t == TYPE_STRING or t == TYPE_SCRIPT:
- build.set_flag(pk, pv)
+ build[pk] = pv
elif t == TYPE_BOOL:
- build.set_flag(pk, _decode_bool(pv))
+ build[pk] = _decode_bool(pv)
def parse_buildline(lines):
v = "".join(lines)
parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
if len(parts) < 3:
- raise MetaDataException("Invalid build format: " + v + " in " + mf.name)
+ warn_or_exception("Invalid build format: " + v + " in " + mf.name)
build = Build()
- build.version = parts[0]
- build.vercode = parts[1]
+ build.versionName = parts[0]
+ build.versionCode = parts[1]
+ check_versionCode(build.versionCode)
+
if parts[2].startswith('!'):
# For backwards compatibility, handle old-style disabling,
# including attempting to extract the commit from the message
return build
+ def check_versionCode(versionCode):
+ try:
+ int(versionCode)
+ except ValueError:
+ warn_or_exception('Invalid versionCode: "' + versionCode + '" is not an integer!')
+
def add_comments(key):
if not curcomments:
return
build = None
vc_seen = set()
+ app.builds = []
+
c = 0
for line in mf:
c += 1
del buildlines[:]
else:
if not build.commit and not build.disable:
- raise MetaDataException("No commit specified for {0} in {1}"
- .format(build.version, linedesc))
+ warn_or_exception("No commit specified for {0} in {1}"
+ .format(build.versionName, linedesc))
app.builds.append(build)
- add_comments('build:' + build.vercode)
+ add_comments('build:' + build.versionCode)
mode = 0
if mode == 0:
try:
f, v = line.split(':', 1)
except ValueError:
- raise MetaDataException("Invalid metadata in " + linedesc)
+ warn_or_exception("Invalid metadata in " + linedesc)
+
+ if f not in app_fields:
+ warn_or_exception('Unrecognised app field: ' + f)
# Translate obsolete fields...
if f == 'Market Version':
if f == 'Market Version Code':
f = 'Current Version Code'
+ f = f.replace(' ', '')
+
ftype = fieldtype(f)
if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
add_comments(f)
if ftype == TYPE_MULTILINE:
mode = 1
if v:
- raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
+ warn_or_exception("Unexpected text on same line as "
+ + f + " in " + linedesc)
elif ftype == TYPE_STRING:
- app.set_field(f, v)
+ app[f] = v
elif ftype == TYPE_LIST:
- app.set_field(f, split_list_values(v))
+ app[f] = split_list_values(v)
elif ftype == TYPE_BUILD:
if v.endswith("\\"):
mode = 2
else:
build = parse_buildline([v])
app.builds.append(build)
- add_comments('build:' + app.builds[-1].vercode)
+ add_comments('build:' + app.builds[-1].versionCode)
elif ftype == TYPE_BUILD_V2:
vv = v.split(',')
if len(vv) != 2:
- raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
- .format(v, linedesc))
+ warn_or_exception('Build should have comma-separated',
+ 'versionName and versionCode,',
+ 'not "{0}", in {1}'.format(v, linedesc))
build = Build()
- build.version = vv[0]
- build.vercode = vv[1]
- if build.vercode in vc_seen:
- raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
- build.vercode, linedesc))
- vc_seen.add(build.vercode)
+ build.versionName = vv[0]
+ build.versionCode = vv[1]
+ check_versionCode(build.versionCode)
+
+ if build.versionCode in vc_seen:
+ warn_or_exception('Duplicate build recipe found for versionCode %s in %s'
+ % (build.versionCode, linedesc))
+ vc_seen.add(build.versionCode)
del buildlines[:]
mode = 3
elif ftype == TYPE_OBSOLETE:
pass # Just throw it away!
else:
- raise MetaDataException("Unrecognised field '" + f + "' in " + linedesc)
+ warn_or_exception("Unrecognised field '" + f + "' in " + linedesc)
elif mode == 1: # Multiline field
if line == '.':
mode = 0
- app.set_field(f, '\n'.join(multiline_lines))
+ app[f] = '\n'.join(multiline_lines)
del multiline_lines[:]
else:
multiline_lines.append(line)
buildlines.append(line)
build = parse_buildline(buildlines)
app.builds.append(build)
- add_comments('build:' + app.builds[-1].vercode)
+ add_comments('build:' + app.builds[-1].versionCode)
mode = 0
add_comments(None)
# Mode at end of file should always be 0
if mode == 1:
- raise MetaDataException(f + " not terminated in " + mf.name)
+ warn_or_exception(f + " not terminated in " + mf.name)
if mode == 2:
- raise MetaDataException("Unterminated continuation in " + mf.name)
+ warn_or_exception("Unterminated continuation in " + mf.name)
if mode == 3:
- raise MetaDataException("Unterminated build in " + mf.name)
+ warn_or_exception("Unterminated build in " + mf.name)
return app
def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
+ def field_to_attr(f):
+ """
+ Translates human-readable field names to attribute names, e.g.
+ 'Auto Name' to 'AutoName'
+ """
+ return f.replace(' ', '')
+
+ def attr_to_field(k):
+ """
+ Translates attribute names to human-readable field names, e.g.
+ 'AutoName' to 'Auto Name'
+ """
+ if k in app_fields:
+ return k
+ f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
+ return f
+
def w_comments(key):
if key not in app.comments:
return
w_comment(line)
def w_field_always(f, v=None):
+ key = field_to_attr(f)
if v is None:
- v = app.get_field(f)
- w_comments(f)
+ v = app.get(key)
+ w_comments(key)
w_field(f, v)
def w_field_nonempty(f, v=None):
+ key = field_to_attr(f)
if v is None:
- v = app.get_field(f)
- w_comments(f)
+ v = app.get(key)
+ w_comments(key)
if v:
w_field(f, v)
w_field_always('License')
w_field_nonempty('Author Name')
w_field_nonempty('Author Email')
+ w_field_nonempty('Author Web Site')
w_field_always('Web Site')
w_field_always('Source Code')
w_field_always('Issue Tracker')
mf.write('\n')
w_field_nonempty('Name')
w_field_nonempty('Auto Name')
- w_field_always('Summary')
- w_field_always('Description', description_txt(app.Description))
+ w_field_nonempty('Summary')
+ w_field_nonempty('Description', description_txt(app.Description))
mf.write('\n')
if app.RequiresRoot:
w_field_always('Requires Root', 'yes')
for build in app.builds:
- if build.version == "Ignore":
+ if build.versionName == "Ignore":
continue
- w_comments('build:' + build.vercode)
+ w_comments('build:%s' % build.versionCode)
w_build(build)
mf.write('\n')
mf.write("%s:%s\n" % (f, v))
def w_build(build):
- mf.write("Build:%s,%s\n" % (build.version, build.vercode))
+ mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
for f in build_flags_order:
- v = build.get_flag(f)
+ v = build.get(f)
if not v:
continue
t = flagtype(f)
+ if f == 'androidupdate':
+ f = 'update' # avoid conflicting with Build(dict).update()
mf.write(' %s=' % f)
if t == TYPE_STRING:
mf.write(v)
first = False
else:
mf.write(' && \\\n ')
- mf.write(s)
+ mf.write(s.strip())
elif t == TYPE_LIST:
mf.write(','.join(v))
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-def write_yaml(mf, app):
-
- def w_comment(line):
- mf.write("# %s\n" % line)
-
- def escape(v):
- if not v:
- return ''
- if any(c in v for c in [': ', '%', '@', '*']):
- return "'" + v.replace("'", "''") + "'"
- return v
-
- def w_field(f, v, prefix='', t=None):
- if t is None:
- t = fieldtype(f)
- v = ''
- if t == TYPE_LIST:
- v = '\n'
- for e in v:
- v += prefix + ' - ' + escape(e) + '\n'
- elif t == TYPE_MULTILINE:
- v = ' |\n'
- for l in v.splitlines():
- if l:
- v += prefix + ' ' + l + '\n'
- else:
- v += '\n'
- elif t == TYPE_BOOL:
- v = ' yes\n'
- elif t == TYPE_SCRIPT:
- cmds = [s + '&& \\' for s in v.split('&& ')]
- if len(cmds) > 0:
- cmds[-1] = cmds[-1][:-len('&& \\')]
- w_field(f, cmds, prefix, 'multiline')
- return
- else:
- v = ' ' + escape(v) + '\n'
-
- mf.write(prefix)
- mf.write(f)
- mf.write(":")
- mf.write(v)
-
- global first_build
- first_build = True
-
- def w_build(build):
- global first_build
- if first_build:
- mf.write("builds:\n")
- first_build = False
-
- w_field('versionName', build.version, ' - ', TYPE_STRING)
- w_field('versionCode', build.vercode, ' ', TYPE_STRING)
- for f in build_flags_order:
- v = build.get_flag(f)
- if not v:
- continue
-
- w_field(f, v, ' ', flagtype(f))
-
- write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-
-
def write_metadata(metadatapath, app):
_, ext = fdroidserver.common.get_extension(metadatapath)
accepted = fdroidserver.common.config['accepted_formats']
if ext not in accepted:
- raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
- metadatapath, ', '.join(accepted)))
-
- with open(metadatapath, 'w', encoding='utf8') as mf:
- if ext == 'txt':
- return write_txt(mf, app)
- elif ext == 'yml':
- return write_yaml(mf, app)
- raise MetaDataException('Unknown metadata format: %s' % metadatapath)
+ warn_or_exception('Cannot write "%s", not an accepted format, use: %s'
+ % (metadatapath, ', '.join(accepted)))
+
+ try:
+ with open(metadatapath, 'w', encoding='utf8') as mf:
+ if ext == 'txt':
+ return write_txt(mf, app)
+ elif ext == 'yml':
+ return write_yaml(mf, app)
+ except FDroidException as e:
+ os.remove(metadatapath)
+ raise e
+
+ warn_or_exception('Unknown metadata format: %s' % metadatapath)
+
+
+def add_metadata_arguments(parser):
+ '''add common command line flags related to metadata processing'''
+ parser.add_argument("-W", default='error',
+ help="force errors to be warnings, or ignore")
~ianmdlvl / fdroidserver.git / blobdiff