import os
import re
import glob
-import cgi
+import html
import logging
import textwrap
import io
-
import yaml
# use libyaml if it is available
try:
YamlLoader = Loader
import fdroidserver.common
+from fdroidserver.exception import MetaDataException, FDroidException
srclibs = None
warnings_action = None
-class MetaDataException(Exception):
-
- def __init__(self, value):
- self.value = value
-
- def __str__(self):
- return self.value
-
-
def warn_or_exception(value):
'''output warning or Exception depending on -W'''
if warnings_action == 'ignore':
elif warnings_action == 'error':
raise MetaDataException(value)
else:
- logging.warn(value)
+ logging.warning(value)
# To filter which ones should be written to the metadata files if
'License',
'Author Name',
'Author Email',
+ 'Author Web Site',
'Web Site',
'Source Code',
'Issue Tracker',
self.License = 'Unknown'
self.AuthorName = None
self.AuthorEmail = None
+ self.AuthorWebSite = None
self.WebSite = ''
self.SourceCode = ''
self.IssueTracker = ''
TYPE_MULTILINE = 6
TYPE_BUILD = 7
TYPE_BUILD_V2 = 8
+TYPE_INT = 9
fieldtypes = {
'Description': TYPE_MULTILINE,
'commit',
'subdir',
'submodules',
+ 'sudo',
'init',
'patch',
'gradle',
'maven',
'kivy',
+ 'buildozer',
'output',
'srclibs',
'oldsdkloc',
'gradleprops',
'antcommands',
'novcheck',
+ 'antifeatures',
]
-
-build_flags = set(build_flags_order + ['versionName', 'versionCode'])
+# old .txt format has version name/code inline in the 'Build:' line
+# but YAML and JSON have a explicit key for them
+build_flags = ['versionName', 'versionCode'] + build_flags_order
class Build(dict):
self.commit = None
self.subdir = None
self.submodules = False
+ self.sudo = ''
self.init = ''
self.patch = []
self.gradle = []
self.maven = False
self.kivy = False
+ self.buildozer = False
self.output = None
self.srclibs = []
self.oldsdkloc = False
self.gradleprops = []
self.antcommands = []
self.novcheck = False
+ self.antifeatures = []
if copydict:
super().__init__(copydict)
return
raise AttributeError("No such attribute: " + name)
def build_method(self):
- for f in ['maven', 'gradle', 'kivy']:
+ for f in ['maven', 'gradle', 'kivy', 'buildozer']:
if self.get(f):
return f
if self.output:
def output_method(self):
if self.output:
return 'raw'
- for f in ['maven', 'gradle', 'kivy']:
+ for f in ['maven', 'gradle', 'kivy', 'buildozer']:
if self.get(f):
return f
return 'ant'
flagtypes = {
+ 'versionCode': TYPE_INT,
'extlibs': TYPE_LIST,
'srclibs': TYPE_LIST,
'patch': TYPE_LIST,
'gradle': TYPE_LIST,
'antcommands': TYPE_LIST,
'gradleprops': TYPE_LIST,
+ 'sudo': TYPE_SCRIPT,
'init': TYPE_SCRIPT,
'prebuild': TYPE_SCRIPT,
'build': TYPE_SCRIPT,
'forceversion': TYPE_BOOL,
'forcevercode': TYPE_BOOL,
'novcheck': TYPE_BOOL,
+ 'antifeatures': TYPE_LIST,
}
# Generic value types
valuetypes = {
- FieldValidator("Hexadecimal",
- r'^[0-9a-f]+$',
+ FieldValidator("Flattr ID",
+ r'^[0-9a-z]+$',
['FlattrID']),
FieldValidator("HTTP link",
["ArchivePolicy"]),
FieldValidator("Anti-Feature",
- r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln)$',
+ r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
["AntiFeatures"]),
FieldValidator("Auto Update Mode",
self.laststate = self.state
self.state = self.stNONE
- def formatted(self, txt, html):
+ def formatted(self, txt, htmlbody):
res = ''
- if html:
- txt = cgi.escape(txt)
+ if htmlbody:
+ txt = html.escape(txt, quote=False)
while True:
index = txt.find("''")
if index == -1:
res += txt[:index]
txt = txt[index:]
if txt.startswith("'''"):
- if html:
+ if htmlbody:
if self.bold:
res += '</b>'
else:
self.bold = not self.bold
txt = txt[3:]
else:
- if html:
+ if htmlbody:
if self.ital:
res += '</i>'
else:
url, urltext = self.linkResolver(url)
else:
urltext = url
- res_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
+ res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
res_plain += urltext
txt = txt[index + 2:]
else:
url = url[:index2]
if url == urltxt:
warn_or_exception("Url title is just the URL - use [url]")
- res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
+ res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
res_plain += urltxt
if urltxt != url:
res_plain += ' (' + url + ')'
for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
+ glob.glob(os.path.join('metadata', '*.json'))
+ glob.glob(os.path.join('metadata', '*.yml'))
+ + glob.glob('.fdroid.txt')
+ glob.glob('.fdroid.json')
+ glob.glob('.fdroid.yml')):
packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
if type(v) in (float, int):
app[k] = str(v)
+ if 'Builds' in app:
+ app['builds'] = app.pop('Builds')
+
+ if 'flavours' in app and app['flavours'] == [True]:
+ app['flavours'] = 'yes'
+
if isinstance(app.Categories, str):
app.Categories = [app.Categories]
elif app.Categories is None:
else:
app.Categories = [str(i) for i in app.Categories]
+ def _yaml_bool_unmapable(v):
+ return v in (True, False, [True], [False])
+
+ def _yaml_bool_unmap(v):
+ if v is True:
+ return 'yes'
+ elif v is False:
+ return 'no'
+ elif v == [True]:
+ return ['yes']
+ elif v == [False]:
+ return ['no']
+
+ _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
+
builds = []
if 'builds' in app:
for build in app['builds']:
if not isinstance(build, Build):
build = Build(build)
for k, v in build.items():
- if flagtype(k) == TYPE_LIST:
- if isinstance(v, str):
- build[k] = [v]
- elif isinstance(v, bool):
- if v:
- build[k] = ['yes']
+ if not (v is None):
+ if flagtype(k) == TYPE_LIST:
+ if _yaml_bool_unmapable(v):
+ build[k] = _yaml_bool_unmap(v)
+
+ if isinstance(v, str):
+ build[k] = [v]
+ elif isinstance(v, bool):
+ if v:
+ build[k] = ['yes']
+ else:
+ build[k] = []
+ elif flagtype(k) is TYPE_INT:
+ build[k] = str(v)
+ elif flagtype(k) is TYPE_STRING:
+ if isinstance(v, bool) and k in _bool_allowed:
+ build[k] = v
else:
- build[k] = []
- elif flagtype(k) == TYPE_STRING and type(v) in (float, int):
- build[k] = str(v)
+ if _yaml_bool_unmapable(v):
+ build[k] = _yaml_bool_unmap(v)
+ else:
+ build[k] = str(v)
builds.append(build)
app.builds = sorted_builds(builds)
def parse_yaml_metadata(mf, app):
-
- yamlinfo = yaml.load(mf, Loader=YamlLoader)
- app.update(yamlinfo)
+ yamldata = yaml.load(mf, Loader=YamlLoader)
+ if yamldata:
+ app.update(yamldata)
return app
+def write_yaml(mf, app):
+
+ # import rumael.yaml and check version
+ try:
+ import ruamel.yaml
+ except ImportError as e:
+ raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
+ if not ruamel.yaml.__version__:
+ raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
+ m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
+ ruamel.yaml.__version__)
+ if not m:
+ raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
+ if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
+ raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
+ # suiteable version ruamel.yaml imported successfully
+
+ _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
+ 'true', 'True', 'TRUE',
+ 'on', 'On', 'ON')
+ _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
+ 'false', 'False', 'FALSE',
+ 'off', 'Off', 'OFF')
+ _yaml_bools_plus_lists = []
+ _yaml_bools_plus_lists.extend(_yaml_bools_true)
+ _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
+ _yaml_bools_plus_lists.extend(_yaml_bools_false)
+ _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
+
+ def _class_as_dict_representer(dumper, data):
+ '''Creates a YAML representation of a App/Build instance'''
+ return dumper.represent_dict(data)
+
+ def _field_to_yaml(typ, value):
+ if typ is TYPE_STRING:
+ if value in _yaml_bools_plus_lists:
+ return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
+ return str(value)
+ elif typ is TYPE_INT:
+ return int(value)
+ elif typ is TYPE_MULTILINE:
+ if '\n' in value:
+ return ruamel.yaml.scalarstring.preserve_literal(str(value))
+ else:
+ return str(value)
+ elif typ is TYPE_SCRIPT:
+ if len(value) > 50:
+ return ruamel.yaml.scalarstring.preserve_literal(value)
+ else:
+ return value
+ else:
+ return value
+
+ def _app_to_yaml(app):
+ cm = ruamel.yaml.comments.CommentedMap()
+ insert_newline = False
+ for field in yaml_app_field_order:
+ if field is '\n':
+ # next iteration will need to insert a newline
+ insert_newline = True
+ else:
+ if app.get(field) or field is 'Builds':
+ # .txt calls it 'builds' internally, everywhere else its 'Builds'
+ if field is 'Builds':
+ if app.get('builds'):
+ cm.update({field: _builds_to_yaml(app)})
+ elif field is 'CurrentVersionCode':
+ cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
+ else:
+ cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
+
+ if insert_newline:
+ # we need to prepend a newline in front of this field
+ insert_newline = False
+ # inserting empty lines is not supported so we add a
+ # bogus comment and over-write its value
+ cm.yaml_set_comment_before_after_key(field, 'bogus')
+ cm.ca.items[field][1][-1].value = '\n'
+ return cm
+
+ def _builds_to_yaml(app):
+ fields = ['versionName', 'versionCode']
+ fields.extend(build_flags_order)
+ builds = ruamel.yaml.comments.CommentedSeq()
+ for build in app.builds:
+ b = ruamel.yaml.comments.CommentedMap()
+ for field in fields:
+ if hasattr(build, field) and getattr(build, field):
+ value = getattr(build, field)
+ if field == 'gradle' and value == ['off']:
+ value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
+ if field in ('disable', 'kivy', 'maven', 'buildozer'):
+ if value == 'no':
+ continue
+ elif value == 'yes':
+ value = 'yes'
+ b.update({field: _field_to_yaml(flagtype(field), value)})
+ builds.append(b)
+
+ # insert extra empty lines between build entries
+ for i in range(1, len(builds)):
+ builds.yaml_set_comment_before_after_key(i, 'bogus')
+ builds.ca.items[i][1][-1].value = '\n'
+
+ return builds
+
+ yaml_app_field_order = [
+ 'Disabled',
+ 'AntiFeatures',
+ 'Provides',
+ 'Categories',
+ 'License',
+ 'AuthorName',
+ 'AuthorEmail',
+ 'AuthorWebSite',
+ 'WebSite',
+ 'SourceCode',
+ 'IssueTracker',
+ 'Changelog',
+ 'Donate',
+ 'FlattrID',
+ 'Bitcoin',
+ 'Litecoin',
+ '\n',
+ 'Name',
+ 'AutoName',
+ 'Summary',
+ 'Description',
+ '\n',
+ 'RequiresRoot',
+ '\n',
+ 'RepoType',
+ 'Repo',
+ 'Binaries',
+ '\n',
+ 'Builds',
+ '\n',
+ 'MaintainerNotes',
+ '\n',
+ 'ArchivePolicy',
+ 'AutoUpdateMode',
+ 'UpdateCheckMode',
+ 'UpdateCheckIgnore',
+ 'VercodeOperation',
+ 'UpdateCheckName',
+ 'UpdateCheckData',
+ 'CurrentVersion',
+ 'CurrentVersionCode',
+ '\n',
+ 'NoSourceSince',
+ ]
+
+ yaml_app = _app_to_yaml(app)
+ ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
+
+
build_line_sep = re.compile(r'(?<!\\),')
build_cont = re.compile(r'^[ \t]')
w_field_always('License')
w_field_nonempty('Author Name')
w_field_nonempty('Author Email')
+ w_field_nonempty('Author Web Site')
w_field_always('Web Site')
w_field_always('Source Code')
w_field_always('Issue Tracker')
mf.write('\n')
w_field_nonempty('Name')
w_field_nonempty('Auto Name')
- w_field_always('Summary')
- w_field_always('Description', description_txt(app.Description))
+ w_field_nonempty('Summary')
+ w_field_nonempty('Description', description_txt(app.Description))
mf.write('\n')
if app.RequiresRoot:
w_field_always('Requires Root', 'yes')
first = False
else:
mf.write(' && \\\n ')
- mf.write(s)
+ mf.write(s.strip())
elif t == TYPE_LIST:
mf.write(','.join(v))
write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-def write_yaml(mf, app):
-
- def w_comment(line):
- mf.write("# %s\n" % line)
-
- def escape(v):
- if not v:
- return ''
- if any(c in v for c in [': ', '%', '@', '*']):
- return "'" + v.replace("'", "''") + "'"
- return v
-
- def w_field(f, v, prefix='', t=None):
- if t is None:
- t = fieldtype(f)
- v = ''
- if t == TYPE_LIST:
- v = '\n'
- for e in v:
- v += prefix + ' - ' + escape(e) + '\n'
- elif t == TYPE_MULTILINE:
- v = ' |\n'
- for l in v.splitlines():
- if l:
- v += prefix + ' ' + l + '\n'
- else:
- v += '\n'
- elif t == TYPE_BOOL:
- v = ' yes\n'
- elif t == TYPE_SCRIPT:
- cmds = [s + '&& \\' for s in v.split('&& ')]
- if len(cmds) > 0:
- cmds[-1] = cmds[-1][:-len('&& \\')]
- w_field(f, cmds, prefix, 'multiline')
- return
- else:
- v = ' ' + escape(v) + '\n'
-
- mf.write(prefix)
- mf.write(f)
- mf.write(":")
- mf.write(v)
-
- global first_build
- first_build = True
-
- def w_build(build):
- global first_build
- if first_build:
- mf.write("builds:\n")
- first_build = False
-
- w_field('versionName', build.versionName, ' - ', TYPE_STRING)
- w_field('versionCode', build.versionCode, ' ', TYPE_STRING)
- for f in build_flags_order:
- v = build.get(f)
- if not v:
- continue
-
- w_field(f, v, ' ', flagtype(f))
-
- write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
-
-
def write_metadata(metadatapath, app):
_, ext = fdroidserver.common.get_extension(metadatapath)
accepted = fdroidserver.common.config['accepted_formats']
warn_or_exception('Cannot write "%s", not an accepted format, use: %s'
% (metadatapath, ', '.join(accepted)))
- with open(metadatapath, 'w', encoding='utf8') as mf:
- if ext == 'txt':
- return write_txt(mf, app)
- elif ext == 'yml':
- return write_yaml(mf, app)
+ try:
+ with open(metadatapath, 'w', encoding='utf8') as mf:
+ if ext == 'txt':
+ return write_txt(mf, app)
+ elif ext == 'yml':
+ return write_yaml(mf, app)
+ except FDroidException as e:
+ os.remove(metadatapath)
+ raise e
+
warn_or_exception('Unknown metadata format: %s' % metadatapath)
~ianmdlvl / fdroidserver.git / blobdiff