3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 from collections import OrderedDict
30 # use libyaml if it is available
32 from yaml import CLoader
35 from yaml import Loader
38 import fdroidserver.common
39 from fdroidserver import _
40 from fdroidserver.exception import MetaDataException, FDroidException
43 warnings_action = None
46 def warn_or_exception(value):
47 '''output warning or Exception depending on -W'''
48 if warnings_action == 'ignore':
50 elif warnings_action == 'error':
51 raise MetaDataException(value)
53 logging.warning(value)
56 # To filter which ones should be written to the metadata files if
88 'Update Check Ignore',
93 'Current Version Code',
97 'comments', # For formats that don't do inline comments
98 'builds', # For formats that do builds as a list
104 def __init__(self, copydict=None):
106 super().__init__(copydict)
111 self.AntiFeatures = []
113 self.Categories = ['None']
114 self.License = 'Unknown'
115 self.AuthorName = None
116 self.AuthorEmail = None
117 self.AuthorWebSite = None
120 self.IssueTracker = ''
124 self.LiberapayID = None
130 self.Description = ''
131 self.RequiresRoot = False
135 self.MaintainerNotes = ''
136 self.ArchivePolicy = None
137 self.AutoUpdateMode = 'None'
138 self.UpdateCheckMode = 'None'
139 self.UpdateCheckIgnore = None
140 self.VercodeOperation = None
141 self.UpdateCheckName = None
142 self.UpdateCheckData = None
143 self.CurrentVersion = ''
144 self.CurrentVersionCode = None
145 self.NoSourceSince = ''
148 self.metadatapath = None
152 self.lastUpdated = None
154 def __getattr__(self, name):
158 raise AttributeError("No such attribute: " + name)
160 def __setattr__(self, name, value):
163 def __delattr__(self, name):
167 raise AttributeError("No such attribute: " + name)
169 def get_last_build(self):
170 if len(self.builds) > 0:
171 return self.builds[-1]
188 'Description': TYPE_MULTILINE,
189 'MaintainerNotes': TYPE_MULTILINE,
190 'Categories': TYPE_LIST,
191 'AntiFeatures': TYPE_LIST,
192 'BuildVersion': TYPE_BUILD,
193 'Build': TYPE_BUILD_V2,
194 'UseBuilt': TYPE_OBSOLETE,
199 name = name.replace(' ', '')
200 if name in fieldtypes:
201 return fieldtypes[name]
205 # In the order in which they are laid out on files
206 build_flags_order = [
240 # old .txt format has version name/code inline in the 'Build:' line
241 # but YAML and JSON have a explicit key for them
242 build_flags = ['versionName', 'versionCode'] + build_flags_order
247 def __init__(self, copydict=None):
252 self.submodules = False
258 self.buildozer = False
261 self.oldsdkloc = False
263 self.forceversion = False
264 self.forcevercode = False
268 self.androidupdate = []
275 self.preassemble = []
276 self.gradleprops = []
277 self.antcommands = []
278 self.novcheck = False
279 self.antifeatures = []
281 super().__init__(copydict)
284 def __getattr__(self, name):
288 raise AttributeError("No such attribute: " + name)
290 def __setattr__(self, name, value):
293 def __delattr__(self, name):
297 raise AttributeError("No such attribute: " + name)
299 def build_method(self):
300 for f in ['maven', 'gradle', 'buildozer']:
307 # like build_method, but prioritize output=
308 def output_method(self):
311 for f in ['maven', 'gradle', 'buildozer']:
319 version = 'r12b' # falls back to latest
320 paths = fdroidserver.common.config['ndk_paths']
321 if version not in paths:
323 return paths[version]
327 'versionCode': TYPE_INT,
328 'extlibs': TYPE_LIST,
329 'srclibs': TYPE_LIST,
332 'buildjni': TYPE_LIST,
333 'preassemble': TYPE_LIST,
334 'androidupdate': TYPE_LIST,
335 'scanignore': TYPE_LIST,
336 'scandelete': TYPE_LIST,
338 'antcommands': TYPE_LIST,
339 'gradleprops': TYPE_LIST,
342 'prebuild': TYPE_SCRIPT,
343 'build': TYPE_SCRIPT,
344 'submodules': TYPE_BOOL,
345 'oldsdkloc': TYPE_BOOL,
346 'forceversion': TYPE_BOOL,
347 'forcevercode': TYPE_BOOL,
348 'novcheck': TYPE_BOOL,
349 'antifeatures': TYPE_LIST,
354 if name in flagtypes:
355 return flagtypes[name]
359 class FieldValidator():
361 Designates App metadata field types and checks that it matches
363 'name' - The long name of the field type
364 'matching' - List of possible values or regex expression
365 'sep' - Separator to use if value may be a list
366 'fields' - Metadata fields (Field:Value) of this type
369 def __init__(self, name, matching, fields):
371 self.matching = matching
372 self.compiled = re.compile(matching)
375 def check(self, v, appid):
383 if not self.compiled.match(v):
384 warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
385 .format(value=v, field=self.name, appid=appid, pattern=self.matching))
388 # Generic value types
390 FieldValidator("Flattr ID",
394 FieldValidator("Liberapay ID",
398 FieldValidator("HTTP link",
400 ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
402 FieldValidator("Email",
403 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
406 FieldValidator("Bitcoin address",
407 r'^[a-zA-Z0-9]{27,34}$',
410 FieldValidator("Litecoin address",
411 r'^L[a-zA-Z0-9]{33}$',
414 FieldValidator("Repo Type",
415 r'^(git|git-svn|svn|hg|bzr|srclib)$',
418 FieldValidator("Binaries",
422 FieldValidator("Archive Policy",
423 r'^[0-9]+ versions$',
426 FieldValidator("Anti-Feature",
427 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
430 FieldValidator("Auto Update Mode",
431 r"^(Version .+|None)$",
434 FieldValidator("Update Check Mode",
435 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
440 # Check an app's metadata information for integrity errors
441 def check_metadata(app):
444 v.check(app[k], app.id)
447 # Formatter for descriptions. Create an instance, and call parseline() with
448 # each line of the description source from the metadata. At the end, call
449 # end() and then text_txt and text_html will contain the result.
450 class DescriptionFormatter:
457 def __init__(self, linkres):
460 self.state = self.stNONE
461 self.laststate = self.stNONE
464 self.html = io.StringIO()
465 self.text = io.StringIO()
467 self.linkResolver = None
468 self.linkResolver = linkres
470 def endcur(self, notstates=None):
471 if notstates and self.state in notstates:
473 if self.state == self.stPARA:
475 elif self.state == self.stUL:
477 elif self.state == self.stOL:
481 self.laststate = self.state
482 self.state = self.stNONE
483 whole_para = ' '.join(self.para_lines)
484 self.addtext(whole_para)
485 wrapped = textwrap.fill(whole_para, 80,
486 break_long_words=False,
487 break_on_hyphens=False)
488 self.text.write(wrapped)
489 self.html.write('</p>')
490 del self.para_lines[:]
493 self.html.write('</ul>')
494 self.laststate = self.state
495 self.state = self.stNONE
498 self.html.write('</ol>')
499 self.laststate = self.state
500 self.state = self.stNONE
502 def formatted(self, txt, htmlbody):
505 txt = html.escape(txt, quote=False)
507 index = txt.find("''")
512 if txt.startswith("'''"):
518 self.bold = not self.bold
526 self.ital = not self.ital
529 def linkify(self, txt):
533 index = txt.find("[")
535 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
536 res_plain += self.formatted(txt[:index], False)
537 res_html += self.formatted(txt[:index], True)
539 if txt.startswith("[["):
540 index = txt.find("]]")
542 warn_or_exception(_("Unterminated ]]"))
544 if self.linkResolver:
545 url, urltext = self.linkResolver(url)
548 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
550 txt = txt[index + 2:]
552 index = txt.find("]")
554 warn_or_exception(_("Unterminated ]"))
556 index2 = url.find(' ')
560 urltxt = url[index2 + 1:]
563 warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
564 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
567 res_plain += ' (' + url + ')'
568 txt = txt[index + 1:]
570 def addtext(self, txt):
571 p, h = self.linkify(txt)
574 def parseline(self, line):
577 elif line.startswith('* '):
578 self.endcur([self.stUL])
579 if self.state != self.stUL:
580 self.html.write('<ul>')
581 self.state = self.stUL
582 if self.laststate != self.stNONE:
583 self.text.write('\n\n')
585 self.text.write('\n')
586 self.text.write(line)
587 self.html.write('<li>')
588 self.addtext(line[1:])
589 self.html.write('</li>')
590 elif line.startswith('# '):
591 self.endcur([self.stOL])
592 if self.state != self.stOL:
593 self.html.write('<ol>')
594 self.state = self.stOL
595 if self.laststate != self.stNONE:
596 self.text.write('\n\n')
598 self.text.write('\n')
599 self.text.write(line)
600 self.html.write('<li>')
601 self.addtext(line[1:])
602 self.html.write('</li>')
604 self.para_lines.append(line)
605 self.endcur([self.stPARA])
606 if self.state == self.stNONE:
607 self.state = self.stPARA
608 if self.laststate != self.stNONE:
609 self.text.write('\n\n')
610 self.html.write('<p>')
614 self.text_txt = self.text.getvalue()
615 self.text_html = self.html.getvalue()
620 # Parse multiple lines of description as written in a metadata file, returning
621 # a single string in text format and wrapped to 80 columns.
622 def description_txt(s):
623 ps = DescriptionFormatter(None)
624 for line in s.splitlines():
630 # Parse multiple lines of description as written in a metadata file, returning
631 # a single string in wiki format. Used for the Maintainer Notes field as well,
632 # because it's the same format.
633 def description_wiki(s):
637 # Parse multiple lines of description as written in a metadata file, returning
638 # a single string in HTML format.
639 def description_html(s, linkres):
640 ps = DescriptionFormatter(linkres)
641 for line in s.splitlines():
647 def parse_srclib(metadatapath):
651 # Defaults for fields that come from metadata
652 thisinfo['Repo Type'] = ''
653 thisinfo['Repo'] = ''
654 thisinfo['Subdir'] = None
655 thisinfo['Prepare'] = None
657 if not os.path.exists(metadatapath):
660 metafile = open(metadatapath, "r", encoding='utf-8')
663 for line in metafile:
665 line = line.rstrip('\r\n')
666 if not line or line.startswith("#"):
670 f, v = line.split(':', 1)
672 warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
675 thisinfo[f] = v.split(',')
685 """Read all srclib metadata.
687 The information read will be accessible as metadata.srclibs, which is a
688 dictionary, keyed on srclib name, with the values each being a dictionary
689 in the same format as that returned by the parse_srclib function.
691 A MetaDataException is raised if there are any problems with the srclib
696 # They were already loaded
697 if srclibs is not None:
703 if not os.path.exists(srcdir):
706 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
707 srclibname = os.path.basename(metadatapath[:-4])
708 srclibs[srclibname] = parse_srclib(metadatapath)
711 def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
712 """Return a list of App instances sorted newest first
714 This reads all of the metadata files in a 'data' repository, then
715 builds a list of App instances from those files. The list is
716 sorted based on creation time, newest first. Most of the time,
717 the newer files are the most interesting.
719 If there are multiple metadata files for a single appid, then the first
720 file that is parsed wins over all the others, and the rest throw an
721 exception. So the original .txt format is parsed first, at least until
722 newer formats stabilize.
724 check_vcs is the list of appids to check for .fdroid.yml in source
728 # Always read the srclibs before the apps, since they can use a srlib as
729 # their source repository.
734 for basedir in ('metadata', 'tmp'):
735 if not os.path.exists(basedir):
738 metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
739 + glob.glob(os.path.join('metadata', '*.json'))
740 + glob.glob(os.path.join('metadata', '*.yml'))
741 + glob.glob('.fdroid.txt')
742 + glob.glob('.fdroid.json')
743 + glob.glob('.fdroid.yml'))
746 entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
748 for _ignored, path in sorted(entries, reverse=True):
749 metadatafiles.append(path)
751 # most things want the index alpha sorted for stability
752 metadatafiles = sorted(metadatafiles)
754 for metadatapath in metadatafiles:
755 if metadatapath == '.fdroid.txt':
756 warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'))
757 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
759 warn_or_exception(_("Found multiple metadata files for {appid}")
760 .format(appid=appid))
761 app = parse_metadata(metadatapath, appid in check_vcs, refresh)
766 # Parse all descriptions at load time, just to ensure cross-referencing
767 # errors are caught early rather than when they hit the build server.
770 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
771 warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid))
773 for appid, app in apps.items():
775 description_html(app.Description, linkres)
776 except MetaDataException as e:
777 warn_or_exception(_("Problem with description of {appid}: {error}")
778 .format(appid=appid, error=str(e)))
783 # Port legacy ';' separators
784 list_sep = re.compile(r'[,;]')
787 def split_list_values(s):
789 for v in re.split(list_sep, s):
799 def get_default_app_info(metadatapath=None):
800 if metadatapath is None:
803 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
805 if appid == '.fdroid': # we have local metadata in the app's source
806 if os.path.exists('AndroidManifest.xml'):
807 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
809 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
810 for root, dirs, files in os.walk(os.getcwd()):
811 if 'build.gradle' in files:
812 p = os.path.join(root, 'build.gradle')
813 with open(p, 'rb') as f:
815 m = pattern.search(data)
817 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
818 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
820 if manifestroot is None:
821 warn_or_exception(_("Cannot find an appid for {path}!")
822 .format(path=metadatapath))
823 appid = manifestroot.attrib['package']
826 app.metadatapath = metadatapath
827 if appid is not None:
833 def sorted_builds(builds):
834 return sorted(builds, key=lambda build: int(build.versionCode))
837 esc_newlines = re.compile(r'\\( |\n)')
840 def post_metadata_parse(app):
841 # TODO keep native types, convert only for .txt metadata
842 for k, v in app.items():
843 if type(v) in (float, int):
847 app['builds'] = app.pop('Builds')
849 if 'flavours' in app and app['flavours'] == [True]:
850 app['flavours'] = 'yes'
852 if isinstance(app.Categories, str):
853 app.Categories = [app.Categories]
854 elif app.Categories is None:
855 app.Categories = ['None']
857 app.Categories = [str(i) for i in app.Categories]
859 def _yaml_bool_unmapable(v):
860 return v in (True, False, [True], [False])
862 def _yaml_bool_unmap(v):
872 _bool_allowed = ('disable', 'maven', 'buildozer')
876 for build in app['builds']:
877 if not isinstance(build, Build):
879 for k, v in build.items():
881 if flagtype(k) == TYPE_LIST:
882 if _yaml_bool_unmapable(v):
883 build[k] = _yaml_bool_unmap(v)
885 if isinstance(v, str):
887 elif isinstance(v, bool):
892 elif flagtype(k) is TYPE_INT:
894 elif flagtype(k) is TYPE_STRING:
895 if isinstance(v, bool) and k in _bool_allowed:
898 if _yaml_bool_unmapable(v):
899 build[k] = _yaml_bool_unmap(v)
904 app.builds = sorted_builds(builds)
907 # Parse metadata for a single application.
909 # 'metadatapath' - the filename to read. The "Application ID" aka
910 # "Package Name" for the application comes from this
911 # filename. Pass None to get a blank entry.
913 # Returns a dictionary containing all the details of the application. There are
914 # two major kinds of information in the dictionary. Keys beginning with capital
915 # letters correspond directory to identically named keys in the metadata file.
916 # Keys beginning with lower case letters are generated in one way or another,
917 # and are not found verbatim in the metadata.
919 # Known keys not originating from the metadata are:
921 # 'builds' - a list of dictionaries containing build information
922 # for each defined build
923 # 'comments' - a list of comments from the metadata file. Each is
924 # a list of the form [field, comment] where field is
925 # the name of the field it preceded in the metadata
926 # file. Where field is None, the comment goes at the
927 # end of the file. Alternatively, 'build:version' is
928 # for a comment before a particular build version.
929 # 'descriptionlines' - original lines of description as formatted in the
934 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
935 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
939 if bool_true.match(s):
941 if bool_false.match(s):
943 warn_or_exception(_("Invalid boolean '%s'") % s)
946 def parse_metadata(metadatapath, check_vcs=False, refresh=True):
947 '''parse metadata file, optionally checking the git repo for metadata first'''
949 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
950 accepted = fdroidserver.common.config['accepted_formats']
951 if ext not in accepted:
952 warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
953 .format(path=metadatapath, formats=', '.join(accepted)))
956 app.metadatapath = metadatapath
957 name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
958 if name == '.fdroid':
963 with open(metadatapath, 'r', encoding='utf-8') as mf:
965 parse_txt_metadata(mf, app)
967 parse_json_metadata(mf, app)
969 parse_yaml_metadata(mf, app)
971 warn_or_exception(_('Unknown metadata format: {path}')
972 .format(path=metadatapath))
974 if check_vcs and app.Repo:
975 build_dir = fdroidserver.common.get_build_dir(app)
976 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
977 if not os.path.isfile(metadata_in_repo):
978 vcs, build_dir = fdroidserver.common.setup_vcs(app)
979 if isinstance(vcs, fdroidserver.common.vcs_git):
980 vcs.gotorevision('HEAD', refresh) # HEAD since we can't know where else to go
981 if os.path.isfile(metadata_in_repo):
982 logging.debug('Including metadata from ' + metadata_in_repo)
983 # do not include fields already provided by main metadata file
984 app_in_repo = parse_metadata(metadata_in_repo)
985 for k, v in app_in_repo.items():
989 post_metadata_parse(app)
993 build = app.builds[-1]
995 root_dir = build.subdir
998 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
999 _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
1004 def parse_json_metadata(mf, app):
1006 # fdroid metadata is only strings and booleans, no floats or ints.
1007 # TODO create schema using https://pypi.python.org/pypi/jsonschema
1008 jsoninfo = json.load(mf, parse_int=lambda s: s,
1009 parse_float=lambda s: s)
1010 app.update(jsoninfo)
1011 for f in ['Description', 'Maintainer Notes']:
1014 app[f] = '\n'.join(v)
1018 def parse_yaml_metadata(mf, app):
1019 yamldata = yaml.load(mf, Loader=YamlLoader)
1021 app.update(yamldata)
1025 def write_yaml(mf, app):
1027 # import rumael.yaml and check version
1030 except ImportError as e:
1031 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1032 if not ruamel.yaml.__version__:
1033 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1034 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1035 ruamel.yaml.__version__)
1037 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1038 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1039 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1040 # suiteable version ruamel.yaml imported successfully
1042 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1043 'true', 'True', 'TRUE',
1045 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1046 'false', 'False', 'FALSE',
1047 'off', 'Off', 'OFF')
1048 _yaml_bools_plus_lists = []
1049 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1050 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1051 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1052 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1054 def _class_as_dict_representer(dumper, data):
1055 '''Creates a YAML representation of a App/Build instance'''
1056 return dumper.represent_dict(data)
1058 def _field_to_yaml(typ, value):
1059 if typ is TYPE_STRING:
1060 if value in _yaml_bools_plus_lists:
1061 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1063 elif typ is TYPE_INT:
1065 elif typ is TYPE_MULTILINE:
1067 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1070 elif typ is TYPE_SCRIPT:
1072 return ruamel.yaml.scalarstring.preserve_literal(value)
1078 def _app_to_yaml(app):
1079 cm = ruamel.yaml.comments.CommentedMap()
1080 insert_newline = False
1081 for field in yaml_app_field_order:
1083 # next iteration will need to insert a newline
1084 insert_newline = True
1086 if app.get(field) or field is 'Builds':
1087 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1088 if field is 'Builds':
1089 if app.get('builds'):
1090 cm.update({field: _builds_to_yaml(app)})
1091 elif field is 'CurrentVersionCode':
1092 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1094 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1097 # we need to prepend a newline in front of this field
1098 insert_newline = False
1099 # inserting empty lines is not supported so we add a
1100 # bogus comment and over-write its value
1101 cm.yaml_set_comment_before_after_key(field, 'bogus')
1102 cm.ca.items[field][1][-1].value = '\n'
1105 def _builds_to_yaml(app):
1106 fields = ['versionName', 'versionCode']
1107 fields.extend(build_flags_order)
1108 builds = ruamel.yaml.comments.CommentedSeq()
1109 for build in app.builds:
1110 b = ruamel.yaml.comments.CommentedMap()
1111 for field in fields:
1112 if hasattr(build, field) and getattr(build, field):
1113 value = getattr(build, field)
1114 if field == 'gradle' and value == ['off']:
1115 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1116 if field in ('disable', 'maven', 'buildozer'):
1119 elif value == 'yes':
1121 b.update({field: _field_to_yaml(flagtype(field), value)})
1124 # insert extra empty lines between build entries
1125 for i in range(1, len(builds)):
1126 builds.yaml_set_comment_before_after_key(i, 'bogus')
1127 builds.ca.items[i][1][-1].value = '\n'
1131 yaml_app_field_order = [
1168 'UpdateCheckIgnore',
1173 'CurrentVersionCode',
1178 yaml_app = _app_to_yaml(app)
1179 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1182 build_line_sep = re.compile(r'(?<!\\),')
1183 build_cont = re.compile(r'^[ \t]')
1186 def parse_txt_metadata(mf, app):
1190 def add_buildflag(p, build):
1192 warn_or_exception(_("Empty build flag at {linedesc}")
1193 .format(linedesc=linedesc))
1194 bv = p.split('=', 1)
1196 warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
1197 .format(line=buildlines[0], linedesc=linedesc))
1202 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1205 pv = split_list_values(pv)
1207 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1209 elif t == TYPE_BOOL:
1210 build[pk] = _decode_bool(pv)
1212 def parse_buildline(lines):
1214 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1216 warn_or_exception(_("Invalid build format: {value} in {name}")
1217 .format(value=v, name=mf.name))
1219 build.versionName = parts[0]
1220 build.versionCode = parts[1]
1221 check_versionCode(build.versionCode)
1223 if parts[2].startswith('!'):
1224 # For backwards compatibility, handle old-style disabling,
1225 # including attempting to extract the commit from the message
1226 build.disable = parts[2][1:]
1227 commit = 'unknown - see disabled'
1228 index = parts[2].rfind('at ')
1230 commit = parts[2][index + 3:]
1231 if commit.endswith(')'):
1232 commit = commit[:-1]
1233 build.commit = commit
1235 build.commit = parts[2]
1237 add_buildflag(p, build)
1241 def check_versionCode(versionCode):
1245 warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
1246 .format(versionCode=versionCode))
1248 def add_comments(key):
1251 app.comments[key] = list(curcomments)
1256 multiline_lines = []
1266 linedesc = "%s:%d" % (mf.name, c)
1267 line = line.rstrip('\r\n')
1269 if build_cont.match(line):
1270 if line.endswith('\\'):
1271 buildlines.append(line[:-1].lstrip())
1273 buildlines.append(line.lstrip())
1274 bl = ''.join(buildlines)
1275 add_buildflag(bl, build)
1278 if not build.commit and not build.disable:
1279 warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
1280 .format(versionName=build.versionName, linedesc=linedesc))
1282 app.builds.append(build)
1283 add_comments('build:' + build.versionCode)
1289 if line.startswith("#"):
1290 curcomments.append(line[1:].strip())
1293 f, v = line.split(':', 1)
1295 warn_or_exception(_("Invalid metadata in: ") + linedesc)
1297 if f not in app_fields:
1298 warn_or_exception(_('Unrecognised app field: ') + f)
1300 # Translate obsolete fields...
1301 if f == 'Market Version':
1302 f = 'Current Version'
1303 if f == 'Market Version Code':
1304 f = 'Current Version Code'
1306 f = f.replace(' ', '')
1308 ftype = fieldtype(f)
1309 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1311 if ftype == TYPE_MULTILINE:
1314 warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
1315 .format(field=f, linedesc=linedesc))
1316 elif ftype == TYPE_STRING:
1318 elif ftype == TYPE_LIST:
1319 app[f] = split_list_values(v)
1320 elif ftype == TYPE_BUILD:
1321 if v.endswith("\\"):
1324 buildlines.append(v[:-1])
1326 build = parse_buildline([v])
1327 app.builds.append(build)
1328 add_comments('build:' + app.builds[-1].versionCode)
1329 elif ftype == TYPE_BUILD_V2:
1332 warn_or_exception(_('Build should have comma-separated '
1333 'versionName and versionCode, '
1334 'not "{value}", in {linedesc}')
1335 .format(value=v, linedesc=linedesc))
1337 build.versionName = vv[0]
1338 build.versionCode = vv[1]
1339 check_versionCode(build.versionCode)
1341 if build.versionCode in vc_seen:
1342 warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
1343 .format(versionCode=build.versionCode, linedesc=linedesc))
1344 vc_seen.add(build.versionCode)
1347 elif ftype == TYPE_OBSOLETE:
1348 pass # Just throw it away!
1350 warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
1351 .format(field=f, linedesc=linedesc))
1352 elif mode == 1: # Multiline field
1355 app[f] = '\n'.join(multiline_lines)
1356 del multiline_lines[:]
1358 multiline_lines.append(line)
1359 elif mode == 2: # Line continuation mode in Build Version
1360 if line.endswith("\\"):
1361 buildlines.append(line[:-1])
1363 buildlines.append(line)
1364 build = parse_buildline(buildlines)
1365 app.builds.append(build)
1366 add_comments('build:' + app.builds[-1].versionCode)
1370 # Mode at end of file should always be 0
1372 warn_or_exception(_("{field} not terminated in {name}")
1373 .format(field=f, name=mf.name))
1375 warn_or_exception(_("Unterminated continuation in {name}")
1376 .format(name=mf.name))
1378 warn_or_exception(_("Unterminated build in {name}")
1379 .format(name=mf.name))
1384 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1386 def field_to_attr(f):
1388 Translates human-readable field names to attribute names, e.g.
1389 'Auto Name' to 'AutoName'
1391 return f.replace(' ', '')
1393 def attr_to_field(k):
1395 Translates attribute names to human-readable field names, e.g.
1396 'AutoName' to 'Auto Name'
1400 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1403 def w_comments(key):
1404 if key not in app.comments:
1406 for line in app.comments[key]:
1409 def w_field_always(f, v=None):
1410 key = field_to_attr(f)
1416 def w_field_nonempty(f, v=None):
1417 key = field_to_attr(f)
1424 w_field_nonempty('Disabled')
1425 w_field_nonempty('AntiFeatures')
1426 w_field_nonempty('Provides')
1427 w_field_always('Categories')
1428 w_field_always('License')
1429 w_field_nonempty('Author Name')
1430 w_field_nonempty('Author Email')
1431 w_field_nonempty('Author Web Site')
1432 w_field_always('Web Site')
1433 w_field_always('Source Code')
1434 w_field_always('Issue Tracker')
1435 w_field_nonempty('Changelog')
1436 w_field_nonempty('Donate')
1437 w_field_nonempty('FlattrID')
1438 w_field_nonempty('LiberapayID')
1439 w_field_nonempty('Bitcoin')
1440 w_field_nonempty('Litecoin')
1442 w_field_nonempty('Name')
1443 w_field_nonempty('Auto Name')
1444 w_field_nonempty('Summary')
1445 w_field_nonempty('Description', description_txt(app.Description))
1447 if app.RequiresRoot:
1448 w_field_always('Requires Root', 'yes')
1451 w_field_always('Repo Type')
1452 w_field_always('Repo')
1454 w_field_always('Binaries')
1457 for build in app.builds:
1459 if build.versionName == "Ignore":
1462 w_comments('build:%s' % build.versionCode)
1466 if app.MaintainerNotes:
1467 w_field_always('Maintainer Notes', app.MaintainerNotes)
1470 w_field_nonempty('Archive Policy')
1471 w_field_always('Auto Update Mode')
1472 w_field_always('Update Check Mode')
1473 w_field_nonempty('Update Check Ignore')
1474 w_field_nonempty('Vercode Operation')
1475 w_field_nonempty('Update Check Name')
1476 w_field_nonempty('Update Check Data')
1477 if app.CurrentVersion:
1478 w_field_always('Current Version')
1479 w_field_always('Current Version Code')
1480 if app.NoSourceSince:
1482 w_field_always('No Source Since')
1486 # Write a metadata file in txt format.
1488 # 'mf' - Writer interface (file, StringIO, ...)
1489 # 'app' - The app data
1490 def write_txt(mf, app):
1492 def w_comment(line):
1493 mf.write("# %s\n" % line)
1499 elif t == TYPE_MULTILINE:
1500 v = '\n' + v + '\n.'
1501 mf.write("%s:%s\n" % (f, v))
1504 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1506 for f in build_flags_order:
1512 if f == 'androidupdate':
1513 f = 'update' # avoid conflicting with Build(dict).update()
1514 mf.write(' %s=' % f)
1515 if t == TYPE_STRING:
1517 elif t == TYPE_BOOL:
1519 elif t == TYPE_SCRIPT:
1521 for s in v.split(' && '):
1525 mf.write(' && \\\n ')
1527 elif t == TYPE_LIST:
1528 mf.write(','.join(v))
1532 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1535 def write_metadata(metadatapath, app):
1536 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
1537 accepted = fdroidserver.common.config['accepted_formats']
1538 if ext not in accepted:
1539 warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
1540 .format(path=metadatapath, formats=', '.join(accepted)))
1543 with open(metadatapath, 'w', encoding='utf8') as mf:
1545 return write_txt(mf, app)
1547 return write_yaml(mf, app)
1548 except FDroidException as e:
1549 os.remove(metadatapath)
1552 warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
1555 def add_metadata_arguments(parser):
1556 '''add common command line flags related to metadata processing'''
1557 parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
1558 help=_("force metadata errors (default) to be warnings, or to be ignored."))