3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 from collections import OrderedDict
30 # use libyaml if it is available
32 from yaml import CLoader
35 from yaml import Loader
38 import fdroidserver.common
39 from fdroidserver import _
40 from fdroidserver.exception import MetaDataException, FDroidException
43 warnings_action = None
46 def warn_or_exception(value):
47 '''output warning or Exception depending on -W'''
48 if warnings_action == 'ignore':
50 elif warnings_action == 'error':
51 raise MetaDataException(value)
53 logging.warning(value)
56 # To filter which ones should be written to the metadata files if
87 'Update Check Ignore',
92 'Current Version Code',
96 'comments', # For formats that don't do inline comments
97 'builds', # For formats that do builds as a list
103 def __init__(self, copydict=None):
105 super().__init__(copydict)
110 self.AntiFeatures = []
112 self.Categories = ['None']
113 self.License = 'Unknown'
114 self.AuthorName = None
115 self.AuthorEmail = None
116 self.AuthorWebSite = None
119 self.IssueTracker = ''
128 self.Description = ''
129 self.RequiresRoot = False
133 self.MaintainerNotes = ''
134 self.ArchivePolicy = None
135 self.AutoUpdateMode = 'None'
136 self.UpdateCheckMode = 'None'
137 self.UpdateCheckIgnore = None
138 self.VercodeOperation = None
139 self.UpdateCheckName = None
140 self.UpdateCheckData = None
141 self.CurrentVersion = ''
142 self.CurrentVersionCode = None
143 self.NoSourceSince = ''
146 self.metadatapath = None
150 self.lastUpdated = None
152 def __getattr__(self, name):
156 raise AttributeError("No such attribute: " + name)
158 def __setattr__(self, name, value):
161 def __delattr__(self, name):
165 raise AttributeError("No such attribute: " + name)
167 def get_last_build(self):
168 if len(self.builds) > 0:
169 return self.builds[-1]
186 'Description': TYPE_MULTILINE,
187 'MaintainerNotes': TYPE_MULTILINE,
188 'Categories': TYPE_LIST,
189 'AntiFeatures': TYPE_LIST,
190 'BuildVersion': TYPE_BUILD,
191 'Build': TYPE_BUILD_V2,
192 'UseBuilt': TYPE_OBSOLETE,
197 name = name.replace(' ', '')
198 if name in fieldtypes:
199 return fieldtypes[name]
203 # In the order in which they are laid out on files
204 build_flags_order = [
239 # old .txt format has version name/code inline in the 'Build:' line
240 # but YAML and JSON have a explicit key for them
241 build_flags = ['versionName', 'versionCode'] + build_flags_order
246 def __init__(self, copydict=None):
251 self.submodules = False
258 self.buildozer = False
261 self.oldsdkloc = False
263 self.forceversion = False
264 self.forcevercode = False
268 self.androidupdate = []
275 self.preassemble = []
276 self.gradleprops = []
277 self.antcommands = []
278 self.novcheck = False
279 self.antifeatures = []
281 super().__init__(copydict)
284 def __getattr__(self, name):
288 raise AttributeError("No such attribute: " + name)
290 def __setattr__(self, name, value):
293 def __delattr__(self, name):
297 raise AttributeError("No such attribute: " + name)
299 def build_method(self):
300 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
307 # like build_method, but prioritize output=
308 def output_method(self):
311 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
319 version = 'r12b' # falls back to latest
320 paths = fdroidserver.common.config['ndk_paths']
321 if version not in paths:
323 return paths[version]
327 'versionCode': TYPE_INT,
328 'extlibs': TYPE_LIST,
329 'srclibs': TYPE_LIST,
332 'buildjni': TYPE_LIST,
333 'preassemble': TYPE_LIST,
334 'androidupdate': TYPE_LIST,
335 'scanignore': TYPE_LIST,
336 'scandelete': TYPE_LIST,
338 'antcommands': TYPE_LIST,
339 'gradleprops': TYPE_LIST,
342 'prebuild': TYPE_SCRIPT,
343 'build': TYPE_SCRIPT,
344 'submodules': TYPE_BOOL,
345 'oldsdkloc': TYPE_BOOL,
346 'forceversion': TYPE_BOOL,
347 'forcevercode': TYPE_BOOL,
348 'novcheck': TYPE_BOOL,
349 'antifeatures': TYPE_LIST,
354 if name in flagtypes:
355 return flagtypes[name]
359 class FieldValidator():
361 Designates App metadata field types and checks that it matches
363 'name' - The long name of the field type
364 'matching' - List of possible values or regex expression
365 'sep' - Separator to use if value may be a list
366 'fields' - Metadata fields (Field:Value) of this type
369 def __init__(self, name, matching, fields):
371 self.matching = matching
372 self.compiled = re.compile(matching)
375 def check(self, v, appid):
383 if not self.compiled.match(v):
384 warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
385 .format(value=v, field=self.name, appid=appid, pattern=self.matching))
388 # Generic value types
390 FieldValidator("Flattr ID",
394 FieldValidator("HTTP link",
396 ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
398 FieldValidator("Email",
399 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
402 FieldValidator("Bitcoin address",
403 r'^[a-zA-Z0-9]{27,34}$',
406 FieldValidator("Litecoin address",
407 r'^L[a-zA-Z0-9]{33}$',
410 FieldValidator("Repo Type",
411 r'^(git|git-svn|svn|hg|bzr|srclib)$',
414 FieldValidator("Binaries",
418 FieldValidator("Archive Policy",
419 r'^[0-9]+ versions$',
422 FieldValidator("Anti-Feature",
423 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
426 FieldValidator("Auto Update Mode",
427 r"^(Version .+|None)$",
430 FieldValidator("Update Check Mode",
431 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
436 # Check an app's metadata information for integrity errors
437 def check_metadata(app):
440 v.check(app[k], app.id)
443 # Formatter for descriptions. Create an instance, and call parseline() with
444 # each line of the description source from the metadata. At the end, call
445 # end() and then text_txt and text_html will contain the result.
446 class DescriptionFormatter:
453 def __init__(self, linkres):
456 self.state = self.stNONE
457 self.laststate = self.stNONE
460 self.html = io.StringIO()
461 self.text = io.StringIO()
463 self.linkResolver = None
464 self.linkResolver = linkres
466 def endcur(self, notstates=None):
467 if notstates and self.state in notstates:
469 if self.state == self.stPARA:
471 elif self.state == self.stUL:
473 elif self.state == self.stOL:
477 self.laststate = self.state
478 self.state = self.stNONE
479 whole_para = ' '.join(self.para_lines)
480 self.addtext(whole_para)
481 wrapped = textwrap.fill(whole_para, 80,
482 break_long_words=False,
483 break_on_hyphens=False)
484 self.text.write(wrapped)
485 self.html.write('</p>')
486 del self.para_lines[:]
489 self.html.write('</ul>')
490 self.laststate = self.state
491 self.state = self.stNONE
494 self.html.write('</ol>')
495 self.laststate = self.state
496 self.state = self.stNONE
498 def formatted(self, txt, htmlbody):
501 txt = html.escape(txt, quote=False)
503 index = txt.find("''")
508 if txt.startswith("'''"):
514 self.bold = not self.bold
522 self.ital = not self.ital
525 def linkify(self, txt):
529 index = txt.find("[")
531 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
532 res_plain += self.formatted(txt[:index], False)
533 res_html += self.formatted(txt[:index], True)
535 if txt.startswith("[["):
536 index = txt.find("]]")
538 warn_or_exception(_("Unterminated ]]"))
540 if self.linkResolver:
541 url, urltext = self.linkResolver(url)
544 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
546 txt = txt[index + 2:]
548 index = txt.find("]")
550 warn_or_exception(_("Unterminated ]"))
552 index2 = url.find(' ')
556 urltxt = url[index2 + 1:]
559 warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
560 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
563 res_plain += ' (' + url + ')'
564 txt = txt[index + 1:]
566 def addtext(self, txt):
567 p, h = self.linkify(txt)
570 def parseline(self, line):
573 elif line.startswith('* '):
574 self.endcur([self.stUL])
575 if self.state != self.stUL:
576 self.html.write('<ul>')
577 self.state = self.stUL
578 if self.laststate != self.stNONE:
579 self.text.write('\n\n')
581 self.text.write('\n')
582 self.text.write(line)
583 self.html.write('<li>')
584 self.addtext(line[1:])
585 self.html.write('</li>')
586 elif line.startswith('# '):
587 self.endcur([self.stOL])
588 if self.state != self.stOL:
589 self.html.write('<ol>')
590 self.state = self.stOL
591 if self.laststate != self.stNONE:
592 self.text.write('\n\n')
594 self.text.write('\n')
595 self.text.write(line)
596 self.html.write('<li>')
597 self.addtext(line[1:])
598 self.html.write('</li>')
600 self.para_lines.append(line)
601 self.endcur([self.stPARA])
602 if self.state == self.stNONE:
603 self.state = self.stPARA
604 if self.laststate != self.stNONE:
605 self.text.write('\n\n')
606 self.html.write('<p>')
610 self.text_txt = self.text.getvalue()
611 self.text_html = self.html.getvalue()
616 # Parse multiple lines of description as written in a metadata file, returning
617 # a single string in text format and wrapped to 80 columns.
618 def description_txt(s):
619 ps = DescriptionFormatter(None)
620 for line in s.splitlines():
626 # Parse multiple lines of description as written in a metadata file, returning
627 # a single string in wiki format. Used for the Maintainer Notes field as well,
628 # because it's the same format.
629 def description_wiki(s):
633 # Parse multiple lines of description as written in a metadata file, returning
634 # a single string in HTML format.
635 def description_html(s, linkres):
636 ps = DescriptionFormatter(linkres)
637 for line in s.splitlines():
643 def parse_srclib(metadatapath):
647 # Defaults for fields that come from metadata
648 thisinfo['Repo Type'] = ''
649 thisinfo['Repo'] = ''
650 thisinfo['Subdir'] = None
651 thisinfo['Prepare'] = None
653 if not os.path.exists(metadatapath):
656 metafile = open(metadatapath, "r", encoding='utf-8')
659 for line in metafile:
661 line = line.rstrip('\r\n')
662 if not line or line.startswith("#"):
666 f, v = line.split(':', 1)
668 warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
671 thisinfo[f] = v.split(',')
681 """Read all srclib metadata.
683 The information read will be accessible as metadata.srclibs, which is a
684 dictionary, keyed on srclib name, with the values each being a dictionary
685 in the same format as that returned by the parse_srclib function.
687 A MetaDataException is raised if there are any problems with the srclib
692 # They were already loaded
693 if srclibs is not None:
699 if not os.path.exists(srcdir):
702 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
703 srclibname = os.path.basename(metadatapath[:-4])
704 srclibs[srclibname] = parse_srclib(metadatapath)
707 def read_metadata(xref=True, check_vcs=[], sort_by_time=False):
708 """Return a list of App instances sorted newest first
710 This reads all of the metadata files in a 'data' repository, then
711 builds a list of App instances from those files. The list is
712 sorted based on creation time, newest first. Most of the time,
713 the newer files are the most interesting.
715 If there are multiple metadata files for a single appid, then the first
716 file that is parsed wins over all the others, and the rest throw an
717 exception. So the original .txt format is parsed first, at least until
718 newer formats stabilize.
720 check_vcs is the list of packageNames to check for .fdroid.yml in source
724 # Always read the srclibs before the apps, since they can use a srlib as
725 # their source repository.
730 for basedir in ('metadata', 'tmp'):
731 if not os.path.exists(basedir):
734 metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
735 + glob.glob(os.path.join('metadata', '*.json'))
736 + glob.glob(os.path.join('metadata', '*.yml'))
737 + glob.glob('.fdroid.txt')
738 + glob.glob('.fdroid.json')
739 + glob.glob('.fdroid.yml'))
742 entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
744 for _ignored, path in sorted(entries, reverse=True):
745 metadatafiles.append(path)
747 # most things want the index alpha sorted for stability
748 metadatafiles = sorted(metadatafiles)
750 for metadatapath in metadatafiles:
751 packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
752 if packageName in apps:
753 warn_or_exception(_("Found multiple metadata files for {appid}")
754 .format(path=packageName))
755 app = parse_metadata(metadatapath, packageName in check_vcs)
760 # Parse all descriptions at load time, just to ensure cross-referencing
761 # errors are caught early rather than when they hit the build server.
764 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
765 warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid))
767 for appid, app in apps.items():
769 description_html(app.Description, linkres)
770 except MetaDataException as e:
771 warn_or_exception(_("Problem with description of {appid}: {error}")
772 .format(appid=appid, error=str(e)))
777 # Port legacy ';' separators
778 list_sep = re.compile(r'[,;]')
781 def split_list_values(s):
783 for v in re.split(list_sep, s):
793 def get_default_app_info(metadatapath=None):
794 if metadatapath is None:
797 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
799 if appid == '.fdroid': # we have local metadata in the app's source
800 if os.path.exists('AndroidManifest.xml'):
801 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
803 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
804 for root, dirs, files in os.walk(os.getcwd()):
805 if 'build.gradle' in files:
806 p = os.path.join(root, 'build.gradle')
807 with open(p, 'rb') as f:
809 m = pattern.search(data)
811 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
812 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
814 if manifestroot is None:
815 warn_or_exception(_("Cannot find a packageName for {path}!")
816 .format(path=metadatapath))
817 appid = manifestroot.attrib['package']
820 app.metadatapath = metadatapath
821 if appid is not None:
827 def sorted_builds(builds):
828 return sorted(builds, key=lambda build: int(build.versionCode))
831 esc_newlines = re.compile(r'\\( |\n)')
834 def post_metadata_parse(app):
835 # TODO keep native types, convert only for .txt metadata
836 for k, v in app.items():
837 if type(v) in (float, int):
841 app['builds'] = app.pop('Builds')
843 if 'flavours' in app and app['flavours'] == [True]:
844 app['flavours'] = 'yes'
846 if isinstance(app.Categories, str):
847 app.Categories = [app.Categories]
848 elif app.Categories is None:
849 app.Categories = ['None']
851 app.Categories = [str(i) for i in app.Categories]
853 def _yaml_bool_unmapable(v):
854 return v in (True, False, [True], [False])
856 def _yaml_bool_unmap(v):
866 _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
870 for build in app['builds']:
871 if not isinstance(build, Build):
873 for k, v in build.items():
875 if flagtype(k) == TYPE_LIST:
876 if _yaml_bool_unmapable(v):
877 build[k] = _yaml_bool_unmap(v)
879 if isinstance(v, str):
881 elif isinstance(v, bool):
886 elif flagtype(k) is TYPE_INT:
888 elif flagtype(k) is TYPE_STRING:
889 if isinstance(v, bool) and k in _bool_allowed:
892 if _yaml_bool_unmapable(v):
893 build[k] = _yaml_bool_unmap(v)
898 app.builds = sorted_builds(builds)
901 # Parse metadata for a single application.
903 # 'metadatapath' - the filename to read. The package id for the application comes
904 # from this filename. Pass None to get a blank entry.
906 # Returns a dictionary containing all the details of the application. There are
907 # two major kinds of information in the dictionary. Keys beginning with capital
908 # letters correspond directory to identically named keys in the metadata file.
909 # Keys beginning with lower case letters are generated in one way or another,
910 # and are not found verbatim in the metadata.
912 # Known keys not originating from the metadata are:
914 # 'builds' - a list of dictionaries containing build information
915 # for each defined build
916 # 'comments' - a list of comments from the metadata file. Each is
917 # a list of the form [field, comment] where field is
918 # the name of the field it preceded in the metadata
919 # file. Where field is None, the comment goes at the
920 # end of the file. Alternatively, 'build:version' is
921 # for a comment before a particular build version.
922 # 'descriptionlines' - original lines of description as formatted in the
927 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
928 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
932 if bool_true.match(s):
934 if bool_false.match(s):
936 warn_or_exception(_("Invalid boolean '%s'") % s)
939 def parse_metadata(metadatapath, check_vcs=False):
940 '''parse metadata file, optionally checking the git repo for metadata first'''
942 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
943 accepted = fdroidserver.common.config['accepted_formats']
944 if ext not in accepted:
945 warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
946 .format(path=metadatapath, formats=', '.join(accepted)))
949 app.metadatapath = metadatapath
950 name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
951 if name == '.fdroid':
956 with open(metadatapath, 'r', encoding='utf-8') as mf:
958 parse_txt_metadata(mf, app)
960 parse_json_metadata(mf, app)
962 parse_yaml_metadata(mf, app)
964 warn_or_exception(_('Unknown metadata format: {path}')
965 .format(path=metadatapath))
967 if check_vcs and app.Repo:
968 build_dir = fdroidserver.common.get_build_dir(app)
969 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
970 if not os.path.isfile(metadata_in_repo):
971 vcs, build_dir = fdroidserver.common.setup_vcs(app)
972 if isinstance(vcs, fdroidserver.common.vcs_git):
973 vcs.gotorevision('HEAD') # HEAD since we can't know where else to go
974 if os.path.isfile(metadata_in_repo):
975 logging.debug('Including metadata from ' + metadata_in_repo)
976 # do not include fields already provided by main metadata file
977 app_in_repo = parse_metadata(metadata_in_repo)
978 for k, v in app_in_repo.items():
982 post_metadata_parse(app)
986 build = app.builds[-1]
988 root_dir = build.subdir
991 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
992 _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
997 def parse_json_metadata(mf, app):
999 # fdroid metadata is only strings and booleans, no floats or ints.
1000 # TODO create schema using https://pypi.python.org/pypi/jsonschema
1001 jsoninfo = json.load(mf, parse_int=lambda s: s,
1002 parse_float=lambda s: s)
1003 app.update(jsoninfo)
1004 for f in ['Description', 'Maintainer Notes']:
1007 app[f] = '\n'.join(v)
1011 def parse_yaml_metadata(mf, app):
1012 yamldata = yaml.load(mf, Loader=YamlLoader)
1014 app.update(yamldata)
1018 def write_yaml(mf, app):
1020 # import rumael.yaml and check version
1023 except ImportError as e:
1024 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1025 if not ruamel.yaml.__version__:
1026 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1027 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1028 ruamel.yaml.__version__)
1030 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1031 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1032 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1033 # suiteable version ruamel.yaml imported successfully
1035 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1036 'true', 'True', 'TRUE',
1038 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1039 'false', 'False', 'FALSE',
1040 'off', 'Off', 'OFF')
1041 _yaml_bools_plus_lists = []
1042 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1043 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1044 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1045 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1047 def _class_as_dict_representer(dumper, data):
1048 '''Creates a YAML representation of a App/Build instance'''
1049 return dumper.represent_dict(data)
1051 def _field_to_yaml(typ, value):
1052 if typ is TYPE_STRING:
1053 if value in _yaml_bools_plus_lists:
1054 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1056 elif typ is TYPE_INT:
1058 elif typ is TYPE_MULTILINE:
1060 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1063 elif typ is TYPE_SCRIPT:
1065 return ruamel.yaml.scalarstring.preserve_literal(value)
1071 def _app_to_yaml(app):
1072 cm = ruamel.yaml.comments.CommentedMap()
1073 insert_newline = False
1074 for field in yaml_app_field_order:
1076 # next iteration will need to insert a newline
1077 insert_newline = True
1079 if app.get(field) or field is 'Builds':
1080 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1081 if field is 'Builds':
1082 if app.get('builds'):
1083 cm.update({field: _builds_to_yaml(app)})
1084 elif field is 'CurrentVersionCode':
1085 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1087 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1090 # we need to prepend a newline in front of this field
1091 insert_newline = False
1092 # inserting empty lines is not supported so we add a
1093 # bogus comment and over-write its value
1094 cm.yaml_set_comment_before_after_key(field, 'bogus')
1095 cm.ca.items[field][1][-1].value = '\n'
1098 def _builds_to_yaml(app):
1099 fields = ['versionName', 'versionCode']
1100 fields.extend(build_flags_order)
1101 builds = ruamel.yaml.comments.CommentedSeq()
1102 for build in app.builds:
1103 b = ruamel.yaml.comments.CommentedMap()
1104 for field in fields:
1105 if hasattr(build, field) and getattr(build, field):
1106 value = getattr(build, field)
1107 if field == 'gradle' and value == ['off']:
1108 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1109 if field in ('disable', 'kivy', 'maven', 'buildozer'):
1112 elif value == 'yes':
1114 b.update({field: _field_to_yaml(flagtype(field), value)})
1117 # insert extra empty lines between build entries
1118 for i in range(1, len(builds)):
1119 builds.yaml_set_comment_before_after_key(i, 'bogus')
1120 builds.ca.items[i][1][-1].value = '\n'
1124 yaml_app_field_order = [
1160 'UpdateCheckIgnore',
1165 'CurrentVersionCode',
1170 yaml_app = _app_to_yaml(app)
1171 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1174 build_line_sep = re.compile(r'(?<!\\),')
1175 build_cont = re.compile(r'^[ \t]')
1178 def parse_txt_metadata(mf, app):
1182 def add_buildflag(p, build):
1184 warn_or_exception(_("Empty build flag at {linedesc}")
1185 .format(linedesc=linedesc))
1186 bv = p.split('=', 1)
1188 warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
1189 .format(line=buildlines[0], linedesc=linedesc))
1194 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1197 pv = split_list_values(pv)
1199 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1201 elif t == TYPE_BOOL:
1202 build[pk] = _decode_bool(pv)
1204 def parse_buildline(lines):
1206 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1208 warn_or_exception(_("Invalid build format: {value} in {name}")
1209 .format(value=v, name=mf.name))
1211 build.versionName = parts[0]
1212 build.versionCode = parts[1]
1213 check_versionCode(build.versionCode)
1215 if parts[2].startswith('!'):
1216 # For backwards compatibility, handle old-style disabling,
1217 # including attempting to extract the commit from the message
1218 build.disable = parts[2][1:]
1219 commit = 'unknown - see disabled'
1220 index = parts[2].rfind('at ')
1222 commit = parts[2][index + 3:]
1223 if commit.endswith(')'):
1224 commit = commit[:-1]
1225 build.commit = commit
1227 build.commit = parts[2]
1229 add_buildflag(p, build)
1233 def check_versionCode(versionCode):
1237 warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
1238 .format(versionCode=versionCode))
1240 def add_comments(key):
1243 app.comments[key] = list(curcomments)
1248 multiline_lines = []
1258 linedesc = "%s:%d" % (mf.name, c)
1259 line = line.rstrip('\r\n')
1261 if build_cont.match(line):
1262 if line.endswith('\\'):
1263 buildlines.append(line[:-1].lstrip())
1265 buildlines.append(line.lstrip())
1266 bl = ''.join(buildlines)
1267 add_buildflag(bl, build)
1270 if not build.commit and not build.disable:
1271 warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
1272 .format(versionName=build.versionName, linedesc=linedesc))
1274 app.builds.append(build)
1275 add_comments('build:' + build.versionCode)
1281 if line.startswith("#"):
1282 curcomments.append(line[1:].strip())
1285 f, v = line.split(':', 1)
1287 warn_or_exception(_("Invalid metadata in: ") + linedesc)
1289 if f not in app_fields:
1290 warn_or_exception(_('Unrecognised app field: ') + f)
1292 # Translate obsolete fields...
1293 if f == 'Market Version':
1294 f = 'Current Version'
1295 if f == 'Market Version Code':
1296 f = 'Current Version Code'
1298 f = f.replace(' ', '')
1300 ftype = fieldtype(f)
1301 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1303 if ftype == TYPE_MULTILINE:
1306 warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
1307 .format(field=f, linedesc=linedesc))
1308 elif ftype == TYPE_STRING:
1310 elif ftype == TYPE_LIST:
1311 app[f] = split_list_values(v)
1312 elif ftype == TYPE_BUILD:
1313 if v.endswith("\\"):
1316 buildlines.append(v[:-1])
1318 build = parse_buildline([v])
1319 app.builds.append(build)
1320 add_comments('build:' + app.builds[-1].versionCode)
1321 elif ftype == TYPE_BUILD_V2:
1324 warn_or_exception(_('Build should have comma-separated '
1325 'versionName and versionCode, '
1326 'not "{value}", in {linedesc}')
1327 .format(value=v, linedesc=linedesc))
1329 build.versionName = vv[0]
1330 build.versionCode = vv[1]
1331 check_versionCode(build.versionCode)
1333 if build.versionCode in vc_seen:
1334 warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
1335 .format(versionCode=build.versionCode, linedesc=linedesc))
1336 vc_seen.add(build.versionCode)
1339 elif ftype == TYPE_OBSOLETE:
1340 pass # Just throw it away!
1342 warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
1343 .format(field=f, linedesc=linedesc))
1344 elif mode == 1: # Multiline field
1347 app[f] = '\n'.join(multiline_lines)
1348 del multiline_lines[:]
1350 multiline_lines.append(line)
1351 elif mode == 2: # Line continuation mode in Build Version
1352 if line.endswith("\\"):
1353 buildlines.append(line[:-1])
1355 buildlines.append(line)
1356 build = parse_buildline(buildlines)
1357 app.builds.append(build)
1358 add_comments('build:' + app.builds[-1].versionCode)
1362 # Mode at end of file should always be 0
1364 warn_or_exception(_("{field} not terminated in {name}")
1365 .format(field=f, name=mf.name))
1367 warn_or_exception(_("Unterminated continuation in {name}")
1368 .format(name=mf.name))
1370 warn_or_exception(_("Unterminated build in {name}")
1371 .format(name=mf.name))
1376 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1378 def field_to_attr(f):
1380 Translates human-readable field names to attribute names, e.g.
1381 'Auto Name' to 'AutoName'
1383 return f.replace(' ', '')
1385 def attr_to_field(k):
1387 Translates attribute names to human-readable field names, e.g.
1388 'AutoName' to 'Auto Name'
1392 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1395 def w_comments(key):
1396 if key not in app.comments:
1398 for line in app.comments[key]:
1401 def w_field_always(f, v=None):
1402 key = field_to_attr(f)
1408 def w_field_nonempty(f, v=None):
1409 key = field_to_attr(f)
1416 w_field_nonempty('Disabled')
1417 w_field_nonempty('AntiFeatures')
1418 w_field_nonempty('Provides')
1419 w_field_always('Categories')
1420 w_field_always('License')
1421 w_field_nonempty('Author Name')
1422 w_field_nonempty('Author Email')
1423 w_field_nonempty('Author Web Site')
1424 w_field_always('Web Site')
1425 w_field_always('Source Code')
1426 w_field_always('Issue Tracker')
1427 w_field_nonempty('Changelog')
1428 w_field_nonempty('Donate')
1429 w_field_nonempty('FlattrID')
1430 w_field_nonempty('Bitcoin')
1431 w_field_nonempty('Litecoin')
1433 w_field_nonempty('Name')
1434 w_field_nonempty('Auto Name')
1435 w_field_nonempty('Summary')
1436 w_field_nonempty('Description', description_txt(app.Description))
1438 if app.RequiresRoot:
1439 w_field_always('Requires Root', 'yes')
1442 w_field_always('Repo Type')
1443 w_field_always('Repo')
1445 w_field_always('Binaries')
1448 for build in app.builds:
1450 if build.versionName == "Ignore":
1453 w_comments('build:%s' % build.versionCode)
1457 if app.MaintainerNotes:
1458 w_field_always('Maintainer Notes', app.MaintainerNotes)
1461 w_field_nonempty('Archive Policy')
1462 w_field_always('Auto Update Mode')
1463 w_field_always('Update Check Mode')
1464 w_field_nonempty('Update Check Ignore')
1465 w_field_nonempty('Vercode Operation')
1466 w_field_nonempty('Update Check Name')
1467 w_field_nonempty('Update Check Data')
1468 if app.CurrentVersion:
1469 w_field_always('Current Version')
1470 w_field_always('Current Version Code')
1471 if app.NoSourceSince:
1473 w_field_always('No Source Since')
1477 # Write a metadata file in txt format.
1479 # 'mf' - Writer interface (file, StringIO, ...)
1480 # 'app' - The app data
1481 def write_txt(mf, app):
1483 def w_comment(line):
1484 mf.write("# %s\n" % line)
1490 elif t == TYPE_MULTILINE:
1491 v = '\n' + v + '\n.'
1492 mf.write("%s:%s\n" % (f, v))
1495 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1497 for f in build_flags_order:
1503 if f == 'androidupdate':
1504 f = 'update' # avoid conflicting with Build(dict).update()
1505 mf.write(' %s=' % f)
1506 if t == TYPE_STRING:
1508 elif t == TYPE_BOOL:
1510 elif t == TYPE_SCRIPT:
1512 for s in v.split(' && '):
1516 mf.write(' && \\\n ')
1518 elif t == TYPE_LIST:
1519 mf.write(','.join(v))
1523 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1526 def write_metadata(metadatapath, app):
1527 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
1528 accepted = fdroidserver.common.config['accepted_formats']
1529 if ext not in accepted:
1530 warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
1531 .format(path=metadatapath, formats=', '.join(accepted)))
1534 with open(metadatapath, 'w', encoding='utf8') as mf:
1536 return write_txt(mf, app)
1538 return write_yaml(mf, app)
1539 except FDroidException as e:
1540 os.remove(metadatapath)
1543 warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
1546 def add_metadata_arguments(parser):
1547 '''add common command line flags related to metadata processing'''
1548 parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
1549 help=_("force metadata errors (default) to be warnings, or to be ignored."))