3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 from collections import OrderedDict
30 # use libyaml if it is available
32 from yaml import CLoader
35 from yaml import Loader
38 import fdroidserver.common
39 from fdroidserver import _
40 from fdroidserver.exception import MetaDataException, FDroidException
43 warnings_action = None
46 def warn_or_exception(value):
47 '''output warning or Exception depending on -W'''
48 if warnings_action == 'ignore':
50 elif warnings_action == 'error':
51 raise MetaDataException(value)
53 logging.warning(value)
56 # To filter which ones should be written to the metadata files if
87 'Update Check Ignore',
92 'Current Version Code',
96 'comments', # For formats that don't do inline comments
97 'builds', # For formats that do builds as a list
103 def __init__(self, copydict=None):
105 super().__init__(copydict)
110 self.AntiFeatures = []
112 self.Categories = ['None']
113 self.License = 'Unknown'
114 self.AuthorName = None
115 self.AuthorEmail = None
116 self.AuthorWebSite = None
119 self.IssueTracker = ''
128 self.Description = ''
129 self.RequiresRoot = False
133 self.MaintainerNotes = ''
134 self.ArchivePolicy = None
135 self.AutoUpdateMode = 'None'
136 self.UpdateCheckMode = 'None'
137 self.UpdateCheckIgnore = None
138 self.VercodeOperation = None
139 self.UpdateCheckName = None
140 self.UpdateCheckData = None
141 self.CurrentVersion = ''
142 self.CurrentVersionCode = None
143 self.NoSourceSince = ''
146 self.metadatapath = None
150 self.lastUpdated = None
152 def __getattr__(self, name):
156 raise AttributeError("No such attribute: " + name)
158 def __setattr__(self, name, value):
161 def __delattr__(self, name):
165 raise AttributeError("No such attribute: " + name)
167 def get_last_build(self):
168 if len(self.builds) > 0:
169 return self.builds[-1]
186 'Description': TYPE_MULTILINE,
187 'MaintainerNotes': TYPE_MULTILINE,
188 'Categories': TYPE_LIST,
189 'AntiFeatures': TYPE_LIST,
190 'BuildVersion': TYPE_BUILD,
191 'Build': TYPE_BUILD_V2,
192 'UseBuilt': TYPE_OBSOLETE,
197 name = name.replace(' ', '')
198 if name in fieldtypes:
199 return fieldtypes[name]
203 # In the order in which they are laid out on files
204 build_flags_order = [
239 # old .txt format has version name/code inline in the 'Build:' line
240 # but YAML and JSON have a explicit key for them
241 build_flags = ['versionName', 'versionCode'] + build_flags_order
246 def __init__(self, copydict=None):
251 self.submodules = False
258 self.buildozer = False
261 self.oldsdkloc = False
263 self.forceversion = False
264 self.forcevercode = False
268 self.androidupdate = []
275 self.preassemble = []
276 self.gradleprops = []
277 self.antcommands = []
278 self.novcheck = False
279 self.antifeatures = []
281 super().__init__(copydict)
284 def __getattr__(self, name):
288 raise AttributeError("No such attribute: " + name)
290 def __setattr__(self, name, value):
293 def __delattr__(self, name):
297 raise AttributeError("No such attribute: " + name)
299 def build_method(self):
300 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
307 # like build_method, but prioritize output=
308 def output_method(self):
311 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
319 version = 'r12b' # falls back to latest
320 paths = fdroidserver.common.config['ndk_paths']
321 if version not in paths:
323 return paths[version]
327 'versionCode': TYPE_INT,
328 'extlibs': TYPE_LIST,
329 'srclibs': TYPE_LIST,
332 'buildjni': TYPE_LIST,
333 'preassemble': TYPE_LIST,
334 'androidupdate': TYPE_LIST,
335 'scanignore': TYPE_LIST,
336 'scandelete': TYPE_LIST,
338 'antcommands': TYPE_LIST,
339 'gradleprops': TYPE_LIST,
342 'prebuild': TYPE_SCRIPT,
343 'build': TYPE_SCRIPT,
344 'submodules': TYPE_BOOL,
345 'oldsdkloc': TYPE_BOOL,
346 'forceversion': TYPE_BOOL,
347 'forcevercode': TYPE_BOOL,
348 'novcheck': TYPE_BOOL,
349 'antifeatures': TYPE_LIST,
354 if name in flagtypes:
355 return flagtypes[name]
359 class FieldValidator():
361 Designates App metadata field types and checks that it matches
363 'name' - The long name of the field type
364 'matching' - List of possible values or regex expression
365 'sep' - Separator to use if value may be a list
366 'fields' - Metadata fields (Field:Value) of this type
369 def __init__(self, name, matching, fields):
371 self.matching = matching
372 self.compiled = re.compile(matching)
375 def check(self, v, appid):
383 if not self.compiled.match(v):
384 warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
385 .format(value=v, field=self.name, appid=appid, pattern=self.matching))
388 # Generic value types
390 FieldValidator("Flattr ID",
394 FieldValidator("HTTP link",
396 ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
398 FieldValidator("Email",
399 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
402 FieldValidator("Bitcoin address",
403 r'^[a-zA-Z0-9]{27,34}$',
406 FieldValidator("Litecoin address",
407 r'^L[a-zA-Z0-9]{33}$',
410 FieldValidator("Repo Type",
411 r'^(git|git-svn|svn|hg|bzr|srclib)$',
414 FieldValidator("Binaries",
418 FieldValidator("Archive Policy",
419 r'^[0-9]+ versions$',
422 FieldValidator("Anti-Feature",
423 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
426 FieldValidator("Auto Update Mode",
427 r"^(Version .+|None)$",
430 FieldValidator("Update Check Mode",
431 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
436 # Check an app's metadata information for integrity errors
437 def check_metadata(app):
440 v.check(app[k], app.id)
443 # Formatter for descriptions. Create an instance, and call parseline() with
444 # each line of the description source from the metadata. At the end, call
445 # end() and then text_txt and text_html will contain the result.
446 class DescriptionFormatter:
453 def __init__(self, linkres):
456 self.state = self.stNONE
457 self.laststate = self.stNONE
460 self.html = io.StringIO()
461 self.text = io.StringIO()
463 self.linkResolver = None
464 self.linkResolver = linkres
466 def endcur(self, notstates=None):
467 if notstates and self.state in notstates:
469 if self.state == self.stPARA:
471 elif self.state == self.stUL:
473 elif self.state == self.stOL:
477 self.laststate = self.state
478 self.state = self.stNONE
479 whole_para = ' '.join(self.para_lines)
480 self.addtext(whole_para)
481 wrapped = textwrap.fill(whole_para, 80,
482 break_long_words=False,
483 break_on_hyphens=False)
484 self.text.write(wrapped)
485 self.html.write('</p>')
486 del self.para_lines[:]
489 self.html.write('</ul>')
490 self.laststate = self.state
491 self.state = self.stNONE
494 self.html.write('</ol>')
495 self.laststate = self.state
496 self.state = self.stNONE
498 def formatted(self, txt, htmlbody):
501 txt = html.escape(txt, quote=False)
503 index = txt.find("''")
508 if txt.startswith("'''"):
514 self.bold = not self.bold
522 self.ital = not self.ital
525 def linkify(self, txt):
529 index = txt.find("[")
531 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
532 res_plain += self.formatted(txt[:index], False)
533 res_html += self.formatted(txt[:index], True)
535 if txt.startswith("[["):
536 index = txt.find("]]")
538 warn_or_exception(_("Unterminated ]]"))
540 if self.linkResolver:
541 url, urltext = self.linkResolver(url)
544 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
546 txt = txt[index + 2:]
548 index = txt.find("]")
550 warn_or_exception(_("Unterminated ]"))
552 index2 = url.find(' ')
556 urltxt = url[index2 + 1:]
559 warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
560 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
563 res_plain += ' (' + url + ')'
564 txt = txt[index + 1:]
566 def addtext(self, txt):
567 p, h = self.linkify(txt)
570 def parseline(self, line):
573 elif line.startswith('* '):
574 self.endcur([self.stUL])
575 if self.state != self.stUL:
576 self.html.write('<ul>')
577 self.state = self.stUL
578 if self.laststate != self.stNONE:
579 self.text.write('\n\n')
581 self.text.write('\n')
582 self.text.write(line)
583 self.html.write('<li>')
584 self.addtext(line[1:])
585 self.html.write('</li>')
586 elif line.startswith('# '):
587 self.endcur([self.stOL])
588 if self.state != self.stOL:
589 self.html.write('<ol>')
590 self.state = self.stOL
591 if self.laststate != self.stNONE:
592 self.text.write('\n\n')
594 self.text.write('\n')
595 self.text.write(line)
596 self.html.write('<li>')
597 self.addtext(line[1:])
598 self.html.write('</li>')
600 self.para_lines.append(line)
601 self.endcur([self.stPARA])
602 if self.state == self.stNONE:
603 self.state = self.stPARA
604 if self.laststate != self.stNONE:
605 self.text.write('\n\n')
606 self.html.write('<p>')
610 self.text_txt = self.text.getvalue()
611 self.text_html = self.html.getvalue()
616 # Parse multiple lines of description as written in a metadata file, returning
617 # a single string in text format and wrapped to 80 columns.
618 def description_txt(s):
619 ps = DescriptionFormatter(None)
620 for line in s.splitlines():
626 # Parse multiple lines of description as written in a metadata file, returning
627 # a single string in wiki format. Used for the Maintainer Notes field as well,
628 # because it's the same format.
629 def description_wiki(s):
633 # Parse multiple lines of description as written in a metadata file, returning
634 # a single string in HTML format.
635 def description_html(s, linkres):
636 ps = DescriptionFormatter(linkres)
637 for line in s.splitlines():
643 def parse_srclib(metadatapath):
647 # Defaults for fields that come from metadata
648 thisinfo['Repo Type'] = ''
649 thisinfo['Repo'] = ''
650 thisinfo['Subdir'] = None
651 thisinfo['Prepare'] = None
653 if not os.path.exists(metadatapath):
656 metafile = open(metadatapath, "r", encoding='utf-8')
659 for line in metafile:
661 line = line.rstrip('\r\n')
662 if not line or line.startswith("#"):
666 f, v = line.split(':', 1)
668 warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
671 thisinfo[f] = v.split(',')
681 """Read all srclib metadata.
683 The information read will be accessible as metadata.srclibs, which is a
684 dictionary, keyed on srclib name, with the values each being a dictionary
685 in the same format as that returned by the parse_srclib function.
687 A MetaDataException is raised if there are any problems with the srclib
692 # They were already loaded
693 if srclibs is not None:
699 if not os.path.exists(srcdir):
702 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
703 srclibname = os.path.basename(metadatapath[:-4])
704 srclibs[srclibname] = parse_srclib(metadatapath)
707 def read_metadata(xref=True, check_vcs=[], sort_by_time=False):
708 """Return a list of App instances sorted newest first
710 This reads all of the metadata files in a 'data' repository, then
711 builds a list of App instances from those files. The list is
712 sorted based on creation time, newest first. Most of the time,
713 the newer files are the most interesting.
715 If there are multiple metadata files for a single appid, then the first
716 file that is parsed wins over all the others, and the rest throw an
717 exception. So the original .txt format is parsed first, at least until
718 newer formats stabilize.
720 check_vcs is the list of packageNames to check for .fdroid.yml in source
724 # Always read the srclibs before the apps, since they can use a srlib as
725 # their source repository.
730 for basedir in ('metadata', 'tmp'):
731 if not os.path.exists(basedir):
734 metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
735 + glob.glob(os.path.join('metadata', '*.json'))
736 + glob.glob(os.path.join('metadata', '*.yml'))
737 + glob.glob('.fdroid.txt')
738 + glob.glob('.fdroid.json')
739 + glob.glob('.fdroid.yml'))
742 entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
744 for _ignored, path in sorted(entries, reverse=True):
745 metadatafiles.append(path)
747 # most things want the index alpha sorted for stability
748 metadatafiles = sorted(metadatafiles)
750 for metadatapath in metadatafiles:
751 if metadatapath == '.fdroid.txt':
752 warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'))
753 packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
754 if packageName in apps:
755 warn_or_exception(_("Found multiple metadata files for {appid}")
756 .format(path=packageName))
757 app = parse_metadata(metadatapath, packageName in check_vcs)
762 # Parse all descriptions at load time, just to ensure cross-referencing
763 # errors are caught early rather than when they hit the build server.
766 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
767 warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid))
769 for appid, app in apps.items():
771 description_html(app.Description, linkres)
772 except MetaDataException as e:
773 warn_or_exception(_("Problem with description of {appid}: {error}")
774 .format(appid=appid, error=str(e)))
779 # Port legacy ';' separators
780 list_sep = re.compile(r'[,;]')
783 def split_list_values(s):
785 for v in re.split(list_sep, s):
795 def get_default_app_info(metadatapath=None):
796 if metadatapath is None:
799 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
801 if appid == '.fdroid': # we have local metadata in the app's source
802 if os.path.exists('AndroidManifest.xml'):
803 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
805 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
806 for root, dirs, files in os.walk(os.getcwd()):
807 if 'build.gradle' in files:
808 p = os.path.join(root, 'build.gradle')
809 with open(p, 'rb') as f:
811 m = pattern.search(data)
813 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
814 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
816 if manifestroot is None:
817 warn_or_exception(_("Cannot find a packageName for {path}!")
818 .format(path=metadatapath))
819 appid = manifestroot.attrib['package']
822 app.metadatapath = metadatapath
823 if appid is not None:
829 def sorted_builds(builds):
830 return sorted(builds, key=lambda build: int(build.versionCode))
833 esc_newlines = re.compile(r'\\( |\n)')
836 def post_metadata_parse(app):
837 # TODO keep native types, convert only for .txt metadata
838 for k, v in app.items():
839 if type(v) in (float, int):
843 app['builds'] = app.pop('Builds')
845 if 'flavours' in app and app['flavours'] == [True]:
846 app['flavours'] = 'yes'
848 if isinstance(app.Categories, str):
849 app.Categories = [app.Categories]
850 elif app.Categories is None:
851 app.Categories = ['None']
853 app.Categories = [str(i) for i in app.Categories]
855 def _yaml_bool_unmapable(v):
856 return v in (True, False, [True], [False])
858 def _yaml_bool_unmap(v):
868 _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
872 for build in app['builds']:
873 if not isinstance(build, Build):
875 for k, v in build.items():
877 if flagtype(k) == TYPE_LIST:
878 if _yaml_bool_unmapable(v):
879 build[k] = _yaml_bool_unmap(v)
881 if isinstance(v, str):
883 elif isinstance(v, bool):
888 elif flagtype(k) is TYPE_INT:
890 elif flagtype(k) is TYPE_STRING:
891 if isinstance(v, bool) and k in _bool_allowed:
894 if _yaml_bool_unmapable(v):
895 build[k] = _yaml_bool_unmap(v)
900 app.builds = sorted_builds(builds)
903 # Parse metadata for a single application.
905 # 'metadatapath' - the filename to read. The package id for the application comes
906 # from this filename. Pass None to get a blank entry.
908 # Returns a dictionary containing all the details of the application. There are
909 # two major kinds of information in the dictionary. Keys beginning with capital
910 # letters correspond directory to identically named keys in the metadata file.
911 # Keys beginning with lower case letters are generated in one way or another,
912 # and are not found verbatim in the metadata.
914 # Known keys not originating from the metadata are:
916 # 'builds' - a list of dictionaries containing build information
917 # for each defined build
918 # 'comments' - a list of comments from the metadata file. Each is
919 # a list of the form [field, comment] where field is
920 # the name of the field it preceded in the metadata
921 # file. Where field is None, the comment goes at the
922 # end of the file. Alternatively, 'build:version' is
923 # for a comment before a particular build version.
924 # 'descriptionlines' - original lines of description as formatted in the
929 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
930 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
934 if bool_true.match(s):
936 if bool_false.match(s):
938 warn_or_exception(_("Invalid boolean '%s'") % s)
941 def parse_metadata(metadatapath, check_vcs=False):
942 '''parse metadata file, optionally checking the git repo for metadata first'''
944 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
945 accepted = fdroidserver.common.config['accepted_formats']
946 if ext not in accepted:
947 warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
948 .format(path=metadatapath, formats=', '.join(accepted)))
951 app.metadatapath = metadatapath
952 name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
953 if name == '.fdroid':
958 with open(metadatapath, 'r', encoding='utf-8') as mf:
960 parse_txt_metadata(mf, app)
962 parse_json_metadata(mf, app)
964 parse_yaml_metadata(mf, app)
966 warn_or_exception(_('Unknown metadata format: {path}')
967 .format(path=metadatapath))
969 if check_vcs and app.Repo:
970 build_dir = fdroidserver.common.get_build_dir(app)
971 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
972 if not os.path.isfile(metadata_in_repo):
973 vcs, build_dir = fdroidserver.common.setup_vcs(app)
974 if isinstance(vcs, fdroidserver.common.vcs_git):
975 vcs.gotorevision('HEAD') # HEAD since we can't know where else to go
976 if os.path.isfile(metadata_in_repo):
977 logging.debug('Including metadata from ' + metadata_in_repo)
978 # do not include fields already provided by main metadata file
979 app_in_repo = parse_metadata(metadata_in_repo)
980 for k, v in app_in_repo.items():
984 post_metadata_parse(app)
988 build = app.builds[-1]
990 root_dir = build.subdir
993 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
994 _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
999 def parse_json_metadata(mf, app):
1001 # fdroid metadata is only strings and booleans, no floats or ints.
1002 # TODO create schema using https://pypi.python.org/pypi/jsonschema
1003 jsoninfo = json.load(mf, parse_int=lambda s: s,
1004 parse_float=lambda s: s)
1005 app.update(jsoninfo)
1006 for f in ['Description', 'Maintainer Notes']:
1009 app[f] = '\n'.join(v)
1013 def parse_yaml_metadata(mf, app):
1014 yamldata = yaml.load(mf, Loader=YamlLoader)
1016 app.update(yamldata)
1020 def write_yaml(mf, app):
1022 # import rumael.yaml and check version
1025 except ImportError as e:
1026 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1027 if not ruamel.yaml.__version__:
1028 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1029 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1030 ruamel.yaml.__version__)
1032 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1033 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1034 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1035 # suiteable version ruamel.yaml imported successfully
1037 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1038 'true', 'True', 'TRUE',
1040 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1041 'false', 'False', 'FALSE',
1042 'off', 'Off', 'OFF')
1043 _yaml_bools_plus_lists = []
1044 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1045 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1046 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1047 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1049 def _class_as_dict_representer(dumper, data):
1050 '''Creates a YAML representation of a App/Build instance'''
1051 return dumper.represent_dict(data)
1053 def _field_to_yaml(typ, value):
1054 if typ is TYPE_STRING:
1055 if value in _yaml_bools_plus_lists:
1056 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1058 elif typ is TYPE_INT:
1060 elif typ is TYPE_MULTILINE:
1062 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1065 elif typ is TYPE_SCRIPT:
1067 return ruamel.yaml.scalarstring.preserve_literal(value)
1073 def _app_to_yaml(app):
1074 cm = ruamel.yaml.comments.CommentedMap()
1075 insert_newline = False
1076 for field in yaml_app_field_order:
1078 # next iteration will need to insert a newline
1079 insert_newline = True
1081 if app.get(field) or field is 'Builds':
1082 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1083 if field is 'Builds':
1084 if app.get('builds'):
1085 cm.update({field: _builds_to_yaml(app)})
1086 elif field is 'CurrentVersionCode':
1087 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1089 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1092 # we need to prepend a newline in front of this field
1093 insert_newline = False
1094 # inserting empty lines is not supported so we add a
1095 # bogus comment and over-write its value
1096 cm.yaml_set_comment_before_after_key(field, 'bogus')
1097 cm.ca.items[field][1][-1].value = '\n'
1100 def _builds_to_yaml(app):
1101 fields = ['versionName', 'versionCode']
1102 fields.extend(build_flags_order)
1103 builds = ruamel.yaml.comments.CommentedSeq()
1104 for build in app.builds:
1105 b = ruamel.yaml.comments.CommentedMap()
1106 for field in fields:
1107 if hasattr(build, field) and getattr(build, field):
1108 value = getattr(build, field)
1109 if field == 'gradle' and value == ['off']:
1110 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1111 if field in ('disable', 'kivy', 'maven', 'buildozer'):
1114 elif value == 'yes':
1116 b.update({field: _field_to_yaml(flagtype(field), value)})
1119 # insert extra empty lines between build entries
1120 for i in range(1, len(builds)):
1121 builds.yaml_set_comment_before_after_key(i, 'bogus')
1122 builds.ca.items[i][1][-1].value = '\n'
1126 yaml_app_field_order = [
1162 'UpdateCheckIgnore',
1167 'CurrentVersionCode',
1172 yaml_app = _app_to_yaml(app)
1173 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1176 build_line_sep = re.compile(r'(?<!\\),')
1177 build_cont = re.compile(r'^[ \t]')
1180 def parse_txt_metadata(mf, app):
1184 def add_buildflag(p, build):
1186 warn_or_exception(_("Empty build flag at {linedesc}")
1187 .format(linedesc=linedesc))
1188 bv = p.split('=', 1)
1190 warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
1191 .format(line=buildlines[0], linedesc=linedesc))
1196 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1199 pv = split_list_values(pv)
1201 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1203 elif t == TYPE_BOOL:
1204 build[pk] = _decode_bool(pv)
1206 def parse_buildline(lines):
1208 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1210 warn_or_exception(_("Invalid build format: {value} in {name}")
1211 .format(value=v, name=mf.name))
1213 build.versionName = parts[0]
1214 build.versionCode = parts[1]
1215 check_versionCode(build.versionCode)
1217 if parts[2].startswith('!'):
1218 # For backwards compatibility, handle old-style disabling,
1219 # including attempting to extract the commit from the message
1220 build.disable = parts[2][1:]
1221 commit = 'unknown - see disabled'
1222 index = parts[2].rfind('at ')
1224 commit = parts[2][index + 3:]
1225 if commit.endswith(')'):
1226 commit = commit[:-1]
1227 build.commit = commit
1229 build.commit = parts[2]
1231 add_buildflag(p, build)
1235 def check_versionCode(versionCode):
1239 warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
1240 .format(versionCode=versionCode))
1242 def add_comments(key):
1245 app.comments[key] = list(curcomments)
1250 multiline_lines = []
1260 linedesc = "%s:%d" % (mf.name, c)
1261 line = line.rstrip('\r\n')
1263 if build_cont.match(line):
1264 if line.endswith('\\'):
1265 buildlines.append(line[:-1].lstrip())
1267 buildlines.append(line.lstrip())
1268 bl = ''.join(buildlines)
1269 add_buildflag(bl, build)
1272 if not build.commit and not build.disable:
1273 warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
1274 .format(versionName=build.versionName, linedesc=linedesc))
1276 app.builds.append(build)
1277 add_comments('build:' + build.versionCode)
1283 if line.startswith("#"):
1284 curcomments.append(line[1:].strip())
1287 f, v = line.split(':', 1)
1289 warn_or_exception(_("Invalid metadata in: ") + linedesc)
1291 if f not in app_fields:
1292 warn_or_exception(_('Unrecognised app field: ') + f)
1294 # Translate obsolete fields...
1295 if f == 'Market Version':
1296 f = 'Current Version'
1297 if f == 'Market Version Code':
1298 f = 'Current Version Code'
1300 f = f.replace(' ', '')
1302 ftype = fieldtype(f)
1303 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1305 if ftype == TYPE_MULTILINE:
1308 warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
1309 .format(field=f, linedesc=linedesc))
1310 elif ftype == TYPE_STRING:
1312 elif ftype == TYPE_LIST:
1313 app[f] = split_list_values(v)
1314 elif ftype == TYPE_BUILD:
1315 if v.endswith("\\"):
1318 buildlines.append(v[:-1])
1320 build = parse_buildline([v])
1321 app.builds.append(build)
1322 add_comments('build:' + app.builds[-1].versionCode)
1323 elif ftype == TYPE_BUILD_V2:
1326 warn_or_exception(_('Build should have comma-separated '
1327 'versionName and versionCode, '
1328 'not "{value}", in {linedesc}')
1329 .format(value=v, linedesc=linedesc))
1331 build.versionName = vv[0]
1332 build.versionCode = vv[1]
1333 check_versionCode(build.versionCode)
1335 if build.versionCode in vc_seen:
1336 warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
1337 .format(versionCode=build.versionCode, linedesc=linedesc))
1338 vc_seen.add(build.versionCode)
1341 elif ftype == TYPE_OBSOLETE:
1342 pass # Just throw it away!
1344 warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
1345 .format(field=f, linedesc=linedesc))
1346 elif mode == 1: # Multiline field
1349 app[f] = '\n'.join(multiline_lines)
1350 del multiline_lines[:]
1352 multiline_lines.append(line)
1353 elif mode == 2: # Line continuation mode in Build Version
1354 if line.endswith("\\"):
1355 buildlines.append(line[:-1])
1357 buildlines.append(line)
1358 build = parse_buildline(buildlines)
1359 app.builds.append(build)
1360 add_comments('build:' + app.builds[-1].versionCode)
1364 # Mode at end of file should always be 0
1366 warn_or_exception(_("{field} not terminated in {name}")
1367 .format(field=f, name=mf.name))
1369 warn_or_exception(_("Unterminated continuation in {name}")
1370 .format(name=mf.name))
1372 warn_or_exception(_("Unterminated build in {name}")
1373 .format(name=mf.name))
1378 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1380 def field_to_attr(f):
1382 Translates human-readable field names to attribute names, e.g.
1383 'Auto Name' to 'AutoName'
1385 return f.replace(' ', '')
1387 def attr_to_field(k):
1389 Translates attribute names to human-readable field names, e.g.
1390 'AutoName' to 'Auto Name'
1394 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1397 def w_comments(key):
1398 if key not in app.comments:
1400 for line in app.comments[key]:
1403 def w_field_always(f, v=None):
1404 key = field_to_attr(f)
1410 def w_field_nonempty(f, v=None):
1411 key = field_to_attr(f)
1418 w_field_nonempty('Disabled')
1419 w_field_nonempty('AntiFeatures')
1420 w_field_nonempty('Provides')
1421 w_field_always('Categories')
1422 w_field_always('License')
1423 w_field_nonempty('Author Name')
1424 w_field_nonempty('Author Email')
1425 w_field_nonempty('Author Web Site')
1426 w_field_always('Web Site')
1427 w_field_always('Source Code')
1428 w_field_always('Issue Tracker')
1429 w_field_nonempty('Changelog')
1430 w_field_nonempty('Donate')
1431 w_field_nonempty('FlattrID')
1432 w_field_nonempty('Bitcoin')
1433 w_field_nonempty('Litecoin')
1435 w_field_nonempty('Name')
1436 w_field_nonempty('Auto Name')
1437 w_field_nonempty('Summary')
1438 w_field_nonempty('Description', description_txt(app.Description))
1440 if app.RequiresRoot:
1441 w_field_always('Requires Root', 'yes')
1444 w_field_always('Repo Type')
1445 w_field_always('Repo')
1447 w_field_always('Binaries')
1450 for build in app.builds:
1452 if build.versionName == "Ignore":
1455 w_comments('build:%s' % build.versionCode)
1459 if app.MaintainerNotes:
1460 w_field_always('Maintainer Notes', app.MaintainerNotes)
1463 w_field_nonempty('Archive Policy')
1464 w_field_always('Auto Update Mode')
1465 w_field_always('Update Check Mode')
1466 w_field_nonempty('Update Check Ignore')
1467 w_field_nonempty('Vercode Operation')
1468 w_field_nonempty('Update Check Name')
1469 w_field_nonempty('Update Check Data')
1470 if app.CurrentVersion:
1471 w_field_always('Current Version')
1472 w_field_always('Current Version Code')
1473 if app.NoSourceSince:
1475 w_field_always('No Source Since')
1479 # Write a metadata file in txt format.
1481 # 'mf' - Writer interface (file, StringIO, ...)
1482 # 'app' - The app data
1483 def write_txt(mf, app):
1485 def w_comment(line):
1486 mf.write("# %s\n" % line)
1492 elif t == TYPE_MULTILINE:
1493 v = '\n' + v + '\n.'
1494 mf.write("%s:%s\n" % (f, v))
1497 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1499 for f in build_flags_order:
1505 if f == 'androidupdate':
1506 f = 'update' # avoid conflicting with Build(dict).update()
1507 mf.write(' %s=' % f)
1508 if t == TYPE_STRING:
1510 elif t == TYPE_BOOL:
1512 elif t == TYPE_SCRIPT:
1514 for s in v.split(' && '):
1518 mf.write(' && \\\n ')
1520 elif t == TYPE_LIST:
1521 mf.write(','.join(v))
1525 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1528 def write_metadata(metadatapath, app):
1529 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
1530 accepted = fdroidserver.common.config['accepted_formats']
1531 if ext not in accepted:
1532 warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
1533 .format(path=metadatapath, formats=', '.join(accepted)))
1536 with open(metadatapath, 'w', encoding='utf8') as mf:
1538 return write_txt(mf, app)
1540 return write_yaml(mf, app)
1541 except FDroidException as e:
1542 os.remove(metadatapath)
1545 warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
1548 def add_metadata_arguments(parser):
1549 '''add common command line flags related to metadata processing'''
1550 parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
1551 help=_("force metadata errors (default) to be warnings, or to be ignored."))