3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 # use libyaml if it is available
31 from yaml import CLoader
34 from yaml import Loader
37 import fdroidserver.common
38 from fdroidserver.exception import MetaDataException, FDroidException
41 warnings_action = None
44 def warn_or_exception(value):
45 '''output warning or Exception depending on -W'''
46 if warnings_action == 'ignore':
48 elif warnings_action == 'error':
49 raise MetaDataException(value)
51 logging.warning(value)
54 # To filter which ones should be written to the metadata files if
85 'Update Check Ignore',
90 'Current Version Code',
94 'comments', # For formats that don't do inline comments
95 'builds', # For formats that do builds as a list
101 def __init__(self, copydict=None):
103 super().__init__(copydict)
108 self.AntiFeatures = []
110 self.Categories = ['None']
111 self.License = 'Unknown'
112 self.AuthorName = None
113 self.AuthorEmail = None
114 self.AuthorWebSite = None
117 self.IssueTracker = ''
126 self.Description = ''
127 self.RequiresRoot = False
131 self.MaintainerNotes = ''
132 self.ArchivePolicy = None
133 self.AutoUpdateMode = 'None'
134 self.UpdateCheckMode = 'None'
135 self.UpdateCheckIgnore = None
136 self.VercodeOperation = None
137 self.UpdateCheckName = None
138 self.UpdateCheckData = None
139 self.CurrentVersion = ''
140 self.CurrentVersionCode = None
141 self.NoSourceSince = ''
144 self.metadatapath = None
148 self.lastUpdated = None
150 def __getattr__(self, name):
154 raise AttributeError("No such attribute: " + name)
156 def __setattr__(self, name, value):
159 def __delattr__(self, name):
163 raise AttributeError("No such attribute: " + name)
165 def get_last_build(self):
166 if len(self.builds) > 0:
167 return self.builds[-1]
184 'Description': TYPE_MULTILINE,
185 'MaintainerNotes': TYPE_MULTILINE,
186 'Categories': TYPE_LIST,
187 'AntiFeatures': TYPE_LIST,
188 'BuildVersion': TYPE_BUILD,
189 'Build': TYPE_BUILD_V2,
190 'UseBuilt': TYPE_OBSOLETE,
195 name = name.replace(' ', '')
196 if name in fieldtypes:
197 return fieldtypes[name]
201 # In the order in which they are laid out on files
202 build_flags_order = [
237 # old .txt format has version name/code inline in the 'Build:' line
238 # but YAML and JSON have a explicit key for them
239 build_flags = ['versionName', 'versionCode'] + build_flags_order
244 def __init__(self, copydict=None):
249 self.submodules = False
256 self.buildozer = False
259 self.oldsdkloc = False
261 self.forceversion = False
262 self.forcevercode = False
266 self.androidupdate = []
273 self.preassemble = []
274 self.gradleprops = []
275 self.antcommands = []
276 self.novcheck = False
277 self.antifeatures = []
279 super().__init__(copydict)
282 def __getattr__(self, name):
286 raise AttributeError("No such attribute: " + name)
288 def __setattr__(self, name, value):
291 def __delattr__(self, name):
295 raise AttributeError("No such attribute: " + name)
297 def build_method(self):
298 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
305 # like build_method, but prioritize output=
306 def output_method(self):
309 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
317 version = 'r12b' # falls back to latest
318 paths = fdroidserver.common.config['ndk_paths']
319 if version not in paths:
321 return paths[version]
325 'versionCode': TYPE_INT,
326 'extlibs': TYPE_LIST,
327 'srclibs': TYPE_LIST,
330 'buildjni': TYPE_LIST,
331 'preassemble': TYPE_LIST,
332 'androidupdate': TYPE_LIST,
333 'scanignore': TYPE_LIST,
334 'scandelete': TYPE_LIST,
336 'antcommands': TYPE_LIST,
337 'gradleprops': TYPE_LIST,
340 'prebuild': TYPE_SCRIPT,
341 'build': TYPE_SCRIPT,
342 'submodules': TYPE_BOOL,
343 'oldsdkloc': TYPE_BOOL,
344 'forceversion': TYPE_BOOL,
345 'forcevercode': TYPE_BOOL,
346 'novcheck': TYPE_BOOL,
347 'antifeatures': TYPE_LIST,
352 if name in flagtypes:
353 return flagtypes[name]
357 class FieldValidator():
359 Designates App metadata field types and checks that it matches
361 'name' - The long name of the field type
362 'matching' - List of possible values or regex expression
363 'sep' - Separator to use if value may be a list
364 'fields' - Metadata fields (Field:Value) of this type
367 def __init__(self, name, matching, fields):
369 self.matching = matching
370 self.compiled = re.compile(matching)
373 def check(self, v, appid):
381 if not self.compiled.match(v):
382 warn_or_exception("'%s' is not a valid %s in %s. Regex pattern: %s"
383 % (v, self.name, appid, self.matching))
386 # Generic value types
388 FieldValidator("Flattr ID",
392 FieldValidator("HTTP link",
394 ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
396 FieldValidator("Email",
397 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
400 FieldValidator("Bitcoin address",
401 r'^[a-zA-Z0-9]{27,34}$',
404 FieldValidator("Litecoin address",
405 r'^L[a-zA-Z0-9]{33}$',
408 FieldValidator("Repo Type",
409 r'^(git|git-svn|svn|hg|bzr|srclib)$',
412 FieldValidator("Binaries",
416 FieldValidator("Archive Policy",
417 r'^[0-9]+ versions$',
420 FieldValidator("Anti-Feature",
421 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
424 FieldValidator("Auto Update Mode",
425 r"^(Version .+|None)$",
428 FieldValidator("Update Check Mode",
429 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
434 # Check an app's metadata information for integrity errors
435 def check_metadata(app):
438 v.check(app[k], app.id)
441 # Formatter for descriptions. Create an instance, and call parseline() with
442 # each line of the description source from the metadata. At the end, call
443 # end() and then text_txt and text_html will contain the result.
444 class DescriptionFormatter:
451 def __init__(self, linkres):
454 self.state = self.stNONE
455 self.laststate = self.stNONE
458 self.html = io.StringIO()
459 self.text = io.StringIO()
461 self.linkResolver = None
462 self.linkResolver = linkres
464 def endcur(self, notstates=None):
465 if notstates and self.state in notstates:
467 if self.state == self.stPARA:
469 elif self.state == self.stUL:
471 elif self.state == self.stOL:
475 self.laststate = self.state
476 self.state = self.stNONE
477 whole_para = ' '.join(self.para_lines)
478 self.addtext(whole_para)
479 wrapped = textwrap.fill(whole_para, 80,
480 break_long_words=False,
481 break_on_hyphens=False)
482 self.text.write(wrapped)
483 self.html.write('</p>')
484 del self.para_lines[:]
487 self.html.write('</ul>')
488 self.laststate = self.state
489 self.state = self.stNONE
492 self.html.write('</ol>')
493 self.laststate = self.state
494 self.state = self.stNONE
496 def formatted(self, txt, htmlbody):
499 txt = html.escape(txt, quote=False)
501 index = txt.find("''")
506 if txt.startswith("'''"):
512 self.bold = not self.bold
520 self.ital = not self.ital
523 def linkify(self, txt):
527 index = txt.find("[")
529 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
530 res_plain += self.formatted(txt[:index], False)
531 res_html += self.formatted(txt[:index], True)
533 if txt.startswith("[["):
534 index = txt.find("]]")
536 warn_or_exception("Unterminated ]]")
538 if self.linkResolver:
539 url, urltext = self.linkResolver(url)
542 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
544 txt = txt[index + 2:]
546 index = txt.find("]")
548 warn_or_exception("Unterminated ]")
550 index2 = url.find(' ')
554 urltxt = url[index2 + 1:]
557 warn_or_exception("Url title is just the URL - use [url]")
558 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
561 res_plain += ' (' + url + ')'
562 txt = txt[index + 1:]
564 def addtext(self, txt):
565 p, h = self.linkify(txt)
568 def parseline(self, line):
571 elif line.startswith('* '):
572 self.endcur([self.stUL])
573 if self.state != self.stUL:
574 self.html.write('<ul>')
575 self.state = self.stUL
576 if self.laststate != self.stNONE:
577 self.text.write('\n\n')
579 self.text.write('\n')
580 self.text.write(line)
581 self.html.write('<li>')
582 self.addtext(line[1:])
583 self.html.write('</li>')
584 elif line.startswith('# '):
585 self.endcur([self.stOL])
586 if self.state != self.stOL:
587 self.html.write('<ol>')
588 self.state = self.stOL
589 if self.laststate != self.stNONE:
590 self.text.write('\n\n')
592 self.text.write('\n')
593 self.text.write(line)
594 self.html.write('<li>')
595 self.addtext(line[1:])
596 self.html.write('</li>')
598 self.para_lines.append(line)
599 self.endcur([self.stPARA])
600 if self.state == self.stNONE:
601 self.state = self.stPARA
602 if self.laststate != self.stNONE:
603 self.text.write('\n\n')
604 self.html.write('<p>')
608 self.text_txt = self.text.getvalue()
609 self.text_html = self.html.getvalue()
614 # Parse multiple lines of description as written in a metadata file, returning
615 # a single string in text format and wrapped to 80 columns.
616 def description_txt(s):
617 ps = DescriptionFormatter(None)
618 for line in s.splitlines():
624 # Parse multiple lines of description as written in a metadata file, returning
625 # a single string in wiki format. Used for the Maintainer Notes field as well,
626 # because it's the same format.
627 def description_wiki(s):
631 # Parse multiple lines of description as written in a metadata file, returning
632 # a single string in HTML format.
633 def description_html(s, linkres):
634 ps = DescriptionFormatter(linkres)
635 for line in s.splitlines():
641 def parse_srclib(metadatapath):
645 # Defaults for fields that come from metadata
646 thisinfo['Repo Type'] = ''
647 thisinfo['Repo'] = ''
648 thisinfo['Subdir'] = None
649 thisinfo['Prepare'] = None
651 if not os.path.exists(metadatapath):
654 metafile = open(metadatapath, "r", encoding='utf-8')
657 for line in metafile:
659 line = line.rstrip('\r\n')
660 if not line or line.startswith("#"):
664 f, v = line.split(':', 1)
666 warn_or_exception("Invalid metadata in %s:%d" % (line, n))
669 thisinfo[f] = v.split(',')
679 """Read all srclib metadata.
681 The information read will be accessible as metadata.srclibs, which is a
682 dictionary, keyed on srclib name, with the values each being a dictionary
683 in the same format as that returned by the parse_srclib function.
685 A MetaDataException is raised if there are any problems with the srclib
690 # They were already loaded
691 if srclibs is not None:
697 if not os.path.exists(srcdir):
700 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
701 srclibname = os.path.basename(metadatapath[:-4])
702 srclibs[srclibname] = parse_srclib(metadatapath)
705 def read_metadata(xref=True, check_vcs=[]):
707 Read all metadata. Returns a list of 'app' objects (which are dictionaries as
708 returned by the parse_txt_metadata function.
710 check_vcs is the list of packageNames to check for .fdroid.yml in source
713 # Always read the srclibs before the apps, since they can use a srlib as
714 # their source repository.
719 for basedir in ('metadata', 'tmp'):
720 if not os.path.exists(basedir):
723 # If there are multiple metadata files for a single appid, then the first
724 # file that is parsed wins over all the others, and the rest throw an
725 # exception. So the original .txt format is parsed first, at least until
726 # newer formats stabilize.
728 for metadatapath in sorted(glob.glob(os.path.join('metadata', '*.txt'))
729 + glob.glob(os.path.join('metadata', '*.json'))
730 + glob.glob(os.path.join('metadata', '*.yml'))
731 + glob.glob('.fdroid.json')
732 + glob.glob('.fdroid.yml')):
733 packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
734 if packageName in apps:
735 warn_or_exception("Found multiple metadata files for " + packageName)
736 app = parse_metadata(metadatapath, packageName in check_vcs)
741 # Parse all descriptions at load time, just to ensure cross-referencing
742 # errors are caught early rather than when they hit the build server.
745 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
746 warn_or_exception("Cannot resolve app id " + appid)
748 for appid, app in apps.items():
750 description_html(app.Description, linkres)
751 except MetaDataException as e:
752 warn_or_exception("Problem with description of " + appid +
758 # Port legacy ';' separators
759 list_sep = re.compile(r'[,;]')
762 def split_list_values(s):
764 for v in re.split(list_sep, s):
774 def get_default_app_info(metadatapath=None):
775 if metadatapath is None:
778 appid, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
780 if appid == '.fdroid': # we have local metadata in the app's source
781 if os.path.exists('AndroidManifest.xml'):
782 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
784 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
785 for root, dirs, files in os.walk(os.getcwd()):
786 if 'build.gradle' in files:
787 p = os.path.join(root, 'build.gradle')
788 with open(p, 'rb') as f:
790 m = pattern.search(data)
792 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
793 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
795 if manifestroot is None:
796 warn_or_exception("Cannot find a packageName for {0}!".format(metadatapath))
797 appid = manifestroot.attrib['package']
800 app.metadatapath = metadatapath
801 if appid is not None:
807 def sorted_builds(builds):
808 return sorted(builds, key=lambda build: int(build.versionCode))
811 esc_newlines = re.compile(r'\\( |\n)')
814 def post_metadata_parse(app):
815 # TODO keep native types, convert only for .txt metadata
816 for k, v in app.items():
817 if type(v) in (float, int):
821 app['builds'] = app.pop('Builds')
823 if 'flavours' in app and app['flavours'] == [True]:
824 app['flavours'] = 'yes'
826 if isinstance(app.Categories, str):
827 app.Categories = [app.Categories]
828 elif app.Categories is None:
829 app.Categories = ['None']
831 app.Categories = [str(i) for i in app.Categories]
833 def _yaml_bool_unmapable(v):
834 return v in (True, False, [True], [False])
836 def _yaml_bool_unmap(v):
846 _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
850 for build in app['builds']:
851 if not isinstance(build, Build):
853 for k, v in build.items():
855 if flagtype(k) == TYPE_LIST:
856 if _yaml_bool_unmapable(v):
857 build[k] = _yaml_bool_unmap(v)
859 if isinstance(v, str):
861 elif isinstance(v, bool):
866 elif flagtype(k) is TYPE_INT:
868 elif flagtype(k) is TYPE_STRING:
869 if isinstance(v, bool) and k in _bool_allowed:
872 if _yaml_bool_unmapable(v):
873 build[k] = _yaml_bool_unmap(v)
878 app.builds = sorted_builds(builds)
881 # Parse metadata for a single application.
883 # 'metadatapath' - the filename to read. The package id for the application comes
884 # from this filename. Pass None to get a blank entry.
886 # Returns a dictionary containing all the details of the application. There are
887 # two major kinds of information in the dictionary. Keys beginning with capital
888 # letters correspond directory to identically named keys in the metadata file.
889 # Keys beginning with lower case letters are generated in one way or another,
890 # and are not found verbatim in the metadata.
892 # Known keys not originating from the metadata are:
894 # 'builds' - a list of dictionaries containing build information
895 # for each defined build
896 # 'comments' - a list of comments from the metadata file. Each is
897 # a list of the form [field, comment] where field is
898 # the name of the field it preceded in the metadata
899 # file. Where field is None, the comment goes at the
900 # end of the file. Alternatively, 'build:version' is
901 # for a comment before a particular build version.
902 # 'descriptionlines' - original lines of description as formatted in the
907 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
908 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
912 if bool_true.match(s):
914 if bool_false.match(s):
916 warn_or_exception("Invalid bool '%s'" % s)
919 def parse_metadata(metadatapath, check_vcs=False):
920 '''parse metadata file, optionally checking the git repo for metadata first'''
922 _, ext = fdroidserver.common.get_extension(metadatapath)
923 accepted = fdroidserver.common.config['accepted_formats']
924 if ext not in accepted:
925 warn_or_exception('"%s" is not an accepted format, convert to: %s' % (
926 metadatapath, ', '.join(accepted)))
929 app.metadatapath = metadatapath
930 name, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
931 if name == '.fdroid':
936 with open(metadatapath, 'r', encoding='utf-8') as mf:
938 parse_txt_metadata(mf, app)
940 parse_json_metadata(mf, app)
942 parse_yaml_metadata(mf, app)
944 warn_or_exception('Unknown metadata format: %s' % metadatapath)
946 if check_vcs and app.Repo:
947 build_dir = fdroidserver.common.get_build_dir(app)
948 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
949 if not os.path.isfile(metadata_in_repo):
950 vcs, build_dir = fdroidserver.common.setup_vcs(app)
951 if isinstance(vcs, fdroidserver.common.vcs_git):
952 vcs.gotorevision('HEAD') # HEAD since we can't know where else to go
953 if os.path.isfile(metadata_in_repo):
954 logging.debug('Including metadata from ' + metadata_in_repo)
955 # do not include fields already provided by main metadata file
956 app_in_repo = parse_metadata(metadata_in_repo)
957 for k, v in app_in_repo.items():
961 post_metadata_parse(app)
965 build = app.builds[-1]
967 root_dir = build.subdir
970 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
971 _, _, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
976 def parse_json_metadata(mf, app):
978 # fdroid metadata is only strings and booleans, no floats or ints.
979 # TODO create schema using https://pypi.python.org/pypi/jsonschema
980 jsoninfo = json.load(mf, parse_int=lambda s: s,
981 parse_float=lambda s: s)
983 for f in ['Description', 'Maintainer Notes']:
986 app[f] = '\n'.join(v)
990 def parse_yaml_metadata(mf, app):
991 yamldata = yaml.load(mf, Loader=YamlLoader)
996 def write_yaml(mf, app):
998 # import rumael.yaml and check version
1001 except ImportError as e:
1002 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1003 if not ruamel.yaml.__version__:
1004 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1005 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1006 ruamel.yaml.__version__)
1008 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1009 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1010 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1011 # suiteable version ruamel.yaml imported successfully
1013 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1014 'true', 'True', 'TRUE',
1016 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1017 'false', 'False', 'FALSE',
1018 'off', 'Off', 'OFF')
1019 _yaml_bools_plus_lists = []
1020 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1021 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1022 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1023 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1025 def _class_as_dict_representer(dumper, data):
1026 '''Creates a YAML representation of a App/Build instance'''
1027 return dumper.represent_dict(data)
1029 def _field_to_yaml(typ, value):
1030 if typ is TYPE_STRING:
1031 if value in _yaml_bools_plus_lists:
1032 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1034 elif typ is TYPE_INT:
1036 elif typ is TYPE_MULTILINE:
1038 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1041 elif typ is TYPE_SCRIPT:
1043 return ruamel.yaml.scalarstring.preserve_literal(value)
1049 def _app_to_yaml(app):
1050 cm = ruamel.yaml.comments.CommentedMap()
1051 insert_newline = False
1052 for field in yaml_app_field_order:
1054 # next iteration will need to insert a newline
1055 insert_newline = True
1057 if app.get(field) or field is 'Builds':
1058 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1059 if field is 'Builds':
1060 if app.get('builds'):
1061 cm.update({field: _builds_to_yaml(app)})
1062 elif field is 'CurrentVersionCode':
1063 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1065 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1068 # we need to prepend a newline in front of this field
1069 insert_newline = False
1070 # inserting empty lines is not supported so we add a
1071 # bogus comment and over-write its value
1072 cm.yaml_set_comment_before_after_key(field, 'bogus')
1073 cm.ca.items[field][1][-1].value = '\n'
1076 def _builds_to_yaml(app):
1077 fields = ['versionName', 'versionCode']
1078 fields.extend(build_flags_order)
1079 builds = ruamel.yaml.comments.CommentedSeq()
1080 for build in app.builds:
1081 b = ruamel.yaml.comments.CommentedMap()
1082 for field in fields:
1083 if hasattr(build, field) and getattr(build, field):
1084 value = getattr(build, field)
1085 if field == 'gradle' and value == ['off']:
1086 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1087 if field in ('disable', 'kivy', 'maven', 'buildozer'):
1090 elif value == 'yes':
1092 b.update({field: _field_to_yaml(flagtype(field), value)})
1095 # insert extra empty lines between build entries
1096 for i in range(1, len(builds)):
1097 builds.yaml_set_comment_before_after_key(i, 'bogus')
1098 builds.ca.items[i][1][-1].value = '\n'
1102 yaml_app_field_order = [
1138 'UpdateCheckIgnore',
1143 'CurrentVersionCode',
1148 yaml_app = _app_to_yaml(app)
1149 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1152 build_line_sep = re.compile(r'(?<!\\),')
1153 build_cont = re.compile(r'^[ \t]')
1156 def parse_txt_metadata(mf, app):
1160 def add_buildflag(p, build):
1162 warn_or_exception("Empty build flag at {1}"
1163 .format(buildlines[0], linedesc))
1164 bv = p.split('=', 1)
1166 warn_or_exception("Invalid build flag at {0} in {1}"
1167 .format(buildlines[0], linedesc))
1172 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1175 pv = split_list_values(pv)
1177 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1179 elif t == TYPE_BOOL:
1180 build[pk] = _decode_bool(pv)
1182 def parse_buildline(lines):
1184 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1186 warn_or_exception("Invalid build format: " + v + " in " + mf.name)
1188 build.versionName = parts[0]
1189 build.versionCode = parts[1]
1190 check_versionCode(build.versionCode)
1192 if parts[2].startswith('!'):
1193 # For backwards compatibility, handle old-style disabling,
1194 # including attempting to extract the commit from the message
1195 build.disable = parts[2][1:]
1196 commit = 'unknown - see disabled'
1197 index = parts[2].rfind('at ')
1199 commit = parts[2][index + 3:]
1200 if commit.endswith(')'):
1201 commit = commit[:-1]
1202 build.commit = commit
1204 build.commit = parts[2]
1206 add_buildflag(p, build)
1210 def check_versionCode(versionCode):
1214 warn_or_exception('Invalid versionCode: "' + versionCode + '" is not an integer!')
1216 def add_comments(key):
1219 app.comments[key] = list(curcomments)
1224 multiline_lines = []
1234 linedesc = "%s:%d" % (mf.name, c)
1235 line = line.rstrip('\r\n')
1237 if build_cont.match(line):
1238 if line.endswith('\\'):
1239 buildlines.append(line[:-1].lstrip())
1241 buildlines.append(line.lstrip())
1242 bl = ''.join(buildlines)
1243 add_buildflag(bl, build)
1246 if not build.commit and not build.disable:
1247 warn_or_exception("No commit specified for {0} in {1}"
1248 .format(build.versionName, linedesc))
1250 app.builds.append(build)
1251 add_comments('build:' + build.versionCode)
1257 if line.startswith("#"):
1258 curcomments.append(line[1:].strip())
1261 f, v = line.split(':', 1)
1263 warn_or_exception("Invalid metadata in " + linedesc)
1265 if f not in app_fields:
1266 warn_or_exception('Unrecognised app field: ' + f)
1268 # Translate obsolete fields...
1269 if f == 'Market Version':
1270 f = 'Current Version'
1271 if f == 'Market Version Code':
1272 f = 'Current Version Code'
1274 f = f.replace(' ', '')
1276 ftype = fieldtype(f)
1277 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1279 if ftype == TYPE_MULTILINE:
1282 warn_or_exception("Unexpected text on same line as "
1283 + f + " in " + linedesc)
1284 elif ftype == TYPE_STRING:
1286 elif ftype == TYPE_LIST:
1287 app[f] = split_list_values(v)
1288 elif ftype == TYPE_BUILD:
1289 if v.endswith("\\"):
1292 buildlines.append(v[:-1])
1294 build = parse_buildline([v])
1295 app.builds.append(build)
1296 add_comments('build:' + app.builds[-1].versionCode)
1297 elif ftype == TYPE_BUILD_V2:
1300 warn_or_exception('Build should have comma-separated',
1301 'versionName and versionCode,',
1302 'not "{0}", in {1}'.format(v, linedesc))
1304 build.versionName = vv[0]
1305 build.versionCode = vv[1]
1306 check_versionCode(build.versionCode)
1308 if build.versionCode in vc_seen:
1309 warn_or_exception('Duplicate build recipe found for versionCode %s in %s'
1310 % (build.versionCode, linedesc))
1311 vc_seen.add(build.versionCode)
1314 elif ftype == TYPE_OBSOLETE:
1315 pass # Just throw it away!
1317 warn_or_exception("Unrecognised field '" + f + "' in " + linedesc)
1318 elif mode == 1: # Multiline field
1321 app[f] = '\n'.join(multiline_lines)
1322 del multiline_lines[:]
1324 multiline_lines.append(line)
1325 elif mode == 2: # Line continuation mode in Build Version
1326 if line.endswith("\\"):
1327 buildlines.append(line[:-1])
1329 buildlines.append(line)
1330 build = parse_buildline(buildlines)
1331 app.builds.append(build)
1332 add_comments('build:' + app.builds[-1].versionCode)
1336 # Mode at end of file should always be 0
1338 warn_or_exception(f + " not terminated in " + mf.name)
1340 warn_or_exception("Unterminated continuation in " + mf.name)
1342 warn_or_exception("Unterminated build in " + mf.name)
1347 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1349 def field_to_attr(f):
1351 Translates human-readable field names to attribute names, e.g.
1352 'Auto Name' to 'AutoName'
1354 return f.replace(' ', '')
1356 def attr_to_field(k):
1358 Translates attribute names to human-readable field names, e.g.
1359 'AutoName' to 'Auto Name'
1363 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1366 def w_comments(key):
1367 if key not in app.comments:
1369 for line in app.comments[key]:
1372 def w_field_always(f, v=None):
1373 key = field_to_attr(f)
1379 def w_field_nonempty(f, v=None):
1380 key = field_to_attr(f)
1387 w_field_nonempty('Disabled')
1388 w_field_nonempty('AntiFeatures')
1389 w_field_nonempty('Provides')
1390 w_field_always('Categories')
1391 w_field_always('License')
1392 w_field_nonempty('Author Name')
1393 w_field_nonempty('Author Email')
1394 w_field_nonempty('Author Web Site')
1395 w_field_always('Web Site')
1396 w_field_always('Source Code')
1397 w_field_always('Issue Tracker')
1398 w_field_nonempty('Changelog')
1399 w_field_nonempty('Donate')
1400 w_field_nonempty('FlattrID')
1401 w_field_nonempty('Bitcoin')
1402 w_field_nonempty('Litecoin')
1404 w_field_nonempty('Name')
1405 w_field_nonempty('Auto Name')
1406 w_field_nonempty('Summary')
1407 w_field_nonempty('Description', description_txt(app.Description))
1409 if app.RequiresRoot:
1410 w_field_always('Requires Root', 'yes')
1413 w_field_always('Repo Type')
1414 w_field_always('Repo')
1416 w_field_always('Binaries')
1419 for build in app.builds:
1421 if build.versionName == "Ignore":
1424 w_comments('build:%s' % build.versionCode)
1428 if app.MaintainerNotes:
1429 w_field_always('Maintainer Notes', app.MaintainerNotes)
1432 w_field_nonempty('Archive Policy')
1433 w_field_always('Auto Update Mode')
1434 w_field_always('Update Check Mode')
1435 w_field_nonempty('Update Check Ignore')
1436 w_field_nonempty('Vercode Operation')
1437 w_field_nonempty('Update Check Name')
1438 w_field_nonempty('Update Check Data')
1439 if app.CurrentVersion:
1440 w_field_always('Current Version')
1441 w_field_always('Current Version Code')
1442 if app.NoSourceSince:
1444 w_field_always('No Source Since')
1448 # Write a metadata file in txt format.
1450 # 'mf' - Writer interface (file, StringIO, ...)
1451 # 'app' - The app data
1452 def write_txt(mf, app):
1454 def w_comment(line):
1455 mf.write("# %s\n" % line)
1461 elif t == TYPE_MULTILINE:
1462 v = '\n' + v + '\n.'
1463 mf.write("%s:%s\n" % (f, v))
1466 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1468 for f in build_flags_order:
1474 if f == 'androidupdate':
1475 f = 'update' # avoid conflicting with Build(dict).update()
1476 mf.write(' %s=' % f)
1477 if t == TYPE_STRING:
1479 elif t == TYPE_BOOL:
1481 elif t == TYPE_SCRIPT:
1483 for s in v.split(' && '):
1487 mf.write(' && \\\n ')
1489 elif t == TYPE_LIST:
1490 mf.write(','.join(v))
1494 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1497 def write_metadata(metadatapath, app):
1498 _, ext = fdroidserver.common.get_extension(metadatapath)
1499 accepted = fdroidserver.common.config['accepted_formats']
1500 if ext not in accepted:
1501 warn_or_exception('Cannot write "%s", not an accepted format, use: %s'
1502 % (metadatapath, ', '.join(accepted)))
1505 with open(metadatapath, 'w', encoding='utf8') as mf:
1507 return write_txt(mf, app)
1509 return write_yaml(mf, app)
1510 except FDroidException as e:
1511 os.remove(metadatapath)
1514 warn_or_exception('Unknown metadata format: %s' % metadatapath)
1517 def add_metadata_arguments(parser):
1518 '''add common command line flags related to metadata processing'''
1519 parser.add_argument("-W", default='error',
1520 help="force errors to be warnings, or ignore")