3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 from collections import OrderedDict
30 # use libyaml if it is available
32 from yaml import CLoader
35 from yaml import Loader
38 import fdroidserver.common
39 from fdroidserver import _
40 from fdroidserver.exception import MetaDataException, FDroidException
43 warnings_action = None
46 def warn_or_exception(value):
47 '''output warning or Exception depending on -W'''
48 if warnings_action == 'ignore':
50 elif warnings_action == 'error':
51 raise MetaDataException(value)
53 logging.warning(value)
56 # To filter which ones should be written to the metadata files if
89 'Update Check Ignore',
94 'Current Version Code',
98 'comments', # For formats that don't do inline comments
99 'builds', # For formats that do builds as a list
105 def __init__(self, copydict=None):
107 super().__init__(copydict)
112 self.AntiFeatures = []
114 self.Categories = ['None']
115 self.License = 'Unknown'
116 self.AuthorName = None
117 self.AuthorEmail = None
118 self.AuthorWebSite = None
121 self.IssueTracker = ''
122 self.Translation = ''
126 self.LiberapayID = None
132 self.Description = ''
133 self.RequiresRoot = False
137 self.MaintainerNotes = ''
138 self.ArchivePolicy = None
139 self.AutoUpdateMode = 'None'
140 self.UpdateCheckMode = 'None'
141 self.UpdateCheckIgnore = None
142 self.VercodeOperation = None
143 self.UpdateCheckName = None
144 self.UpdateCheckData = None
145 self.CurrentVersion = ''
146 self.CurrentVersionCode = None
147 self.NoSourceSince = ''
150 self.metadatapath = None
154 self.lastUpdated = None
156 def __getattr__(self, name):
160 raise AttributeError("No such attribute: " + name)
162 def __setattr__(self, name, value):
165 def __delattr__(self, name):
169 raise AttributeError("No such attribute: " + name)
171 def get_last_build(self):
172 if len(self.builds) > 0:
173 return self.builds[-1]
190 'Description': TYPE_MULTILINE,
191 'MaintainerNotes': TYPE_MULTILINE,
192 'Categories': TYPE_LIST,
193 'AntiFeatures': TYPE_LIST,
194 'BuildVersion': TYPE_BUILD,
195 'Build': TYPE_BUILD_V2,
196 'UseBuilt': TYPE_OBSOLETE,
201 name = name.replace(' ', '')
202 if name in fieldtypes:
203 return fieldtypes[name]
207 # In the order in which they are laid out on files
208 build_flags_order = [
243 # old .txt format has version name/code inline in the 'Build:' line
244 # but YAML and JSON have a explicit key for them
245 build_flags = ['versionName', 'versionCode'] + build_flags_order
250 def __init__(self, copydict=None):
256 self.submodules = False
262 self.buildozer = False
265 self.oldsdkloc = False
267 self.forceversion = False
268 self.forcevercode = False
272 self.androidupdate = []
279 self.preassemble = []
280 self.gradleprops = []
281 self.antcommands = []
282 self.novcheck = False
283 self.antifeatures = []
285 super().__init__(copydict)
288 def __getattr__(self, name):
292 raise AttributeError("No such attribute: " + name)
294 def __setattr__(self, name, value):
297 def __delattr__(self, name):
301 raise AttributeError("No such attribute: " + name)
303 def build_method(self):
304 for f in ['maven', 'gradle', 'buildozer']:
311 # like build_method, but prioritize output=
312 def output_method(self):
315 for f in ['maven', 'gradle', 'buildozer']:
323 version = 'r12b' # falls back to latest
324 paths = fdroidserver.common.config['ndk_paths']
325 if version not in paths:
327 return paths[version]
331 'versionCode': TYPE_INT,
332 'extlibs': TYPE_LIST,
333 'srclibs': TYPE_LIST,
336 'buildjni': TYPE_LIST,
337 'preassemble': TYPE_LIST,
338 'androidupdate': TYPE_LIST,
339 'scanignore': TYPE_LIST,
340 'scandelete': TYPE_LIST,
342 'antcommands': TYPE_LIST,
343 'gradleprops': TYPE_LIST,
346 'prebuild': TYPE_SCRIPT,
347 'build': TYPE_SCRIPT,
348 'submodules': TYPE_BOOL,
349 'oldsdkloc': TYPE_BOOL,
350 'forceversion': TYPE_BOOL,
351 'forcevercode': TYPE_BOOL,
352 'novcheck': TYPE_BOOL,
353 'antifeatures': TYPE_LIST,
359 if name in flagtypes:
360 return flagtypes[name]
364 class FieldValidator():
366 Designates App metadata field types and checks that it matches
368 'name' - The long name of the field type
369 'matching' - List of possible values or regex expression
370 'sep' - Separator to use if value may be a list
371 'fields' - Metadata fields (Field:Value) of this type
374 def __init__(self, name, matching, fields):
376 self.matching = matching
377 self.compiled = re.compile(matching)
380 def check(self, v, appid):
388 if not self.compiled.match(v):
389 warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
390 .format(value=v, field=self.name, appid=appid, pattern=self.matching))
393 # Generic value types
395 FieldValidator("Flattr ID",
399 FieldValidator("Liberapay ID",
403 FieldValidator("HTTP link",
405 ["WebSite", "SourceCode", "IssueTracker", "Translation", "Changelog", "Donate"]),
407 FieldValidator("Email",
408 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
411 FieldValidator("Bitcoin address",
412 r'^[a-zA-Z0-9]{27,34}$',
415 FieldValidator("Litecoin address",
416 r'^L[a-zA-Z0-9]{33}$',
419 FieldValidator("Repo Type",
420 r'^(git|git-svn|svn|hg|bzr|srclib)$',
423 FieldValidator("Binaries",
427 FieldValidator("Archive Policy",
428 r'^[0-9]+ versions$',
431 FieldValidator("Anti-Feature",
432 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
435 FieldValidator("Auto Update Mode",
436 r"^(Version .+|None)$",
439 FieldValidator("Update Check Mode",
440 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
445 # Check an app's metadata information for integrity errors
446 def check_metadata(app):
449 v.check(app[k], app.id)
452 # Formatter for descriptions. Create an instance, and call parseline() with
453 # each line of the description source from the metadata. At the end, call
454 # end() and then text_txt and text_html will contain the result.
455 class DescriptionFormatter:
462 def __init__(self, linkres):
465 self.state = self.stNONE
466 self.laststate = self.stNONE
469 self.html = io.StringIO()
470 self.text = io.StringIO()
472 self.linkResolver = None
473 self.linkResolver = linkres
475 def endcur(self, notstates=None):
476 if notstates and self.state in notstates:
478 if self.state == self.stPARA:
480 elif self.state == self.stUL:
482 elif self.state == self.stOL:
486 self.laststate = self.state
487 self.state = self.stNONE
488 whole_para = ' '.join(self.para_lines)
489 self.addtext(whole_para)
490 wrapped = textwrap.fill(whole_para, 80,
491 break_long_words=False,
492 break_on_hyphens=False)
493 self.text.write(wrapped)
494 self.html.write('</p>')
495 del self.para_lines[:]
498 self.html.write('</ul>')
499 self.laststate = self.state
500 self.state = self.stNONE
503 self.html.write('</ol>')
504 self.laststate = self.state
505 self.state = self.stNONE
507 def formatted(self, txt, htmlbody):
510 txt = html.escape(txt, quote=False)
512 index = txt.find("''")
517 if txt.startswith("'''"):
523 self.bold = not self.bold
531 self.ital = not self.ital
534 def linkify(self, txt):
538 index = txt.find("[")
540 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
541 res_plain += self.formatted(txt[:index], False)
542 res_html += self.formatted(txt[:index], True)
544 if txt.startswith("[["):
545 index = txt.find("]]")
547 warn_or_exception(_("Unterminated ]]"))
549 if self.linkResolver:
550 url, urltext = self.linkResolver(url)
553 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
555 txt = txt[index + 2:]
557 index = txt.find("]")
559 warn_or_exception(_("Unterminated ]"))
561 index2 = url.find(' ')
565 urltxt = url[index2 + 1:]
568 warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
569 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
572 res_plain += ' (' + url + ')'
573 txt = txt[index + 1:]
575 def addtext(self, txt):
576 p, h = self.linkify(txt)
579 def parseline(self, line):
582 elif line.startswith('* '):
583 self.endcur([self.stUL])
584 if self.state != self.stUL:
585 self.html.write('<ul>')
586 self.state = self.stUL
587 if self.laststate != self.stNONE:
588 self.text.write('\n\n')
590 self.text.write('\n')
591 self.text.write(line)
592 self.html.write('<li>')
593 self.addtext(line[1:])
594 self.html.write('</li>')
595 elif line.startswith('# '):
596 self.endcur([self.stOL])
597 if self.state != self.stOL:
598 self.html.write('<ol>')
599 self.state = self.stOL
600 if self.laststate != self.stNONE:
601 self.text.write('\n\n')
603 self.text.write('\n')
604 self.text.write(line)
605 self.html.write('<li>')
606 self.addtext(line[1:])
607 self.html.write('</li>')
609 self.para_lines.append(line)
610 self.endcur([self.stPARA])
611 if self.state == self.stNONE:
612 self.state = self.stPARA
613 if self.laststate != self.stNONE:
614 self.text.write('\n\n')
615 self.html.write('<p>')
619 self.text_txt = self.text.getvalue()
620 self.text_html = self.html.getvalue()
625 # Parse multiple lines of description as written in a metadata file, returning
626 # a single string in text format and wrapped to 80 columns.
627 def description_txt(s):
628 ps = DescriptionFormatter(None)
629 for line in s.splitlines():
635 # Parse multiple lines of description as written in a metadata file, returning
636 # a single string in wiki format. Used for the Maintainer Notes field as well,
637 # because it's the same format.
638 def description_wiki(s):
642 # Parse multiple lines of description as written in a metadata file, returning
643 # a single string in HTML format.
644 def description_html(s, linkres):
645 ps = DescriptionFormatter(linkres)
646 for line in s.splitlines():
652 def parse_srclib(metadatapath):
656 # Defaults for fields that come from metadata
657 thisinfo['Repo Type'] = ''
658 thisinfo['Repo'] = ''
659 thisinfo['Subdir'] = None
660 thisinfo['Prepare'] = None
662 if not os.path.exists(metadatapath):
665 metafile = open(metadatapath, "r", encoding='utf-8')
668 for line in metafile:
670 line = line.rstrip('\r\n')
671 if not line or line.startswith("#"):
675 f, v = line.split(':', 1)
677 warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
680 thisinfo[f] = v.split(',')
690 """Read all srclib metadata.
692 The information read will be accessible as metadata.srclibs, which is a
693 dictionary, keyed on srclib name, with the values each being a dictionary
694 in the same format as that returned by the parse_srclib function.
696 A MetaDataException is raised if there are any problems with the srclib
701 # They were already loaded
702 if srclibs is not None:
708 if not os.path.exists(srcdir):
711 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
712 srclibname = os.path.basename(metadatapath[:-4])
713 srclibs[srclibname] = parse_srclib(metadatapath)
716 def read_metadata(xref=True, check_vcs=[], refresh=True, sort_by_time=False):
717 """Return a list of App instances sorted newest first
719 This reads all of the metadata files in a 'data' repository, then
720 builds a list of App instances from those files. The list is
721 sorted based on creation time, newest first. Most of the time,
722 the newer files are the most interesting.
724 If there are multiple metadata files for a single appid, then the first
725 file that is parsed wins over all the others, and the rest throw an
726 exception. So the original .txt format is parsed first, at least until
727 newer formats stabilize.
729 check_vcs is the list of appids to check for .fdroid.yml in source
733 # Always read the srclibs before the apps, since they can use a srlib as
734 # their source repository.
739 for basedir in ('metadata', 'tmp'):
740 if not os.path.exists(basedir):
743 metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
744 + glob.glob(os.path.join('metadata', '*.json'))
745 + glob.glob(os.path.join('metadata', '*.yml'))
746 + glob.glob('.fdroid.txt')
747 + glob.glob('.fdroid.json')
748 + glob.glob('.fdroid.yml'))
751 entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
753 for _ignored, path in sorted(entries, reverse=True):
754 metadatafiles.append(path)
756 # most things want the index alpha sorted for stability
757 metadatafiles = sorted(metadatafiles)
759 for metadatapath in metadatafiles:
760 if metadatapath == '.fdroid.txt':
761 warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'))
762 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
764 warn_or_exception(_("Found multiple metadata files for {appid}")
765 .format(appid=appid))
766 app = parse_metadata(metadatapath, appid in check_vcs, refresh)
771 # Parse all descriptions at load time, just to ensure cross-referencing
772 # errors are caught early rather than when they hit the build server.
775 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
776 warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid))
778 for appid, app in apps.items():
780 description_html(app.Description, linkres)
781 except MetaDataException as e:
782 warn_or_exception(_("Problem with description of {appid}: {error}")
783 .format(appid=appid, error=str(e)))
788 # Port legacy ';' separators
789 list_sep = re.compile(r'[,;]')
792 def split_list_values(s):
794 for v in re.split(list_sep, s):
804 def get_default_app_info(metadatapath=None):
805 if metadatapath is None:
808 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
810 if appid == '.fdroid': # we have local metadata in the app's source
811 if os.path.exists('AndroidManifest.xml'):
812 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
814 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
815 for root, dirs, files in os.walk(os.getcwd()):
816 if 'build.gradle' in files:
817 p = os.path.join(root, 'build.gradle')
818 with open(p, 'rb') as f:
820 m = pattern.search(data)
822 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
823 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
825 if manifestroot is None:
826 warn_or_exception(_("Cannot find an appid for {path}!")
827 .format(path=metadatapath))
828 appid = manifestroot.attrib['package']
831 app.metadatapath = metadatapath
832 if appid is not None:
838 def sorted_builds(builds):
839 return sorted(builds, key=lambda build: int(build.versionCode))
842 esc_newlines = re.compile(r'\\( |\n)')
845 def post_metadata_parse(app):
846 # TODO keep native types, convert only for .txt metadata
847 for k, v in app.items():
848 if type(v) in (float, int):
852 app['builds'] = app.pop('Builds')
854 if 'flavours' in app and app['flavours'] == [True]:
855 app['flavours'] = 'yes'
857 if isinstance(app.Categories, str):
858 app.Categories = [app.Categories]
859 elif app.Categories is None:
860 app.Categories = ['None']
862 app.Categories = [str(i) for i in app.Categories]
864 def _yaml_bool_unmapable(v):
865 return v in (True, False, [True], [False])
867 def _yaml_bool_unmap(v):
877 _bool_allowed = ('disable', 'maven', 'buildozer')
881 for build in app['builds']:
882 if not isinstance(build, Build):
884 for k, v in build.items():
886 if flagtype(k) == TYPE_LIST:
887 if _yaml_bool_unmapable(v):
888 build[k] = _yaml_bool_unmap(v)
890 if isinstance(v, str):
892 elif isinstance(v, bool):
897 elif flagtype(k) is TYPE_INT:
899 elif flagtype(k) is TYPE_STRING:
900 if isinstance(v, bool) and k in _bool_allowed:
903 if _yaml_bool_unmapable(v):
904 build[k] = _yaml_bool_unmap(v)
909 app.builds = sorted_builds(builds)
912 # Parse metadata for a single application.
914 # 'metadatapath' - the filename to read. The "Application ID" aka
915 # "Package Name" for the application comes from this
916 # filename. Pass None to get a blank entry.
918 # Returns a dictionary containing all the details of the application. There are
919 # two major kinds of information in the dictionary. Keys beginning with capital
920 # letters correspond directory to identically named keys in the metadata file.
921 # Keys beginning with lower case letters are generated in one way or another,
922 # and are not found verbatim in the metadata.
924 # Known keys not originating from the metadata are:
926 # 'builds' - a list of dictionaries containing build information
927 # for each defined build
928 # 'comments' - a list of comments from the metadata file. Each is
929 # a list of the form [field, comment] where field is
930 # the name of the field it preceded in the metadata
931 # file. Where field is None, the comment goes at the
932 # end of the file. Alternatively, 'build:version' is
933 # for a comment before a particular build version.
934 # 'descriptionlines' - original lines of description as formatted in the
939 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
940 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
944 if bool_true.match(s):
946 if bool_false.match(s):
948 warn_or_exception(_("Invalid boolean '%s'") % s)
951 def parse_metadata(metadatapath, check_vcs=False, refresh=True):
952 '''parse metadata file, optionally checking the git repo for metadata first'''
954 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
955 accepted = fdroidserver.common.config['accepted_formats']
956 if ext not in accepted:
957 warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
958 .format(path=metadatapath, formats=', '.join(accepted)))
961 app.metadatapath = metadatapath
962 name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
963 if name == '.fdroid':
968 with open(metadatapath, 'r', encoding='utf-8') as mf:
970 parse_txt_metadata(mf, app)
972 parse_json_metadata(mf, app)
974 parse_yaml_metadata(mf, app)
976 warn_or_exception(_('Unknown metadata format: {path}')
977 .format(path=metadatapath))
979 if check_vcs and app.Repo:
980 build_dir = fdroidserver.common.get_build_dir(app)
981 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
982 if not os.path.isfile(metadata_in_repo):
983 vcs, build_dir = fdroidserver.common.setup_vcs(app)
984 if isinstance(vcs, fdroidserver.common.vcs_git):
985 vcs.gotorevision('HEAD', refresh) # HEAD since we can't know where else to go
986 if os.path.isfile(metadata_in_repo):
987 logging.debug('Including metadata from ' + metadata_in_repo)
988 # do not include fields already provided by main metadata file
989 app_in_repo = parse_metadata(metadata_in_repo)
990 for k, v in app_in_repo.items():
994 post_metadata_parse(app)
998 build = app.builds[-1]
1000 root_dir = build.subdir
1003 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
1004 _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
1009 def parse_json_metadata(mf, app):
1011 # fdroid metadata is only strings and booleans, no floats or ints.
1012 # TODO create schema using https://pypi.python.org/pypi/jsonschema
1013 jsoninfo = json.load(mf, parse_int=lambda s: s,
1014 parse_float=lambda s: s)
1015 app.update(jsoninfo)
1016 for f in ['Description', 'Maintainer Notes']:
1019 app[f] = '\n'.join(v)
1023 def parse_yaml_metadata(mf, app):
1024 yamldata = yaml.load(mf, Loader=YamlLoader)
1026 app.update(yamldata)
1030 def write_yaml(mf, app):
1032 # import rumael.yaml and check version
1035 except ImportError as e:
1036 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1037 if not ruamel.yaml.__version__:
1038 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1039 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1040 ruamel.yaml.__version__)
1042 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1043 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1044 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1045 # suiteable version ruamel.yaml imported successfully
1047 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1048 'true', 'True', 'TRUE',
1050 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1051 'false', 'False', 'FALSE',
1052 'off', 'Off', 'OFF')
1053 _yaml_bools_plus_lists = []
1054 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1055 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1056 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1057 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1059 def _class_as_dict_representer(dumper, data):
1060 '''Creates a YAML representation of a App/Build instance'''
1061 return dumper.represent_dict(data)
1063 def _field_to_yaml(typ, value):
1064 if typ is TYPE_STRING:
1065 if value in _yaml_bools_plus_lists:
1066 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1068 elif typ is TYPE_INT:
1070 elif typ is TYPE_MULTILINE:
1072 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1075 elif typ is TYPE_SCRIPT:
1077 return ruamel.yaml.scalarstring.preserve_literal(value)
1083 def _app_to_yaml(app):
1084 cm = ruamel.yaml.comments.CommentedMap()
1085 insert_newline = False
1086 for field in yaml_app_field_order:
1088 # next iteration will need to insert a newline
1089 insert_newline = True
1091 if app.get(field) or field is 'Builds':
1092 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1093 if field is 'Builds':
1094 if app.get('builds'):
1095 cm.update({field: _builds_to_yaml(app)})
1096 elif field is 'CurrentVersionCode':
1097 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1099 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1102 # we need to prepend a newline in front of this field
1103 insert_newline = False
1104 # inserting empty lines is not supported so we add a
1105 # bogus comment and over-write its value
1106 cm.yaml_set_comment_before_after_key(field, 'bogus')
1107 cm.ca.items[field][1][-1].value = '\n'
1110 def _builds_to_yaml(app):
1111 fields = ['versionName', 'versionCode']
1112 fields.extend(build_flags_order)
1113 builds = ruamel.yaml.comments.CommentedSeq()
1114 for build in app.builds:
1115 b = ruamel.yaml.comments.CommentedMap()
1116 for field in fields:
1117 if hasattr(build, field) and getattr(build, field):
1118 value = getattr(build, field)
1119 if field == 'gradle' and value == ['off']:
1120 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1121 if field in ('disable', 'maven', 'buildozer'):
1124 elif value == 'yes':
1126 b.update({field: _field_to_yaml(flagtype(field), value)})
1129 # insert extra empty lines between build entries
1130 for i in range(1, len(builds)):
1131 builds.yaml_set_comment_before_after_key(i, 'bogus')
1132 builds.ca.items[i][1][-1].value = '\n'
1136 yaml_app_field_order = [
1174 'UpdateCheckIgnore',
1179 'CurrentVersionCode',
1184 yaml_app = _app_to_yaml(app)
1185 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1188 build_line_sep = re.compile(r'(?<!\\),')
1189 build_cont = re.compile(r'^[ \t]')
1192 def parse_txt_metadata(mf, app):
1196 def add_buildflag(p, build):
1198 warn_or_exception(_("Empty build flag at {linedesc}")
1199 .format(linedesc=linedesc))
1200 bv = p.split('=', 1)
1202 warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
1203 .format(line=buildlines[0], linedesc=linedesc))
1208 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1211 pv = split_list_values(pv)
1213 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1215 elif t == TYPE_BOOL:
1216 build[pk] = _decode_bool(pv)
1220 def parse_buildline(lines):
1222 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1224 warn_or_exception(_("Invalid build format: {value} in {name}")
1225 .format(value=v, name=mf.name))
1227 build.versionName = parts[0]
1228 build.versionCode = parts[1]
1229 check_versionCode(build.versionCode)
1231 if parts[2].startswith('!'):
1232 # For backwards compatibility, handle old-style disabling,
1233 # including attempting to extract the commit from the message
1234 build.disable = parts[2][1:]
1235 commit = 'unknown - see disabled'
1236 index = parts[2].rfind('at ')
1238 commit = parts[2][index + 3:]
1239 if commit.endswith(')'):
1240 commit = commit[:-1]
1241 build.commit = commit
1243 build.commit = parts[2]
1245 add_buildflag(p, build)
1249 def check_versionCode(versionCode):
1253 warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
1254 .format(versionCode=versionCode))
1256 def add_comments(key):
1259 app.comments[key] = list(curcomments)
1264 multiline_lines = []
1274 linedesc = "%s:%d" % (mf.name, c)
1275 line = line.rstrip('\r\n')
1277 if build_cont.match(line):
1278 if line.endswith('\\'):
1279 buildlines.append(line[:-1].lstrip())
1281 buildlines.append(line.lstrip())
1282 bl = ''.join(buildlines)
1283 add_buildflag(bl, build)
1286 if not build.commit and not build.disable:
1287 warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
1288 .format(versionName=build.versionName, linedesc=linedesc))
1290 app.builds.append(build)
1291 add_comments('build:' + build.versionCode)
1297 if line.startswith("#"):
1298 curcomments.append(line[1:].strip())
1301 f, v = line.split(':', 1)
1303 warn_or_exception(_("Invalid metadata in: ") + linedesc)
1305 if f not in app_fields:
1306 warn_or_exception(_('Unrecognised app field: ') + f)
1308 # Translate obsolete fields...
1309 if f == 'Market Version':
1310 f = 'Current Version'
1311 if f == 'Market Version Code':
1312 f = 'Current Version Code'
1314 f = f.replace(' ', '')
1316 ftype = fieldtype(f)
1317 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1319 if ftype == TYPE_MULTILINE:
1322 warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
1323 .format(field=f, linedesc=linedesc))
1324 elif ftype == TYPE_STRING:
1326 elif ftype == TYPE_LIST:
1327 app[f] = split_list_values(v)
1328 elif ftype == TYPE_BUILD:
1329 if v.endswith("\\"):
1332 buildlines.append(v[:-1])
1334 build = parse_buildline([v])
1335 app.builds.append(build)
1336 add_comments('build:' + app.builds[-1].versionCode)
1337 elif ftype == TYPE_BUILD_V2:
1340 warn_or_exception(_('Build should have comma-separated '
1341 'versionName and versionCode, '
1342 'not "{value}", in {linedesc}')
1343 .format(value=v, linedesc=linedesc))
1345 build.versionName = vv[0]
1346 build.versionCode = vv[1]
1347 check_versionCode(build.versionCode)
1349 if build.versionCode in vc_seen:
1350 warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
1351 .format(versionCode=build.versionCode, linedesc=linedesc))
1352 vc_seen.add(build.versionCode)
1355 elif ftype == TYPE_OBSOLETE:
1356 pass # Just throw it away!
1358 warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
1359 .format(field=f, linedesc=linedesc))
1360 elif mode == 1: # Multiline field
1363 app[f] = '\n'.join(multiline_lines)
1364 del multiline_lines[:]
1366 multiline_lines.append(line)
1367 elif mode == 2: # Line continuation mode in Build Version
1368 if line.endswith("\\"):
1369 buildlines.append(line[:-1])
1371 buildlines.append(line)
1372 build = parse_buildline(buildlines)
1373 app.builds.append(build)
1374 add_comments('build:' + app.builds[-1].versionCode)
1378 # Mode at end of file should always be 0
1380 warn_or_exception(_("{field} not terminated in {name}")
1381 .format(field=f, name=mf.name))
1383 warn_or_exception(_("Unterminated continuation in {name}")
1384 .format(name=mf.name))
1386 warn_or_exception(_("Unterminated build in {name}")
1387 .format(name=mf.name))
1392 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1394 def field_to_attr(f):
1396 Translates human-readable field names to attribute names, e.g.
1397 'Auto Name' to 'AutoName'
1399 return f.replace(' ', '')
1401 def attr_to_field(k):
1403 Translates attribute names to human-readable field names, e.g.
1404 'AutoName' to 'Auto Name'
1408 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1411 def w_comments(key):
1412 if key not in app.comments:
1414 for line in app.comments[key]:
1417 def w_field_always(f, v=None):
1418 key = field_to_attr(f)
1424 def w_field_nonempty(f, v=None):
1425 key = field_to_attr(f)
1432 w_field_nonempty('Disabled')
1433 w_field_nonempty('AntiFeatures')
1434 w_field_nonempty('Provides')
1435 w_field_always('Categories')
1436 w_field_always('License')
1437 w_field_nonempty('Author Name')
1438 w_field_nonempty('Author Email')
1439 w_field_nonempty('Author Web Site')
1440 w_field_always('Web Site')
1441 w_field_always('Source Code')
1442 w_field_always('Issue Tracker')
1443 w_field_nonempty('Translation')
1444 w_field_nonempty('Changelog')
1445 w_field_nonempty('Donate')
1446 w_field_nonempty('FlattrID')
1447 w_field_nonempty('LiberapayID')
1448 w_field_nonempty('Bitcoin')
1449 w_field_nonempty('Litecoin')
1451 w_field_nonempty('Name')
1452 w_field_nonempty('Auto Name')
1453 w_field_nonempty('Summary')
1454 w_field_nonempty('Description', description_txt(app.Description))
1456 if app.RequiresRoot:
1457 w_field_always('Requires Root', 'yes')
1460 w_field_always('Repo Type')
1461 w_field_always('Repo')
1463 w_field_always('Binaries')
1466 for build in app.builds:
1468 if build.versionName == "Ignore":
1471 w_comments('build:%s' % build.versionCode)
1475 if app.MaintainerNotes:
1476 w_field_always('Maintainer Notes', app.MaintainerNotes)
1479 w_field_nonempty('Archive Policy')
1480 w_field_always('Auto Update Mode')
1481 w_field_always('Update Check Mode')
1482 w_field_nonempty('Update Check Ignore')
1483 w_field_nonempty('Vercode Operation')
1484 w_field_nonempty('Update Check Name')
1485 w_field_nonempty('Update Check Data')
1486 if app.CurrentVersion:
1487 w_field_always('Current Version')
1488 w_field_always('Current Version Code')
1489 if app.NoSourceSince:
1491 w_field_always('No Source Since')
1495 # Write a metadata file in txt format.
1497 # 'mf' - Writer interface (file, StringIO, ...)
1498 # 'app' - The app data
1499 def write_txt(mf, app):
1501 def w_comment(line):
1502 mf.write("# %s\n" % line)
1508 elif t == TYPE_MULTILINE:
1509 v = '\n' + v + '\n.'
1510 mf.write("%s:%s\n" % (f, v))
1513 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1515 for f in build_flags_order:
1521 if f == 'androidupdate':
1522 f = 'update' # avoid conflicting with Build(dict).update()
1523 mf.write(' %s=' % f)
1524 if t == TYPE_STRING or t == TYPE_INT:
1526 elif t == TYPE_BOOL:
1528 elif t == TYPE_SCRIPT:
1530 for s in v.split(' && '):
1534 mf.write(' && \\\n ')
1536 elif t == TYPE_LIST:
1537 mf.write(','.join(v))
1541 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1544 def write_metadata(metadatapath, app):
1545 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
1546 accepted = fdroidserver.common.config['accepted_formats']
1547 if ext not in accepted:
1548 warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
1549 .format(path=metadatapath, formats=', '.join(accepted)))
1552 with open(metadatapath, 'w', encoding='utf8') as mf:
1554 return write_txt(mf, app)
1556 return write_yaml(mf, app)
1557 except FDroidException as e:
1558 os.remove(metadatapath)
1561 warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
1564 def add_metadata_arguments(parser):
1565 '''add common command line flags related to metadata processing'''
1566 parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
1567 help=_("force metadata errors (default) to be warnings, or to be ignored."))