3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
29 from collections import OrderedDict
30 # use libyaml if it is available
32 from yaml import CLoader
35 from yaml import Loader
38 import fdroidserver.common
39 from fdroidserver import _
40 from fdroidserver.exception import MetaDataException, FDroidException
43 warnings_action = None
46 def warn_or_exception(value):
47 '''output warning or Exception depending on -W'''
48 if warnings_action == 'ignore':
50 elif warnings_action == 'error':
51 raise MetaDataException(value)
53 logging.warning(value)
56 # To filter which ones should be written to the metadata files if
88 'Update Check Ignore',
93 'Current Version Code',
97 'comments', # For formats that don't do inline comments
98 'builds', # For formats that do builds as a list
104 def __init__(self, copydict=None):
106 super().__init__(copydict)
111 self.AntiFeatures = []
113 self.Categories = ['None']
114 self.License = 'Unknown'
115 self.AuthorName = None
116 self.AuthorEmail = None
117 self.AuthorWebSite = None
120 self.IssueTracker = ''
124 self.LiberapayID = None
130 self.Description = ''
131 self.RequiresRoot = False
135 self.MaintainerNotes = ''
136 self.ArchivePolicy = None
137 self.AutoUpdateMode = 'None'
138 self.UpdateCheckMode = 'None'
139 self.UpdateCheckIgnore = None
140 self.VercodeOperation = None
141 self.UpdateCheckName = None
142 self.UpdateCheckData = None
143 self.CurrentVersion = ''
144 self.CurrentVersionCode = None
145 self.NoSourceSince = ''
148 self.metadatapath = None
152 self.lastUpdated = None
154 def __getattr__(self, name):
158 raise AttributeError("No such attribute: " + name)
160 def __setattr__(self, name, value):
163 def __delattr__(self, name):
167 raise AttributeError("No such attribute: " + name)
169 def get_last_build(self):
170 if len(self.builds) > 0:
171 return self.builds[-1]
188 'Description': TYPE_MULTILINE,
189 'MaintainerNotes': TYPE_MULTILINE,
190 'Categories': TYPE_LIST,
191 'AntiFeatures': TYPE_LIST,
192 'BuildVersion': TYPE_BUILD,
193 'Build': TYPE_BUILD_V2,
194 'UseBuilt': TYPE_OBSOLETE,
199 name = name.replace(' ', '')
200 if name in fieldtypes:
201 return fieldtypes[name]
205 # In the order in which they are laid out on files
206 build_flags_order = [
241 # old .txt format has version name/code inline in the 'Build:' line
242 # but YAML and JSON have a explicit key for them
243 build_flags = ['versionName', 'versionCode'] + build_flags_order
248 def __init__(self, copydict=None):
253 self.submodules = False
260 self.buildozer = False
263 self.oldsdkloc = False
265 self.forceversion = False
266 self.forcevercode = False
270 self.androidupdate = []
277 self.preassemble = []
278 self.gradleprops = []
279 self.antcommands = []
280 self.novcheck = False
281 self.antifeatures = []
283 super().__init__(copydict)
286 def __getattr__(self, name):
290 raise AttributeError("No such attribute: " + name)
292 def __setattr__(self, name, value):
295 def __delattr__(self, name):
299 raise AttributeError("No such attribute: " + name)
301 def build_method(self):
302 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
309 # like build_method, but prioritize output=
310 def output_method(self):
313 for f in ['maven', 'gradle', 'kivy', 'buildozer']:
321 version = 'r12b' # falls back to latest
322 paths = fdroidserver.common.config['ndk_paths']
323 if version not in paths:
325 return paths[version]
329 'versionCode': TYPE_INT,
330 'extlibs': TYPE_LIST,
331 'srclibs': TYPE_LIST,
334 'buildjni': TYPE_LIST,
335 'preassemble': TYPE_LIST,
336 'androidupdate': TYPE_LIST,
337 'scanignore': TYPE_LIST,
338 'scandelete': TYPE_LIST,
340 'antcommands': TYPE_LIST,
341 'gradleprops': TYPE_LIST,
344 'prebuild': TYPE_SCRIPT,
345 'build': TYPE_SCRIPT,
346 'submodules': TYPE_BOOL,
347 'oldsdkloc': TYPE_BOOL,
348 'forceversion': TYPE_BOOL,
349 'forcevercode': TYPE_BOOL,
350 'novcheck': TYPE_BOOL,
351 'antifeatures': TYPE_LIST,
356 if name in flagtypes:
357 return flagtypes[name]
361 class FieldValidator():
363 Designates App metadata field types and checks that it matches
365 'name' - The long name of the field type
366 'matching' - List of possible values or regex expression
367 'sep' - Separator to use if value may be a list
368 'fields' - Metadata fields (Field:Value) of this type
371 def __init__(self, name, matching, fields):
373 self.matching = matching
374 self.compiled = re.compile(matching)
377 def check(self, v, appid):
385 if not self.compiled.match(v):
386 warn_or_exception(_("'{value}' is not a valid {field} in {appid}. Regex pattern: {pattern}")
387 .format(value=v, field=self.name, appid=appid, pattern=self.matching))
390 # Generic value types
392 FieldValidator("Flattr ID",
396 FieldValidator("Liberapay ID",
400 FieldValidator("HTTP link",
402 ["WebSite", "SourceCode", "IssueTracker", "Changelog", "Donate"]),
404 FieldValidator("Email",
405 r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
408 FieldValidator("Bitcoin address",
409 r'^[a-zA-Z0-9]{27,34}$',
412 FieldValidator("Litecoin address",
413 r'^L[a-zA-Z0-9]{33}$',
416 FieldValidator("Repo Type",
417 r'^(git|git-svn|svn|hg|bzr|srclib)$',
420 FieldValidator("Binaries",
424 FieldValidator("Archive Policy",
425 r'^[0-9]+ versions$',
428 FieldValidator("Anti-Feature",
429 r'^(Ads|Tracking|NonFreeNet|NonFreeDep|NonFreeAdd|UpstreamNonFree|NonFreeAssets|KnownVuln|ApplicationDebuggable)$',
432 FieldValidator("Auto Update Mode",
433 r"^(Version .+|None)$",
436 FieldValidator("Update Check Mode",
437 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$",
442 # Check an app's metadata information for integrity errors
443 def check_metadata(app):
446 v.check(app[k], app.id)
449 # Formatter for descriptions. Create an instance, and call parseline() with
450 # each line of the description source from the metadata. At the end, call
451 # end() and then text_txt and text_html will contain the result.
452 class DescriptionFormatter:
459 def __init__(self, linkres):
462 self.state = self.stNONE
463 self.laststate = self.stNONE
466 self.html = io.StringIO()
467 self.text = io.StringIO()
469 self.linkResolver = None
470 self.linkResolver = linkres
472 def endcur(self, notstates=None):
473 if notstates and self.state in notstates:
475 if self.state == self.stPARA:
477 elif self.state == self.stUL:
479 elif self.state == self.stOL:
483 self.laststate = self.state
484 self.state = self.stNONE
485 whole_para = ' '.join(self.para_lines)
486 self.addtext(whole_para)
487 wrapped = textwrap.fill(whole_para, 80,
488 break_long_words=False,
489 break_on_hyphens=False)
490 self.text.write(wrapped)
491 self.html.write('</p>')
492 del self.para_lines[:]
495 self.html.write('</ul>')
496 self.laststate = self.state
497 self.state = self.stNONE
500 self.html.write('</ol>')
501 self.laststate = self.state
502 self.state = self.stNONE
504 def formatted(self, txt, htmlbody):
507 txt = html.escape(txt, quote=False)
509 index = txt.find("''")
514 if txt.startswith("'''"):
520 self.bold = not self.bold
528 self.ital = not self.ital
531 def linkify(self, txt):
535 index = txt.find("[")
537 return (res_plain + self.formatted(txt, False), res_html + self.formatted(txt, True))
538 res_plain += self.formatted(txt[:index], False)
539 res_html += self.formatted(txt[:index], True)
541 if txt.startswith("[["):
542 index = txt.find("]]")
544 warn_or_exception(_("Unterminated ]]"))
546 if self.linkResolver:
547 url, urltext = self.linkResolver(url)
550 res_html += '<a href="' + url + '">' + html.escape(urltext, quote=False) + '</a>'
552 txt = txt[index + 2:]
554 index = txt.find("]")
556 warn_or_exception(_("Unterminated ]"))
558 index2 = url.find(' ')
562 urltxt = url[index2 + 1:]
565 warn_or_exception(_("URL title is just the URL, use brackets: [URL]"))
566 res_html += '<a href="' + url + '">' + html.escape(urltxt, quote=False) + '</a>'
569 res_plain += ' (' + url + ')'
570 txt = txt[index + 1:]
572 def addtext(self, txt):
573 p, h = self.linkify(txt)
576 def parseline(self, line):
579 elif line.startswith('* '):
580 self.endcur([self.stUL])
581 if self.state != self.stUL:
582 self.html.write('<ul>')
583 self.state = self.stUL
584 if self.laststate != self.stNONE:
585 self.text.write('\n\n')
587 self.text.write('\n')
588 self.text.write(line)
589 self.html.write('<li>')
590 self.addtext(line[1:])
591 self.html.write('</li>')
592 elif line.startswith('# '):
593 self.endcur([self.stOL])
594 if self.state != self.stOL:
595 self.html.write('<ol>')
596 self.state = self.stOL
597 if self.laststate != self.stNONE:
598 self.text.write('\n\n')
600 self.text.write('\n')
601 self.text.write(line)
602 self.html.write('<li>')
603 self.addtext(line[1:])
604 self.html.write('</li>')
606 self.para_lines.append(line)
607 self.endcur([self.stPARA])
608 if self.state == self.stNONE:
609 self.state = self.stPARA
610 if self.laststate != self.stNONE:
611 self.text.write('\n\n')
612 self.html.write('<p>')
616 self.text_txt = self.text.getvalue()
617 self.text_html = self.html.getvalue()
622 # Parse multiple lines of description as written in a metadata file, returning
623 # a single string in text format and wrapped to 80 columns.
624 def description_txt(s):
625 ps = DescriptionFormatter(None)
626 for line in s.splitlines():
632 # Parse multiple lines of description as written in a metadata file, returning
633 # a single string in wiki format. Used for the Maintainer Notes field as well,
634 # because it's the same format.
635 def description_wiki(s):
639 # Parse multiple lines of description as written in a metadata file, returning
640 # a single string in HTML format.
641 def description_html(s, linkres):
642 ps = DescriptionFormatter(linkres)
643 for line in s.splitlines():
649 def parse_srclib(metadatapath):
653 # Defaults for fields that come from metadata
654 thisinfo['Repo Type'] = ''
655 thisinfo['Repo'] = ''
656 thisinfo['Subdir'] = None
657 thisinfo['Prepare'] = None
659 if not os.path.exists(metadatapath):
662 metafile = open(metadatapath, "r", encoding='utf-8')
665 for line in metafile:
667 line = line.rstrip('\r\n')
668 if not line or line.startswith("#"):
672 f, v = line.split(':', 1)
674 warn_or_exception(_("Invalid metadata in %s:%d") % (line, n))
677 thisinfo[f] = v.split(',')
687 """Read all srclib metadata.
689 The information read will be accessible as metadata.srclibs, which is a
690 dictionary, keyed on srclib name, with the values each being a dictionary
691 in the same format as that returned by the parse_srclib function.
693 A MetaDataException is raised if there are any problems with the srclib
698 # They were already loaded
699 if srclibs is not None:
705 if not os.path.exists(srcdir):
708 for metadatapath in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
709 srclibname = os.path.basename(metadatapath[:-4])
710 srclibs[srclibname] = parse_srclib(metadatapath)
713 def read_metadata(xref=True, check_vcs=[], sort_by_time=False):
714 """Return a list of App instances sorted newest first
716 This reads all of the metadata files in a 'data' repository, then
717 builds a list of App instances from those files. The list is
718 sorted based on creation time, newest first. Most of the time,
719 the newer files are the most interesting.
721 If there are multiple metadata files for a single appid, then the first
722 file that is parsed wins over all the others, and the rest throw an
723 exception. So the original .txt format is parsed first, at least until
724 newer formats stabilize.
726 check_vcs is the list of packageNames to check for .fdroid.yml in source
730 # Always read the srclibs before the apps, since they can use a srlib as
731 # their source repository.
736 for basedir in ('metadata', 'tmp'):
737 if not os.path.exists(basedir):
740 metadatafiles = (glob.glob(os.path.join('metadata', '*.txt'))
741 + glob.glob(os.path.join('metadata', '*.json'))
742 + glob.glob(os.path.join('metadata', '*.yml'))
743 + glob.glob('.fdroid.txt')
744 + glob.glob('.fdroid.json')
745 + glob.glob('.fdroid.yml'))
748 entries = ((os.stat(path).st_mtime, path) for path in metadatafiles)
750 for _ignored, path in sorted(entries, reverse=True):
751 metadatafiles.append(path)
753 # most things want the index alpha sorted for stability
754 metadatafiles = sorted(metadatafiles)
756 for metadatapath in metadatafiles:
757 if metadatapath == '.fdroid.txt':
758 warn_or_exception(_('.fdroid.txt is not supported! Convert to .fdroid.yml or .fdroid.json.'))
759 packageName, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
760 if packageName in apps:
761 warn_or_exception(_("Found multiple metadata files for {appid}")
762 .format(path=packageName))
763 app = parse_metadata(metadatapath, packageName in check_vcs)
768 # Parse all descriptions at load time, just to ensure cross-referencing
769 # errors are caught early rather than when they hit the build server.
772 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
773 warn_or_exception(_("Cannot resolve app id {appid}").format(appid=appid))
775 for appid, app in apps.items():
777 description_html(app.Description, linkres)
778 except MetaDataException as e:
779 warn_or_exception(_("Problem with description of {appid}: {error}")
780 .format(appid=appid, error=str(e)))
785 # Port legacy ';' separators
786 list_sep = re.compile(r'[,;]')
789 def split_list_values(s):
791 for v in re.split(list_sep, s):
801 def get_default_app_info(metadatapath=None):
802 if metadatapath is None:
805 appid, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
807 if appid == '.fdroid': # we have local metadata in the app's source
808 if os.path.exists('AndroidManifest.xml'):
809 manifestroot = fdroidserver.common.parse_xml('AndroidManifest.xml')
811 pattern = re.compile(""".*manifest\.srcFile\s+'AndroidManifest\.xml'.*""")
812 for root, dirs, files in os.walk(os.getcwd()):
813 if 'build.gradle' in files:
814 p = os.path.join(root, 'build.gradle')
815 with open(p, 'rb') as f:
817 m = pattern.search(data)
819 logging.debug('Using: ' + os.path.join(root, 'AndroidManifest.xml'))
820 manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
822 if manifestroot is None:
823 warn_or_exception(_("Cannot find a packageName for {path}!")
824 .format(path=metadatapath))
825 appid = manifestroot.attrib['package']
828 app.metadatapath = metadatapath
829 if appid is not None:
835 def sorted_builds(builds):
836 return sorted(builds, key=lambda build: int(build.versionCode))
839 esc_newlines = re.compile(r'\\( |\n)')
842 def post_metadata_parse(app):
843 # TODO keep native types, convert only for .txt metadata
844 for k, v in app.items():
845 if type(v) in (float, int):
849 app['builds'] = app.pop('Builds')
851 if 'flavours' in app and app['flavours'] == [True]:
852 app['flavours'] = 'yes'
854 if isinstance(app.Categories, str):
855 app.Categories = [app.Categories]
856 elif app.Categories is None:
857 app.Categories = ['None']
859 app.Categories = [str(i) for i in app.Categories]
861 def _yaml_bool_unmapable(v):
862 return v in (True, False, [True], [False])
864 def _yaml_bool_unmap(v):
874 _bool_allowed = ('disable', 'kivy', 'maven', 'buildozer')
878 for build in app['builds']:
879 if not isinstance(build, Build):
881 for k, v in build.items():
883 if flagtype(k) == TYPE_LIST:
884 if _yaml_bool_unmapable(v):
885 build[k] = _yaml_bool_unmap(v)
887 if isinstance(v, str):
889 elif isinstance(v, bool):
894 elif flagtype(k) is TYPE_INT:
896 elif flagtype(k) is TYPE_STRING:
897 if isinstance(v, bool) and k in _bool_allowed:
900 if _yaml_bool_unmapable(v):
901 build[k] = _yaml_bool_unmap(v)
906 app.builds = sorted_builds(builds)
909 # Parse metadata for a single application.
911 # 'metadatapath' - the filename to read. The package id for the application comes
912 # from this filename. Pass None to get a blank entry.
914 # Returns a dictionary containing all the details of the application. There are
915 # two major kinds of information in the dictionary. Keys beginning with capital
916 # letters correspond directory to identically named keys in the metadata file.
917 # Keys beginning with lower case letters are generated in one way or another,
918 # and are not found verbatim in the metadata.
920 # Known keys not originating from the metadata are:
922 # 'builds' - a list of dictionaries containing build information
923 # for each defined build
924 # 'comments' - a list of comments from the metadata file. Each is
925 # a list of the form [field, comment] where field is
926 # the name of the field it preceded in the metadata
927 # file. Where field is None, the comment goes at the
928 # end of the file. Alternatively, 'build:version' is
929 # for a comment before a particular build version.
930 # 'descriptionlines' - original lines of description as formatted in the
935 bool_true = re.compile(r'([Yy]es|[Tt]rue)')
936 bool_false = re.compile(r'([Nn]o|[Ff]alse)')
940 if bool_true.match(s):
942 if bool_false.match(s):
944 warn_or_exception(_("Invalid boolean '%s'") % s)
947 def parse_metadata(metadatapath, check_vcs=False):
948 '''parse metadata file, optionally checking the git repo for metadata first'''
950 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
951 accepted = fdroidserver.common.config['accepted_formats']
952 if ext not in accepted:
953 warn_or_exception(_('"{path}" is not an accepted format, convert to: {formats}')
954 .format(path=metadatapath, formats=', '.join(accepted)))
957 app.metadatapath = metadatapath
958 name, _ignored = fdroidserver.common.get_extension(os.path.basename(metadatapath))
959 if name == '.fdroid':
964 with open(metadatapath, 'r', encoding='utf-8') as mf:
966 parse_txt_metadata(mf, app)
968 parse_json_metadata(mf, app)
970 parse_yaml_metadata(mf, app)
972 warn_or_exception(_('Unknown metadata format: {path}')
973 .format(path=metadatapath))
975 if check_vcs and app.Repo:
976 build_dir = fdroidserver.common.get_build_dir(app)
977 metadata_in_repo = os.path.join(build_dir, '.fdroid.yml')
978 if not os.path.isfile(metadata_in_repo):
979 vcs, build_dir = fdroidserver.common.setup_vcs(app)
980 if isinstance(vcs, fdroidserver.common.vcs_git):
981 vcs.gotorevision('HEAD') # HEAD since we can't know where else to go
982 if os.path.isfile(metadata_in_repo):
983 logging.debug('Including metadata from ' + metadata_in_repo)
984 # do not include fields already provided by main metadata file
985 app_in_repo = parse_metadata(metadata_in_repo)
986 for k, v in app_in_repo.items():
990 post_metadata_parse(app)
994 build = app.builds[-1]
996 root_dir = build.subdir
999 paths = fdroidserver.common.manifest_paths(root_dir, build.gradle)
1000 _ignored, _ignored, app.id = fdroidserver.common.parse_androidmanifests(paths, app)
1005 def parse_json_metadata(mf, app):
1007 # fdroid metadata is only strings and booleans, no floats or ints.
1008 # TODO create schema using https://pypi.python.org/pypi/jsonschema
1009 jsoninfo = json.load(mf, parse_int=lambda s: s,
1010 parse_float=lambda s: s)
1011 app.update(jsoninfo)
1012 for f in ['Description', 'Maintainer Notes']:
1015 app[f] = '\n'.join(v)
1019 def parse_yaml_metadata(mf, app):
1020 yamldata = yaml.load(mf, Loader=YamlLoader)
1022 app.update(yamldata)
1026 def write_yaml(mf, app):
1028 # import rumael.yaml and check version
1031 except ImportError as e:
1032 raise FDroidException('ruamel.yaml not instlled, can not write metadata.') from e
1033 if not ruamel.yaml.__version__:
1034 raise FDroidException('ruamel.yaml.__version__ not accessible. Please make sure a ruamel.yaml >= 0.13 is installed..')
1035 m = re.match('(?P<major>[0-9]+)\.(?P<minor>[0-9]+)\.(?P<patch>[0-9]+)(-.+)?',
1036 ruamel.yaml.__version__)
1038 raise FDroidException('ruamel.yaml version malfored, please install an upstream version of ruamel.yaml')
1039 if int(m.group('major')) < 0 or int(m.group('minor')) < 13:
1040 raise FDroidException('currently installed version of ruamel.yaml ({}) is too old, >= 1.13 required.'.format(ruamel.yaml.__version__))
1041 # suiteable version ruamel.yaml imported successfully
1043 _yaml_bools_true = ('y', 'Y', 'yes', 'Yes', 'YES',
1044 'true', 'True', 'TRUE',
1046 _yaml_bools_false = ('n', 'N', 'no', 'No', 'NO',
1047 'false', 'False', 'FALSE',
1048 'off', 'Off', 'OFF')
1049 _yaml_bools_plus_lists = []
1050 _yaml_bools_plus_lists.extend(_yaml_bools_true)
1051 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_true])
1052 _yaml_bools_plus_lists.extend(_yaml_bools_false)
1053 _yaml_bools_plus_lists.extend([[x] for x in _yaml_bools_false])
1055 def _class_as_dict_representer(dumper, data):
1056 '''Creates a YAML representation of a App/Build instance'''
1057 return dumper.represent_dict(data)
1059 def _field_to_yaml(typ, value):
1060 if typ is TYPE_STRING:
1061 if value in _yaml_bools_plus_lists:
1062 return ruamel.yaml.scalarstring.SingleQuotedScalarString(str(value))
1064 elif typ is TYPE_INT:
1066 elif typ is TYPE_MULTILINE:
1068 return ruamel.yaml.scalarstring.preserve_literal(str(value))
1071 elif typ is TYPE_SCRIPT:
1073 return ruamel.yaml.scalarstring.preserve_literal(value)
1079 def _app_to_yaml(app):
1080 cm = ruamel.yaml.comments.CommentedMap()
1081 insert_newline = False
1082 for field in yaml_app_field_order:
1084 # next iteration will need to insert a newline
1085 insert_newline = True
1087 if app.get(field) or field is 'Builds':
1088 # .txt calls it 'builds' internally, everywhere else its 'Builds'
1089 if field is 'Builds':
1090 if app.get('builds'):
1091 cm.update({field: _builds_to_yaml(app)})
1092 elif field is 'CurrentVersionCode':
1093 cm.update({field: _field_to_yaml(TYPE_INT, getattr(app, field))})
1095 cm.update({field: _field_to_yaml(fieldtype(field), getattr(app, field))})
1098 # we need to prepend a newline in front of this field
1099 insert_newline = False
1100 # inserting empty lines is not supported so we add a
1101 # bogus comment and over-write its value
1102 cm.yaml_set_comment_before_after_key(field, 'bogus')
1103 cm.ca.items[field][1][-1].value = '\n'
1106 def _builds_to_yaml(app):
1107 fields = ['versionName', 'versionCode']
1108 fields.extend(build_flags_order)
1109 builds = ruamel.yaml.comments.CommentedSeq()
1110 for build in app.builds:
1111 b = ruamel.yaml.comments.CommentedMap()
1112 for field in fields:
1113 if hasattr(build, field) and getattr(build, field):
1114 value = getattr(build, field)
1115 if field == 'gradle' and value == ['off']:
1116 value = [ruamel.yaml.scalarstring.SingleQuotedScalarString('off')]
1117 if field in ('disable', 'kivy', 'maven', 'buildozer'):
1120 elif value == 'yes':
1122 b.update({field: _field_to_yaml(flagtype(field), value)})
1125 # insert extra empty lines between build entries
1126 for i in range(1, len(builds)):
1127 builds.yaml_set_comment_before_after_key(i, 'bogus')
1128 builds.ca.items[i][1][-1].value = '\n'
1132 yaml_app_field_order = [
1169 'UpdateCheckIgnore',
1174 'CurrentVersionCode',
1179 yaml_app = _app_to_yaml(app)
1180 ruamel.yaml.round_trip_dump(yaml_app, mf, indent=4, block_seq_indent=2)
1183 build_line_sep = re.compile(r'(?<!\\),')
1184 build_cont = re.compile(r'^[ \t]')
1187 def parse_txt_metadata(mf, app):
1191 def add_buildflag(p, build):
1193 warn_or_exception(_("Empty build flag at {linedesc}")
1194 .format(linedesc=linedesc))
1195 bv = p.split('=', 1)
1197 warn_or_exception(_("Invalid build flag at {line} in {linedesc}")
1198 .format(line=buildlines[0], linedesc=linedesc))
1203 pk = 'androidupdate' # avoid conflicting with Build(dict).update()
1206 pv = split_list_values(pv)
1208 elif t == TYPE_STRING or t == TYPE_SCRIPT:
1210 elif t == TYPE_BOOL:
1211 build[pk] = _decode_bool(pv)
1213 def parse_buildline(lines):
1215 parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
1217 warn_or_exception(_("Invalid build format: {value} in {name}")
1218 .format(value=v, name=mf.name))
1220 build.versionName = parts[0]
1221 build.versionCode = parts[1]
1222 check_versionCode(build.versionCode)
1224 if parts[2].startswith('!'):
1225 # For backwards compatibility, handle old-style disabling,
1226 # including attempting to extract the commit from the message
1227 build.disable = parts[2][1:]
1228 commit = 'unknown - see disabled'
1229 index = parts[2].rfind('at ')
1231 commit = parts[2][index + 3:]
1232 if commit.endswith(')'):
1233 commit = commit[:-1]
1234 build.commit = commit
1236 build.commit = parts[2]
1238 add_buildflag(p, build)
1242 def check_versionCode(versionCode):
1246 warn_or_exception(_('Invalid versionCode: "{versionCode}" is not an integer!')
1247 .format(versionCode=versionCode))
1249 def add_comments(key):
1252 app.comments[key] = list(curcomments)
1257 multiline_lines = []
1267 linedesc = "%s:%d" % (mf.name, c)
1268 line = line.rstrip('\r\n')
1270 if build_cont.match(line):
1271 if line.endswith('\\'):
1272 buildlines.append(line[:-1].lstrip())
1274 buildlines.append(line.lstrip())
1275 bl = ''.join(buildlines)
1276 add_buildflag(bl, build)
1279 if not build.commit and not build.disable:
1280 warn_or_exception(_("No commit specified for {versionName} in {linedesc}")
1281 .format(versionName=build.versionName, linedesc=linedesc))
1283 app.builds.append(build)
1284 add_comments('build:' + build.versionCode)
1290 if line.startswith("#"):
1291 curcomments.append(line[1:].strip())
1294 f, v = line.split(':', 1)
1296 warn_or_exception(_("Invalid metadata in: ") + linedesc)
1298 if f not in app_fields:
1299 warn_or_exception(_('Unrecognised app field: ') + f)
1301 # Translate obsolete fields...
1302 if f == 'Market Version':
1303 f = 'Current Version'
1304 if f == 'Market Version Code':
1305 f = 'Current Version Code'
1307 f = f.replace(' ', '')
1309 ftype = fieldtype(f)
1310 if ftype not in [TYPE_BUILD, TYPE_BUILD_V2]:
1312 if ftype == TYPE_MULTILINE:
1315 warn_or_exception(_("Unexpected text on same line as {field} in {linedesc}")
1316 .format(field=f, linedesc=linedesc))
1317 elif ftype == TYPE_STRING:
1319 elif ftype == TYPE_LIST:
1320 app[f] = split_list_values(v)
1321 elif ftype == TYPE_BUILD:
1322 if v.endswith("\\"):
1325 buildlines.append(v[:-1])
1327 build = parse_buildline([v])
1328 app.builds.append(build)
1329 add_comments('build:' + app.builds[-1].versionCode)
1330 elif ftype == TYPE_BUILD_V2:
1333 warn_or_exception(_('Build should have comma-separated '
1334 'versionName and versionCode, '
1335 'not "{value}", in {linedesc}')
1336 .format(value=v, linedesc=linedesc))
1338 build.versionName = vv[0]
1339 build.versionCode = vv[1]
1340 check_versionCode(build.versionCode)
1342 if build.versionCode in vc_seen:
1343 warn_or_exception(_('Duplicate build recipe found for versionCode {versionCode} in {linedesc}')
1344 .format(versionCode=build.versionCode, linedesc=linedesc))
1345 vc_seen.add(build.versionCode)
1348 elif ftype == TYPE_OBSOLETE:
1349 pass # Just throw it away!
1351 warn_or_exception(_("Unrecognised field '{field}' in {linedesc}")
1352 .format(field=f, linedesc=linedesc))
1353 elif mode == 1: # Multiline field
1356 app[f] = '\n'.join(multiline_lines)
1357 del multiline_lines[:]
1359 multiline_lines.append(line)
1360 elif mode == 2: # Line continuation mode in Build Version
1361 if line.endswith("\\"):
1362 buildlines.append(line[:-1])
1364 buildlines.append(line)
1365 build = parse_buildline(buildlines)
1366 app.builds.append(build)
1367 add_comments('build:' + app.builds[-1].versionCode)
1371 # Mode at end of file should always be 0
1373 warn_or_exception(_("{field} not terminated in {name}")
1374 .format(field=f, name=mf.name))
1376 warn_or_exception(_("Unterminated continuation in {name}")
1377 .format(name=mf.name))
1379 warn_or_exception(_("Unterminated build in {name}")
1380 .format(name=mf.name))
1385 def write_plaintext_metadata(mf, app, w_comment, w_field, w_build):
1387 def field_to_attr(f):
1389 Translates human-readable field names to attribute names, e.g.
1390 'Auto Name' to 'AutoName'
1392 return f.replace(' ', '')
1394 def attr_to_field(k):
1396 Translates attribute names to human-readable field names, e.g.
1397 'AutoName' to 'Auto Name'
1401 f = re.sub(r'([a-z])([A-Z])', r'\1 \2', k)
1404 def w_comments(key):
1405 if key not in app.comments:
1407 for line in app.comments[key]:
1410 def w_field_always(f, v=None):
1411 key = field_to_attr(f)
1417 def w_field_nonempty(f, v=None):
1418 key = field_to_attr(f)
1425 w_field_nonempty('Disabled')
1426 w_field_nonempty('AntiFeatures')
1427 w_field_nonempty('Provides')
1428 w_field_always('Categories')
1429 w_field_always('License')
1430 w_field_nonempty('Author Name')
1431 w_field_nonempty('Author Email')
1432 w_field_nonempty('Author Web Site')
1433 w_field_always('Web Site')
1434 w_field_always('Source Code')
1435 w_field_always('Issue Tracker')
1436 w_field_nonempty('Changelog')
1437 w_field_nonempty('Donate')
1438 w_field_nonempty('FlattrID')
1439 w_field_nonempty('LiberapayID')
1440 w_field_nonempty('Bitcoin')
1441 w_field_nonempty('Litecoin')
1443 w_field_nonempty('Name')
1444 w_field_nonempty('Auto Name')
1445 w_field_nonempty('Summary')
1446 w_field_nonempty('Description', description_txt(app.Description))
1448 if app.RequiresRoot:
1449 w_field_always('Requires Root', 'yes')
1452 w_field_always('Repo Type')
1453 w_field_always('Repo')
1455 w_field_always('Binaries')
1458 for build in app.builds:
1460 if build.versionName == "Ignore":
1463 w_comments('build:%s' % build.versionCode)
1467 if app.MaintainerNotes:
1468 w_field_always('Maintainer Notes', app.MaintainerNotes)
1471 w_field_nonempty('Archive Policy')
1472 w_field_always('Auto Update Mode')
1473 w_field_always('Update Check Mode')
1474 w_field_nonempty('Update Check Ignore')
1475 w_field_nonempty('Vercode Operation')
1476 w_field_nonempty('Update Check Name')
1477 w_field_nonempty('Update Check Data')
1478 if app.CurrentVersion:
1479 w_field_always('Current Version')
1480 w_field_always('Current Version Code')
1481 if app.NoSourceSince:
1483 w_field_always('No Source Since')
1487 # Write a metadata file in txt format.
1489 # 'mf' - Writer interface (file, StringIO, ...)
1490 # 'app' - The app data
1491 def write_txt(mf, app):
1493 def w_comment(line):
1494 mf.write("# %s\n" % line)
1500 elif t == TYPE_MULTILINE:
1501 v = '\n' + v + '\n.'
1502 mf.write("%s:%s\n" % (f, v))
1505 mf.write("Build:%s,%s\n" % (build.versionName, build.versionCode))
1507 for f in build_flags_order:
1513 if f == 'androidupdate':
1514 f = 'update' # avoid conflicting with Build(dict).update()
1515 mf.write(' %s=' % f)
1516 if t == TYPE_STRING:
1518 elif t == TYPE_BOOL:
1520 elif t == TYPE_SCRIPT:
1522 for s in v.split(' && '):
1526 mf.write(' && \\\n ')
1528 elif t == TYPE_LIST:
1529 mf.write(','.join(v))
1533 write_plaintext_metadata(mf, app, w_comment, w_field, w_build)
1536 def write_metadata(metadatapath, app):
1537 _ignored, ext = fdroidserver.common.get_extension(metadatapath)
1538 accepted = fdroidserver.common.config['accepted_formats']
1539 if ext not in accepted:
1540 warn_or_exception(_('Cannot write "{path}", not an accepted format, use: {formats}')
1541 .format(path=metadatapath, formats=', '.join(accepted)))
1544 with open(metadatapath, 'w', encoding='utf8') as mf:
1546 return write_txt(mf, app)
1548 return write_yaml(mf, app)
1549 except FDroidException as e:
1550 os.remove(metadatapath)
1553 warn_or_exception(_('Unknown metadata format: %s') % metadatapath)
1556 def add_metadata_arguments(parser):
1557 '''add common command line flags related to metadata processing'''
1558 parser.add_argument("-W", choices=['error', 'warn', 'ignore'], default='error',
1559 help=_("force metadata errors (default) to be warnings, or to be ignored."))