1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from collections import OrderedDict
34 class MetaDataException(Exception):
36 def __init__(self, value):
42 # In the order in which they are laid out on files
43 app_defaults = OrderedDict([
45 ('AntiFeatures', None),
47 ('Categories', ['None']),
48 ('License', 'Unknown'),
51 ('Issue Tracker', ''),
62 ('Requires Root', False),
66 ('Maintainer Notes', []),
67 ('Archive Policy', None),
68 ('Auto Update Mode', 'None'),
69 ('Update Check Mode', 'None'),
70 ('Update Check Ignore', None),
71 ('Vercode Operation', None),
72 ('Update Check Name', None),
73 ('Update Check Data', None),
74 ('Current Version', ''),
75 ('Current Version Code', '0'),
76 ('No Source Since', ''),
80 # In the order in which they are laid out on files
81 # Sorted by their action and their place in the build timeline
82 flag_defaults = OrderedDict([
86 ('submodules', False),
96 ('forceversion', False),
97 ('forcevercode', False),
101 ('update', ['auto']),
107 ('ndk', 'r10e'), # defaults to latest
110 ('antcommands', None),
115 # Designates a metadata field type and checks that it matches
117 # 'name' - The long name of the field type
118 # 'matching' - List of possible values or regex expression
119 # 'sep' - Separator to use if value may be a list
120 # 'fields' - Metadata fields (Field:Value) of this type
121 # 'attrs' - Build attributes (attr=value) of this type
123 class FieldValidator():
125 def __init__(self, name, matching, sep, fields, attrs):
127 self.matching = matching
128 if type(matching) is str:
129 self.compiled = re.compile(matching)
134 def _assert_regex(self, values, appid):
136 if not self.compiled.match(v):
137 raise MetaDataException("'%s' is not a valid %s in %s. "
138 % (v, self.name, appid) +
139 "Regex pattern: %s" % (self.matching))
141 def _assert_list(self, values, appid):
143 if v not in self.matching:
144 raise MetaDataException("'%s' is not a valid %s in %s. "
145 % (v, self.name, appid) +
146 "Possible values: %s" % (", ".join(self.matching)))
148 def check(self, value, appid):
149 if type(value) is not str or not value:
151 if self.sep is not None:
152 values = value.split(self.sep)
155 if type(self.matching) is list:
156 self._assert_list(values, appid)
158 self._assert_regex(values, appid)
161 # Generic value types
163 FieldValidator("Integer",
164 r'^[1-9][0-9]*$', None,
168 FieldValidator("Hexadecimal",
169 r'^[0-9a-f]+$', None,
173 FieldValidator("HTTP link",
174 r'^http[s]?://', None,
175 ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
177 FieldValidator("Bitcoin address",
178 r'^[a-zA-Z0-9]{27,34}$', None,
182 FieldValidator("Litecoin address",
183 r'^L[a-zA-Z0-9]{33}$', None,
187 FieldValidator("Dogecoin address",
188 r'^D[a-zA-Z0-9]{33}$', None,
192 FieldValidator("Boolean",
197 FieldValidator("bool",
200 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
203 FieldValidator("Repo Type",
204 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
208 FieldValidator("Binaries",
209 r'^http[s]?://', None,
213 FieldValidator("Archive Policy",
214 r'^[0-9]+ versions$', None,
218 FieldValidator("Anti-Feature",
219 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
223 FieldValidator("Auto Update Mode",
224 r"^(Version .+|None)$", None,
225 ["Auto Update Mode"],
228 FieldValidator("Update Check Mode",
229 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
230 ["Update Check Mode"],
235 # Check an app's metadata information for integrity errors
236 def check_metadata(info):
238 for field in v.fields:
239 v.check(info[field], info['id'])
240 for build in info['builds']:
242 v.check(build[attr], info['id'])
245 # Formatter for descriptions. Create an instance, and call parseline() with
246 # each line of the description source from the metadata. At the end, call
247 # end() and then text_wiki and text_html will contain the result.
248 class DescriptionFormatter:
260 def __init__(self, linkres):
261 self.linkResolver = linkres
263 def endcur(self, notstates=None):
264 if notstates and self.state in notstates:
266 if self.state == self.stPARA:
268 elif self.state == self.stUL:
270 elif self.state == self.stOL:
274 self.text_html += '</p>'
275 self.state = self.stNONE
278 self.text_html += '</ul>'
279 self.state = self.stNONE
282 self.text_html += '</ol>'
283 self.state = self.stNONE
285 def formatted(self, txt, html):
288 txt = cgi.escape(txt)
290 index = txt.find("''")
292 return formatted + txt
293 formatted += txt[:index]
295 if txt.startswith("'''"):
301 self.bold = not self.bold
309 self.ital = not self.ital
312 def linkify(self, txt):
316 index = txt.find("[")
318 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
319 linkified_plain += self.formatted(txt[:index], False)
320 linkified_html += self.formatted(txt[:index], True)
322 if txt.startswith("[["):
323 index = txt.find("]]")
325 raise MetaDataException("Unterminated ]]")
327 if self.linkResolver:
328 url, urltext = self.linkResolver(url)
331 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
332 linkified_plain += urltext
333 txt = txt[index + 2:]
335 index = txt.find("]")
337 raise MetaDataException("Unterminated ]")
339 index2 = url.find(' ')
343 urltxt = url[index2 + 1:]
346 raise MetaDataException("Url title is just the URL - use [url]")
347 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
348 linkified_plain += urltxt
350 linkified_plain += ' (' + url + ')'
351 txt = txt[index + 1:]
353 def addtext(self, txt):
354 p, h = self.linkify(txt)
357 def parseline(self, line):
358 self.text_wiki += "%s\n" % line
361 elif line.startswith('* '):
362 self.endcur([self.stUL])
363 if self.state != self.stUL:
364 self.text_html += '<ul>'
365 self.state = self.stUL
366 self.text_html += '<li>'
367 self.addtext(line[1:])
368 self.text_html += '</li>'
369 elif line.startswith('# '):
370 self.endcur([self.stOL])
371 if self.state != self.stOL:
372 self.text_html += '<ol>'
373 self.state = self.stOL
374 self.text_html += '<li>'
375 self.addtext(line[1:])
376 self.text_html += '</li>'
378 self.endcur([self.stPARA])
379 if self.state == self.stNONE:
380 self.text_html += '<p>'
381 self.state = self.stPARA
382 elif self.state == self.stPARA:
383 self.text_html += ' '
390 # Parse multiple lines of description as written in a metadata file, returning
391 # a single string in wiki format. Used for the Maintainer Notes field as well,
392 # because it's the same format.
393 def description_wiki(lines):
394 ps = DescriptionFormatter(None)
401 # Parse multiple lines of description as written in a metadata file, returning
402 # a single string in HTML format.
403 def description_html(lines, linkres):
404 ps = DescriptionFormatter(linkres)
411 def parse_srclib(metafile):
414 if metafile and not isinstance(metafile, file):
415 metafile = open(metafile, "r")
417 # Defaults for fields that come from metadata
418 thisinfo['Repo Type'] = ''
419 thisinfo['Repo'] = ''
420 thisinfo['Subdir'] = None
421 thisinfo['Prepare'] = None
427 for line in metafile:
429 line = line.rstrip('\r\n')
430 if not line or line.startswith("#"):
434 field, value = line.split(':', 1)
436 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
438 if field == "Subdir":
439 thisinfo[field] = value.split(',')
441 thisinfo[field] = value
447 """Read all srclib metadata.
449 The information read will be accessible as metadata.srclibs, which is a
450 dictionary, keyed on srclib name, with the values each being a dictionary
451 in the same format as that returned by the parse_srclib function.
453 A MetaDataException is raised if there are any problems with the srclib
458 # They were already loaded
459 if srclibs is not None:
465 if not os.path.exists(srcdir):
468 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
469 srclibname = os.path.basename(metafile[:-4])
470 srclibs[srclibname] = parse_srclib(metafile)
473 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
474 # returned by the parse_txt_metadata function.
475 def read_metadata(xref=True):
477 # Always read the srclibs before the apps, since they can use a srlib as
478 # their source repository.
483 for basedir in ('metadata', 'tmp'):
484 if not os.path.exists(basedir):
487 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
488 appid, appinfo = parse_txt_metadata(metafile)
489 check_metadata(appinfo)
490 apps[appid] = appinfo
492 for metafile in sorted(glob.glob(os.path.join('metadata', '*.json'))):
493 appid, appinfo = parse_json_metadata(metafile)
494 check_metadata(appinfo)
495 apps[appid] = appinfo
498 # Parse all descriptions at load time, just to ensure cross-referencing
499 # errors are caught early rather than when they hit the build server.
502 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
503 raise MetaDataException("Cannot resolve app id " + appid)
505 for appid, app in apps.iteritems():
507 description_html(app['Description'], linkres)
508 except MetaDataException, e:
509 raise MetaDataException("Problem with description of " + appid +
515 # Get the type expected for a given metadata field.
516 def metafieldtype(name):
517 if name in ['Description', 'Maintainer Notes']:
519 if name in ['Categories']:
521 if name == 'Build Version':
525 if name == 'Use Built':
527 if name not in app_defaults:
533 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
534 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands',
537 if name in ['init', 'prebuild', 'build']:
539 if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
545 def fill_build_defaults(build):
547 def get_build_type():
548 for t in ['maven', 'gradle', 'kivy']:
555 for flag, value in flag_defaults.iteritems():
559 build['type'] = get_build_type()
560 build['ndk_path'] = common.get_ndk_path(build['ndk'])
563 def split_list_values(s):
564 # Port legacy ';' separators
565 l = [v.strip() for v in s.replace(';', ',').split(',')]
566 return [v for v in l if v]
569 def get_default_app_info_list():
571 thisinfo.update(app_defaults)
573 # General defaults...
574 thisinfo['builds'] = []
575 thisinfo['comments'] = []
580 def post_metadata_parse(thisinfo):
582 if not thisinfo['Description']:
583 thisinfo['Description'].append('No description available')
585 for build in thisinfo['builds']:
586 fill_build_defaults(build)
588 thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
591 # Parse metadata for a single application.
593 # 'metafile' - the filename to read. The package id for the application comes
594 # from this filename. Pass None to get a blank entry.
596 # Returns a dictionary containing all the details of the application. There are
597 # two major kinds of information in the dictionary. Keys beginning with capital
598 # letters correspond directory to identically named keys in the metadata file.
599 # Keys beginning with lower case letters are generated in one way or another,
600 # and are not found verbatim in the metadata.
602 # Known keys not originating from the metadata are:
604 # 'builds' - a list of dictionaries containing build information
605 # for each defined build
606 # 'comments' - a list of comments from the metadata file. Each is
607 # a list of the form [field, comment] where field is
608 # the name of the field it preceded in the metadata
609 # file. Where field is None, the comment goes at the
610 # end of the file. Alternatively, 'build:version' is
611 # for a comment before a particular build version.
612 # 'descriptionlines' - original lines of description as formatted in the
617 def _decode_list(data):
618 '''convert items in a list from unicode to basestring'''
621 if isinstance(item, unicode):
622 item = item.encode('utf-8')
623 elif isinstance(item, list):
624 item = _decode_list(item)
625 elif isinstance(item, dict):
626 item = _decode_dict(item)
631 def _decode_dict(data):
632 '''convert items in a dict from unicode to basestring'''
634 for key, value in data.iteritems():
635 if isinstance(key, unicode):
636 key = key.encode('utf-8')
637 if isinstance(value, unicode):
638 value = value.encode('utf-8')
639 elif isinstance(value, list):
640 value = _decode_list(value)
641 elif isinstance(value, dict):
642 value = _decode_dict(value)
647 def parse_json_metadata(metafile):
649 appid = os.path.basename(metafile)[0:-5] # strip path and .json
650 thisinfo = get_default_app_info_list()
651 thisinfo['id'] = appid
653 # fdroid metadata is only strings and booleans, no floats or ints. And
654 # json returns unicode, and fdroidserver still uses plain python strings
655 jsoninfo = json.load(open(metafile, 'r'),
656 object_hook=_decode_dict,
657 parse_int=lambda s: s,
658 parse_float=lambda s: s)
659 supported_metadata = app_defaults.keys() + ['builds', 'comments']
660 for k, v in jsoninfo.iteritems():
661 if k == 'Requires Root':
662 if isinstance(v, basestring):
663 if re.match('^\s*(yes|true).*', v, flags=re.IGNORECASE):
665 elif re.match('^\s*(no|false).*', v, flags=re.IGNORECASE):
667 if isinstance(v, bool):
672 if k not in supported_metadata:
673 logging.warn(metafile + ' contains unknown metadata key, ignoring: ' + k)
674 thisinfo.update(jsoninfo)
676 for build in thisinfo['builds']:
677 for k, v in build.iteritems():
678 if k in ('buildjni', 'gradle', 'maven', 'kivy'):
679 # convert standard types to mixed datatype legacy format
680 if isinstance(v, bool):
685 elif k == 'versionCode':
687 del build['versionCode']
688 elif k == 'versionName':
690 del build['versionName']
692 # TODO create schema using https://pypi.python.org/pypi/jsonschema
693 post_metadata_parse(thisinfo)
695 return (appid, thisinfo)
698 def parse_txt_metadata(metafile):
703 def add_buildflag(p, thisbuild):
705 raise MetaDataException("Empty build flag at {1}"
706 .format(buildlines[0], linedesc))
709 raise MetaDataException("Invalid build flag at {0} in {1}"
710 .format(buildlines[0], linedesc))
713 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
714 .format(pk, thisbuild['version'], linedesc))
717 if pk not in flag_defaults:
718 raise MetaDataException("Unrecognised build flag at {0} in {1}"
719 .format(p, linedesc))
722 pv = split_list_values(pv)
724 if len(pv) == 1 and pv[0] in ['main', 'yes']:
727 elif t == 'string' or t == 'script':
734 logging.debug("...ignoring bool flag %s" % p)
737 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
740 def parse_buildline(lines):
741 value = "".join(lines)
742 parts = [p.replace("\\,", ",")
743 for p in re.split(r"(?<!\\),", value)]
745 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
747 thisbuild['origlines'] = lines
748 thisbuild['version'] = parts[0]
749 thisbuild['vercode'] = parts[1]
750 if parts[2].startswith('!'):
751 # For backwards compatibility, handle old-style disabling,
752 # including attempting to extract the commit from the message
753 thisbuild['disable'] = parts[2][1:]
754 commit = 'unknown - see disabled'
755 index = parts[2].rfind('at ')
757 commit = parts[2][index + 3:]
758 if commit.endswith(')'):
760 thisbuild['commit'] = commit
762 thisbuild['commit'] = parts[2]
764 add_buildflag(p, thisbuild)
768 def add_comments(key):
771 for comment in curcomments:
772 thisinfo['comments'].append([key, comment])
775 thisinfo = get_default_app_info_list()
777 if not isinstance(metafile, file):
778 metafile = open(metafile, "r")
779 appid = metafile.name[9:-4]
780 thisinfo['id'] = appid
782 return appid, thisinfo
791 for line in metafile:
793 linedesc = "%s:%d" % (metafile.name, c)
794 line = line.rstrip('\r\n')
796 if not any(line.startswith(s) for s in (' ', '\t')):
797 commit = curbuild['commit'] if 'commit' in curbuild else None
798 if not commit and 'disable' not in curbuild:
799 raise MetaDataException("No commit specified for {0} in {1}"
800 .format(curbuild['version'], linedesc))
802 thisinfo['builds'].append(curbuild)
803 add_comments('build:' + curbuild['vercode'])
806 if line.endswith('\\'):
807 buildlines.append(line[:-1].lstrip())
809 buildlines.append(line.lstrip())
810 bl = ''.join(buildlines)
811 add_buildflag(bl, curbuild)
817 if line.startswith("#"):
818 curcomments.append(line)
821 field, value = line.split(':', 1)
823 raise MetaDataException("Invalid metadata in " + linedesc)
824 if field != field.strip() or value != value.strip():
825 raise MetaDataException("Extra spacing found in " + linedesc)
827 # Translate obsolete fields...
828 if field == 'Market Version':
829 field = 'Current Version'
830 if field == 'Market Version Code':
831 field = 'Current Version Code'
833 fieldtype = metafieldtype(field)
834 if fieldtype not in ['build', 'buildv2']:
836 if fieldtype == 'multiline':
840 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
841 elif fieldtype == 'string':
842 thisinfo[field] = value
843 elif fieldtype == 'list':
844 thisinfo[field] = split_list_values(value)
845 elif fieldtype == 'build':
846 if value.endswith("\\"):
848 buildlines = [value[:-1]]
850 curbuild = parse_buildline([value])
851 thisinfo['builds'].append(curbuild)
852 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
853 elif fieldtype == 'buildv2':
855 vv = value.split(',')
857 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
858 .format(value, linedesc))
859 curbuild['version'] = vv[0]
860 curbuild['vercode'] = vv[1]
861 if curbuild['vercode'] in vc_seen:
862 raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
863 curbuild['vercode'], linedesc))
864 vc_seen[curbuild['vercode']] = True
867 elif fieldtype == 'obsolete':
868 pass # Just throw it away!
870 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
871 elif mode == 1: # Multiline field
875 thisinfo[field].append(line)
876 elif mode == 2: # Line continuation mode in Build Version
877 if line.endswith("\\"):
878 buildlines.append(line[:-1])
880 buildlines.append(line)
881 curbuild = parse_buildline(buildlines)
882 thisinfo['builds'].append(curbuild)
883 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
887 # Mode at end of file should always be 0...
889 raise MetaDataException(field + " not terminated in " + metafile.name)
891 raise MetaDataException("Unterminated continuation in " + metafile.name)
893 raise MetaDataException("Unterminated build in " + metafile.name)
895 post_metadata_parse(thisinfo)
897 return (appid, thisinfo)
900 # Write a metadata file.
902 # 'dest' - The path to the output file
903 # 'app' - The app data
904 def write_metadata(dest, app):
906 def writecomments(key):
908 for pf, comment in app['comments']:
910 mf.write("%s\n" % comment)
913 logging.debug("...writing comments for " + (key or 'EOF'))
915 def writefield(field, value=None):
919 t = metafieldtype(field)
921 value = ','.join(value)
922 mf.write("%s:%s\n" % (field, value))
924 def writefield_nonempty(field, value=None):
928 writefield(field, value)
931 writefield_nonempty('Disabled')
932 writefield_nonempty('AntiFeatures')
933 writefield_nonempty('Provides')
934 writefield('Categories')
935 writefield('License')
936 writefield('Web Site')
937 writefield('Source Code')
938 writefield('Issue Tracker')
939 writefield_nonempty('Changelog')
940 writefield_nonempty('Donate')
941 writefield_nonempty('FlattrID')
942 writefield_nonempty('Bitcoin')
943 writefield_nonempty('Litecoin')
944 writefield_nonempty('Dogecoin')
946 writefield_nonempty('Name')
947 writefield_nonempty('Auto Name')
948 writefield('Summary')
949 writefield('Description', '')
950 for line in app['Description']:
951 mf.write("%s\n" % line)
954 if app['Requires Root']:
955 writefield('Requires Root', 'Yes')
958 writefield('Repo Type')
961 writefield('Binaries')
963 for build in app['builds']:
965 if build['version'] == "Ignore":
968 writecomments('build:' + build['vercode'])
969 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
971 def write_builditem(key, value):
973 if key in ['version', 'vercode']:
976 if value == flag_defaults[key]:
981 logging.debug("...writing {0} : {1}".format(key, value))
982 outline = ' %s=' % key
989 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
991 outline += ','.join(value) if type(value) == list else value
996 for flag in flag_defaults:
999 write_builditem(flag, value)
1002 if app['Maintainer Notes']:
1003 writefield('Maintainer Notes', '')
1004 for line in app['Maintainer Notes']:
1005 mf.write("%s\n" % line)
1009 writefield_nonempty('Archive Policy')
1010 writefield('Auto Update Mode')
1011 writefield('Update Check Mode')
1012 writefield_nonempty('Update Check Ignore')
1013 writefield_nonempty('Vercode Operation')
1014 writefield_nonempty('Update Check Name')
1015 writefield_nonempty('Update Check Data')
1016 if app['Current Version']:
1017 writefield('Current Version')
1018 writefield('Current Version Code')
1020 if app['No Source Since']:
1021 writefield('No Source Since')