1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from collections import OrderedDict
31 class MetaDataException(Exception):
32 def __init__(self, value):
38 # In the order in which they are laid out on files
39 app_defaults = OrderedDict([
41 ('AntiFeatures', None),
43 ('Categories', ['None']),
44 ('License', 'Unknown'),
47 ('Issue Tracker', ''),
57 ('Requires Root', False),
60 ('Maintainer Notes', []),
61 ('Archive Policy', None),
62 ('Auto Update Mode', 'None'),
63 ('Update Check Mode', 'None'),
64 ('Update Check Ignore', None),
65 ('Vercode Operation', None),
66 ('Update Check Name', None),
67 ('Update Check Data', None),
68 ('Current Version', ''),
69 ('Current Version Code', '0'),
70 ('No Source Since', ''),
74 # In the order in which they are laid out on files
75 # Sorted by their action and their place in the build timeline
76 flag_defaults = OrderedDict([
80 ('submodules', False),
90 ('forceversion', False),
91 ('forcevercode', False),
102 ('antcommand', None),
107 # Designates a metadata field type and checks that it matches
109 # 'name' - The long name of the field type
110 # 'matching' - List of possible values or regex expression
111 # 'sep' - Separator to use if value may be a list
112 # 'fields' - Metadata fields (Field:Value) of this type
113 # 'attrs' - Build attributes (attr=value) of this type
115 class FieldValidator():
117 def __init__(self, name, matching, sep, fields, attrs):
119 self.matching = matching
120 if type(matching) is str:
121 self.compiled = re.compile(matching)
126 def _assert_regex(self, values, appid):
128 if not self.compiled.match(v):
129 raise MetaDataException("'%s' is not a valid %s in %s. "
130 % (v, self.name, appid) +
131 "Regex pattern: %s" % (self.matching))
133 def _assert_list(self, values, appid):
135 if v not in self.matching:
136 raise MetaDataException("'%s' is not a valid %s in %s. "
137 % (v, self.name, appid) +
138 "Possible values: %s" % (", ".join(self.matching)))
140 def check(self, value, appid):
141 if type(value) is not str or not value:
143 if self.sep is not None:
144 values = value.split(self.sep)
147 if type(self.matching) is list:
148 self._assert_list(values, appid)
150 self._assert_regex(values, appid)
153 # Generic value types
155 FieldValidator("Integer",
156 r'^[1-9][0-9]*$', None,
160 FieldValidator("HTTP link",
161 r'^http[s]?://', None,
162 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
164 FieldValidator("Bitcoin address",
165 r'^[a-zA-Z0-9]{27,34}$', None,
169 FieldValidator("Litecoin address",
170 r'^L[a-zA-Z0-9]{33}$', None,
174 FieldValidator("Dogecoin address",
175 r'^D[a-zA-Z0-9]{33}$', None,
179 FieldValidator("Boolean",
184 FieldValidator("bool",
187 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
190 FieldValidator("Repo Type",
191 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
195 FieldValidator("Archive Policy",
196 r'^[0-9]+ versions$', None,
200 FieldValidator("Anti-Feature",
201 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
205 FieldValidator("Auto Update Mode",
206 r"^(Version .+|None)$", None,
207 ["Auto Update Mode"],
210 FieldValidator("Update Check Mode",
211 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
212 ["Update Check Mode"],
217 # Check an app's metadata information for integrity errors
218 def check_metadata(info):
220 for field in v.fields:
221 v.check(info[field], info['id'])
222 for build in info['builds']:
224 v.check(build[attr], info['id'])
227 # Formatter for descriptions. Create an instance, and call parseline() with
228 # each line of the description source from the metadata. At the end, call
229 # end() and then text_plain, text_wiki and text_html will contain the result.
230 class DescriptionFormatter:
243 def __init__(self, linkres):
244 self.linkResolver = linkres
246 def endcur(self, notstates=None):
247 if notstates and self.state in notstates:
249 if self.state == self.stPARA:
251 elif self.state == self.stUL:
253 elif self.state == self.stOL:
257 self.text_plain += '\n'
258 self.text_html += '</p>'
259 self.state = self.stNONE
262 self.text_html += '</ul>'
263 self.state = self.stNONE
266 self.text_html += '</ol>'
267 self.state = self.stNONE
269 def formatted(self, txt, html):
272 txt = cgi.escape(txt)
274 index = txt.find("''")
276 return formatted + txt
277 formatted += txt[:index]
279 if txt.startswith("'''"):
285 self.bold = not self.bold
293 self.ital = not self.ital
296 def linkify(self, txt):
300 index = txt.find("[")
302 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
303 linkified_plain += self.formatted(txt[:index], False)
304 linkified_html += self.formatted(txt[:index], True)
306 if txt.startswith("[["):
307 index = txt.find("]]")
309 raise MetaDataException("Unterminated ]]")
311 if self.linkResolver:
312 url, urltext = self.linkResolver(url)
315 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
316 linkified_plain += urltext
317 txt = txt[index + 2:]
319 index = txt.find("]")
321 raise MetaDataException("Unterminated ]")
323 index2 = url.find(' ')
327 urltxt = url[index2 + 1:]
329 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
330 linkified_plain += urltxt
332 linkified_plain += ' (' + url + ')'
333 txt = txt[index + 1:]
335 def addtext(self, txt):
336 p, h = self.linkify(txt)
340 def parseline(self, line):
341 self.text_wiki += "%s\n" % line
344 elif line.startswith('* '):
345 self.endcur([self.stUL])
346 if self.state != self.stUL:
347 self.text_html += '<ul>'
348 self.state = self.stUL
349 self.text_html += '<li>'
350 self.text_plain += '* '
351 self.addtext(line[1:])
352 self.text_html += '</li>'
353 elif line.startswith('# '):
354 self.endcur([self.stOL])
355 if self.state != self.stOL:
356 self.text_html += '<ol>'
357 self.state = self.stOL
358 self.text_html += '<li>'
359 self.text_plain += '* ' # TODO: lazy - put the numbers in!
360 self.addtext(line[1:])
361 self.text_html += '</li>'
363 self.endcur([self.stPARA])
364 if self.state == self.stNONE:
365 self.text_html += '<p>'
366 self.state = self.stPARA
367 elif self.state == self.stPARA:
368 self.text_html += ' '
369 self.text_plain += ' '
376 # Parse multiple lines of description as written in a metadata file, returning
377 # a single string in plain text format.
378 def description_plain(lines, linkres):
379 ps = DescriptionFormatter(linkres)
386 # Parse multiple lines of description as written in a metadata file, returning
387 # a single string in wiki format. Used for the Maintainer Notes field as well,
388 # because it's the same format.
389 def description_wiki(lines):
390 ps = DescriptionFormatter(None)
397 # Parse multiple lines of description as written in a metadata file, returning
398 # a single string in HTML format.
399 def description_html(lines, linkres):
400 ps = DescriptionFormatter(linkres)
407 def parse_srclib(metafile):
410 if metafile and not isinstance(metafile, file):
411 metafile = open(metafile, "r")
413 # Defaults for fields that come from metadata
414 thisinfo['Repo Type'] = ''
415 thisinfo['Repo'] = ''
416 thisinfo['Subdir'] = None
417 thisinfo['Prepare'] = None
418 thisinfo['Srclibs'] = None
424 for line in metafile:
426 line = line.rstrip('\r\n')
427 if not line or line.startswith("#"):
431 field, value = line.split(':', 1)
433 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
435 if field == "Subdir":
436 thisinfo[field] = value.split(',')
438 thisinfo[field] = value
444 """Read all srclib metadata.
446 The information read will be accessible as metadata.srclibs, which is a
447 dictionary, keyed on srclib name, with the values each being a dictionary
448 in the same format as that returned by the parse_srclib function.
450 A MetaDataException is raised if there are any problems with the srclib
455 # They were already loaded
456 if srclibs is not None:
462 if not os.path.exists(srcdir):
465 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
466 srclibname = os.path.basename(metafile[:-4])
467 srclibs[srclibname] = parse_srclib(metafile)
470 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
471 # returned by the parse_metadata function.
472 def read_metadata(xref=True):
474 # Always read the srclibs before the apps, since they can use a srlib as
475 # their source repository.
480 for basedir in ('metadata', 'tmp'):
481 if not os.path.exists(basedir):
484 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
485 appinfo = parse_metadata(metafile)
486 check_metadata(appinfo)
490 # Parse all descriptions at load time, just to ensure cross-referencing
491 # errors are caught early rather than when they hit the build server.
494 if app['id'] == link:
495 return ("fdroid.app:" + link, "Dummy name - don't know yet")
496 raise MetaDataException("Cannot resolve app id " + link)
499 description_html(app['Description'], linkres)
501 raise MetaDataException("Problem with description of " + app['id'] +
507 # Get the type expected for a given metadata field.
508 def metafieldtype(name):
509 if name in ['Description', 'Maintainer Notes']:
511 if name in ['Categories']:
513 if name == 'Build Version':
517 if name == 'Use Built':
519 if name not in app_defaults:
525 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
526 'update', 'scanignore', 'scandelete']:
528 if name in ['init', 'prebuild', 'build']:
530 if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
536 # Parse metadata for a single application.
538 # 'metafile' - the filename to read. The package id for the application comes
539 # from this filename. Pass None to get a blank entry.
541 # Returns a dictionary containing all the details of the application. There are
542 # two major kinds of information in the dictionary. Keys beginning with capital
543 # letters correspond directory to identically named keys in the metadata file.
544 # Keys beginning with lower case letters are generated in one way or another,
545 # and are not found verbatim in the metadata.
547 # Known keys not originating from the metadata are:
549 # 'id' - the application's package ID
550 # 'builds' - a list of dictionaries containing build information
551 # for each defined build
552 # 'comments' - a list of comments from the metadata file. Each is
553 # a tuple of the form (field, comment) where field is
554 # the name of the field it preceded in the metadata
555 # file. Where field is None, the comment goes at the
556 # end of the file. Alternatively, 'build:version' is
557 # for a comment before a particular build version.
558 # 'descriptionlines' - original lines of description as formatted in the
561 def parse_metadata(metafile):
565 def add_buildflag(p, thisbuild):
568 raise MetaDataException("Invalid build flag at {0} in {1}"
569 .format(buildlines[0], linedesc))
572 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
573 .format(pk, thisbuild['version'], linedesc))
576 if pk not in flag_defaults:
577 raise MetaDataException("Unrecognised build flag at {0} in {1}"
578 .format(p, linedesc))
581 # Port legacy ';' separators
582 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
583 elif t == 'string' or t == 'script':
590 logging.debug("...ignoring bool flag %s" % p)
593 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
596 def parse_buildline(lines):
597 value = "".join(lines)
598 parts = [p.replace("\\,", ",")
599 for p in re.split(r"(?<!\\),", value)]
601 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
603 thisbuild['origlines'] = lines
604 thisbuild['version'] = parts[0]
605 thisbuild['vercode'] = parts[1]
606 if parts[2].startswith('!'):
607 # For backwards compatibility, handle old-style disabling,
608 # including attempting to extract the commit from the message
609 thisbuild['disable'] = parts[2][1:]
610 commit = 'unknown - see disabled'
611 index = parts[2].rfind('at ')
613 commit = parts[2][index + 3:]
614 if commit.endswith(')'):
616 thisbuild['commit'] = commit
618 thisbuild['commit'] = parts[2]
620 add_buildflag(p, thisbuild)
624 def add_comments(key):
627 for comment in curcomments:
628 thisinfo['comments'].append((key, comment))
631 def get_build_type(build):
632 for t in ['maven', 'gradle', 'kivy']:
641 if not isinstance(metafile, file):
642 metafile = open(metafile, "r")
643 thisinfo['id'] = metafile.name[9:-4]
645 thisinfo['id'] = None
647 thisinfo.update(app_defaults)
649 # General defaults...
650 thisinfo['builds'] = []
651 thisinfo['comments'] = []
663 for line in metafile:
665 linedesc = "%s:%d" % (metafile.name, c)
666 line = line.rstrip('\r\n')
668 if not any(line.startswith(s) for s in (' ', '\t')):
669 if 'commit' not in curbuild and 'disable' not in curbuild:
670 raise MetaDataException("No commit specified for {0} in {1}"
671 .format(curbuild['version'], linedesc))
673 thisinfo['builds'].append(curbuild)
674 add_comments('build:' + curbuild['vercode'])
677 if line.endswith('\\'):
678 buildlines.append(line[:-1].lstrip())
680 buildlines.append(line.lstrip())
681 bl = ''.join(buildlines)
682 add_buildflag(bl, curbuild)
688 if line.startswith("#"):
689 curcomments.append(line)
692 field, value = line.split(':', 1)
694 raise MetaDataException("Invalid metadata in " + linedesc)
695 if field != field.strip() or value != value.strip():
696 raise MetaDataException("Extra spacing found in " + linedesc)
698 # Translate obsolete fields...
699 if field == 'Market Version':
700 field = 'Current Version'
701 if field == 'Market Version Code':
702 field = 'Current Version Code'
704 fieldtype = metafieldtype(field)
705 if fieldtype not in ['build', 'buildv2']:
707 if fieldtype == 'multiline':
711 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
712 elif fieldtype == 'string':
713 thisinfo[field] = value
714 elif fieldtype == 'list':
715 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
716 elif fieldtype == 'build':
717 if value.endswith("\\"):
719 buildlines = [value[:-1]]
721 curbuild = parse_buildline([value])
722 thisinfo['builds'].append(curbuild)
723 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
724 elif fieldtype == 'buildv2':
726 vv = value.split(',')
728 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
729 .format(value, linedesc))
730 curbuild['version'] = vv[0]
731 curbuild['vercode'] = vv[1]
732 if curbuild['vercode'] in vc_seen:
733 raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
734 curbuild['vercode'], linedesc))
735 vc_seen[curbuild['vercode']] = True
738 elif fieldtype == 'obsolete':
739 pass # Just throw it away!
741 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
742 elif mode == 1: # Multiline field
746 thisinfo[field].append(line)
747 elif mode == 2: # Line continuation mode in Build Version
748 if line.endswith("\\"):
749 buildlines.append(line[:-1])
751 buildlines.append(line)
752 curbuild = parse_buildline(buildlines)
753 thisinfo['builds'].append(curbuild)
754 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
758 # Mode at end of file should always be 0...
760 raise MetaDataException(field + " not terminated in " + metafile.name)
762 raise MetaDataException("Unterminated continuation in " + metafile.name)
764 raise MetaDataException("Unterminated build in " + metafile.name)
766 if not thisinfo['Description']:
767 thisinfo['Description'].append('No description available')
769 for build in thisinfo['builds']:
770 for flag, value in flag_defaults.iteritems():
774 build['type'] = get_build_type(build)
779 # Write a metadata file.
781 # 'dest' - The path to the output file
782 # 'app' - The app data
783 def write_metadata(dest, app):
785 def writecomments(key):
787 for pf, comment in app['comments']:
789 mf.write("%s\n" % comment)
792 logging.debug("...writing comments for " + (key if key else 'EOF'))
794 def writefield(field, value=None):
798 t = metafieldtype(field)
800 value = ','.join(value)
801 mf.write("%s:%s\n" % (field, value))
805 writefield('Disabled')
806 if app['AntiFeatures']:
807 writefield('AntiFeatures')
809 writefield('Provides')
810 writefield('Categories')
811 writefield('License')
812 writefield('Web Site')
813 writefield('Source Code')
814 writefield('Issue Tracker')
818 writefield('FlattrID')
820 writefield('Bitcoin')
822 writefield('Litecoin')
824 writefield('Dogecoin')
829 writefield('Auto Name')
830 writefield('Summary')
831 writefield('Description', '')
832 for line in app['Description']:
833 mf.write("%s\n" % line)
836 if app['Requires Root']:
837 writefield('Requires Root', 'Yes')
840 writefield('Repo Type')
843 for build in app['builds']:
844 writecomments('build:' + build['vercode'])
845 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
847 def write_builditem(key, value):
849 if key in ['version', 'vercode']:
852 if value == flag_defaults[key]:
857 logging.debug("...writing {0} : {1}".format(key, value))
858 outline = ' %s=' % key
865 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
867 outline += ','.join(value) if type(value) == list else value
872 for flag in flag_defaults:
875 write_builditem(flag, value)
878 if app['Maintainer Notes']:
879 writefield('Maintainer Notes', '')
880 for line in app['Maintainer Notes']:
881 mf.write("%s\n" % line)
885 if app['Archive Policy']:
886 writefield('Archive Policy')
887 writefield('Auto Update Mode')
888 writefield('Update Check Mode')
889 if app['Update Check Ignore']:
890 writefield('Update Check Ignore')
891 if app['Vercode Operation']:
892 writefield('Vercode Operation')
893 if app['Update Check Name']:
894 writefield('Update Check Name')
895 if app['Update Check Data']:
896 writefield('Update Check Data')
897 if app['Current Version']:
898 writefield('Current Version')
899 writefield('Current Version Code')
901 if app['No Source Since']:
902 writefield('No Source Since')