1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from collections import OrderedDict
31 class MetaDataException(Exception):
32 def __init__(self, value):
38 # In the order in which they are laid out on files
39 app_defaults = OrderedDict([
41 ('AntiFeatures', None),
43 ('Categories', ['None']),
44 ('License', 'Unknown'),
47 ('Issue Tracker', ''),
57 ('Requires Root', False),
61 ('Maintainer Notes', []),
62 ('Archive Policy', None),
63 ('Auto Update Mode', 'None'),
64 ('Update Check Mode', 'None'),
65 ('Update Check Ignore', None),
66 ('Vercode Operation', None),
67 ('Update Check Name', None),
68 ('Update Check Data', None),
69 ('Current Version', ''),
70 ('Current Version Code', '0'),
71 ('No Source Since', ''),
75 # In the order in which they are laid out on files
76 # Sorted by their action and their place in the build timeline
77 flag_defaults = OrderedDict([
81 ('submodules', False),
91 ('forceversion', False),
92 ('forcevercode', False),
103 ('antcommands', None),
108 # Designates a metadata field type and checks that it matches
110 # 'name' - The long name of the field type
111 # 'matching' - List of possible values or regex expression
112 # 'sep' - Separator to use if value may be a list
113 # 'fields' - Metadata fields (Field:Value) of this type
114 # 'attrs' - Build attributes (attr=value) of this type
116 class FieldValidator():
118 def __init__(self, name, matching, sep, fields, attrs):
120 self.matching = matching
121 if type(matching) is str:
122 self.compiled = re.compile(matching)
127 def _assert_regex(self, values, appid):
129 if not self.compiled.match(v):
130 raise MetaDataException("'%s' is not a valid %s in %s. "
131 % (v, self.name, appid) +
132 "Regex pattern: %s" % (self.matching))
134 def _assert_list(self, values, appid):
136 if v not in self.matching:
137 raise MetaDataException("'%s' is not a valid %s in %s. "
138 % (v, self.name, appid) +
139 "Possible values: %s" % (", ".join(self.matching)))
141 def check(self, value, appid):
142 if type(value) is not str or not value:
144 if self.sep is not None:
145 values = value.split(self.sep)
148 if type(self.matching) is list:
149 self._assert_list(values, appid)
151 self._assert_regex(values, appid)
154 # Generic value types
156 FieldValidator("Integer",
157 r'^[1-9][0-9]*$', None,
161 FieldValidator("Hexadecimal",
162 r'^[0-9a-f]+$', None,
166 FieldValidator("HTTP link",
167 r'^http[s]?://', None,
168 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
170 FieldValidator("Bitcoin address",
171 r'^[a-zA-Z0-9]{27,34}$', None,
175 FieldValidator("Litecoin address",
176 r'^L[a-zA-Z0-9]{33}$', None,
180 FieldValidator("Dogecoin address",
181 r'^D[a-zA-Z0-9]{33}$', None,
185 FieldValidator("Boolean",
190 FieldValidator("bool",
193 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
196 FieldValidator("Repo Type",
197 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
201 FieldValidator("Binaries",
202 r'^http[s]?://', None,
206 FieldValidator("Archive Policy",
207 r'^[0-9]+ versions$', None,
211 FieldValidator("Anti-Feature",
212 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
216 FieldValidator("Auto Update Mode",
217 r"^(Version .+|None)$", None,
218 ["Auto Update Mode"],
221 FieldValidator("Update Check Mode",
222 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
223 ["Update Check Mode"],
228 # Check an app's metadata information for integrity errors
229 def check_metadata(info):
231 for field in v.fields:
232 v.check(info[field], info['id'])
233 for build in info['builds']:
235 v.check(build[attr], info['id'])
238 # Formatter for descriptions. Create an instance, and call parseline() with
239 # each line of the description source from the metadata. At the end, call
240 # end() and then text_plain, text_wiki and text_html will contain the result.
241 class DescriptionFormatter:
254 def __init__(self, linkres):
255 self.linkResolver = linkres
257 def endcur(self, notstates=None):
258 if notstates and self.state in notstates:
260 if self.state == self.stPARA:
262 elif self.state == self.stUL:
264 elif self.state == self.stOL:
268 self.text_plain += '\n'
269 self.text_html += '</p>'
270 self.state = self.stNONE
273 self.text_html += '</ul>'
274 self.state = self.stNONE
277 self.text_html += '</ol>'
278 self.state = self.stNONE
280 def formatted(self, txt, html):
283 txt = cgi.escape(txt)
285 index = txt.find("''")
287 return formatted + txt
288 formatted += txt[:index]
290 if txt.startswith("'''"):
296 self.bold = not self.bold
304 self.ital = not self.ital
307 def linkify(self, txt):
311 index = txt.find("[")
313 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
314 linkified_plain += self.formatted(txt[:index], False)
315 linkified_html += self.formatted(txt[:index], True)
317 if txt.startswith("[["):
318 index = txt.find("]]")
320 raise MetaDataException("Unterminated ]]")
322 if self.linkResolver:
323 url, urltext = self.linkResolver(url)
326 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
327 linkified_plain += urltext
328 txt = txt[index + 2:]
330 index = txt.find("]")
332 raise MetaDataException("Unterminated ]")
334 index2 = url.find(' ')
338 urltxt = url[index2 + 1:]
340 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
341 linkified_plain += urltxt
343 linkified_plain += ' (' + url + ')'
344 txt = txt[index + 1:]
346 def addtext(self, txt):
347 p, h = self.linkify(txt)
351 def parseline(self, line):
352 self.text_wiki += "%s\n" % line
355 elif line.startswith('* '):
356 self.endcur([self.stUL])
357 if self.state != self.stUL:
358 self.text_html += '<ul>'
359 self.state = self.stUL
360 self.text_html += '<li>'
361 self.text_plain += '* '
362 self.addtext(line[1:])
363 self.text_html += '</li>'
364 elif line.startswith('# '):
365 self.endcur([self.stOL])
366 if self.state != self.stOL:
367 self.text_html += '<ol>'
368 self.state = self.stOL
369 self.text_html += '<li>'
370 self.text_plain += '* ' # TODO: lazy - put the numbers in!
371 self.addtext(line[1:])
372 self.text_html += '</li>'
374 self.endcur([self.stPARA])
375 if self.state == self.stNONE:
376 self.text_html += '<p>'
377 self.state = self.stPARA
378 elif self.state == self.stPARA:
379 self.text_html += ' '
380 self.text_plain += ' '
387 # Parse multiple lines of description as written in a metadata file, returning
388 # a single string in plain text format.
389 def description_plain(lines, linkres):
390 ps = DescriptionFormatter(linkres)
397 # Parse multiple lines of description as written in a metadata file, returning
398 # a single string in wiki format. Used for the Maintainer Notes field as well,
399 # because it's the same format.
400 def description_wiki(lines):
401 ps = DescriptionFormatter(None)
408 # Parse multiple lines of description as written in a metadata file, returning
409 # a single string in HTML format.
410 def description_html(lines, linkres):
411 ps = DescriptionFormatter(linkres)
418 def parse_srclib(metafile):
421 if metafile and not isinstance(metafile, file):
422 metafile = open(metafile, "r")
424 # Defaults for fields that come from metadata
425 thisinfo['Repo Type'] = ''
426 thisinfo['Repo'] = ''
427 thisinfo['Subdir'] = None
428 thisinfo['Prepare'] = None
429 thisinfo['Srclibs'] = None
435 for line in metafile:
437 line = line.rstrip('\r\n')
438 if not line or line.startswith("#"):
442 field, value = line.split(':', 1)
444 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
446 if field == "Subdir":
447 thisinfo[field] = value.split(',')
449 thisinfo[field] = value
455 """Read all srclib metadata.
457 The information read will be accessible as metadata.srclibs, which is a
458 dictionary, keyed on srclib name, with the values each being a dictionary
459 in the same format as that returned by the parse_srclib function.
461 A MetaDataException is raised if there are any problems with the srclib
466 # They were already loaded
467 if srclibs is not None:
473 if not os.path.exists(srcdir):
476 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
477 srclibname = os.path.basename(metafile[:-4])
478 srclibs[srclibname] = parse_srclib(metafile)
481 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
482 # returned by the parse_metadata function.
483 def read_metadata(xref=True):
485 # Always read the srclibs before the apps, since they can use a srlib as
486 # their source repository.
491 for basedir in ('metadata', 'tmp'):
492 if not os.path.exists(basedir):
495 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
496 appid, appinfo = parse_metadata(metafile)
497 check_metadata(appinfo)
498 apps[appid] = appinfo
501 # Parse all descriptions at load time, just to ensure cross-referencing
502 # errors are caught early rather than when they hit the build server.
505 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
506 raise MetaDataException("Cannot resolve app id " + appid)
508 for appid, app in apps.iteritems():
510 description_html(app['Description'], linkres)
511 except MetaDataException, e:
512 raise MetaDataException("Problem with description of " + appid +
518 # Get the type expected for a given metadata field.
519 def metafieldtype(name):
520 if name in ['Description', 'Maintainer Notes']:
522 if name in ['Categories']:
524 if name == 'Build Version':
528 if name == 'Use Built':
530 if name not in app_defaults:
536 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
537 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands']:
539 if name in ['init', 'prebuild', 'build']:
541 if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
547 def fill_build_defaults(build):
549 def get_build_type():
550 for t in ['maven', 'gradle', 'kivy']:
557 for flag, value in flag_defaults.iteritems():
561 build['type'] = get_build_type()
564 # Parse metadata for a single application.
566 # 'metafile' - the filename to read. The package id for the application comes
567 # from this filename. Pass None to get a blank entry.
569 # Returns a dictionary containing all the details of the application. There are
570 # two major kinds of information in the dictionary. Keys beginning with capital
571 # letters correspond directory to identically named keys in the metadata file.
572 # Keys beginning with lower case letters are generated in one way or another,
573 # and are not found verbatim in the metadata.
575 # Known keys not originating from the metadata are:
577 # 'builds' - a list of dictionaries containing build information
578 # for each defined build
579 # 'comments' - a list of comments from the metadata file. Each is
580 # a tuple of the form (field, comment) where field is
581 # the name of the field it preceded in the metadata
582 # file. Where field is None, the comment goes at the
583 # end of the file. Alternatively, 'build:version' is
584 # for a comment before a particular build version.
585 # 'descriptionlines' - original lines of description as formatted in the
588 def parse_metadata(metafile):
593 def add_buildflag(p, thisbuild):
596 raise MetaDataException("Invalid build flag at {0} in {1}"
597 .format(buildlines[0], linedesc))
600 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
601 .format(pk, thisbuild['version'], linedesc))
604 if pk not in flag_defaults:
605 raise MetaDataException("Unrecognised build flag at {0} in {1}"
606 .format(p, linedesc))
609 # Port legacy ';' separators
610 pv = [v.strip() for v in pv.replace(';', ',').split(',')]
612 if len(pv) == 1 and pv[0] in ['main', 'yes']:
615 elif t == 'string' or t == 'script':
622 logging.debug("...ignoring bool flag %s" % p)
625 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
628 def parse_buildline(lines):
629 value = "".join(lines)
630 parts = [p.replace("\\,", ",")
631 for p in re.split(r"(?<!\\),", value)]
633 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
635 thisbuild['origlines'] = lines
636 thisbuild['version'] = parts[0]
637 thisbuild['vercode'] = parts[1]
638 if parts[2].startswith('!'):
639 # For backwards compatibility, handle old-style disabling,
640 # including attempting to extract the commit from the message
641 thisbuild['disable'] = parts[2][1:]
642 commit = 'unknown - see disabled'
643 index = parts[2].rfind('at ')
645 commit = parts[2][index + 3:]
646 if commit.endswith(')'):
648 thisbuild['commit'] = commit
650 thisbuild['commit'] = parts[2]
652 add_buildflag(p, thisbuild)
656 def add_comments(key):
659 for comment in curcomments:
660 thisinfo['comments'].append((key, comment))
665 if not isinstance(metafile, file):
666 metafile = open(metafile, "r")
667 appid = metafile.name[9:-4]
669 thisinfo.update(app_defaults)
670 thisinfo['id'] = appid
672 # General defaults...
673 thisinfo['builds'] = []
674 thisinfo['comments'] = []
677 return appid, thisinfo
686 for line in metafile:
688 linedesc = "%s:%d" % (metafile.name, c)
689 line = line.rstrip('\r\n')
691 if not any(line.startswith(s) for s in (' ', '\t')):
692 commit = curbuild['commit'] if 'commit' in curbuild else None
693 if not commit and 'disable' not in curbuild:
694 raise MetaDataException("No commit specified for {0} in {1}"
695 .format(curbuild['version'], linedesc))
697 thisinfo['builds'].append(curbuild)
698 add_comments('build:' + curbuild['vercode'])
701 if line.endswith('\\'):
702 buildlines.append(line[:-1].lstrip())
704 buildlines.append(line.lstrip())
705 bl = ''.join(buildlines)
706 add_buildflag(bl, curbuild)
712 if line.startswith("#"):
713 curcomments.append(line)
716 field, value = line.split(':', 1)
718 raise MetaDataException("Invalid metadata in " + linedesc)
719 if field != field.strip() or value != value.strip():
720 raise MetaDataException("Extra spacing found in " + linedesc)
722 # Translate obsolete fields...
723 if field == 'Market Version':
724 field = 'Current Version'
725 if field == 'Market Version Code':
726 field = 'Current Version Code'
728 fieldtype = metafieldtype(field)
729 if fieldtype not in ['build', 'buildv2']:
731 if fieldtype == 'multiline':
735 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
736 elif fieldtype == 'string':
737 thisinfo[field] = value
738 elif fieldtype == 'list':
739 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
740 elif fieldtype == 'build':
741 if value.endswith("\\"):
743 buildlines = [value[:-1]]
745 curbuild = parse_buildline([value])
746 thisinfo['builds'].append(curbuild)
747 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
748 elif fieldtype == 'buildv2':
750 vv = value.split(',')
752 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
753 .format(value, linedesc))
754 curbuild['version'] = vv[0]
755 curbuild['vercode'] = vv[1]
756 if curbuild['vercode'] in vc_seen:
757 raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
758 curbuild['vercode'], linedesc))
759 vc_seen[curbuild['vercode']] = True
762 elif fieldtype == 'obsolete':
763 pass # Just throw it away!
765 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
766 elif mode == 1: # Multiline field
770 thisinfo[field].append(line)
771 elif mode == 2: # Line continuation mode in Build Version
772 if line.endswith("\\"):
773 buildlines.append(line[:-1])
775 buildlines.append(line)
776 curbuild = parse_buildline(buildlines)
777 thisinfo['builds'].append(curbuild)
778 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
782 # Mode at end of file should always be 0...
784 raise MetaDataException(field + " not terminated in " + metafile.name)
786 raise MetaDataException("Unterminated continuation in " + metafile.name)
788 raise MetaDataException("Unterminated build in " + metafile.name)
790 if not thisinfo['Description']:
791 thisinfo['Description'].append('No description available')
793 for build in thisinfo['builds']:
794 fill_build_defaults(build)
796 thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
798 return (appid, thisinfo)
801 # Write a metadata file.
803 # 'dest' - The path to the output file
804 # 'app' - The app data
805 def write_metadata(dest, app):
807 def writecomments(key):
809 for pf, comment in app['comments']:
811 mf.write("%s\n" % comment)
814 logging.debug("...writing comments for " + (key or 'EOF'))
816 def writefield(field, value=None):
820 t = metafieldtype(field)
822 value = ','.join(value)
823 mf.write("%s:%s\n" % (field, value))
825 def writefield_nonempty(field, value=None):
829 writefield(field, value)
832 writefield_nonempty('Disabled')
833 writefield_nonempty('AntiFeatures')
834 writefield_nonempty('Provides')
835 writefield('Categories')
836 writefield('License')
837 writefield('Web Site')
838 writefield('Source Code')
839 writefield('Issue Tracker')
840 writefield_nonempty('Donate')
841 writefield_nonempty('FlattrID')
842 writefield_nonempty('Bitcoin')
843 writefield_nonempty('Litecoin')
844 writefield_nonempty('Dogecoin')
846 writefield_nonempty('Name')
847 writefield_nonempty('Auto Name')
848 writefield('Summary')
849 writefield('Description', '')
850 for line in app['Description']:
851 mf.write("%s\n" % line)
854 if app['Requires Root']:
855 writefield('Requires Root', 'Yes')
858 writefield('Repo Type')
861 writefield('Binaries')
863 for build in app['builds']:
865 if build['version'] == "Ignore":
868 writecomments('build:' + build['vercode'])
869 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
871 def write_builditem(key, value):
873 if key in ['version', 'vercode']:
876 if value == flag_defaults[key]:
881 logging.debug("...writing {0} : {1}".format(key, value))
882 outline = ' %s=' % key
889 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
891 outline += ','.join(value) if type(value) == list else value
896 for flag in flag_defaults:
899 write_builditem(flag, value)
902 if app['Maintainer Notes']:
903 writefield('Maintainer Notes', '')
904 for line in app['Maintainer Notes']:
905 mf.write("%s\n" % line)
909 writefield_nonempty('Archive Policy')
910 writefield('Auto Update Mode')
911 writefield('Update Check Mode')
912 writefield_nonempty('Update Check Ignore')
913 writefield_nonempty('Vercode Operation')
914 writefield_nonempty('Update Check Name')
915 writefield_nonempty('Update Check Data')
916 if app['Current Version']:
917 writefield('Current Version')
918 writefield('Current Version Code')
920 if app['No Source Since']:
921 writefield('No Source Since')