1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from collections import OrderedDict
33 class MetaDataException(Exception):
35 def __init__(self, value):
41 # In the order in which they are laid out on files
42 app_defaults = OrderedDict([
44 ('AntiFeatures', None),
46 ('Categories', ['None']),
47 ('License', 'Unknown'),
50 ('Issue Tracker', ''),
61 ('Requires Root', False),
65 ('Maintainer Notes', []),
66 ('Archive Policy', None),
67 ('Auto Update Mode', 'None'),
68 ('Update Check Mode', 'None'),
69 ('Update Check Ignore', None),
70 ('Vercode Operation', None),
71 ('Update Check Name', None),
72 ('Update Check Data', None),
73 ('Current Version', ''),
74 ('Current Version Code', '0'),
75 ('No Source Since', ''),
79 # In the order in which they are laid out on files
80 # Sorted by their action and their place in the build timeline
81 flag_defaults = OrderedDict([
85 ('submodules', False),
95 ('forceversion', False),
96 ('forcevercode', False),
100 ('update', ['auto']),
106 ('ndk', 'r10e'), # defaults to latest
108 ('antcommands', None),
113 # Designates a metadata field type and checks that it matches
115 # 'name' - The long name of the field type
116 # 'matching' - List of possible values or regex expression
117 # 'sep' - Separator to use if value may be a list
118 # 'fields' - Metadata fields (Field:Value) of this type
119 # 'attrs' - Build attributes (attr=value) of this type
121 class FieldValidator():
123 def __init__(self, name, matching, sep, fields, attrs):
125 self.matching = matching
126 if type(matching) is str:
127 self.compiled = re.compile(matching)
132 def _assert_regex(self, values, appid):
134 if not self.compiled.match(v):
135 raise MetaDataException("'%s' is not a valid %s in %s. "
136 % (v, self.name, appid) +
137 "Regex pattern: %s" % (self.matching))
139 def _assert_list(self, values, appid):
141 if v not in self.matching:
142 raise MetaDataException("'%s' is not a valid %s in %s. "
143 % (v, self.name, appid) +
144 "Possible values: %s" % (", ".join(self.matching)))
146 def check(self, value, appid):
147 if type(value) is not str or not value:
149 if self.sep is not None:
150 values = value.split(self.sep)
153 if type(self.matching) is list:
154 self._assert_list(values, appid)
156 self._assert_regex(values, appid)
159 # Generic value types
161 FieldValidator("Integer",
162 r'^[1-9][0-9]*$', None,
166 FieldValidator("Hexadecimal",
167 r'^[0-9a-f]+$', None,
171 FieldValidator("HTTP link",
172 r'^http[s]?://', None,
173 ["Web Site", "Source Code", "Issue Tracker", "Changelog", "Donate"], []),
175 FieldValidator("Bitcoin address",
176 r'^[a-zA-Z0-9]{27,34}$', None,
180 FieldValidator("Litecoin address",
181 r'^L[a-zA-Z0-9]{33}$', None,
185 FieldValidator("Dogecoin address",
186 r'^D[a-zA-Z0-9]{33}$', None,
190 FieldValidator("Boolean",
195 FieldValidator("bool",
198 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
201 FieldValidator("Repo Type",
202 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
206 FieldValidator("Binaries",
207 r'^http[s]?://', None,
211 FieldValidator("Archive Policy",
212 r'^[0-9]+ versions$', None,
216 FieldValidator("Anti-Feature",
217 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
221 FieldValidator("Auto Update Mode",
222 r"^(Version .+|None)$", None,
223 ["Auto Update Mode"],
226 FieldValidator("Update Check Mode",
227 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
228 ["Update Check Mode"],
233 # Check an app's metadata information for integrity errors
234 def check_metadata(info):
236 for field in v.fields:
237 v.check(info[field], info['id'])
238 for build in info['builds']:
240 v.check(build[attr], info['id'])
243 # Formatter for descriptions. Create an instance, and call parseline() with
244 # each line of the description source from the metadata. At the end, call
245 # end() and then text_wiki and text_html will contain the result.
246 class DescriptionFormatter:
258 def __init__(self, linkres):
259 self.linkResolver = linkres
261 def endcur(self, notstates=None):
262 if notstates and self.state in notstates:
264 if self.state == self.stPARA:
266 elif self.state == self.stUL:
268 elif self.state == self.stOL:
272 self.text_html += '</p>'
273 self.state = self.stNONE
276 self.text_html += '</ul>'
277 self.state = self.stNONE
280 self.text_html += '</ol>'
281 self.state = self.stNONE
283 def formatted(self, txt, html):
286 txt = cgi.escape(txt)
288 index = txt.find("''")
290 return formatted + txt
291 formatted += txt[:index]
293 if txt.startswith("'''"):
299 self.bold = not self.bold
307 self.ital = not self.ital
310 def linkify(self, txt):
314 index = txt.find("[")
316 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
317 linkified_plain += self.formatted(txt[:index], False)
318 linkified_html += self.formatted(txt[:index], True)
320 if txt.startswith("[["):
321 index = txt.find("]]")
323 raise MetaDataException("Unterminated ]]")
325 if self.linkResolver:
326 url, urltext = self.linkResolver(url)
329 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
330 linkified_plain += urltext
331 txt = txt[index + 2:]
333 index = txt.find("]")
335 raise MetaDataException("Unterminated ]")
337 index2 = url.find(' ')
341 urltxt = url[index2 + 1:]
343 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
344 linkified_plain += urltxt
346 linkified_plain += ' (' + url + ')'
347 txt = txt[index + 1:]
349 def addtext(self, txt):
350 p, h = self.linkify(txt)
353 def parseline(self, line):
354 self.text_wiki += "%s\n" % line
357 elif line.startswith('* '):
358 self.endcur([self.stUL])
359 if self.state != self.stUL:
360 self.text_html += '<ul>'
361 self.state = self.stUL
362 self.text_html += '<li>'
363 self.addtext(line[1:])
364 self.text_html += '</li>'
365 elif line.startswith('# '):
366 self.endcur([self.stOL])
367 if self.state != self.stOL:
368 self.text_html += '<ol>'
369 self.state = self.stOL
370 self.text_html += '<li>'
371 self.addtext(line[1:])
372 self.text_html += '</li>'
374 self.endcur([self.stPARA])
375 if self.state == self.stNONE:
376 self.text_html += '<p>'
377 self.state = self.stPARA
378 elif self.state == self.stPARA:
379 self.text_html += ' '
386 # Parse multiple lines of description as written in a metadata file, returning
387 # a single string in wiki format. Used for the Maintainer Notes field as well,
388 # because it's the same format.
389 def description_wiki(lines):
390 ps = DescriptionFormatter(None)
397 # Parse multiple lines of description as written in a metadata file, returning
398 # a single string in HTML format.
399 def description_html(lines, linkres):
400 ps = DescriptionFormatter(linkres)
407 def parse_srclib(metafile):
410 if metafile and not isinstance(metafile, file):
411 metafile = open(metafile, "r")
413 # Defaults for fields that come from metadata
414 thisinfo['Repo Type'] = ''
415 thisinfo['Repo'] = ''
416 thisinfo['Subdir'] = None
417 thisinfo['Prepare'] = None
423 for line in metafile:
425 line = line.rstrip('\r\n')
426 if not line or line.startswith("#"):
430 field, value = line.split(':', 1)
432 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
434 if field == "Subdir":
435 thisinfo[field] = value.split(',')
437 thisinfo[field] = value
443 """Read all srclib metadata.
445 The information read will be accessible as metadata.srclibs, which is a
446 dictionary, keyed on srclib name, with the values each being a dictionary
447 in the same format as that returned by the parse_srclib function.
449 A MetaDataException is raised if there are any problems with the srclib
454 # They were already loaded
455 if srclibs is not None:
461 if not os.path.exists(srcdir):
464 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
465 srclibname = os.path.basename(metafile[:-4])
466 srclibs[srclibname] = parse_srclib(metafile)
469 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
470 # returned by the parse_metadata function.
471 def read_metadata(xref=True):
473 # Always read the srclibs before the apps, since they can use a srlib as
474 # their source repository.
479 for basedir in ('metadata', 'tmp'):
480 if not os.path.exists(basedir):
483 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
484 appid, appinfo = parse_metadata(metafile)
485 check_metadata(appinfo)
486 apps[appid] = appinfo
489 # Parse all descriptions at load time, just to ensure cross-referencing
490 # errors are caught early rather than when they hit the build server.
493 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
494 raise MetaDataException("Cannot resolve app id " + appid)
496 for appid, app in apps.iteritems():
498 description_html(app['Description'], linkres)
499 except MetaDataException, e:
500 raise MetaDataException("Problem with description of " + appid +
506 # Get the type expected for a given metadata field.
507 def metafieldtype(name):
508 if name in ['Description', 'Maintainer Notes']:
510 if name in ['Categories']:
512 if name == 'Build Version':
516 if name == 'Use Built':
518 if name not in app_defaults:
524 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni', 'preassemble',
525 'update', 'scanignore', 'scandelete', 'gradle', 'antcommands']:
527 if name in ['init', 'prebuild', 'build']:
529 if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
535 def fill_build_defaults(build):
537 def get_build_type():
538 for t in ['maven', 'gradle', 'kivy']:
545 for flag, value in flag_defaults.iteritems():
549 build['type'] = get_build_type()
550 build['ndk_path'] = common.get_ndk_path(build['ndk'])
553 def split_list_values(s):
554 # Port legacy ';' separators
555 l = [v.strip() for v in s.replace(';', ',').split(',')]
556 return [v for v in l if v]
559 # Parse metadata for a single application.
561 # 'metafile' - the filename to read. The package id for the application comes
562 # from this filename. Pass None to get a blank entry.
564 # Returns a dictionary containing all the details of the application. There are
565 # two major kinds of information in the dictionary. Keys beginning with capital
566 # letters correspond directory to identically named keys in the metadata file.
567 # Keys beginning with lower case letters are generated in one way or another,
568 # and are not found verbatim in the metadata.
570 # Known keys not originating from the metadata are:
572 # 'builds' - a list of dictionaries containing build information
573 # for each defined build
574 # 'comments' - a list of comments from the metadata file. Each is
575 # a tuple of the form (field, comment) where field is
576 # the name of the field it preceded in the metadata
577 # file. Where field is None, the comment goes at the
578 # end of the file. Alternatively, 'build:version' is
579 # for a comment before a particular build version.
580 # 'descriptionlines' - original lines of description as formatted in the
583 def parse_metadata(metafile):
588 def add_buildflag(p, thisbuild):
590 raise MetaDataException("Empty build flag at {1}"
591 .format(buildlines[0], linedesc))
594 raise MetaDataException("Invalid build flag at {0} in {1}"
595 .format(buildlines[0], linedesc))
598 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
599 .format(pk, thisbuild['version'], linedesc))
602 if pk not in flag_defaults:
603 raise MetaDataException("Unrecognised build flag at {0} in {1}"
604 .format(p, linedesc))
607 pv = split_list_values(pv)
609 if len(pv) == 1 and pv[0] in ['main', 'yes']:
612 elif t == 'string' or t == 'script':
619 logging.debug("...ignoring bool flag %s" % p)
622 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
625 def parse_buildline(lines):
626 value = "".join(lines)
627 parts = [p.replace("\\,", ",")
628 for p in re.split(r"(?<!\\),", value)]
630 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
632 thisbuild['origlines'] = lines
633 thisbuild['version'] = parts[0]
634 thisbuild['vercode'] = parts[1]
635 if parts[2].startswith('!'):
636 # For backwards compatibility, handle old-style disabling,
637 # including attempting to extract the commit from the message
638 thisbuild['disable'] = parts[2][1:]
639 commit = 'unknown - see disabled'
640 index = parts[2].rfind('at ')
642 commit = parts[2][index + 3:]
643 if commit.endswith(')'):
645 thisbuild['commit'] = commit
647 thisbuild['commit'] = parts[2]
649 add_buildflag(p, thisbuild)
653 def add_comments(key):
656 for comment in curcomments:
657 thisinfo['comments'].append((key, comment))
662 if not isinstance(metafile, file):
663 metafile = open(metafile, "r")
664 appid = metafile.name[9:-4]
666 thisinfo.update(app_defaults)
667 thisinfo['id'] = appid
669 # General defaults...
670 thisinfo['builds'] = []
671 thisinfo['comments'] = []
674 return appid, thisinfo
683 for line in metafile:
685 linedesc = "%s:%d" % (metafile.name, c)
686 line = line.rstrip('\r\n')
688 if not any(line.startswith(s) for s in (' ', '\t')):
689 commit = curbuild['commit'] if 'commit' in curbuild else None
690 if not commit and 'disable' not in curbuild:
691 raise MetaDataException("No commit specified for {0} in {1}"
692 .format(curbuild['version'], linedesc))
694 thisinfo['builds'].append(curbuild)
695 add_comments('build:' + curbuild['vercode'])
698 if line.endswith('\\'):
699 buildlines.append(line[:-1].lstrip())
701 buildlines.append(line.lstrip())
702 bl = ''.join(buildlines)
703 add_buildflag(bl, curbuild)
709 if line.startswith("#"):
710 curcomments.append(line)
713 field, value = line.split(':', 1)
715 raise MetaDataException("Invalid metadata in " + linedesc)
716 if field != field.strip() or value != value.strip():
717 raise MetaDataException("Extra spacing found in " + linedesc)
719 # Translate obsolete fields...
720 if field == 'Market Version':
721 field = 'Current Version'
722 if field == 'Market Version Code':
723 field = 'Current Version Code'
725 fieldtype = metafieldtype(field)
726 if fieldtype not in ['build', 'buildv2']:
728 if fieldtype == 'multiline':
732 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
733 elif fieldtype == 'string':
734 thisinfo[field] = value
735 elif fieldtype == 'list':
736 thisinfo[field] = split_list_values(value)
737 elif fieldtype == 'build':
738 if value.endswith("\\"):
740 buildlines = [value[:-1]]
742 curbuild = parse_buildline([value])
743 thisinfo['builds'].append(curbuild)
744 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
745 elif fieldtype == 'buildv2':
747 vv = value.split(',')
749 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
750 .format(value, linedesc))
751 curbuild['version'] = vv[0]
752 curbuild['vercode'] = vv[1]
753 if curbuild['vercode'] in vc_seen:
754 raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
755 curbuild['vercode'], linedesc))
756 vc_seen[curbuild['vercode']] = True
759 elif fieldtype == 'obsolete':
760 pass # Just throw it away!
762 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
763 elif mode == 1: # Multiline field
767 thisinfo[field].append(line)
768 elif mode == 2: # Line continuation mode in Build Version
769 if line.endswith("\\"):
770 buildlines.append(line[:-1])
772 buildlines.append(line)
773 curbuild = parse_buildline(buildlines)
774 thisinfo['builds'].append(curbuild)
775 add_comments('build:' + thisinfo['builds'][-1]['vercode'])
779 # Mode at end of file should always be 0...
781 raise MetaDataException(field + " not terminated in " + metafile.name)
783 raise MetaDataException("Unterminated continuation in " + metafile.name)
785 raise MetaDataException("Unterminated build in " + metafile.name)
787 if not thisinfo['Description']:
788 thisinfo['Description'].append('No description available')
790 for build in thisinfo['builds']:
791 fill_build_defaults(build)
793 thisinfo['builds'] = sorted(thisinfo['builds'], key=lambda build: int(build['vercode']))
795 return (appid, thisinfo)
798 # Write a metadata file.
800 # 'dest' - The path to the output file
801 # 'app' - The app data
802 def write_metadata(dest, app):
804 def writecomments(key):
806 for pf, comment in app['comments']:
808 mf.write("%s\n" % comment)
811 logging.debug("...writing comments for " + (key or 'EOF'))
813 def writefield(field, value=None):
817 t = metafieldtype(field)
819 value = ','.join(value)
820 mf.write("%s:%s\n" % (field, value))
822 def writefield_nonempty(field, value=None):
826 writefield(field, value)
829 writefield_nonempty('Disabled')
830 writefield_nonempty('AntiFeatures')
831 writefield_nonempty('Provides')
832 writefield('Categories')
833 writefield('License')
834 writefield('Web Site')
835 writefield('Source Code')
836 writefield('Issue Tracker')
837 writefield_nonempty('Changelog')
838 writefield_nonempty('Donate')
839 writefield_nonempty('FlattrID')
840 writefield_nonempty('Bitcoin')
841 writefield_nonempty('Litecoin')
842 writefield_nonempty('Dogecoin')
844 writefield_nonempty('Name')
845 writefield_nonempty('Auto Name')
846 writefield('Summary')
847 writefield('Description', '')
848 for line in app['Description']:
849 mf.write("%s\n" % line)
852 if app['Requires Root']:
853 writefield('Requires Root', 'Yes')
856 writefield('Repo Type')
859 writefield('Binaries')
861 for build in app['builds']:
863 if build['version'] == "Ignore":
866 writecomments('build:' + build['vercode'])
867 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
869 def write_builditem(key, value):
871 if key in ['version', 'vercode']:
874 if value == flag_defaults[key]:
879 logging.debug("...writing {0} : {1}".format(key, value))
880 outline = ' %s=' % key
887 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
889 outline += ','.join(value) if type(value) == list else value
894 for flag in flag_defaults:
897 write_builditem(flag, value)
900 if app['Maintainer Notes']:
901 writefield('Maintainer Notes', '')
902 for line in app['Maintainer Notes']:
903 mf.write("%s\n" % line)
907 writefield_nonempty('Archive Policy')
908 writefield('Auto Update Mode')
909 writefield('Update Check Mode')
910 writefield_nonempty('Update Check Ignore')
911 writefield_nonempty('Vercode Operation')
912 writefield_nonempty('Update Check Name')
913 writefield_nonempty('Update Check Data')
914 if app['Current Version']:
915 writefield('Current Version')
916 writefield('Current Version Code')
918 if app['No Source Since']:
919 writefield('No Source Since')