1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from collections import OrderedDict
31 class MetaDataException(Exception):
32 def __init__(self, value):
38 # In the order in which they are laid out on files
39 app_defaults = OrderedDict([
41 ('AntiFeatures', None),
43 ('Categories', ['None']),
44 ('License', 'Unknown'),
47 ('Issue Tracker', ''),
57 ('Requires Root', False),
60 ('Maintainer Notes', []),
61 ('Archive Policy', None),
62 ('Auto Update Mode', 'None'),
63 ('Update Check Mode', 'None'),
64 ('Update Check Ignore', None),
65 ('Vercode Operation', None),
66 ('Update Check Name', None),
67 ('Update Check Data', None),
68 ('Current Version', ''),
69 ('Current Version Code', '0'),
70 ('No Source Since', ''),
74 # In the order in which they are laid out on files
75 # Sorted by their action and their place in the build timeline
76 flag_defaults = OrderedDict([
80 ('submodules', False),
90 ('forceversion', False),
91 ('forcevercode', False),
102 ('antcommand', None),
107 # Designates a metadata field type and checks that it matches
109 # 'name' - The long name of the field type
110 # 'matching' - List of possible values or regex expression
111 # 'sep' - Separator to use if value may be a list
112 # 'fields' - Metadata fields (Field:Value) of this type
113 # 'attrs' - Build attributes (attr=value) of this type
115 class FieldValidator():
117 def __init__(self, name, matching, sep, fields, attrs):
119 self.matching = matching
120 if type(matching) is str:
121 self.compiled = re.compile(matching)
126 def _assert_regex(self, values, appid):
128 if not self.compiled.match(v):
129 raise MetaDataException("'%s' is not a valid %s in %s. "
130 % (v, self.name, appid) +
131 "Regex pattern: %s" % (self.matching))
133 def _assert_list(self, values, appid):
135 if v not in self.matching:
136 raise MetaDataException("'%s' is not a valid %s in %s. "
137 % (v, self.name, appid) +
138 "Possible values: %s" % (", ".join(self.matching)))
140 def check(self, value, appid):
141 if type(value) is not str or not value:
143 if self.sep is not None:
144 values = value.split(self.sep)
147 if type(self.matching) is list:
148 self._assert_list(values, appid)
150 self._assert_regex(values, appid)
153 # Generic value types
155 FieldValidator("Integer",
156 r'^[1-9][0-9]*$', None,
160 FieldValidator("HTTP link",
161 r'^http[s]?://', None,
162 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
164 FieldValidator("Bitcoin address",
165 r'^[a-zA-Z0-9]{27,34}$', None,
169 FieldValidator("Litecoin address",
170 r'^L[a-zA-Z0-9]{33}$', None,
174 FieldValidator("Dogecoin address",
175 r'^D[a-zA-Z0-9]{33}$', None,
179 FieldValidator("Boolean",
184 FieldValidator("bool",
187 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
190 FieldValidator("Repo Type",
191 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
195 FieldValidator("Archive Policy",
196 r'^[0-9]+ versions$', None,
200 FieldValidator("Anti-Feature",
201 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
205 FieldValidator("Auto Update Mode",
206 r"^(Version .+|None)$", None,
207 ["Auto Update Mode"],
210 FieldValidator("Update Check Mode",
211 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
212 ["Update Check Mode"],
217 # Check an app's metadata information for integrity errors
218 def check_metadata(info):
220 for field in v.fields:
222 v.check(info[field], info['id'])
223 for build in info['builds']:
226 v.check(build[attr], info['id'])
229 # Formatter for descriptions. Create an instance, and call parseline() with
230 # each line of the description source from the metadata. At the end, call
231 # end() and then text_plain, text_wiki and text_html will contain the result.
232 class DescriptionFormatter:
245 def __init__(self, linkres):
246 self.linkResolver = linkres
248 def endcur(self, notstates=None):
249 if notstates and self.state in notstates:
251 if self.state == self.stPARA:
253 elif self.state == self.stUL:
255 elif self.state == self.stOL:
259 self.text_plain += '\n'
260 self.text_html += '</p>'
261 self.state = self.stNONE
264 self.text_html += '</ul>'
265 self.state = self.stNONE
268 self.text_html += '</ol>'
269 self.state = self.stNONE
271 def formatted(self, txt, html):
274 txt = cgi.escape(txt)
276 index = txt.find("''")
278 return formatted + txt
279 formatted += txt[:index]
281 if txt.startswith("'''"):
287 self.bold = not self.bold
295 self.ital = not self.ital
298 def linkify(self, txt):
302 index = txt.find("[")
304 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
305 linkified_plain += self.formatted(txt[:index], False)
306 linkified_html += self.formatted(txt[:index], True)
308 if txt.startswith("[["):
309 index = txt.find("]]")
311 raise MetaDataException("Unterminated ]]")
313 if self.linkResolver:
314 url, urltext = self.linkResolver(url)
317 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
318 linkified_plain += urltext
319 txt = txt[index + 2:]
321 index = txt.find("]")
323 raise MetaDataException("Unterminated ]")
325 index2 = url.find(' ')
329 urltxt = url[index2 + 1:]
331 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
332 linkified_plain += urltxt
334 linkified_plain += ' (' + url + ')'
335 txt = txt[index + 1:]
337 def addtext(self, txt):
338 p, h = self.linkify(txt)
342 def parseline(self, line):
343 self.text_wiki += "%s\n" % line
346 elif line.startswith('* '):
347 self.endcur([self.stUL])
348 if self.state != self.stUL:
349 self.text_html += '<ul>'
350 self.state = self.stUL
351 self.text_html += '<li>'
352 self.text_plain += '* '
353 self.addtext(line[1:])
354 self.text_html += '</li>'
355 elif line.startswith('# '):
356 self.endcur([self.stOL])
357 if self.state != self.stOL:
358 self.text_html += '<ol>'
359 self.state = self.stOL
360 self.text_html += '<li>'
361 self.text_plain += '* ' # TODO: lazy - put the numbers in!
362 self.addtext(line[1:])
363 self.text_html += '</li>'
365 self.endcur([self.stPARA])
366 if self.state == self.stNONE:
367 self.text_html += '<p>'
368 self.state = self.stPARA
369 elif self.state == self.stPARA:
370 self.text_html += ' '
371 self.text_plain += ' '
378 # Parse multiple lines of description as written in a metadata file, returning
379 # a single string in plain text format.
380 def description_plain(lines, linkres):
381 ps = DescriptionFormatter(linkres)
388 # Parse multiple lines of description as written in a metadata file, returning
389 # a single string in wiki format. Used for the Maintainer Notes field as well,
390 # because it's the same format.
391 def description_wiki(lines):
392 ps = DescriptionFormatter(None)
399 # Parse multiple lines of description as written in a metadata file, returning
400 # a single string in HTML format.
401 def description_html(lines, linkres):
402 ps = DescriptionFormatter(linkres)
409 def parse_srclib(metafile):
412 if metafile and not isinstance(metafile, file):
413 metafile = open(metafile, "r")
415 # Defaults for fields that come from metadata
416 thisinfo['Repo Type'] = ''
417 thisinfo['Repo'] = ''
418 thisinfo['Subdir'] = None
419 thisinfo['Prepare'] = None
420 thisinfo['Srclibs'] = None
426 for line in metafile:
428 line = line.rstrip('\r\n')
429 if not line or line.startswith("#"):
433 field, value = line.split(':', 1)
435 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
437 if field == "Subdir":
438 thisinfo[field] = value.split(',')
440 thisinfo[field] = value
446 """Read all srclib metadata.
448 The information read will be accessible as metadata.srclibs, which is a
449 dictionary, keyed on srclib name, with the values each being a dictionary
450 in the same format as that returned by the parse_srclib function.
452 A MetaDataException is raised if there are any problems with the srclib
459 if not os.path.exists(srcdir):
462 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
463 srclibname = os.path.basename(metafile[:-4])
464 srclibs[srclibname] = parse_srclib(metafile)
467 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
468 # returned by the parse_metadata function.
469 def read_metadata(xref=True):
472 for basedir in ('metadata', 'tmp'):
473 if not os.path.exists(basedir):
476 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
477 appinfo = parse_metadata(metafile)
478 check_metadata(appinfo)
482 # Parse all descriptions at load time, just to ensure cross-referencing
483 # errors are caught early rather than when they hit the build server.
486 if app['id'] == link:
487 return ("fdroid.app:" + link, "Dummy name - don't know yet")
488 raise MetaDataException("Cannot resolve app id " + link)
491 description_html(app['Description'], linkres)
493 raise MetaDataException("Problem with description of " + app['id'] +
499 # Get the type expected for a given metadata field.
500 def metafieldtype(name):
501 if name in ['Description', 'Maintainer Notes']:
503 if name in ['Categories']:
505 if name == 'Build Version':
509 if name == 'Use Built':
511 if name not in app_defaults:
517 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
518 'update', 'scanignore', 'scandelete']:
520 if name in ['init', 'prebuild', 'build']:
522 if name in ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
528 # Parse metadata for a single application.
530 # 'metafile' - the filename to read. The package id for the application comes
531 # from this filename. Pass None to get a blank entry.
533 # Returns a dictionary containing all the details of the application. There are
534 # two major kinds of information in the dictionary. Keys beginning with capital
535 # letters correspond directory to identically named keys in the metadata file.
536 # Keys beginning with lower case letters are generated in one way or another,
537 # and are not found verbatim in the metadata.
539 # Known keys not originating from the metadata are:
541 # 'id' - the application's package ID
542 # 'builds' - a list of dictionaries containing build information
543 # for each defined build
544 # 'comments' - a list of comments from the metadata file. Each is
545 # a tuple of the form (field, comment) where field is
546 # the name of the field it preceded in the metadata
547 # file. Where field is None, the comment goes at the
548 # end of the file. Alternatively, 'build:version' is
549 # for a comment before a particular build version.
550 # 'descriptionlines' - original lines of description as formatted in the
553 def parse_metadata(metafile):
557 def add_buildflag(p, thisbuild):
560 raise MetaDataException("Invalid build flag at {0} in {1}"
561 .format(buildlines[0], linedesc))
564 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
565 .format(pk, thisbuild['version'], linedesc))
568 if pk not in flag_defaults:
569 raise MetaDataException("Unrecognised build flag at {0} in {1}"
570 .format(p, linedesc))
573 # Port legacy ';' separators
574 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
575 elif t == 'string' or t == 'script':
582 logging.debug("...ignoring bool flag %s" % p)
585 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
588 def parse_buildline(lines):
589 value = "".join(lines)
590 parts = [p.replace("\\,", ",")
591 for p in re.split(r"(?<!\\),", value)]
593 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
595 thisbuild['origlines'] = lines
596 thisbuild['version'] = parts[0]
597 thisbuild['vercode'] = parts[1]
598 if parts[2].startswith('!'):
599 # For backwards compatibility, handle old-style disabling,
600 # including attempting to extract the commit from the message
601 thisbuild['disable'] = parts[2][1:]
602 commit = 'unknown - see disabled'
603 index = parts[2].rfind('at ')
605 commit = parts[2][index + 3:]
606 if commit.endswith(')'):
608 thisbuild['commit'] = commit
610 thisbuild['commit'] = parts[2]
612 add_buildflag(p, thisbuild)
616 def add_comments(key):
619 for comment in curcomments:
620 thisinfo['comments'].append((key, comment))
623 def get_build_type(build):
624 for t in ['maven', 'gradle', 'kivy']:
633 if not isinstance(metafile, file):
634 metafile = open(metafile, "r")
635 thisinfo['id'] = metafile.name[9:-4]
637 thisinfo['id'] = None
639 thisinfo.update(app_defaults)
641 # General defaults...
642 thisinfo['builds'] = []
643 thisinfo['comments'] = []
655 for line in metafile:
657 linedesc = "%s:%d" % (metafile.name, c)
658 line = line.rstrip('\r\n')
660 if not any(line.startswith(s) for s in (' ', '\t')):
661 if 'commit' not in curbuild and 'disable' not in curbuild:
662 raise MetaDataException("No commit specified for {0} in {1}"
663 .format(curbuild['version'], linedesc))
665 thisinfo['builds'].append(curbuild)
666 add_comments('build:' + curbuild['version'])
669 if line.endswith('\\'):
670 buildlines.append(line[:-1].lstrip())
672 buildlines.append(line.lstrip())
673 bl = ''.join(buildlines)
674 add_buildflag(bl, curbuild)
680 if line.startswith("#"):
681 curcomments.append(line)
684 field, value = line.split(':', 1)
686 raise MetaDataException("Invalid metadata in " + linedesc)
687 if field != field.strip() or value != value.strip():
688 raise MetaDataException("Extra spacing found in " + linedesc)
690 # Translate obsolete fields...
691 if field == 'Market Version':
692 field = 'Current Version'
693 if field == 'Market Version Code':
694 field = 'Current Version Code'
696 fieldtype = metafieldtype(field)
697 if fieldtype not in ['build', 'buildv2']:
699 if fieldtype == 'multiline':
703 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
704 elif fieldtype == 'string':
705 thisinfo[field] = value
706 elif fieldtype == 'list':
707 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
708 elif fieldtype == 'build':
709 if value.endswith("\\"):
711 buildlines = [value[:-1]]
713 curbuild = parse_buildline([value])
714 add_comments('build:' + thisinfo['builds'][-1]['version'])
715 elif fieldtype == 'buildv2':
717 vv = value.split(',')
719 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
720 .format(value, linedesc))
721 curbuild['version'] = vv[0]
722 curbuild['vercode'] = vv[1]
723 if curbuild['vercode'] in vc_seen:
724 raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
725 curbuild['vercode'], linedesc))
726 vc_seen[curbuild['vercode']] = True
729 elif fieldtype == 'obsolete':
730 pass # Just throw it away!
732 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
733 elif mode == 1: # Multiline field
737 thisinfo[field].append(line)
738 elif mode == 2: # Line continuation mode in Build Version
739 if line.endswith("\\"):
740 buildlines.append(line[:-1])
742 buildlines.append(line)
743 curbuild = parse_buildline(buildlines)
744 thisinfo['builds'].append(curbuild)
745 add_comments('build:' + thisinfo['builds'][-1]['version'])
749 # Mode at end of file should always be 0...
751 raise MetaDataException(field + " not terminated in " + metafile.name)
753 raise MetaDataException("Unterminated continuation in " + metafile.name)
755 raise MetaDataException("Unterminated build in " + metafile.name)
757 if not thisinfo['Description']:
758 thisinfo['Description'].append('No description available')
760 for build in thisinfo['builds']:
761 for flag, value in flag_defaults.iteritems():
765 build['type'] = get_build_type(build)
770 # Write a metadata file.
772 # 'dest' - The path to the output file
773 # 'app' - The app data
774 def write_metadata(dest, app):
776 def writecomments(key):
778 for pf, comment in app['comments']:
780 mf.write("%s\n" % comment)
783 logging.debug("...writing comments for " + (key if key else 'EOF'))
785 def writefield(field, value=None):
789 t = metafieldtype(field)
791 value = ','.join(value)
792 mf.write("%s:%s\n" % (field, value))
796 writefield('Disabled')
797 if app['AntiFeatures']:
798 writefield('AntiFeatures')
800 writefield('Provides')
801 writefield('Categories')
802 writefield('License')
803 writefield('Web Site')
804 writefield('Source Code')
805 writefield('Issue Tracker')
809 writefield('FlattrID')
811 writefield('Bitcoin')
813 writefield('Litecoin')
815 writefield('Dogecoin')
820 writefield('Auto Name')
821 writefield('Summary')
822 writefield('Description', '')
823 for line in app['Description']:
824 mf.write("%s\n" % line)
827 if app['Requires Root']:
828 writefield('Requires Root', 'Yes')
831 writefield('Repo Type')
834 for build in app['builds']:
835 writecomments('build:' + build['version'])
836 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
838 def write_builditem(key, value):
840 if key in ['version', 'vercode', 'origlines', 'type']:
843 if value == flag_defaults[key]:
848 logging.debug("...writing {0} : {1}".format(key, value))
849 outline = ' %s=' % key
856 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
858 outline += ','.join(value) if type(value) == list else value
863 for flag in flag_defaults:
866 write_builditem(flag, value)
869 if app['Maintainer Notes']:
870 writefield('Maintainer Notes', '')
871 for line in app['Maintainer Notes']:
872 mf.write("%s\n" % line)
876 if app['Archive Policy']:
877 writefield('Archive Policy')
878 writefield('Auto Update Mode')
879 writefield('Update Check Mode')
880 if app['Update Check Ignore']:
881 writefield('Update Check Ignore')
882 if app['Vercode Operation']:
883 writefield('Vercode Operation')
884 if app['Update Check Name']:
885 writefield('Update Check Name')
886 if app['Update Check Data']:
887 writefield('Update Check Data')
888 if app['Current Version']:
889 writefield('Current Version')
890 writefield('Current Version Code')
892 if app['No Source Since']:
893 writefield('No Source Since')