1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 class MetaDataException(Exception):
25 def __init__(self, value):
29 return repr(self.value)
35 'Categories': ['None'],
49 'Archive Policy': None,
50 'Update Check Mode': 'None',
51 'Update Check Name': None,
52 'Update Check Data': None,
53 'Vercode Operation': None,
54 'Auto Update Mode': 'None',
55 'Current Version': '',
56 'Current Version Code': '0',
59 'Requires Root': False,
64 # This defines the preferred order for the build items - as in the
65 # manual, they're roughly in order of application.
67 'disable', 'commit', 'subdir', 'submodules', 'init',
68 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
69 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
70 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
71 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
72 'antcommand', 'novcheck'
76 # Designates a metadata field type and checks that it matches
78 # 'name' - The long name of the field type
79 # 'matching' - List of possible values or regex expression
80 # 'sep' - Separator to use if value may be a list
81 # 'fields' - Metadata fields (Field:Value) of this type
82 # 'attrs' - Build attributes (attr=value) of this type
85 def __init__(self, name, matching, sep, fields, attrs):
87 self.matching = matching
88 if type(matching) is str:
89 self.compiled = re.compile(matching)
94 def _assert_regex(self, values, appid):
96 if not self.compiled.match(v):
97 raise MetaDataException("'%s' is not a valid %s in %s. "
98 % (v, self.name, appid) +
99 "Regex pattern: %s" % (self.matching))
101 def _assert_list(self, values, appid):
103 if v not in self.matching:
104 raise MetaDataException("'%s' is not a valid %s in %s. "
105 % (v, self.name, appid) +
106 "Possible values: %s" % (", ".join(self.matching)))
108 def check(self, value, appid):
109 if type(value) is not str or not value:
111 if self.sep is not None:
112 values = value.split(self.sep)
115 if type(self.matching) is list:
116 self._assert_list(values, appid)
118 self._assert_regex(values, appid)
121 # Generic value types
123 'int': FieldType("Integer",
124 r'^[1-9][0-9]*$', None,
128 'http': FieldType("HTTP link",
129 r'^http[s]?://', None,
130 ["Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
132 'bitcoin': FieldType("Bitcoin address",
133 r'^[a-zA-Z0-9]{27,34}$', None,
137 'litecoin': FieldType("Litecoin address",
138 r'^L[a-zA-Z0-9]{33}$', None,
142 'dogecoin': FieldType("Dogecoin address",
143 r'^D[a-zA-Z0-9]{33}$', None,
147 'Bool': FieldType("Boolean",
152 'bool': FieldType("Boolean",
155 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
158 'Repo Type': FieldType("Repo Type",
159 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
163 'archive': FieldType("Archive Policy",
164 r'^[0-9]+ versions$', None,
168 'antifeatures': FieldType("Anti-Feature",
169 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
173 'autoupdatemodes': FieldType("Auto Update Mode",
174 r"^(Version .+|None)$", None,
175 ["Auto Update Mode" ],
178 'updatecheckmodes': FieldType("Update Check Mode",
179 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
180 ["Update Check Mode" ],
184 # Check an app's metadata information for integrity errors
185 def check_metadata(info):
186 for k, t in valuetypes.iteritems():
187 for field in t.fields:
189 t.check(info[field], info['id'])
191 info[field] = info[field] == "Yes"
192 for build in info['builds']:
195 t.check(build[attr], info['id'])
197 build[attr] = build[attr] == "yes"
201 # Formatter for descriptions. Create an instance, and call parseline() with
202 # each line of the description source from the metadata. At the end, call
203 # end() and then text_plain, text_wiki and text_html will contain the result.
204 class DescriptionFormatter:
216 def __init__(self, linkres):
217 self.linkResolver = linkres
218 def endcur(self, notstates=None):
219 if notstates and self.state in notstates:
221 if self.state == self.stPARA:
223 elif self.state == self.stUL:
225 elif self.state == self.stOL:
228 self.text_plain += '\n'
229 self.text_html += '</p>'
230 self.state = self.stNONE
232 self.text_html += '</ul>'
233 self.state = self.stNONE
235 self.text_html += '</ol>'
236 self.state = self.stNONE
238 def formatted(self, txt, html):
241 txt = cgi.escape(txt)
243 index = txt.find("''")
245 return formatted + txt
246 formatted += txt[:index]
248 if txt.startswith("'''"):
254 self.bold = not self.bold
262 self.ital = not self.ital
266 def linkify(self, txt):
270 index = txt.find("[")
272 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
273 linkified_plain += self.formatted(txt[:index], False)
274 linkified_html += self.formatted(txt[:index], True)
276 if txt.startswith("[["):
277 index = txt.find("]]")
279 raise MetaDataException("Unterminated ]]")
281 if self.linkResolver:
282 url, urltext = self.linkResolver(url)
285 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
286 linkified_plain += urltext
289 index = txt.find("]")
291 raise MetaDataException("Unterminated ]")
293 index2 = url.find(' ')
297 urltxt = url[index2 + 1:]
299 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
300 linkified_plain += urltxt
302 linkified_plain += ' (' + url + ')'
305 def addtext(self, txt):
306 p, h = self.linkify(txt)
310 def parseline(self, line):
311 self.text_wiki += "%s\n" % line
314 elif line.startswith('* '):
315 self.endcur([self.stUL])
316 if self.state != self.stUL:
317 self.text_html += '<ul>'
318 self.state = self.stUL
319 self.text_html += '<li>'
320 self.text_plain += '* '
321 self.addtext(line[1:])
322 self.text_html += '</li>'
323 elif line.startswith('# '):
324 self.endcur([self.stOL])
325 if self.state != self.stOL:
326 self.text_html += '<ol>'
327 self.state = self.stOL
328 self.text_html += '<li>'
329 self.text_plain += '* ' #TODO: lazy - put the numbers in!
330 self.addtext(line[1:])
331 self.text_html += '</li>'
333 self.endcur([self.stPARA])
334 if self.state == self.stNONE:
335 self.text_html += '<p>'
336 self.state = self.stPARA
337 elif self.state == self.stPARA:
338 self.text_html += ' '
339 self.text_plain += ' '
345 # Parse multiple lines of description as written in a metadata file, returning
346 # a single string in plain text format.
347 def description_plain(lines, linkres):
348 ps = DescriptionFormatter(linkres)
354 # Parse multiple lines of description as written in a metadata file, returning
355 # a single string in wiki format. Used for the Maintainer Notes field as well,
356 # because it's the same format.
357 def description_wiki(lines):
358 ps = DescriptionFormatter(None)
364 # Parse multiple lines of description as written in a metadata file, returning
365 # a single string in HTML format.
366 def description_html(lines, linkres):
367 ps = DescriptionFormatter(linkres)
373 def parse_srclib(metafile, **kw):
376 if metafile and not isinstance(metafile, file):
377 metafile = open(metafile, "r")
379 # Defaults for fields that come from metadata
380 thisinfo['Repo Type'] = ''
381 thisinfo['Repo'] = ''
382 thisinfo['Subdir'] = None
383 thisinfo['Prepare'] = None
384 thisinfo['Srclibs'] = None
390 for line in metafile:
392 line = line.rstrip('\r\n')
393 if not line or line.startswith("#"):
397 field, value = line.split(':', 1)
399 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
401 if field == "Subdir":
402 thisinfo[field] = value.split(',')
404 thisinfo[field] = value
408 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
409 # returned by the parse_metadata function.
410 def read_metadata(xref=True, package=None, store=True):
413 for basedir in ('metadata', 'tmp'):
414 if not os.path.exists(basedir):
417 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
418 if package is None or metafile == os.path.join('metadata', package + '.txt'):
419 appinfo = parse_metadata(metafile)
420 check_metadata(appinfo)
424 # Parse all descriptions at load time, just to ensure cross-referencing
425 # errors are caught early rather than when they hit the build server.
428 if app['id'] == link:
429 return ("fdroid.app:" + link, "Dummy name - don't know yet")
430 raise MetaDataException("Cannot resolve app id " + link)
433 description_html(app['Description'], linkres)
435 raise MetaDataException("Problem with description of " + app['id'] +
440 # Get the type expected for a given metadata field.
441 def metafieldtype(name):
442 if name in ['Description', 'Maintainer Notes']:
444 if name in ['Categories']:
446 if name == 'Build Version':
450 if name == 'Use Built':
452 if name not in app_defaults:
457 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
458 'update', 'scanignore', 'scandelete']:
460 if name in ['init', 'prebuild', 'build']:
464 # Parse metadata for a single application.
466 # 'metafile' - the filename to read. The package id for the application comes
467 # from this filename. Pass None to get a blank entry.
469 # Returns a dictionary containing all the details of the application. There are
470 # two major kinds of information in the dictionary. Keys beginning with capital
471 # letters correspond directory to identically named keys in the metadata file.
472 # Keys beginning with lower case letters are generated in one way or another,
473 # and are not found verbatim in the metadata.
475 # Known keys not originating from the metadata are:
477 # 'id' - the application's package ID
478 # 'builds' - a list of dictionaries containing build information
479 # for each defined build
480 # 'comments' - a list of comments from the metadata file. Each is
481 # a tuple of the form (field, comment) where field is
482 # the name of the field it preceded in the metadata
483 # file. Where field is None, the comment goes at the
484 # end of the file. Alternatively, 'build:version' is
485 # for a comment before a particular build version.
486 # 'descriptionlines' - original lines of description as formatted in the
489 def parse_metadata(metafile):
493 def add_buildflag(p, thisbuild):
496 raise MetaDataException("Invalid build flag at {0} in {1}".
497 format(buildlines[0], linedesc))
500 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
501 format(pk, thisbuild['version'], linedesc))
504 if pk not in ordered_flags:
505 raise MetaDataException("Unrecognised build flag at {0} in {1}".
509 # Port legacy ';' separators
510 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
516 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % (
519 def parse_buildline(lines):
520 value = "".join(lines)
521 parts = [p.replace("\\,", ",")
522 for p in re.split(r"(?<!\\),", value)]
524 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
526 thisbuild['origlines'] = lines
527 thisbuild['version'] = parts[0]
528 thisbuild['vercode'] = parts[1]
529 if parts[2].startswith('!'):
530 # For backwards compatibility, handle old-style disabling,
531 # including attempting to extract the commit from the message
532 thisbuild['disable'] = parts[2][1:]
533 commit = 'unknown - see disabled'
534 index = parts[2].rfind('at ')
536 commit = parts[2][index+3:]
537 if commit.endswith(')'):
539 thisbuild['commit'] = commit
541 thisbuild['commit'] = parts[2]
543 add_buildflag(p, thisbuild)
547 def add_comments(key):
550 for comment in curcomments:
551 thisinfo['comments'].append((key, comment))
554 def get_build_type(build):
555 for t in ['maven', 'gradle', 'kivy']:
556 if build.get(t, 'no') != 'no':
558 if 'output' in build:
564 if not isinstance(metafile, file):
565 metafile = open(metafile, "r")
566 thisinfo['id'] = metafile.name[9:-4]
568 thisinfo['id'] = None
570 thisinfo.update(app_defaults)
572 # General defaults...
573 thisinfo['builds'] = []
574 thisinfo['comments'] = []
585 for line in metafile:
587 linedesc = "%s:%d" % (metafile.name, c)
588 line = line.rstrip('\r\n')
590 if not any(line.startswith(s) for s in (' ', '\t')):
591 if 'commit' not in curbuild and 'disable' not in curbuild:
592 raise MetaDataException("No commit specified for {0} in {1}".format(
593 curbuild['version'], linedesc))
594 thisinfo['builds'].append(curbuild)
595 add_comments('build:' + curbuild['version'])
598 if line.endswith('\\'):
599 buildlines.append(line[:-1].lstrip())
601 buildlines.append(line.lstrip())
602 bl = ''.join(buildlines)
603 add_buildflag(bl, curbuild)
609 if line.startswith("#"):
610 curcomments.append(line)
613 field, value = line.split(':', 1)
615 raise MetaDataException("Invalid metadata in "+linedesc)
616 if field != field.strip() or value != value.strip():
617 raise MetaDataException("Extra spacing found in "+linedesc)
619 # Translate obsolete fields...
620 if field == 'Market Version':
621 field = 'Current Version'
622 if field == 'Market Version Code':
623 field = 'Current Version Code'
625 fieldtype = metafieldtype(field)
626 if fieldtype not in ['build', 'buildv2']:
628 if fieldtype == 'multiline':
632 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
633 elif fieldtype == 'string':
634 thisinfo[field] = value
635 elif fieldtype == 'list':
636 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
637 elif fieldtype == 'build':
638 if value.endswith("\\"):
640 buildlines = [value[:-1]]
642 thisinfo['builds'].append(parse_buildline([value]))
643 add_comments('build:' + thisinfo['builds'][-1]['version'])
644 elif fieldtype == 'buildv2':
646 vv = value.split(',')
648 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
649 format(value, linedesc))
650 curbuild['version'] = vv[0]
651 curbuild['vercode'] = vv[1]
654 elif fieldtype == 'obsolete':
655 pass # Just throw it away!
657 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
658 elif mode == 1: # Multiline field
662 thisinfo[field].append(line)
663 elif mode == 2: # Line continuation mode in Build Version
664 if line.endswith("\\"):
665 buildlines.append(line[:-1])
667 buildlines.append(line)
668 thisinfo['builds'].append(
669 parse_buildline(buildlines))
670 add_comments('build:' + thisinfo['builds'][-1]['version'])
674 # Mode at end of file should always be 0...
676 raise MetaDataException(field + " not terminated in " + metafile.name)
678 raise MetaDataException("Unterminated continuation in " + metafile.name)
680 raise MetaDataException("Unterminated build in " + metafile.name)
682 if not thisinfo['Description']:
683 thisinfo['Description'].append('No description available')
685 for build in thisinfo['builds']:
686 build['type'] = get_build_type(build)
690 # Write a metadata file.
692 # 'dest' - The path to the output file
693 # 'app' - The app data
694 def write_metadata(dest, app):
696 def writecomments(key):
698 for pf, comment in app['comments']:
700 mf.write("%s\n" % comment)
703 logging.debug("...writing comments for " + (key if key else 'EOF'))
705 def writefield(field, value=None):
709 t = metafieldtype(field)
711 value = ','.join(value)
712 mf.write("%s:%s\n" % (field, value))
716 writefield('Disabled')
717 if app['AntiFeatures']:
718 writefield('AntiFeatures')
720 writefield('Provides')
721 writefield('Categories')
722 writefield('License')
723 writefield('Web Site')
724 writefield('Source Code')
725 writefield('Issue Tracker')
729 writefield('FlattrID')
731 writefield('Bitcoin')
733 writefield('Litecoin')
735 writefield('Dogecoin')
740 writefield('Auto Name')
741 writefield('Summary')
742 writefield('Description', '')
743 for line in app['Description']:
744 mf.write("%s\n" % line)
747 if app['Requires Root']:
748 writefield('Requires Root', 'Yes')
751 writefield('Repo Type')
754 for build in app['builds']:
755 writecomments('build:' + build['version'])
756 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
758 def write_builditem(key, value):
759 if key in ['version', 'vercode', 'origlines', 'type']:
761 if key in valuetypes['bool'].attrs:
766 logging.debug("...writing {0} : {1}".format(key, value))
767 outline = ' %s=' % key
771 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
773 outline += ','.join(value) if type(value) == list else value
777 for key in ordered_flags:
779 write_builditem(key, build[key])
782 if 'Maintainer Notes' in app:
783 writefield('Maintainer Notes', '')
784 for line in app['Maintainer Notes']:
785 mf.write("%s\n" % line)
790 if app['Archive Policy']:
791 writefield('Archive Policy')
792 writefield('Auto Update Mode')
793 writefield('Update Check Mode')
794 if app['Vercode Operation']:
795 writefield('Vercode Operation')
796 if app['Update Check Data']:
797 writefield('Update Check Data')
798 if app['Current Version']:
799 writefield('Current Version')
800 writefield('Current Version Code')
802 if app['No Source Since']:
803 writefield('No Source Since')