1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 class MetaDataException(Exception):
28 def __init__(self, value):
32 return repr(self.value)
38 'Categories': ['None'],
52 'Archive Policy': None,
53 'Update Check Mode': 'None',
54 'Update Check Name': None,
55 'Update Check Data': None,
56 'Vercode Operation': None,
57 'Auto Update Mode': 'None',
58 'Current Version': '',
59 'Current Version Code': '0',
62 'Requires Root': False,
67 # This defines the preferred order for the build items - as in the
68 # manual, they're roughly in order of application.
70 'disable', 'commit', 'subdir', 'submodules', 'init',
71 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
72 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
73 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
74 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
75 'antcommand', 'novcheck'
79 # Designates a metadata field type and checks that it matches
81 # 'name' - The long name of the field type
82 # 'matching' - List of possible values or regex expression
83 # 'sep' - Separator to use if value may be a list
84 # 'fields' - Metadata fields (Field:Value) of this type
85 # 'attrs' - Build attributes (attr=value) of this type
88 def __init__(self, name, matching, sep, fields, attrs):
90 self.matching = matching
91 if type(matching) is str:
92 self.compiled = re.compile(matching)
97 def _assert_regex(self, values, appid):
99 if not self.compiled.match(v):
100 raise MetaDataException("'%s' is not a valid %s in %s. "
101 % (v, self.name, appid) +
102 "Regex pattern: %s" % (self.matching))
104 def _assert_list(self, values, appid):
106 if v not in self.matching:
107 raise MetaDataException("'%s' is not a valid %s in %s. "
108 % (v, self.name, appid) +
109 "Possible values: %s" % (", ".join(self.matching)))
111 def check(self, value, appid):
112 if type(value) is not str or not value:
114 if self.sep is not None:
115 values = value.split(self.sep)
118 if type(self.matching) is list:
119 self._assert_list(values, appid)
121 self._assert_regex(values, appid)
124 # Generic value types
126 'int': FieldType("Integer",
127 r'^[1-9][0-9]*$', None,
131 'http': FieldType("HTTP link",
132 r'^http[s]?://', None,
133 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
135 'bitcoin': FieldType("Bitcoin address",
136 r'^[a-zA-Z0-9]{27,34}$', None,
140 'litecoin': FieldType("Litecoin address",
141 r'^L[a-zA-Z0-9]{33}$', None,
145 'dogecoin': FieldType("Dogecoin address",
146 r'^D[a-zA-Z0-9]{33}$', None,
150 'Bool': FieldType("Boolean",
155 'bool': FieldType("Boolean",
158 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
161 'Repo Type': FieldType("Repo Type",
162 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
166 'archive': FieldType("Archive Policy",
167 r'^[0-9]+ versions$', None,
171 'antifeatures': FieldType("Anti-Feature",
172 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
176 'autoupdatemodes': FieldType("Auto Update Mode",
177 r"^(Version .+|None)$", None,
178 ["Auto Update Mode"],
181 'updatecheckmodes': FieldType("Update Check Mode",
182 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
183 ["Update Check Mode"],
188 # Check an app's metadata information for integrity errors
189 def check_metadata(info):
190 for k, t in valuetypes.iteritems():
191 for field in t.fields:
193 t.check(info[field], info['id'])
195 info[field] = info[field] == "Yes"
196 for build in info['builds']:
199 t.check(build[attr], info['id'])
201 build[attr] = build[attr] == "yes"
206 # Formatter for descriptions. Create an instance, and call parseline() with
207 # each line of the description source from the metadata. At the end, call
208 # end() and then text_plain, text_wiki and text_html will contain the result.
209 class DescriptionFormatter:
222 def __init__(self, linkres):
223 self.linkResolver = linkres
225 def endcur(self, notstates=None):
226 if notstates and self.state in notstates:
228 if self.state == self.stPARA:
230 elif self.state == self.stUL:
232 elif self.state == self.stOL:
236 self.text_plain += '\n'
237 self.text_html += '</p>'
238 self.state = self.stNONE
241 self.text_html += '</ul>'
242 self.state = self.stNONE
245 self.text_html += '</ol>'
246 self.state = self.stNONE
248 def formatted(self, txt, html):
251 txt = cgi.escape(txt)
253 index = txt.find("''")
255 return formatted + txt
256 formatted += txt[:index]
258 if txt.startswith("'''"):
264 self.bold = not self.bold
272 self.ital = not self.ital
275 def linkify(self, txt):
279 index = txt.find("[")
281 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
282 linkified_plain += self.formatted(txt[:index], False)
283 linkified_html += self.formatted(txt[:index], True)
285 if txt.startswith("[["):
286 index = txt.find("]]")
288 raise MetaDataException("Unterminated ]]")
290 if self.linkResolver:
291 url, urltext = self.linkResolver(url)
294 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
295 linkified_plain += urltext
298 index = txt.find("]")
300 raise MetaDataException("Unterminated ]")
302 index2 = url.find(' ')
306 urltxt = url[index2 + 1:]
308 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
309 linkified_plain += urltxt
311 linkified_plain += ' (' + url + ')'
314 def addtext(self, txt):
315 p, h = self.linkify(txt)
319 def parseline(self, line):
320 self.text_wiki += "%s\n" % line
323 elif line.startswith('* '):
324 self.endcur([self.stUL])
325 if self.state != self.stUL:
326 self.text_html += '<ul>'
327 self.state = self.stUL
328 self.text_html += '<li>'
329 self.text_plain += '* '
330 self.addtext(line[1:])
331 self.text_html += '</li>'
332 elif line.startswith('# '):
333 self.endcur([self.stOL])
334 if self.state != self.stOL:
335 self.text_html += '<ol>'
336 self.state = self.stOL
337 self.text_html += '<li>'
338 self.text_plain += '* ' # TODO: lazy - put the numbers in!
339 self.addtext(line[1:])
340 self.text_html += '</li>'
342 self.endcur([self.stPARA])
343 if self.state == self.stNONE:
344 self.text_html += '<p>'
345 self.state = self.stPARA
346 elif self.state == self.stPARA:
347 self.text_html += ' '
348 self.text_plain += ' '
355 # Parse multiple lines of description as written in a metadata file, returning
356 # a single string in plain text format.
357 def description_plain(lines, linkres):
358 ps = DescriptionFormatter(linkres)
365 # Parse multiple lines of description as written in a metadata file, returning
366 # a single string in wiki format. Used for the Maintainer Notes field as well,
367 # because it's the same format.
368 def description_wiki(lines):
369 ps = DescriptionFormatter(None)
376 # Parse multiple lines of description as written in a metadata file, returning
377 # a single string in HTML format.
378 def description_html(lines, linkres):
379 ps = DescriptionFormatter(linkres)
386 def parse_srclib(metafile, **kw):
389 if metafile and not isinstance(metafile, file):
390 metafile = open(metafile, "r")
392 # Defaults for fields that come from metadata
393 thisinfo['Repo Type'] = ''
394 thisinfo['Repo'] = ''
395 thisinfo['Subdir'] = None
396 thisinfo['Prepare'] = None
397 thisinfo['Srclibs'] = None
403 for line in metafile:
405 line = line.rstrip('\r\n')
406 if not line or line.startswith("#"):
410 field, value = line.split(':', 1)
412 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
414 if field == "Subdir":
415 thisinfo[field] = value.split(',')
417 thisinfo[field] = value
422 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
423 # returned by the parse_metadata function.
424 def read_metadata(xref=True, package=None, store=True):
427 for basedir in ('metadata', 'tmp'):
428 if not os.path.exists(basedir):
431 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
432 if package is None or metafile == os.path.join('metadata', package + '.txt'):
433 appinfo = parse_metadata(metafile)
434 check_metadata(appinfo)
438 # Parse all descriptions at load time, just to ensure cross-referencing
439 # errors are caught early rather than when they hit the build server.
442 if app['id'] == link:
443 return ("fdroid.app:" + link, "Dummy name - don't know yet")
444 raise MetaDataException("Cannot resolve app id " + link)
447 description_html(app['Description'], linkres)
449 raise MetaDataException("Problem with description of " + app['id'] +
455 # Get the type expected for a given metadata field.
456 def metafieldtype(name):
457 if name in ['Description', 'Maintainer Notes']:
459 if name in ['Categories']:
461 if name == 'Build Version':
465 if name == 'Use Built':
467 if name not in app_defaults:
473 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
474 'update', 'scanignore', 'scandelete']:
476 if name in ['init', 'prebuild', 'build']:
481 # Parse metadata for a single application.
483 # 'metafile' - the filename to read. The package id for the application comes
484 # from this filename. Pass None to get a blank entry.
486 # Returns a dictionary containing all the details of the application. There are
487 # two major kinds of information in the dictionary. Keys beginning with capital
488 # letters correspond directory to identically named keys in the metadata file.
489 # Keys beginning with lower case letters are generated in one way or another,
490 # and are not found verbatim in the metadata.
492 # Known keys not originating from the metadata are:
494 # 'id' - the application's package ID
495 # 'builds' - a list of dictionaries containing build information
496 # for each defined build
497 # 'comments' - a list of comments from the metadata file. Each is
498 # a tuple of the form (field, comment) where field is
499 # the name of the field it preceded in the metadata
500 # file. Where field is None, the comment goes at the
501 # end of the file. Alternatively, 'build:version' is
502 # for a comment before a particular build version.
503 # 'descriptionlines' - original lines of description as formatted in the
506 def parse_metadata(metafile):
510 def add_buildflag(p, thisbuild):
513 raise MetaDataException("Invalid build flag at {0} in {1}".
514 format(buildlines[0], linedesc))
517 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
518 format(pk, thisbuild['version'], linedesc))
521 if pk not in ordered_flags:
522 raise MetaDataException("Unrecognised build flag at {0} in {1}".
526 # Port legacy ';' separators
527 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
533 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % (
536 def parse_buildline(lines):
537 value = "".join(lines)
538 parts = [p.replace("\\,", ",")
539 for p in re.split(r"(?<!\\),", value)]
541 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
543 thisbuild['origlines'] = lines
544 thisbuild['version'] = parts[0]
545 thisbuild['vercode'] = parts[1]
546 if parts[2].startswith('!'):
547 # For backwards compatibility, handle old-style disabling,
548 # including attempting to extract the commit from the message
549 thisbuild['disable'] = parts[2][1:]
550 commit = 'unknown - see disabled'
551 index = parts[2].rfind('at ')
553 commit = parts[2][index+3:]
554 if commit.endswith(')'):
556 thisbuild['commit'] = commit
558 thisbuild['commit'] = parts[2]
560 add_buildflag(p, thisbuild)
564 def add_comments(key):
567 for comment in curcomments:
568 thisinfo['comments'].append((key, comment))
571 def get_build_type(build):
572 for t in ['maven', 'gradle', 'kivy']:
573 if build.get(t, 'no') != 'no':
575 if 'output' in build:
581 if not isinstance(metafile, file):
582 metafile = open(metafile, "r")
583 thisinfo['id'] = metafile.name[9:-4]
585 thisinfo['id'] = None
587 thisinfo.update(app_defaults)
589 # General defaults...
590 thisinfo['builds'] = []
591 thisinfo['comments'] = []
602 for line in metafile:
604 linedesc = "%s:%d" % (metafile.name, c)
605 line = line.rstrip('\r\n')
607 if not any(line.startswith(s) for s in (' ', '\t')):
608 if 'commit' not in curbuild and 'disable' not in curbuild:
609 raise MetaDataException("No commit specified for {0} in {1}".format(
610 curbuild['version'], linedesc))
611 thisinfo['builds'].append(curbuild)
612 add_comments('build:' + curbuild['version'])
615 if line.endswith('\\'):
616 buildlines.append(line[:-1].lstrip())
618 buildlines.append(line.lstrip())
619 bl = ''.join(buildlines)
620 add_buildflag(bl, curbuild)
626 if line.startswith("#"):
627 curcomments.append(line)
630 field, value = line.split(':', 1)
632 raise MetaDataException("Invalid metadata in "+linedesc)
633 if field != field.strip() or value != value.strip():
634 raise MetaDataException("Extra spacing found in "+linedesc)
636 # Translate obsolete fields...
637 if field == 'Market Version':
638 field = 'Current Version'
639 if field == 'Market Version Code':
640 field = 'Current Version Code'
642 fieldtype = metafieldtype(field)
643 if fieldtype not in ['build', 'buildv2']:
645 if fieldtype == 'multiline':
649 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
650 elif fieldtype == 'string':
651 thisinfo[field] = value
652 elif fieldtype == 'list':
653 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
654 elif fieldtype == 'build':
655 if value.endswith("\\"):
657 buildlines = [value[:-1]]
659 thisinfo['builds'].append(parse_buildline([value]))
660 add_comments('build:' + thisinfo['builds'][-1]['version'])
661 elif fieldtype == 'buildv2':
663 vv = value.split(',')
665 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
666 format(value, linedesc))
667 curbuild['version'] = vv[0]
668 curbuild['vercode'] = vv[1]
671 elif fieldtype == 'obsolete':
672 pass # Just throw it away!
674 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
675 elif mode == 1: # Multiline field
679 thisinfo[field].append(line)
680 elif mode == 2: # Line continuation mode in Build Version
681 if line.endswith("\\"):
682 buildlines.append(line[:-1])
684 buildlines.append(line)
685 thisinfo['builds'].append(
686 parse_buildline(buildlines))
687 add_comments('build:' + thisinfo['builds'][-1]['version'])
691 # Mode at end of file should always be 0...
693 raise MetaDataException(field + " not terminated in " + metafile.name)
695 raise MetaDataException("Unterminated continuation in " + metafile.name)
697 raise MetaDataException("Unterminated build in " + metafile.name)
699 if not thisinfo['Description']:
700 thisinfo['Description'].append('No description available')
702 for build in thisinfo['builds']:
703 build['type'] = get_build_type(build)
708 # Write a metadata file.
710 # 'dest' - The path to the output file
711 # 'app' - The app data
712 def write_metadata(dest, app):
714 def writecomments(key):
716 for pf, comment in app['comments']:
718 mf.write("%s\n" % comment)
721 logging.debug("...writing comments for " + (key if key else 'EOF'))
723 def writefield(field, value=None):
727 t = metafieldtype(field)
729 value = ','.join(value)
730 mf.write("%s:%s\n" % (field, value))
734 writefield('Disabled')
735 if app['AntiFeatures']:
736 writefield('AntiFeatures')
738 writefield('Provides')
739 writefield('Categories')
740 writefield('License')
741 writefield('Web Site')
742 writefield('Source Code')
743 writefield('Issue Tracker')
747 writefield('FlattrID')
749 writefield('Bitcoin')
751 writefield('Litecoin')
753 writefield('Dogecoin')
758 writefield('Auto Name')
759 writefield('Summary')
760 writefield('Description', '')
761 for line in app['Description']:
762 mf.write("%s\n" % line)
765 if app['Requires Root']:
766 writefield('Requires Root', 'Yes')
769 writefield('Repo Type')
772 for build in app['builds']:
773 writecomments('build:' + build['version'])
774 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
776 def write_builditem(key, value):
777 if key in ['version', 'vercode', 'origlines', 'type']:
779 if key in valuetypes['bool'].attrs:
784 logging.debug("...writing {0} : {1}".format(key, value))
785 outline = ' %s=' % key
789 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
791 outline += ','.join(value) if type(value) == list else value
795 for key in ordered_flags:
797 write_builditem(key, build[key])
800 if 'Maintainer Notes' in app:
801 writefield('Maintainer Notes', '')
802 for line in app['Maintainer Notes']:
803 mf.write("%s\n" % line)
807 if app['Archive Policy']:
808 writefield('Archive Policy')
809 writefield('Auto Update Mode')
810 writefield('Update Check Mode')
811 if app['Vercode Operation']:
812 writefield('Vercode Operation')
813 if app['Update Check Data']:
814 writefield('Update Check Data')
815 if app['Current Version']:
816 writefield('Current Version')
817 writefield('Current Version Code')
819 if app['No Source Since']:
820 writefield('No Source Since')