1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 class MetaDataException(Exception):
28 def __init__(self, value):
32 return repr(self.value)
38 'Categories': ['None'],
52 'Archive Policy': None,
53 'Update Check Mode': 'None',
54 'Update Check Name': None,
55 'Update Check Data': None,
56 'Vercode Operation': None,
57 'Auto Update Mode': 'None',
58 'Current Version': '',
59 'Current Version Code': '0',
62 'Requires Root': False,
67 # This defines the preferred order for the build items - as in the
68 # manual, they're roughly in order of application.
70 'disable', 'commit', 'subdir', 'submodules', 'init',
71 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
72 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
73 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
74 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
75 'antcommand', 'novcheck'
79 # Designates a metadata field type and checks that it matches
81 # 'name' - The long name of the field type
82 # 'matching' - List of possible values or regex expression
83 # 'sep' - Separator to use if value may be a list
84 # 'fields' - Metadata fields (Field:Value) of this type
85 # 'attrs' - Build attributes (attr=value) of this type
88 def __init__(self, name, matching, sep, fields, attrs):
90 self.matching = matching
91 if type(matching) is str:
92 self.compiled = re.compile(matching)
97 def _assert_regex(self, values, appid):
99 if not self.compiled.match(v):
100 raise MetaDataException("'%s' is not a valid %s in %s. "
101 % (v, self.name, appid) +
102 "Regex pattern: %s" % (self.matching))
104 def _assert_list(self, values, appid):
106 if v not in self.matching:
107 raise MetaDataException("'%s' is not a valid %s in %s. "
108 % (v, self.name, appid) +
109 "Possible values: %s" % (", ".join(self.matching)))
111 def check(self, value, appid):
112 if type(value) is not str or not value:
114 if self.sep is not None:
115 values = value.split(self.sep)
118 if type(self.matching) is list:
119 self._assert_list(values, appid)
121 self._assert_regex(values, appid)
124 # Generic value types
126 'int': FieldType("Integer",
127 r'^[1-9][0-9]*$', None,
131 'http': FieldType("HTTP link",
132 r'^http[s]?://', None,
133 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
135 'bitcoin': FieldType("Bitcoin address",
136 r'^[a-zA-Z0-9]{27,34}$', None,
140 'litecoin': FieldType("Litecoin address",
141 r'^L[a-zA-Z0-9]{33}$', None,
145 'dogecoin': FieldType("Dogecoin address",
146 r'^D[a-zA-Z0-9]{33}$', None,
150 'Bool': FieldType("Boolean",
155 'bool': FieldType("Boolean",
158 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
161 'Repo Type': FieldType("Repo Type",
162 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
166 'archive': FieldType("Archive Policy",
167 r'^[0-9]+ versions$', None,
171 'antifeatures': FieldType("Anti-Feature",
172 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
176 'autoupdatemodes': FieldType("Auto Update Mode",
177 r"^(Version .+|None)$", None,
178 ["Auto Update Mode"],
181 'updatecheckmodes': FieldType("Update Check Mode",
182 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
183 ["Update Check Mode"],
188 # Check an app's metadata information for integrity errors
189 def check_metadata(info):
190 for k, t in valuetypes.iteritems():
191 for field in t.fields:
193 t.check(info[field], info['id'])
195 info[field] = info[field] == "Yes"
196 for build in info['builds']:
199 t.check(build[attr], info['id'])
201 build[attr] = build[attr] == "yes"
206 # Formatter for descriptions. Create an instance, and call parseline() with
207 # each line of the description source from the metadata. At the end, call
208 # end() and then text_plain, text_wiki and text_html will contain the result.
209 class DescriptionFormatter:
222 def __init__(self, linkres):
223 self.linkResolver = linkres
225 def endcur(self, notstates=None):
226 if notstates and self.state in notstates:
228 if self.state == self.stPARA:
230 elif self.state == self.stUL:
232 elif self.state == self.stOL:
236 self.text_plain += '\n'
237 self.text_html += '</p>'
238 self.state = self.stNONE
241 self.text_html += '</ul>'
242 self.state = self.stNONE
245 self.text_html += '</ol>'
246 self.state = self.stNONE
248 def formatted(self, txt, html):
251 txt = cgi.escape(txt)
253 index = txt.find("''")
255 return formatted + txt
256 formatted += txt[:index]
258 if txt.startswith("'''"):
264 self.bold = not self.bold
272 self.ital = not self.ital
276 def linkify(self, txt):
280 index = txt.find("[")
282 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
283 linkified_plain += self.formatted(txt[:index], False)
284 linkified_html += self.formatted(txt[:index], True)
286 if txt.startswith("[["):
287 index = txt.find("]]")
289 raise MetaDataException("Unterminated ]]")
291 if self.linkResolver:
292 url, urltext = self.linkResolver(url)
295 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
296 linkified_plain += urltext
299 index = txt.find("]")
301 raise MetaDataException("Unterminated ]")
303 index2 = url.find(' ')
307 urltxt = url[index2 + 1:]
309 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
310 linkified_plain += urltxt
312 linkified_plain += ' (' + url + ')'
315 def addtext(self, txt):
316 p, h = self.linkify(txt)
320 def parseline(self, line):
321 self.text_wiki += "%s\n" % line
324 elif line.startswith('* '):
325 self.endcur([self.stUL])
326 if self.state != self.stUL:
327 self.text_html += '<ul>'
328 self.state = self.stUL
329 self.text_html += '<li>'
330 self.text_plain += '* '
331 self.addtext(line[1:])
332 self.text_html += '</li>'
333 elif line.startswith('# '):
334 self.endcur([self.stOL])
335 if self.state != self.stOL:
336 self.text_html += '<ol>'
337 self.state = self.stOL
338 self.text_html += '<li>'
339 self.text_plain += '* ' #TODO: lazy - put the numbers in!
340 self.addtext(line[1:])
341 self.text_html += '</li>'
343 self.endcur([self.stPARA])
344 if self.state == self.stNONE:
345 self.text_html += '<p>'
346 self.state = self.stPARA
347 elif self.state == self.stPARA:
348 self.text_html += ' '
349 self.text_plain += ' '
356 # Parse multiple lines of description as written in a metadata file, returning
357 # a single string in plain text format.
358 def description_plain(lines, linkres):
359 ps = DescriptionFormatter(linkres)
366 # Parse multiple lines of description as written in a metadata file, returning
367 # a single string in wiki format. Used for the Maintainer Notes field as well,
368 # because it's the same format.
369 def description_wiki(lines):
370 ps = DescriptionFormatter(None)
377 # Parse multiple lines of description as written in a metadata file, returning
378 # a single string in HTML format.
379 def description_html(lines, linkres):
380 ps = DescriptionFormatter(linkres)
387 def parse_srclib(metafile, **kw):
390 if metafile and not isinstance(metafile, file):
391 metafile = open(metafile, "r")
393 # Defaults for fields that come from metadata
394 thisinfo['Repo Type'] = ''
395 thisinfo['Repo'] = ''
396 thisinfo['Subdir'] = None
397 thisinfo['Prepare'] = None
398 thisinfo['Srclibs'] = None
404 for line in metafile:
406 line = line.rstrip('\r\n')
407 if not line or line.startswith("#"):
411 field, value = line.split(':', 1)
413 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
415 if field == "Subdir":
416 thisinfo[field] = value.split(',')
418 thisinfo[field] = value
423 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
424 # returned by the parse_metadata function.
425 def read_metadata(xref=True, package=None, store=True):
428 for basedir in ('metadata', 'tmp'):
429 if not os.path.exists(basedir):
432 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
433 if package is None or metafile == os.path.join('metadata', package + '.txt'):
434 appinfo = parse_metadata(metafile)
435 check_metadata(appinfo)
439 # Parse all descriptions at load time, just to ensure cross-referencing
440 # errors are caught early rather than when they hit the build server.
443 if app['id'] == link:
444 return ("fdroid.app:" + link, "Dummy name - don't know yet")
445 raise MetaDataException("Cannot resolve app id " + link)
448 description_html(app['Description'], linkres)
450 raise MetaDataException("Problem with description of " + app['id'] +
456 # Get the type expected for a given metadata field.
457 def metafieldtype(name):
458 if name in ['Description', 'Maintainer Notes']:
460 if name in ['Categories']:
462 if name == 'Build Version':
466 if name == 'Use Built':
468 if name not in app_defaults:
474 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
475 'update', 'scanignore', 'scandelete']:
477 if name in ['init', 'prebuild', 'build']:
482 # Parse metadata for a single application.
484 # 'metafile' - the filename to read. The package id for the application comes
485 # from this filename. Pass None to get a blank entry.
487 # Returns a dictionary containing all the details of the application. There are
488 # two major kinds of information in the dictionary. Keys beginning with capital
489 # letters correspond directory to identically named keys in the metadata file.
490 # Keys beginning with lower case letters are generated in one way or another,
491 # and are not found verbatim in the metadata.
493 # Known keys not originating from the metadata are:
495 # 'id' - the application's package ID
496 # 'builds' - a list of dictionaries containing build information
497 # for each defined build
498 # 'comments' - a list of comments from the metadata file. Each is
499 # a tuple of the form (field, comment) where field is
500 # the name of the field it preceded in the metadata
501 # file. Where field is None, the comment goes at the
502 # end of the file. Alternatively, 'build:version' is
503 # for a comment before a particular build version.
504 # 'descriptionlines' - original lines of description as formatted in the
507 def parse_metadata(metafile):
511 def add_buildflag(p, thisbuild):
514 raise MetaDataException("Invalid build flag at {0} in {1}".
515 format(buildlines[0], linedesc))
518 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
519 format(pk, thisbuild['version'], linedesc))
522 if pk not in ordered_flags:
523 raise MetaDataException("Unrecognised build flag at {0} in {1}".
527 # Port legacy ';' separators
528 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
534 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % (
537 def parse_buildline(lines):
538 value = "".join(lines)
539 parts = [p.replace("\\,", ",")
540 for p in re.split(r"(?<!\\),", value)]
542 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
544 thisbuild['origlines'] = lines
545 thisbuild['version'] = parts[0]
546 thisbuild['vercode'] = parts[1]
547 if parts[2].startswith('!'):
548 # For backwards compatibility, handle old-style disabling,
549 # including attempting to extract the commit from the message
550 thisbuild['disable'] = parts[2][1:]
551 commit = 'unknown - see disabled'
552 index = parts[2].rfind('at ')
554 commit = parts[2][index+3:]
555 if commit.endswith(')'):
557 thisbuild['commit'] = commit
559 thisbuild['commit'] = parts[2]
561 add_buildflag(p, thisbuild)
565 def add_comments(key):
568 for comment in curcomments:
569 thisinfo['comments'].append((key, comment))
572 def get_build_type(build):
573 for t in ['maven', 'gradle', 'kivy']:
574 if build.get(t, 'no') != 'no':
576 if 'output' in build:
582 if not isinstance(metafile, file):
583 metafile = open(metafile, "r")
584 thisinfo['id'] = metafile.name[9:-4]
586 thisinfo['id'] = None
588 thisinfo.update(app_defaults)
590 # General defaults...
591 thisinfo['builds'] = []
592 thisinfo['comments'] = []
603 for line in metafile:
605 linedesc = "%s:%d" % (metafile.name, c)
606 line = line.rstrip('\r\n')
608 if not any(line.startswith(s) for s in (' ', '\t')):
609 if 'commit' not in curbuild and 'disable' not in curbuild:
610 raise MetaDataException("No commit specified for {0} in {1}".format(
611 curbuild['version'], linedesc))
612 thisinfo['builds'].append(curbuild)
613 add_comments('build:' + curbuild['version'])
616 if line.endswith('\\'):
617 buildlines.append(line[:-1].lstrip())
619 buildlines.append(line.lstrip())
620 bl = ''.join(buildlines)
621 add_buildflag(bl, curbuild)
627 if line.startswith("#"):
628 curcomments.append(line)
631 field, value = line.split(':', 1)
633 raise MetaDataException("Invalid metadata in "+linedesc)
634 if field != field.strip() or value != value.strip():
635 raise MetaDataException("Extra spacing found in "+linedesc)
637 # Translate obsolete fields...
638 if field == 'Market Version':
639 field = 'Current Version'
640 if field == 'Market Version Code':
641 field = 'Current Version Code'
643 fieldtype = metafieldtype(field)
644 if fieldtype not in ['build', 'buildv2']:
646 if fieldtype == 'multiline':
650 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
651 elif fieldtype == 'string':
652 thisinfo[field] = value
653 elif fieldtype == 'list':
654 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
655 elif fieldtype == 'build':
656 if value.endswith("\\"):
658 buildlines = [value[:-1]]
660 thisinfo['builds'].append(parse_buildline([value]))
661 add_comments('build:' + thisinfo['builds'][-1]['version'])
662 elif fieldtype == 'buildv2':
664 vv = value.split(',')
666 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
667 format(value, linedesc))
668 curbuild['version'] = vv[0]
669 curbuild['vercode'] = vv[1]
672 elif fieldtype == 'obsolete':
673 pass # Just throw it away!
675 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
676 elif mode == 1: # Multiline field
680 thisinfo[field].append(line)
681 elif mode == 2: # Line continuation mode in Build Version
682 if line.endswith("\\"):
683 buildlines.append(line[:-1])
685 buildlines.append(line)
686 thisinfo['builds'].append(
687 parse_buildline(buildlines))
688 add_comments('build:' + thisinfo['builds'][-1]['version'])
692 # Mode at end of file should always be 0...
694 raise MetaDataException(field + " not terminated in " + metafile.name)
696 raise MetaDataException("Unterminated continuation in " + metafile.name)
698 raise MetaDataException("Unterminated build in " + metafile.name)
700 if not thisinfo['Description']:
701 thisinfo['Description'].append('No description available')
703 for build in thisinfo['builds']:
704 build['type'] = get_build_type(build)
709 # Write a metadata file.
711 # 'dest' - The path to the output file
712 # 'app' - The app data
713 def write_metadata(dest, app):
715 def writecomments(key):
717 for pf, comment in app['comments']:
719 mf.write("%s\n" % comment)
722 logging.debug("...writing comments for " + (key if key else 'EOF'))
724 def writefield(field, value=None):
728 t = metafieldtype(field)
730 value = ','.join(value)
731 mf.write("%s:%s\n" % (field, value))
735 writefield('Disabled')
736 if app['AntiFeatures']:
737 writefield('AntiFeatures')
739 writefield('Provides')
740 writefield('Categories')
741 writefield('License')
742 writefield('Web Site')
743 writefield('Source Code')
744 writefield('Issue Tracker')
748 writefield('FlattrID')
750 writefield('Bitcoin')
752 writefield('Litecoin')
754 writefield('Dogecoin')
759 writefield('Auto Name')
760 writefield('Summary')
761 writefield('Description', '')
762 for line in app['Description']:
763 mf.write("%s\n" % line)
766 if app['Requires Root']:
767 writefield('Requires Root', 'Yes')
770 writefield('Repo Type')
773 for build in app['builds']:
774 writecomments('build:' + build['version'])
775 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
777 def write_builditem(key, value):
778 if key in ['version', 'vercode', 'origlines', 'type']:
780 if key in valuetypes['bool'].attrs:
785 logging.debug("...writing {0} : {1}".format(key, value))
786 outline = ' %s=' % key
790 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
792 outline += ','.join(value) if type(value) == list else value
796 for key in ordered_flags:
798 write_builditem(key, build[key])
801 if 'Maintainer Notes' in app:
802 writefield('Maintainer Notes', '')
803 for line in app['Maintainer Notes']:
804 mf.write("%s\n" % line)
809 if app['Archive Policy']:
810 writefield('Archive Policy')
811 writefield('Auto Update Mode')
812 writefield('Update Check Mode')
813 if app['Vercode Operation']:
814 writefield('Vercode Operation')
815 if app['Update Check Data']:
816 writefield('Update Check Data')
817 if app['Current Version']:
818 writefield('Current Version')
819 writefield('Current Version Code')
821 if app['No Source Since']:
822 writefield('No Source Since')