1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 class MetaDataException(Exception):
28 def __init__(self, value):
32 return repr(self.value)
38 'Categories': ['None'],
52 'Archive Policy': None,
53 'Update Check Mode': 'None',
54 'Update Check Name': None,
55 'Update Check Data': None,
56 'Vercode Operation': None,
57 'Auto Update Mode': 'None',
58 'Current Version': '',
59 'Current Version Code': '0',
62 'Requires Root': False,
67 # This defines the preferred order for the build items - as in the
68 # manual, they're roughly in order of application.
70 'disable', 'commit', 'subdir', 'submodules', 'init',
71 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
72 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
73 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
74 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
75 'antcommand', 'novcheck'
79 # Designates a metadata field type and checks that it matches
81 # 'name' - The long name of the field type
82 # 'matching' - List of possible values or regex expression
83 # 'sep' - Separator to use if value may be a list
84 # 'fields' - Metadata fields (Field:Value) of this type
85 # 'attrs' - Build attributes (attr=value) of this type
88 def __init__(self, name, matching, sep, fields, attrs):
90 self.matching = matching
91 if type(matching) is str:
92 self.compiled = re.compile(matching)
97 def _assert_regex(self, values, appid):
99 if not self.compiled.match(v):
100 raise MetaDataException("'%s' is not a valid %s in %s. "
101 % (v, self.name, appid) +
102 "Regex pattern: %s" % (self.matching))
104 def _assert_list(self, values, appid):
106 if v not in self.matching:
107 raise MetaDataException("'%s' is not a valid %s in %s. "
108 % (v, self.name, appid) +
109 "Possible values: %s" % (", ".join(self.matching)))
111 def check(self, value, appid):
112 if type(value) is not str or not value:
114 if self.sep is not None:
115 values = value.split(self.sep)
118 if type(self.matching) is list:
119 self._assert_list(values, appid)
121 self._assert_regex(values, appid)
124 # Generic value types
126 'int': FieldType("Integer",
127 r'^[1-9][0-9]*$', None,
131 'http': FieldType("HTTP link",
132 r'^http[s]?://', None,
133 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
135 'bitcoin': FieldType("Bitcoin address",
136 r'^[a-zA-Z0-9]{27,34}$', None,
140 'litecoin': FieldType("Litecoin address",
141 r'^L[a-zA-Z0-9]{33}$', None,
145 'dogecoin': FieldType("Dogecoin address",
146 r'^D[a-zA-Z0-9]{33}$', None,
150 'Bool': FieldType("Boolean",
155 'bool': FieldType("Boolean",
158 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
161 'Repo Type': FieldType("Repo Type",
162 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
166 'archive': FieldType("Archive Policy",
167 r'^[0-9]+ versions$', None,
171 'antifeatures': FieldType("Anti-Feature",
172 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
176 'autoupdatemodes': FieldType("Auto Update Mode",
177 r"^(Version .+|None)$", None,
178 ["Auto Update Mode"],
181 'updatecheckmodes': FieldType("Update Check Mode",
182 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
183 ["Update Check Mode"],
188 # Check an app's metadata information for integrity errors
189 def check_metadata(info):
190 for k, t in valuetypes.iteritems():
191 for field in t.fields:
193 t.check(info[field], info['id'])
195 info[field] = info[field] == "Yes"
196 for build in info['builds']:
199 t.check(build[attr], info['id'])
201 build[attr] = build[attr] == "yes"
206 # Formatter for descriptions. Create an instance, and call parseline() with
207 # each line of the description source from the metadata. At the end, call
208 # end() and then text_plain, text_wiki and text_html will contain the result.
209 class DescriptionFormatter:
221 def __init__(self, linkres):
222 self.linkResolver = linkres
223 def endcur(self, notstates=None):
224 if notstates and self.state in notstates:
226 if self.state == self.stPARA:
228 elif self.state == self.stUL:
230 elif self.state == self.stOL:
233 self.text_plain += '\n'
234 self.text_html += '</p>'
235 self.state = self.stNONE
237 self.text_html += '</ul>'
238 self.state = self.stNONE
240 self.text_html += '</ol>'
241 self.state = self.stNONE
243 def formatted(self, txt, html):
246 txt = cgi.escape(txt)
248 index = txt.find("''")
250 return formatted + txt
251 formatted += txt[:index]
253 if txt.startswith("'''"):
259 self.bold = not self.bold
267 self.ital = not self.ital
271 def linkify(self, txt):
275 index = txt.find("[")
277 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
278 linkified_plain += self.formatted(txt[:index], False)
279 linkified_html += self.formatted(txt[:index], True)
281 if txt.startswith("[["):
282 index = txt.find("]]")
284 raise MetaDataException("Unterminated ]]")
286 if self.linkResolver:
287 url, urltext = self.linkResolver(url)
290 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
291 linkified_plain += urltext
294 index = txt.find("]")
296 raise MetaDataException("Unterminated ]")
298 index2 = url.find(' ')
302 urltxt = url[index2 + 1:]
304 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
305 linkified_plain += urltxt
307 linkified_plain += ' (' + url + ')'
310 def addtext(self, txt):
311 p, h = self.linkify(txt)
315 def parseline(self, line):
316 self.text_wiki += "%s\n" % line
319 elif line.startswith('* '):
320 self.endcur([self.stUL])
321 if self.state != self.stUL:
322 self.text_html += '<ul>'
323 self.state = self.stUL
324 self.text_html += '<li>'
325 self.text_plain += '* '
326 self.addtext(line[1:])
327 self.text_html += '</li>'
328 elif line.startswith('# '):
329 self.endcur([self.stOL])
330 if self.state != self.stOL:
331 self.text_html += '<ol>'
332 self.state = self.stOL
333 self.text_html += '<li>'
334 self.text_plain += '* ' #TODO: lazy - put the numbers in!
335 self.addtext(line[1:])
336 self.text_html += '</li>'
338 self.endcur([self.stPARA])
339 if self.state == self.stNONE:
340 self.text_html += '<p>'
341 self.state = self.stPARA
342 elif self.state == self.stPARA:
343 self.text_html += ' '
344 self.text_plain += ' '
351 # Parse multiple lines of description as written in a metadata file, returning
352 # a single string in plain text format.
353 def description_plain(lines, linkres):
354 ps = DescriptionFormatter(linkres)
361 # Parse multiple lines of description as written in a metadata file, returning
362 # a single string in wiki format. Used for the Maintainer Notes field as well,
363 # because it's the same format.
364 def description_wiki(lines):
365 ps = DescriptionFormatter(None)
372 # Parse multiple lines of description as written in a metadata file, returning
373 # a single string in HTML format.
374 def description_html(lines, linkres):
375 ps = DescriptionFormatter(linkres)
382 def parse_srclib(metafile, **kw):
385 if metafile and not isinstance(metafile, file):
386 metafile = open(metafile, "r")
388 # Defaults for fields that come from metadata
389 thisinfo['Repo Type'] = ''
390 thisinfo['Repo'] = ''
391 thisinfo['Subdir'] = None
392 thisinfo['Prepare'] = None
393 thisinfo['Srclibs'] = None
399 for line in metafile:
401 line = line.rstrip('\r\n')
402 if not line or line.startswith("#"):
406 field, value = line.split(':', 1)
408 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
410 if field == "Subdir":
411 thisinfo[field] = value.split(',')
413 thisinfo[field] = value
418 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
419 # returned by the parse_metadata function.
420 def read_metadata(xref=True, package=None, store=True):
423 for basedir in ('metadata', 'tmp'):
424 if not os.path.exists(basedir):
427 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
428 if package is None or metafile == os.path.join('metadata', package + '.txt'):
429 appinfo = parse_metadata(metafile)
430 check_metadata(appinfo)
434 # Parse all descriptions at load time, just to ensure cross-referencing
435 # errors are caught early rather than when they hit the build server.
438 if app['id'] == link:
439 return ("fdroid.app:" + link, "Dummy name - don't know yet")
440 raise MetaDataException("Cannot resolve app id " + link)
443 description_html(app['Description'], linkres)
445 raise MetaDataException("Problem with description of " + app['id'] +
451 # Get the type expected for a given metadata field.
452 def metafieldtype(name):
453 if name in ['Description', 'Maintainer Notes']:
455 if name in ['Categories']:
457 if name == 'Build Version':
461 if name == 'Use Built':
463 if name not in app_defaults:
469 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
470 'update', 'scanignore', 'scandelete']:
472 if name in ['init', 'prebuild', 'build']:
477 # Parse metadata for a single application.
479 # 'metafile' - the filename to read. The package id for the application comes
480 # from this filename. Pass None to get a blank entry.
482 # Returns a dictionary containing all the details of the application. There are
483 # two major kinds of information in the dictionary. Keys beginning with capital
484 # letters correspond directory to identically named keys in the metadata file.
485 # Keys beginning with lower case letters are generated in one way or another,
486 # and are not found verbatim in the metadata.
488 # Known keys not originating from the metadata are:
490 # 'id' - the application's package ID
491 # 'builds' - a list of dictionaries containing build information
492 # for each defined build
493 # 'comments' - a list of comments from the metadata file. Each is
494 # a tuple of the form (field, comment) where field is
495 # the name of the field it preceded in the metadata
496 # file. Where field is None, the comment goes at the
497 # end of the file. Alternatively, 'build:version' is
498 # for a comment before a particular build version.
499 # 'descriptionlines' - original lines of description as formatted in the
502 def parse_metadata(metafile):
506 def add_buildflag(p, thisbuild):
509 raise MetaDataException("Invalid build flag at {0} in {1}".
510 format(buildlines[0], linedesc))
513 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
514 format(pk, thisbuild['version'], linedesc))
517 if pk not in ordered_flags:
518 raise MetaDataException("Unrecognised build flag at {0} in {1}".
522 # Port legacy ';' separators
523 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
529 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s" % (
532 def parse_buildline(lines):
533 value = "".join(lines)
534 parts = [p.replace("\\,", ",")
535 for p in re.split(r"(?<!\\),", value)]
537 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
539 thisbuild['origlines'] = lines
540 thisbuild['version'] = parts[0]
541 thisbuild['vercode'] = parts[1]
542 if parts[2].startswith('!'):
543 # For backwards compatibility, handle old-style disabling,
544 # including attempting to extract the commit from the message
545 thisbuild['disable'] = parts[2][1:]
546 commit = 'unknown - see disabled'
547 index = parts[2].rfind('at ')
549 commit = parts[2][index+3:]
550 if commit.endswith(')'):
552 thisbuild['commit'] = commit
554 thisbuild['commit'] = parts[2]
556 add_buildflag(p, thisbuild)
560 def add_comments(key):
563 for comment in curcomments:
564 thisinfo['comments'].append((key, comment))
567 def get_build_type(build):
568 for t in ['maven', 'gradle', 'kivy']:
569 if build.get(t, 'no') != 'no':
571 if 'output' in build:
577 if not isinstance(metafile, file):
578 metafile = open(metafile, "r")
579 thisinfo['id'] = metafile.name[9:-4]
581 thisinfo['id'] = None
583 thisinfo.update(app_defaults)
585 # General defaults...
586 thisinfo['builds'] = []
587 thisinfo['comments'] = []
598 for line in metafile:
600 linedesc = "%s:%d" % (metafile.name, c)
601 line = line.rstrip('\r\n')
603 if not any(line.startswith(s) for s in (' ', '\t')):
604 if 'commit' not in curbuild and 'disable' not in curbuild:
605 raise MetaDataException("No commit specified for {0} in {1}".format(
606 curbuild['version'], linedesc))
607 thisinfo['builds'].append(curbuild)
608 add_comments('build:' + curbuild['version'])
611 if line.endswith('\\'):
612 buildlines.append(line[:-1].lstrip())
614 buildlines.append(line.lstrip())
615 bl = ''.join(buildlines)
616 add_buildflag(bl, curbuild)
622 if line.startswith("#"):
623 curcomments.append(line)
626 field, value = line.split(':', 1)
628 raise MetaDataException("Invalid metadata in "+linedesc)
629 if field != field.strip() or value != value.strip():
630 raise MetaDataException("Extra spacing found in "+linedesc)
632 # Translate obsolete fields...
633 if field == 'Market Version':
634 field = 'Current Version'
635 if field == 'Market Version Code':
636 field = 'Current Version Code'
638 fieldtype = metafieldtype(field)
639 if fieldtype not in ['build', 'buildv2']:
641 if fieldtype == 'multiline':
645 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
646 elif fieldtype == 'string':
647 thisinfo[field] = value
648 elif fieldtype == 'list':
649 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
650 elif fieldtype == 'build':
651 if value.endswith("\\"):
653 buildlines = [value[:-1]]
655 thisinfo['builds'].append(parse_buildline([value]))
656 add_comments('build:' + thisinfo['builds'][-1]['version'])
657 elif fieldtype == 'buildv2':
659 vv = value.split(',')
661 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
662 format(value, linedesc))
663 curbuild['version'] = vv[0]
664 curbuild['vercode'] = vv[1]
667 elif fieldtype == 'obsolete':
668 pass # Just throw it away!
670 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
671 elif mode == 1: # Multiline field
675 thisinfo[field].append(line)
676 elif mode == 2: # Line continuation mode in Build Version
677 if line.endswith("\\"):
678 buildlines.append(line[:-1])
680 buildlines.append(line)
681 thisinfo['builds'].append(
682 parse_buildline(buildlines))
683 add_comments('build:' + thisinfo['builds'][-1]['version'])
687 # Mode at end of file should always be 0...
689 raise MetaDataException(field + " not terminated in " + metafile.name)
691 raise MetaDataException("Unterminated continuation in " + metafile.name)
693 raise MetaDataException("Unterminated build in " + metafile.name)
695 if not thisinfo['Description']:
696 thisinfo['Description'].append('No description available')
698 for build in thisinfo['builds']:
699 build['type'] = get_build_type(build)
704 # Write a metadata file.
706 # 'dest' - The path to the output file
707 # 'app' - The app data
708 def write_metadata(dest, app):
710 def writecomments(key):
712 for pf, comment in app['comments']:
714 mf.write("%s\n" % comment)
717 logging.debug("...writing comments for " + (key if key else 'EOF'))
719 def writefield(field, value=None):
723 t = metafieldtype(field)
725 value = ','.join(value)
726 mf.write("%s:%s\n" % (field, value))
730 writefield('Disabled')
731 if app['AntiFeatures']:
732 writefield('AntiFeatures')
734 writefield('Provides')
735 writefield('Categories')
736 writefield('License')
737 writefield('Web Site')
738 writefield('Source Code')
739 writefield('Issue Tracker')
743 writefield('FlattrID')
745 writefield('Bitcoin')
747 writefield('Litecoin')
749 writefield('Dogecoin')
754 writefield('Auto Name')
755 writefield('Summary')
756 writefield('Description', '')
757 for line in app['Description']:
758 mf.write("%s\n" % line)
761 if app['Requires Root']:
762 writefield('Requires Root', 'Yes')
765 writefield('Repo Type')
768 for build in app['builds']:
769 writecomments('build:' + build['version'])
770 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
772 def write_builditem(key, value):
773 if key in ['version', 'vercode', 'origlines', 'type']:
775 if key in valuetypes['bool'].attrs:
780 logging.debug("...writing {0} : {1}".format(key, value))
781 outline = ' %s=' % key
785 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
787 outline += ','.join(value) if type(value) == list else value
791 for key in ordered_flags:
793 write_builditem(key, build[key])
796 if 'Maintainer Notes' in app:
797 writefield('Maintainer Notes', '')
798 for line in app['Maintainer Notes']:
799 mf.write("%s\n" % line)
804 if app['Archive Policy']:
805 writefield('Archive Policy')
806 writefield('Auto Update Mode')
807 writefield('Update Check Mode')
808 if app['Vercode Operation']:
809 writefield('Vercode Operation')
810 if app['Update Check Data']:
811 writefield('Update Check Data')
812 if app['Current Version']:
813 writefield('Current Version')
814 writefield('Current Version Code')
816 if app['No Source Since']:
817 writefield('No Source Since')