1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 self.matching = matching
42 if type(matching) is str:
43 self.compiled = re.compile(matching)
48 def _assert_regex(self, values, appid):
50 if not self.compiled.match(v):
51 raise MetaDataException("'%s' is not a valid %s in %s. "
52 % (v, self.name, appid) +
53 "Regex pattern: %s" % (self.matching))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s. "
59 % (v, self.name, appid) +
60 "Possible values: %s" % (", ".join(self.matching)))
62 def check(self, value, appid):
63 if type(value) is not str or not value:
65 if self.sep is not None:
66 values = value.split(self.sep)
69 if type(self.matching) is list:
70 self._assert_list(values, appid)
72 self._assert_regex(values, appid)
77 'int' : FieldType("Integer",
78 r'^[1-9][0-9]*$', None,
82 'http' : FieldType("HTTP link",
83 r'^http[s]?://', None,
84 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
86 'bitcoin' : FieldType("Bitcoin address",
87 r'^[a-zA-Z0-9]{27,34}$', None,
91 'litecoin' : FieldType("Litecoin address",
92 r'^L[a-zA-Z0-9]{33}$', None,
96 'dogecoin' : FieldType("Dogecoin address",
97 r'^D[a-zA-Z0-9]{33}$', None,
101 'Bool' : FieldType("Boolean",
106 'bool' : FieldType("Boolean",
109 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
110 'fixtrans', 'fixapos', 'novcheck' ]),
112 'Repo Type' : FieldType("Repo Type",
113 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
117 'archive' : FieldType("Archive Policy",
118 r'^[0-9]+ versions$', None,
119 [ "Archive Policy" ],
122 'antifeatures' : FieldType("Anti-Feature",
123 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
127 'autoupdatemodes' : FieldType("Auto Update Mode",
128 r"^(Version .+|None)$", None,
129 [ "Auto Update Mode" ],
132 'updatecheckmodes' : FieldType("Update Check Mode",
133 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
134 [ "Update Check Mode" ],
138 # Check an app's metadata information for integrity errors
139 def check_metadata(info):
140 for k, t in valuetypes.iteritems():
141 for field in t.fields:
143 t.check(info[field], info['id'])
145 info[field] = info[field] == "Yes"
146 for build in info['builds']:
149 t.check(build[attr], info['id'])
151 build[attr] = build[attr] == "yes"
155 # Formatter for descriptions. Create an instance, and call parseline() with
156 # each line of the description source from the metadata. At the end, call
157 # end() and then text_plain, text_wiki and text_html will contain the result.
158 class DescriptionFormatter:
170 def __init__(self, linkres):
171 self.linkResolver = linkres
172 def endcur(self, notstates=None):
173 if notstates and self.state in notstates:
175 if self.state == self.stPARA:
177 elif self.state == self.stUL:
179 elif self.state == self.stOL:
182 self.text_plain += '\n'
183 self.text_html += '</p>'
184 self.state = self.stNONE
186 self.text_html += '</ul>'
187 self.state = self.stNONE
189 self.text_html += '</ol>'
190 self.state = self.stNONE
192 def formatted(self, txt, html):
195 txt = cgi.escape(txt)
197 index = txt.find("''")
199 return formatted + txt
200 formatted += txt[:index]
202 if txt.startswith("'''"):
208 self.bold = not self.bold
216 self.ital = not self.ital
220 def linkify(self, txt):
224 index = txt.find("[")
226 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
227 linkified_plain += self.formatted(txt[:index], False)
228 linkified_html += self.formatted(txt[:index], True)
230 if txt.startswith("[["):
231 index = txt.find("]]")
233 raise MetaDataException("Unterminated ]]")
235 if self.linkResolver:
236 url, urltext = self.linkResolver(url)
239 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
240 linkified_plain += urltext
243 index = txt.find("]")
245 raise MetaDataException("Unterminated ]")
247 index2 = url.find(' ')
251 urltxt = url[index2 + 1:]
253 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
254 linkified_plain += urltxt
256 linkified_plain += ' (' + url + ')'
259 def addtext(self, txt):
260 p, h = self.linkify(txt)
264 def parseline(self, line):
265 self.text_wiki += "%s\n" % line
268 elif line.startswith('*'):
269 self.endcur([self.stUL])
270 if self.state != self.stUL:
271 self.text_html += '<ul>'
272 self.state = self.stUL
273 self.text_html += '<li>'
274 self.text_plain += '*'
275 self.addtext(line[1:])
276 self.text_html += '</li>'
277 elif line.startswith('#'):
278 self.endcur([self.stOL])
279 if self.state != self.stOL:
280 self.text_html += '<ol>'
281 self.state = self.stOL
282 self.text_html += '<li>'
283 self.text_plain += '*' #TODO: lazy - put the numbers in!
284 self.addtext(line[1:])
285 self.text_html += '</li>'
287 self.endcur([self.stPARA])
288 if self.state == self.stNONE:
289 self.text_html += '<p>'
290 self.state = self.stPARA
291 elif self.state == self.stPARA:
292 self.text_html += ' '
293 self.text_plain += ' '
299 # Parse multiple lines of description as written in a metadata file, returning
300 # a single string in plain text format.
301 def description_plain(lines, linkres):
302 ps = DescriptionFormatter(linkres)
308 # Parse multiple lines of description as written in a metadata file, returning
309 # a single string in wiki format. Used for the Maintainer Notes field as well,
310 # because it's the same format.
311 def description_wiki(lines):
312 ps = DescriptionFormatter(None)
318 # Parse multiple lines of description as written in a metadata file, returning
319 # a single string in HTML format.
320 def description_html(lines,linkres):
321 ps = DescriptionFormatter(linkres)
327 def parse_srclib(metafile, **kw):
330 if metafile and not isinstance(metafile, file):
331 metafile = open(metafile, "r")
333 # Defaults for fields that come from metadata
334 thisinfo['Repo Type'] = ''
335 thisinfo['Repo'] = ''
336 thisinfo['Subdir'] = None
337 thisinfo['Prepare'] = None
338 thisinfo['Srclibs'] = None
339 thisinfo['Update Project'] = None
344 for line in metafile:
345 line = line.rstrip('\r\n')
346 if not line or line.startswith("#"):
350 field, value = line.split(':',1)
352 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
354 if field == "Subdir":
355 thisinfo[field] = value.split(',')
357 thisinfo[field] = value
361 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
362 # returned by the parse_metadata function.
363 def read_metadata(xref=True, package=None, store=True):
366 for basedir in ('metadata', 'tmp'):
367 if not os.path.exists(basedir):
370 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
371 if package is None or metafile == os.path.join('metadata', package + '.txt'):
373 appinfo = parse_metadata(metafile)
375 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
376 check_metadata(appinfo)
380 # Parse all descriptions at load time, just to ensure cross-referencing
381 # errors are caught early rather than when they hit the build server.
384 if app['id'] == link:
385 return ("fdroid.app:" + link, "Dummy name - don't know yet")
386 raise MetaDataException("Cannot resolve app id " + link)
389 description_html(app['Description'], linkres)
391 raise MetaDataException("Problem with description of " + app['id'] +
396 # Get the type expected for a given metadata field.
397 def metafieldtype(name):
398 if name in ['Description', 'Maintainer Notes']:
400 if name == 'Build Version':
404 if name == 'Use Built':
408 # Parse metadata for a single application.
410 # 'metafile' - the filename to read. The package id for the application comes
411 # from this filename. Pass None to get a blank entry.
413 # Returns a dictionary containing all the details of the application. There are
414 # two major kinds of information in the dictionary. Keys beginning with capital
415 # letters correspond directory to identically named keys in the metadata file.
416 # Keys beginning with lower case letters are generated in one way or another,
417 # and are not found verbatim in the metadata.
419 # Known keys not originating from the metadata are:
421 # 'id' - the application's package ID
422 # 'builds' - a list of dictionaries containing build information
423 # for each defined build
424 # 'comments' - a list of comments from the metadata file. Each is
425 # a tuple of the form (field, comment) where field is
426 # the name of the field it preceded in the metadata
427 # file. Where field is None, the comment goes at the
428 # end of the file. Alternatively, 'build:version' is
429 # for a comment before a particular build version.
430 # 'descriptionlines' - original lines of description as formatted in the
433 def parse_metadata(metafile):
435 def parse_buildline(lines):
436 value = "".join(lines)
437 parts = [p.replace("\\,", ",")
438 for p in re.split(r"(?<!\\),", value)]
440 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
442 thisbuild['origlines'] = lines
443 thisbuild['version'] = parts[0]
444 thisbuild['vercode'] = parts[1]
445 if parts[2].startswith('!'):
446 # For backwards compatibility, handle old-style disabling,
447 # including attempting to extract the commit from the message
448 thisbuild['disable'] = parts[2][1:]
449 commit = 'unknown - see disabled'
450 index = parts[2].rfind('at ')
452 commit = parts[2][index+3:]
453 if commit.endswith(')'):
455 thisbuild['commit'] = commit
457 thisbuild['commit'] = parts[2]
459 pk, pv = p.split('=', 1)
460 thisbuild[pk.strip()] = pv
464 def add_comments(key):
467 for comment in curcomments:
468 thisinfo['comments'].append((key, comment))
471 def get_build_type(build):
472 for t in ['maven', 'gradle', 'kivy']:
473 if build.get(t, 'no') != 'no':
475 if 'output' in build:
481 if not isinstance(metafile, file):
482 metafile = open(metafile, "r")
483 thisinfo['id'] = metafile.name[9:-4]
485 thisinfo['id'] = None
487 # Defaults for fields that come from metadata...
488 thisinfo['Name'] = None
489 thisinfo['Provides'] = None
490 thisinfo['Auto Name'] = ''
491 thisinfo['Categories'] = 'None'
492 thisinfo['Description'] = []
493 thisinfo['Summary'] = ''
494 thisinfo['License'] = 'Unknown'
495 thisinfo['Web Site'] = ''
496 thisinfo['Source Code'] = ''
497 thisinfo['Issue Tracker'] = ''
498 thisinfo['Donate'] = None
499 thisinfo['FlattrID'] = None
500 thisinfo['Bitcoin'] = None
501 thisinfo['Litecoin'] = None
502 thisinfo['Dogecoin'] = None
503 thisinfo['Disabled'] = None
504 thisinfo['AntiFeatures'] = None
505 thisinfo['Archive Policy'] = None
506 thisinfo['Update Check Mode'] = 'None'
507 thisinfo['Vercode Operation'] = None
508 thisinfo['Auto Update Mode'] = 'None'
509 thisinfo['Current Version'] = ''
510 thisinfo['Current Version Code'] = '0'
511 thisinfo['Repo Type'] = ''
512 thisinfo['Repo'] = ''
513 thisinfo['Requires Root'] = False
514 thisinfo['No Source Since'] = ''
516 # General defaults...
517 thisinfo['builds'] = []
518 thisinfo['comments'] = []
528 for line in metafile:
529 line = line.rstrip('\r\n')
531 if not any(line.startswith(s) for s in (' ', '\t')):
532 if 'commit' not in curbuild and 'disable' not in curbuild:
533 raise MetaDataException("No commit specified for {0} in {1}".format(
534 curbuild['version'], metafile.name))
535 thisinfo['builds'].append(curbuild)
536 add_comments('build:' + curbuild['version'])
539 if line.endswith('\\'):
540 buildlines.append(line[:-1].lstrip())
542 buildlines.append(line.lstrip())
543 bl = ''.join(buildlines)
544 bv = bl.split('=', 1)
546 raise MetaDataException("Invalid build flag at {0} in {1}".
547 format(buildlines[0], metafile.name))
550 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
551 format(name, curbuild['version'], metafile.name))
552 curbuild[name] = val.lstrip()
558 if line.startswith("#"):
559 curcomments.append(line)
562 field, value = line.split(':',1)
564 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
566 # Translate obsolete fields...
567 if field == 'Market Version':
568 field = 'Current Version'
569 if field == 'Market Version Code':
570 field = 'Current Version Code'
572 fieldtype = metafieldtype(field)
573 if fieldtype not in ['build', 'buildv2']:
575 if fieldtype == 'multiline':
579 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
580 elif fieldtype == 'string':
581 if field == 'Category' and thisinfo['Categories'] == 'None':
582 thisinfo['Categories'] = value.replace(';',',')
583 thisinfo[field] = value
584 elif fieldtype == 'build':
585 if value.endswith("\\"):
587 buildlines = [value[:-1]]
589 thisinfo['builds'].append(parse_buildline([value]))
590 add_comments('build:' + thisinfo['builds'][-1]['version'])
591 elif fieldtype == 'buildv2':
593 vv = value.split(',')
595 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
596 format(value, metafile.name))
597 curbuild['version'] = vv[0]
598 curbuild['vercode'] = vv[1]
601 elif fieldtype == 'obsolete':
602 pass # Just throw it away!
604 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
605 elif mode == 1: # Multiline field
609 thisinfo[field].append(line)
610 elif mode == 2: # Line continuation mode in Build Version
611 if line.endswith("\\"):
612 buildlines.append(line[:-1])
614 buildlines.append(line)
615 thisinfo['builds'].append(
616 parse_buildline(buildlines))
617 add_comments('build:' + thisinfo['builds'][-1]['version'])
621 # Mode at end of file should always be 0...
623 raise MetaDataException(field + " not terminated in " + metafile.name)
625 raise MetaDataException("Unterminated continuation in " + metafile.name)
627 raise MetaDataException("Unterminated build in " + metafile.name)
629 if not thisinfo['Description']:
630 thisinfo['Description'].append('No description available')
632 for build in thisinfo['builds']:
633 build['type'] = get_build_type(build)
637 # Write a metadata file.
639 # 'dest' - The path to the output file
640 # 'app' - The app data
641 def write_metadata(dest, app):
643 def writecomments(key):
645 for pf, comment in app['comments']:
647 mf.write("%s\n" % comment)
649 #if options.verbose and written > 0:
650 #print "...writing comments for " + (key if key else 'EOF')
652 def writefield(field, value=None):
656 mf.write("%s:%s\n" % (field, value))
660 writefield('Disabled')
661 if app['AntiFeatures']:
662 writefield('AntiFeatures')
664 writefield('Provides')
665 writefield('Categories')
666 writefield('License')
667 writefield('Web Site')
668 writefield('Source Code')
669 writefield('Issue Tracker')
673 writefield('FlattrID')
675 writefield('Bitcoin')
677 writefield('Litecoin')
679 writefield('Dogecoin')
684 writefield('Auto Name')
685 writefield('Summary')
686 writefield('Description', '')
687 for line in app['Description']:
688 mf.write("%s\n" % line)
691 if app['Requires Root']:
692 writefield('Requires Root', 'Yes')
695 writefield('Repo Type')
698 for build in app['builds']:
699 writecomments('build:' + build['version'])
700 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
702 # This defines the preferred order for the build items - as in the
703 # manual, they're roughly in order of application.
704 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
705 'gradle', 'maven', 'output', 'oldsdkloc', 'target',
706 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
707 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
708 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
709 'preassemble', 'bindir', 'antcommand', 'novcheck']
711 def write_builditem(key, value):
712 if key in ['version', 'vercode', 'origlines', 'type']:
714 if key in valuetypes['bool'].attrs:
719 #print "...writing {0} : {1}".format(key, value)
720 outline = ' %s=' % key
721 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
727 write_builditem(key, build[key])
728 for key, value in build.iteritems():
729 if not key in keyorder:
730 write_builditem(key, value)
733 if 'Maintainer Notes' in app:
734 writefield('Maintainer Notes', '')
735 for line in app['Maintainer Notes']:
736 mf.write("%s\n" % line)
741 if app['Archive Policy']:
742 writefield('Archive Policy')
743 writefield('Auto Update Mode')
744 writefield('Update Check Mode')
745 if app['Vercode Operation']:
746 writefield('Vercode Operation')
747 if 'Update Check Data' in app:
748 writefield('Update Check Data')
749 if app['Current Version']:
750 writefield('Current Version')
751 writefield('Current Version Code')
753 if app['No Source Since']:
754 writefield('No Source Since')