1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 self.matching = matching
42 if type(matching) is str:
43 self.compiled = re.compile(matching)
48 def _assert_regex(self, values, appid):
50 if not self.compiled.match(v):
51 raise MetaDataException("'%s' is not a valid %s in %s. "
52 % (v, self.name, appid) +
53 "Regex pattern: %s" % (self.matching))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s. "
59 % (v, self.name, appid) +
60 "Possible values: %s" % (", ".join(self.matching)))
62 def check(self, value, appid):
63 if type(value) is not str or not value:
65 if self.sep is not None:
66 values = value.split(self.sep)
69 if type(self.matching) is list:
70 self._assert_list(values, appid)
72 self._assert_regex(values, appid)
77 'int' : FieldType("Integer",
82 'http' : FieldType("HTTP link",
83 r'^http[s]?://', None,
84 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
86 'bitcoin' : FieldType("Bitcoin address",
87 r'^[a-zA-Z0-9]{27,34}$', None,
91 'litecoin' : FieldType("Litecoin address",
92 r'^L[a-zA-Z0-9]{33}$', None,
96 'dogecoin' : FieldType("Dogecoin address",
97 r'^D[a-zA-Z0-9]{33}$', None,
101 'Bool' : FieldType("Boolean",
106 'bool' : FieldType("Boolean",
109 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
110 'fixtrans', 'fixapos', 'novcheck' ]),
112 'Repo Type' : FieldType("Repo Type",
113 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
117 'archive' : FieldType("Archive Policy",
118 r'^[0-9]+ versions$', None,
119 [ "Archive Policy" ],
122 'antifeatures' : FieldType("Anti-Feature",
123 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
127 'autoupdatemodes' : FieldType("Auto Update Mode",
128 r"^(Version .+|None)$", None,
129 [ "Auto Update Mode" ],
132 'updatecheckmodes' : FieldType("Update Check Mode",
133 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
134 [ "Update Check Mode" ],
138 # Check an app's metadata information for integrity errors
139 def check_metadata(info):
140 for k, t in valuetypes.iteritems():
141 for field in t.fields:
143 t.check(info[field], info['id'])
145 info[field] = info[field] == "Yes"
146 for build in info['builds']:
149 t.check(build[attr], info['id'])
151 build[attr] = build[attr] == "yes"
155 # Formatter for descriptions. Create an instance, and call parseline() with
156 # each line of the description source from the metadata. At the end, call
157 # end() and then text_plain, text_wiki and text_html will contain the result.
158 class DescriptionFormatter:
170 def __init__(self, linkres):
171 self.linkResolver = linkres
172 def endcur(self, notstates=None):
173 if notstates and self.state in notstates:
175 if self.state == self.stPARA:
177 elif self.state == self.stUL:
179 elif self.state == self.stOL:
182 self.text_plain += '\n'
183 self.text_html += '</p>'
184 self.state = self.stNONE
186 self.text_html += '</ul>'
187 self.state = self.stNONE
189 self.text_html += '</ol>'
190 self.state = self.stNONE
192 def formatted(self, txt, html):
195 txt = cgi.escape(txt)
197 index = txt.find("''")
199 return formatted + txt
200 formatted += txt[:index]
202 if txt.startswith("'''"):
208 self.bold = not self.bold
216 self.ital = not self.ital
220 def linkify(self, txt):
224 index = txt.find("[")
226 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
227 linkified_plain += self.formatted(txt[:index], False)
228 linkified_html += self.formatted(txt[:index], True)
230 if txt.startswith("[["):
231 index = txt.find("]]")
233 raise MetaDataException("Unterminated ]]")
235 if self.linkResolver:
236 url, urltext = self.linkResolver(url)
239 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
240 linkified_plain += urltext
243 index = txt.find("]")
245 raise MetaDataException("Unterminated ]")
247 index2 = url.find(' ')
251 urltxt = url[index2 + 1:]
253 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
254 linkified_plain += urltxt
256 linkified_plain += ' (' + url + ')'
259 def addtext(self, txt):
260 p, h = self.linkify(txt)
264 def parseline(self, line):
265 self.text_wiki += "%s\n" % line
268 elif line.startswith('*'):
269 self.endcur([self.stUL])
270 if self.state != self.stUL:
271 self.text_html += '<ul>'
272 self.state = self.stUL
273 self.text_html += '<li>'
274 self.text_plain += '*'
275 self.addtext(line[1:])
276 self.text_html += '</li>'
277 elif line.startswith('#'):
278 self.endcur([self.stOL])
279 if self.state != self.stOL:
280 self.text_html += '<ol>'
281 self.state = self.stOL
282 self.text_html += '<li>'
283 self.text_plain += '*' #TODO: lazy - put the numbers in!
284 self.addtext(line[1:])
285 self.text_html += '</li>'
287 self.endcur([self.stPARA])
288 if self.state == self.stNONE:
289 self.text_html += '<p>'
290 self.state = self.stPARA
291 elif self.state == self.stPARA:
292 self.text_html += ' '
293 self.text_plain += ' '
299 # Parse multiple lines of description as written in a metadata file, returning
300 # a single string in plain text format.
301 def description_plain(lines, linkres):
302 ps = DescriptionFormatter(linkres)
308 # Parse multiple lines of description as written in a metadata file, returning
309 # a single string in wiki format. Used for the Maintainer Notes field as well,
310 # because it's the same format.
311 def description_wiki(lines):
312 ps = DescriptionFormatter(None)
318 # Parse multiple lines of description as written in a metadata file, returning
319 # a single string in HTML format.
320 def description_html(lines,linkres):
321 ps = DescriptionFormatter(linkres)
327 def parse_srclib(metafile, **kw):
330 if metafile and not isinstance(metafile, file):
331 metafile = open(metafile, "r")
333 # Defaults for fields that come from metadata
334 thisinfo['Repo Type'] = ''
335 thisinfo['Repo'] = ''
336 thisinfo['Subdir'] = None
337 thisinfo['Prepare'] = None
338 thisinfo['Srclibs'] = None
339 thisinfo['Update Project'] = None
344 for line in metafile:
345 line = line.rstrip('\r\n')
346 if not line or line.startswith("#"):
350 field, value = line.split(':',1)
352 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
354 if field == "Subdir":
355 thisinfo[field] = value.split(',')
357 thisinfo[field] = value
361 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
362 # returned by the parse_metadata function.
363 def read_metadata(xref=True, package=None):
365 for basedir in ('metadata', 'tmp'):
366 if not os.path.exists(basedir):
368 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
369 if package is None or metafile == os.path.join('metadata', package + '.txt'):
371 appinfo = parse_metadata(metafile)
373 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
374 check_metadata(appinfo)
378 # Parse all descriptions at load time, just to ensure cross-referencing
379 # errors are caught early rather than when they hit the build server.
382 if app['id'] == link:
383 return ("fdroid.app:" + link, "Dummy name - don't know yet")
384 raise MetaDataException("Cannot resolve app id " + link)
387 description_html(app['Description'], linkres)
389 raise MetaDataException("Problem with description of " + app['id'] +
394 # Get the type expected for a given metadata field.
395 def metafieldtype(name):
396 if name in ['Description', 'Maintainer Notes']:
398 if name == 'Build Version':
402 if name == 'Use Built':
406 # Parse metadata for a single application.
408 # 'metafile' - the filename to read. The package id for the application comes
409 # from this filename. Pass None to get a blank entry.
411 # Returns a dictionary containing all the details of the application. There are
412 # two major kinds of information in the dictionary. Keys beginning with capital
413 # letters correspond directory to identically named keys in the metadata file.
414 # Keys beginning with lower case letters are generated in one way or another,
415 # and are not found verbatim in the metadata.
417 # Known keys not originating from the metadata are:
419 # 'id' - the application's package ID
420 # 'builds' - a list of dictionaries containing build information
421 # for each defined build
422 # 'comments' - a list of comments from the metadata file. Each is
423 # a tuple of the form (field, comment) where field is
424 # the name of the field it preceded in the metadata
425 # file. Where field is None, the comment goes at the
426 # end of the file. Alternatively, 'build:version' is
427 # for a comment before a particular build version.
428 # 'descriptionlines' - original lines of description as formatted in the
431 def parse_metadata(metafile):
433 def parse_buildline(lines):
434 value = "".join(lines)
435 parts = [p.replace("\\,", ",")
436 for p in re.split(r"(?<!\\),", value)]
438 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
440 thisbuild['origlines'] = lines
441 thisbuild['version'] = parts[0]
442 thisbuild['vercode'] = parts[1]
443 if parts[2].startswith('!'):
444 # For backwards compatibility, handle old-style disabling,
445 # including attempting to extract the commit from the message
446 thisbuild['disable'] = parts[2][1:]
447 commit = 'unknown - see disabled'
448 index = parts[2].rfind('at ')
450 commit = parts[2][index+3:]
451 if commit.endswith(')'):
453 thisbuild['commit'] = commit
455 thisbuild['commit'] = parts[2]
457 pk, pv = p.split('=', 1)
458 thisbuild[pk.strip()] = pv
462 def add_comments(key):
465 for comment in curcomments:
466 thisinfo['comments'].append((key, comment))
472 if not isinstance(metafile, file):
473 metafile = open(metafile, "r")
474 thisinfo['id'] = metafile.name[9:-4]
476 thisinfo['id'] = None
478 # Defaults for fields that come from metadata...
479 thisinfo['Name'] = None
480 thisinfo['Provides'] = None
481 thisinfo['Auto Name'] = ''
482 thisinfo['Categories'] = 'None'
483 thisinfo['Description'] = []
484 thisinfo['Summary'] = ''
485 thisinfo['License'] = 'Unknown'
486 thisinfo['Web Site'] = ''
487 thisinfo['Source Code'] = ''
488 thisinfo['Issue Tracker'] = ''
489 thisinfo['Donate'] = None
490 thisinfo['FlattrID'] = None
491 thisinfo['Bitcoin'] = None
492 thisinfo['Litecoin'] = None
493 thisinfo['Dogecoin'] = None
494 thisinfo['Disabled'] = None
495 thisinfo['AntiFeatures'] = None
496 thisinfo['Archive Policy'] = None
497 thisinfo['Update Check Mode'] = 'None'
498 thisinfo['Vercode Operation'] = None
499 thisinfo['Auto Update Mode'] = 'None'
500 thisinfo['Current Version'] = ''
501 thisinfo['Current Version Code'] = '0'
502 thisinfo['Repo Type'] = ''
503 thisinfo['Repo'] = ''
504 thisinfo['Requires Root'] = False
505 thisinfo['No Source Since'] = ''
507 # General defaults...
508 thisinfo['builds'] = []
509 thisinfo['comments'] = []
519 for line in metafile:
520 line = line.rstrip('\r\n')
522 if not any(line.startswith(s) for s in (' ', '\t')):
523 if 'commit' not in curbuild and 'disable' not in curbuild:
524 raise MetaDataException("No commit specified for {0} in {1}".format(
525 curbuild['version'], metafile.name))
526 thisinfo['builds'].append(curbuild)
527 add_comments('build:' + curbuild['version'])
530 if line.endswith('\\'):
531 buildlines.append(line[:-1].lstrip())
533 buildlines.append(line.lstrip())
534 bl = ''.join(buildlines)
535 bv = bl.split('=', 1)
537 raise MetaDataException("Invalid build flag at {0} in {1}".
538 format(buildlines[0], metafile.name))
541 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
542 format(name, curbuild['version'], metafile.name))
543 curbuild[name] = val.lstrip()
549 if line.startswith("#"):
550 curcomments.append(line)
553 field, value = line.split(':',1)
555 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
557 # Translate obsolete fields...
558 if field == 'Market Version':
559 field = 'Current Version'
560 if field == 'Market Version Code':
561 field = 'Current Version Code'
563 fieldtype = metafieldtype(field)
564 if fieldtype not in ['build', 'buildv2']:
566 if fieldtype == 'multiline':
570 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
571 elif fieldtype == 'string':
572 if field == 'Category' and thisinfo['Categories'] == 'None':
573 thisinfo['Categories'] = value.replace(';',',')
574 thisinfo[field] = value
575 elif fieldtype == 'build':
576 if value.endswith("\\"):
578 buildlines = [value[:-1]]
580 thisinfo['builds'].append(parse_buildline([value]))
581 add_comments('build:' + thisinfo['builds'][-1]['version'])
582 elif fieldtype == 'buildv2':
584 vv = value.split(',')
586 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
587 format(value, metafile.name))
588 curbuild['version'] = vv[0]
589 curbuild['vercode'] = vv[1]
592 elif fieldtype == 'obsolete':
593 pass # Just throw it away!
595 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
596 elif mode == 1: # Multiline field
600 thisinfo[field].append(line)
601 elif mode == 2: # Line continuation mode in Build Version
602 if line.endswith("\\"):
603 buildlines.append(line[:-1])
605 buildlines.append(line)
606 thisinfo['builds'].append(
607 parse_buildline(buildlines))
608 add_comments('build:' + thisinfo['builds'][-1]['version'])
612 # Mode at end of file should always be 0...
614 raise MetaDataException(field + " not terminated in " + metafile.name)
616 raise MetaDataException("Unterminated continuation in " + metafile.name)
618 raise MetaDataException("Unterminated build in " + metafile.name)
620 if not thisinfo['Description']:
621 thisinfo['Description'].append('No description available')
625 # Write a metadata file.
627 # 'dest' - The path to the output file
628 # 'app' - The app data
629 def write_metadata(dest, app):
631 def writecomments(key):
633 for pf, comment in app['comments']:
635 mf.write("%s\n" % comment)
637 #if options.verbose and written > 0:
638 #print "...writing comments for " + (key if key else 'EOF')
640 def writefield(field, value=None):
644 mf.write("%s:%s\n" % (field, value))
648 writefield('Disabled')
649 if app['AntiFeatures']:
650 writefield('AntiFeatures')
652 writefield('Provides')
653 writefield('Categories')
654 writefield('License')
655 writefield('Web Site')
656 writefield('Source Code')
657 writefield('Issue Tracker')
661 writefield('FlattrID')
663 writefield('Bitcoin')
665 writefield('Litecoin')
667 writefield('Dogecoin')
672 writefield('Auto Name')
673 writefield('Summary')
674 writefield('Description', '')
675 for line in app['Description']:
676 mf.write("%s\n" % line)
679 if app['Requires Root']:
680 writefield('Requires Root', 'Yes')
683 writefield('Repo Type')
686 for build in app['builds']:
687 writecomments('build:' + build['version'])
688 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
690 # This defines the preferred order for the build items - as in the
691 # manual, they're roughly in order of application.
692 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
693 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
694 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
695 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
696 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
697 'preassemble', 'bindir', 'antcommand', 'novcheck']
699 def write_builditem(key, value):
700 if key in ['version', 'vercode', 'origlines']:
702 if key in valuetypes['bool'].attrs:
707 #print "...writing {0} : {1}".format(key, value)
708 outline = ' %s=' % key
709 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
715 write_builditem(key, build[key])
716 for key, value in build.iteritems():
717 if not key in keyorder:
718 write_builditem(key, value)
721 if 'Maintainer Notes' in app:
722 writefield('Maintainer Notes', '')
723 for line in app['Maintainer Notes']:
724 mf.write("%s\n" % line)
729 if app['Archive Policy']:
730 writefield('Archive Policy')
731 writefield('Auto Update Mode')
732 writefield('Update Check Mode')
733 if app['Vercode Operation']:
734 writefield('Vercode Operation')
735 if 'Update Check Data' in app:
736 writefield('Update Check Data')
737 if app['Current Version']:
738 writefield('Current Version')
739 writefield('Current Version Code')
741 if app['No Source Since']:
742 writefield('No Source Since')