1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 self.matching = matching
42 if type(matching) is str:
43 self.compiled = re.compile(matching)
48 def _assert_regex(self, values, appid):
50 if not self.compiled.match(v):
51 raise MetaDataException("'%s' is not a valid %s in %s. "
52 % (v, self.name, appid) +
53 "Regex pattern: %s" % (self.matching))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s. "
59 % (v, self.name, appid) +
60 "Possible values: %s" % (", ".join(self.matching)))
62 def check(self, value, appid):
63 if type(value) is not str or not value:
65 if self.sep is not None:
66 values = value.split(self.sep)
69 if type(self.matching) is list:
70 self._assert_list(values, appid)
72 self._assert_regex(values, appid)
77 'int' : FieldType("Integer",
78 r'^[1-9][0-9]*$', None,
82 'http' : FieldType("HTTP link",
83 r'^http[s]?://', None,
84 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
86 'bitcoin' : FieldType("Bitcoin address",
87 r'^[a-zA-Z0-9]{27,34}$', None,
91 'litecoin' : FieldType("Litecoin address",
92 r'^L[a-zA-Z0-9]{33}$', None,
96 'dogecoin' : FieldType("Dogecoin address",
97 r'^D[a-zA-Z0-9]{33}$', None,
101 'Bool' : FieldType("Boolean",
106 'bool' : FieldType("Boolean",
109 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
110 'fixtrans', 'fixapos', 'novcheck' ]),
112 'Repo Type' : FieldType("Repo Type",
113 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
117 'archive' : FieldType("Archive Policy",
118 r'^[0-9]+ versions$', None,
119 [ "Archive Policy" ],
122 'antifeatures' : FieldType("Anti-Feature",
123 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
127 'autoupdatemodes' : FieldType("Auto Update Mode",
128 r"^(Version .+|None)$", None,
129 [ "Auto Update Mode" ],
132 'updatecheckmodes' : FieldType("Update Check Mode",
133 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
134 [ "Update Check Mode" ],
138 # Check an app's metadata information for integrity errors
139 def check_metadata(info):
140 for k, t in valuetypes.iteritems():
141 for field in t.fields:
143 t.check(info[field], info['id'])
145 info[field] = info[field] == "Yes"
146 for build in info['builds']:
149 t.check(build[attr], info['id'])
151 build[attr] = build[attr] == "yes"
155 # Formatter for descriptions. Create an instance, and call parseline() with
156 # each line of the description source from the metadata. At the end, call
157 # end() and then text_plain, text_wiki and text_html will contain the result.
158 class DescriptionFormatter:
170 def __init__(self, linkres):
171 self.linkResolver = linkres
172 def endcur(self, notstates=None):
173 if notstates and self.state in notstates:
175 if self.state == self.stPARA:
177 elif self.state == self.stUL:
179 elif self.state == self.stOL:
182 self.text_plain += '\n'
183 self.text_html += '</p>'
184 self.state = self.stNONE
186 self.text_html += '</ul>'
187 self.state = self.stNONE
189 self.text_html += '</ol>'
190 self.state = self.stNONE
192 def formatted(self, txt, html):
195 txt = cgi.escape(txt)
197 index = txt.find("''")
199 return formatted + txt
200 formatted += txt[:index]
202 if txt.startswith("'''"):
208 self.bold = not self.bold
216 self.ital = not self.ital
220 def linkify(self, txt):
224 index = txt.find("[")
226 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
227 linkified_plain += self.formatted(txt[:index], False)
228 linkified_html += self.formatted(txt[:index], True)
230 if txt.startswith("[["):
231 index = txt.find("]]")
233 raise MetaDataException("Unterminated ]]")
235 if self.linkResolver:
236 url, urltext = self.linkResolver(url)
239 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
240 linkified_plain += urltext
243 index = txt.find("]")
245 raise MetaDataException("Unterminated ]")
247 index2 = url.find(' ')
251 urltxt = url[index2 + 1:]
253 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
254 linkified_plain += urltxt
256 linkified_plain += ' (' + url + ')'
259 def addtext(self, txt):
260 p, h = self.linkify(txt)
264 def parseline(self, line):
265 self.text_wiki += "%s\n" % line
268 elif line.startswith('*'):
269 self.endcur([self.stUL])
270 if self.state != self.stUL:
271 self.text_html += '<ul>'
272 self.state = self.stUL
273 self.text_html += '<li>'
274 self.text_plain += '*'
275 self.addtext(line[1:])
276 self.text_html += '</li>'
277 elif line.startswith('#'):
278 self.endcur([self.stOL])
279 if self.state != self.stOL:
280 self.text_html += '<ol>'
281 self.state = self.stOL
282 self.text_html += '<li>'
283 self.text_plain += '*' #TODO: lazy - put the numbers in!
284 self.addtext(line[1:])
285 self.text_html += '</li>'
287 self.endcur([self.stPARA])
288 if self.state == self.stNONE:
289 self.text_html += '<p>'
290 self.state = self.stPARA
291 elif self.state == self.stPARA:
292 self.text_html += ' '
293 self.text_plain += ' '
299 # Parse multiple lines of description as written in a metadata file, returning
300 # a single string in plain text format.
301 def description_plain(lines, linkres):
302 ps = DescriptionFormatter(linkres)
308 # Parse multiple lines of description as written in a metadata file, returning
309 # a single string in wiki format. Used for the Maintainer Notes field as well,
310 # because it's the same format.
311 def description_wiki(lines):
312 ps = DescriptionFormatter(None)
318 # Parse multiple lines of description as written in a metadata file, returning
319 # a single string in HTML format.
320 def description_html(lines,linkres):
321 ps = DescriptionFormatter(linkres)
327 def parse_srclib(metafile, **kw):
330 if metafile and not isinstance(metafile, file):
331 metafile = open(metafile, "r")
333 # Defaults for fields that come from metadata
334 thisinfo['Repo Type'] = ''
335 thisinfo['Repo'] = ''
336 thisinfo['Subdir'] = None
337 thisinfo['Prepare'] = None
338 thisinfo['Srclibs'] = None
339 thisinfo['Update Project'] = None
344 for line in metafile:
345 line = line.rstrip('\r\n')
346 if not line or line.startswith("#"):
350 field, value = line.split(':',1)
352 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
354 if field == "Subdir":
355 thisinfo[field] = value.split(',')
357 thisinfo[field] = value
361 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
362 # returned by the parse_metadata function.
363 def read_metadata(xref=True, package=None):
365 for basedir in ('metadata', 'tmp'):
366 if not os.path.exists(basedir):
368 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
369 if package is None or metafile == os.path.join('metadata', package + '.txt'):
371 appinfo = parse_metadata(metafile)
373 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
374 check_metadata(appinfo)
378 # Parse all descriptions at load time, just to ensure cross-referencing
379 # errors are caught early rather than when they hit the build server.
382 if app['id'] == link:
383 return ("fdroid.app:" + link, "Dummy name - don't know yet")
384 raise MetaDataException("Cannot resolve app id " + link)
387 description_html(app['Description'], linkres)
389 raise MetaDataException("Problem with description of " + app['id'] +
394 # Get the type expected for a given metadata field.
395 def metafieldtype(name):
396 if name in ['Description', 'Maintainer Notes']:
398 if name == 'Build Version':
402 if name == 'Use Built':
406 # Parse metadata for a single application.
408 # 'metafile' - the filename to read. The package id for the application comes
409 # from this filename. Pass None to get a blank entry.
411 # Returns a dictionary containing all the details of the application. There are
412 # two major kinds of information in the dictionary. Keys beginning with capital
413 # letters correspond directory to identically named keys in the metadata file.
414 # Keys beginning with lower case letters are generated in one way or another,
415 # and are not found verbatim in the metadata.
417 # Known keys not originating from the metadata are:
419 # 'id' - the application's package ID
420 # 'builds' - a list of dictionaries containing build information
421 # for each defined build
422 # 'comments' - a list of comments from the metadata file. Each is
423 # a tuple of the form (field, comment) where field is
424 # the name of the field it preceded in the metadata
425 # file. Where field is None, the comment goes at the
426 # end of the file. Alternatively, 'build:version' is
427 # for a comment before a particular build version.
428 # 'descriptionlines' - original lines of description as formatted in the
431 def parse_metadata(metafile):
433 def parse_buildline(lines):
434 value = "".join(lines)
435 parts = [p.replace("\\,", ",")
436 for p in re.split(r"(?<!\\),", value)]
438 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
440 thisbuild['origlines'] = lines
441 thisbuild['version'] = parts[0]
442 thisbuild['vercode'] = parts[1]
443 if parts[2].startswith('!'):
444 # For backwards compatibility, handle old-style disabling,
445 # including attempting to extract the commit from the message
446 thisbuild['disable'] = parts[2][1:]
447 commit = 'unknown - see disabled'
448 index = parts[2].rfind('at ')
450 commit = parts[2][index+3:]
451 if commit.endswith(')'):
453 thisbuild['commit'] = commit
455 thisbuild['commit'] = parts[2]
457 pk, pv = p.split('=', 1)
458 thisbuild[pk.strip()] = pv
462 def add_comments(key):
465 for comment in curcomments:
466 thisinfo['comments'].append((key, comment))
469 def get_build_type(build):
470 for t in ['maven', 'gradle', 'kivy']:
471 if build.get(t, 'no') != 'no':
477 if not isinstance(metafile, file):
478 metafile = open(metafile, "r")
479 thisinfo['id'] = metafile.name[9:-4]
481 thisinfo['id'] = None
483 # Defaults for fields that come from metadata...
484 thisinfo['Name'] = None
485 thisinfo['Provides'] = None
486 thisinfo['Auto Name'] = ''
487 thisinfo['Categories'] = 'None'
488 thisinfo['Description'] = []
489 thisinfo['Summary'] = ''
490 thisinfo['License'] = 'Unknown'
491 thisinfo['Web Site'] = ''
492 thisinfo['Source Code'] = ''
493 thisinfo['Issue Tracker'] = ''
494 thisinfo['Donate'] = None
495 thisinfo['FlattrID'] = None
496 thisinfo['Bitcoin'] = None
497 thisinfo['Litecoin'] = None
498 thisinfo['Dogecoin'] = None
499 thisinfo['Disabled'] = None
500 thisinfo['AntiFeatures'] = None
501 thisinfo['Archive Policy'] = None
502 thisinfo['Update Check Mode'] = 'None'
503 thisinfo['Vercode Operation'] = None
504 thisinfo['Auto Update Mode'] = 'None'
505 thisinfo['Current Version'] = ''
506 thisinfo['Current Version Code'] = '0'
507 thisinfo['Repo Type'] = ''
508 thisinfo['Repo'] = ''
509 thisinfo['Requires Root'] = False
510 thisinfo['No Source Since'] = ''
512 # General defaults...
513 thisinfo['builds'] = []
514 thisinfo['comments'] = []
524 for line in metafile:
525 line = line.rstrip('\r\n')
527 if not any(line.startswith(s) for s in (' ', '\t')):
528 if 'commit' not in curbuild and 'disable' not in curbuild:
529 raise MetaDataException("No commit specified for {0} in {1}".format(
530 curbuild['version'], metafile.name))
531 thisinfo['builds'].append(curbuild)
532 add_comments('build:' + curbuild['version'])
535 if line.endswith('\\'):
536 buildlines.append(line[:-1].lstrip())
538 buildlines.append(line.lstrip())
539 bl = ''.join(buildlines)
540 bv = bl.split('=', 1)
542 raise MetaDataException("Invalid build flag at {0} in {1}".
543 format(buildlines[0], metafile.name))
546 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
547 format(name, curbuild['version'], metafile.name))
548 curbuild[name] = val.lstrip()
554 if line.startswith("#"):
555 curcomments.append(line)
558 field, value = line.split(':',1)
560 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
562 # Translate obsolete fields...
563 if field == 'Market Version':
564 field = 'Current Version'
565 if field == 'Market Version Code':
566 field = 'Current Version Code'
568 fieldtype = metafieldtype(field)
569 if fieldtype not in ['build', 'buildv2']:
571 if fieldtype == 'multiline':
575 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
576 elif fieldtype == 'string':
577 if field == 'Category' and thisinfo['Categories'] == 'None':
578 thisinfo['Categories'] = value.replace(';',',')
579 thisinfo[field] = value
580 elif fieldtype == 'build':
581 if value.endswith("\\"):
583 buildlines = [value[:-1]]
585 thisinfo['builds'].append(parse_buildline([value]))
586 add_comments('build:' + thisinfo['builds'][-1]['version'])
587 elif fieldtype == 'buildv2':
589 vv = value.split(',')
591 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
592 format(value, metafile.name))
593 curbuild['version'] = vv[0]
594 curbuild['vercode'] = vv[1]
597 elif fieldtype == 'obsolete':
598 pass # Just throw it away!
600 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
601 elif mode == 1: # Multiline field
605 thisinfo[field].append(line)
606 elif mode == 2: # Line continuation mode in Build Version
607 if line.endswith("\\"):
608 buildlines.append(line[:-1])
610 buildlines.append(line)
611 thisinfo['builds'].append(
612 parse_buildline(buildlines))
613 add_comments('build:' + thisinfo['builds'][-1]['version'])
617 # Mode at end of file should always be 0...
619 raise MetaDataException(field + " not terminated in " + metafile.name)
621 raise MetaDataException("Unterminated continuation in " + metafile.name)
623 raise MetaDataException("Unterminated build in " + metafile.name)
625 if not thisinfo['Description']:
626 thisinfo['Description'].append('No description available')
628 for build in thisinfo['builds']:
629 build['type'] = get_build_type(build)
633 # Write a metadata file.
635 # 'dest' - The path to the output file
636 # 'app' - The app data
637 def write_metadata(dest, app):
639 def writecomments(key):
641 for pf, comment in app['comments']:
643 mf.write("%s\n" % comment)
645 #if options.verbose and written > 0:
646 #print "...writing comments for " + (key if key else 'EOF')
648 def writefield(field, value=None):
652 mf.write("%s:%s\n" % (field, value))
656 writefield('Disabled')
657 if app['AntiFeatures']:
658 writefield('AntiFeatures')
660 writefield('Provides')
661 writefield('Categories')
662 writefield('License')
663 writefield('Web Site')
664 writefield('Source Code')
665 writefield('Issue Tracker')
669 writefield('FlattrID')
671 writefield('Bitcoin')
673 writefield('Litecoin')
675 writefield('Dogecoin')
680 writefield('Auto Name')
681 writefield('Summary')
682 writefield('Description', '')
683 for line in app['Description']:
684 mf.write("%s\n" % line)
687 if app['Requires Root']:
688 writefield('Requires Root', 'Yes')
691 writefield('Repo Type')
694 for build in app['builds']:
695 writecomments('build:' + build['version'])
696 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
698 # This defines the preferred order for the build items - as in the
699 # manual, they're roughly in order of application.
700 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
701 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
702 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
703 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
704 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
705 'preassemble', 'bindir', 'antcommand', 'novcheck']
707 def write_builditem(key, value):
708 if key in ['version', 'vercode', 'origlines', 'type']:
710 if key in valuetypes['bool'].attrs:
715 #print "...writing {0} : {1}".format(key, value)
716 outline = ' %s=' % key
717 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
723 write_builditem(key, build[key])
724 for key, value in build.iteritems():
725 if not key in keyorder:
726 write_builditem(key, value)
729 if 'Maintainer Notes' in app:
730 writefield('Maintainer Notes', '')
731 for line in app['Maintainer Notes']:
732 mf.write("%s\n" % line)
737 if app['Archive Policy']:
738 writefield('Archive Policy')
739 writefield('Auto Update Mode')
740 writefield('Update Check Mode')
741 if app['Vercode Operation']:
742 writefield('Vercode Operation')
743 if 'Update Check Data' in app:
744 writefield('Update Check Data')
745 if app['Current Version']:
746 writefield('Current Version')
747 writefield('Current Version Code')
749 if app['No Source Since']:
750 writefield('No Source Since')