1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 if type(matching) is str:
42 self.matching = re.compile(matching)
43 elif type(matching) is list:
44 self.matching = matching
49 def _assert_regex(self, values, appid):
51 if not self.matching.match(v):
52 raise MetaDataException("'%s' is not a valid %s in %s"
53 % (v, self.name, appid))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s"
59 % (v, self.name, appid))
61 def check(self, value, appid):
62 if type(value) is not str or not value:
64 if self.sep is not None:
65 values = value.split(self.sep)
68 if type(self.matching) is list:
69 self._assert_list(values, appid)
71 self._assert_regex(values, appid)
76 'int' : FieldType("Integer",
81 'http' : FieldType("HTTP link",
82 r'^http[s]?://', None,
83 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
85 'bitcoin' : FieldType("Bitcoin address",
86 r'^[a-zA-Z0-9]{27,34}$', None,
90 'litecoin' : FieldType("Litecoin address",
91 r'^[a-zA-Z0-9]{27,34}$', None,
95 'archive' : FieldType("Archive Policy",
96 r'^[0-9]+ versions$', None,
100 'bool' : FieldType("Boolean",
103 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
104 'fixtrans', 'fixapos', 'novcheck' ]),
106 'Bool' : FieldType("Boolean",
111 'antifeatures' : FieldType("Anti-Feature",
112 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd" ], ',',
116 'autoupdatemodes' : FieldType("Auto Update Mode",
117 r"^(Version .+|None)$", None,
118 [ "Auto Update Mode" ],
121 'updatecheckmodes' : FieldType("Update Check Mode",
122 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
123 [ "Update Check Mode" ],
127 # Check an app's metadata information for integrity errors
128 def check_metadata(info):
129 for k, t in valuetypes.iteritems():
130 for field in t.fields:
132 t.check(info[field], info['id'])
134 info[field] = info[field] == "Yes"
135 for build in info['builds']:
138 t.check(build[attr], info['id'])
140 build[attr] = build[attr] == "yes"
144 # Formatter for descriptions. Create an instance, and call parseline() with
145 # each line of the description source from the metadata. At the end, call
146 # end() and then text_plain, text_wiki and text_html will contain the result.
147 class DescriptionFormatter:
159 def __init__(self, linkres):
160 self.linkResolver = linkres
161 def endcur(self, notstates=None):
162 if notstates and self.state in notstates:
164 if self.state == self.stPARA:
166 elif self.state == self.stUL:
168 elif self.state == self.stOL:
171 self.text_plain += '\n'
172 self.text_html += '</p>'
173 self.state = self.stNONE
175 self.text_html += '</ul>'
176 self.state = self.stNONE
178 self.text_html += '</ol>'
179 self.state = self.stNONE
181 def formatted(self, txt, html):
184 txt = cgi.escape(txt)
186 index = txt.find("''")
188 return formatted + txt
189 formatted += txt[:index]
191 if txt.startswith("'''"):
197 self.bold = not self.bold
205 self.ital = not self.ital
209 def linkify(self, txt):
213 index = txt.find("[")
215 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
216 linkified_plain += self.formatted(txt[:index], False)
217 linkified_html += self.formatted(txt[:index], True)
219 if txt.startswith("[["):
220 index = txt.find("]]")
222 raise MetaDataException("Unterminated ]]")
224 if self.linkResolver:
225 url, urltext = self.linkResolver(url)
228 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
229 linkified_plain += urltext
232 index = txt.find("]")
234 raise MetaDataException("Unterminated ]")
236 index2 = url.find(' ')
240 urltxt = url[index2 + 1:]
242 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
243 linkified_plain += urltxt
245 linkified_plain += ' (' + url + ')'
248 def addtext(self, txt):
249 p, h = self.linkify(txt)
253 def parseline(self, line):
254 self.text_wiki += "%s\n" % line
257 elif line.startswith('*'):
258 self.endcur([self.stUL])
259 if self.state != self.stUL:
260 self.text_html += '<ul>'
261 self.state = self.stUL
262 self.text_html += '<li>'
263 self.text_plain += '*'
264 self.addtext(line[1:])
265 self.text_html += '</li>'
266 elif line.startswith('#'):
267 self.endcur([self.stOL])
268 if self.state != self.stOL:
269 self.text_html += '<ol>'
270 self.state = self.stOL
271 self.text_html += '<li>'
272 self.text_plain += '*' #TODO: lazy - put the numbers in!
273 self.addtext(line[1:])
274 self.text_html += '</li>'
276 self.endcur([self.stPARA])
277 if self.state == self.stNONE:
278 self.text_html += '<p>'
279 self.state = self.stPARA
280 elif self.state == self.stPARA:
281 self.text_html += ' '
282 self.text_plain += ' '
288 # Parse multiple lines of description as written in a metadata file, returning
289 # a single string in plain text format.
290 def description_plain(lines, linkres):
291 ps = DescriptionFormatter(linkres)
297 # Parse multiple lines of description as written in a metadata file, returning
298 # a single string in wiki format. Used for the Maintainer Notes field as well,
299 # because it's the same format.
300 def description_wiki(lines):
301 ps = DescriptionFormatter(None)
307 # Parse multiple lines of description as written in a metadata file, returning
308 # a single string in HTML format.
309 def description_html(lines,linkres):
310 ps = DescriptionFormatter(linkres)
316 def parse_srclib(metafile, **kw):
319 if metafile and not isinstance(metafile, file):
320 metafile = open(metafile, "r")
322 # Defaults for fields that come from metadata
323 thisinfo['Repo Type'] = ''
324 thisinfo['Repo'] = ''
325 thisinfo['Subdir'] = None
326 thisinfo['Prepare'] = None
327 thisinfo['Srclibs'] = None
328 thisinfo['Update Project'] = None
333 for line in metafile:
334 line = line.rstrip('\r\n')
335 if not line or line.startswith("#"):
338 index = line.find(':')
340 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
342 value = line[index+1:]
344 if field == "Subdir":
345 thisinfo[field] = value.split(',')
347 thisinfo[field] = value
351 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
352 # returned by the parse_metadata function.
353 def read_metadata(xref=True, package=None):
355 for basedir in ('metadata', 'tmp'):
356 if not os.path.exists(basedir):
358 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
359 if package is None or metafile == os.path.join('metadata', package + '.txt'):
361 appinfo = parse_metadata(metafile)
363 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
364 check_metadata(appinfo)
368 # Parse all descriptions at load time, just to ensure cross-referencing
369 # errors are caught early rather than when they hit the build server.
372 if app['id'] == link:
373 return ("fdroid.app:" + link, "Dummy name - don't know yet")
374 raise MetaDataException("Cannot resolve app id " + link)
377 description_html(app['Description'], linkres)
379 raise MetaDataException("Problem with description of " + app['id'] +
384 # Get the type expected for a given metadata field.
385 def metafieldtype(name):
386 if name in ['Description', 'Maintainer Notes']:
388 if name == 'Build Version':
392 if name == 'Use Built':
396 # Parse metadata for a single application.
398 # 'metafile' - the filename to read. The package id for the application comes
399 # from this filename. Pass None to get a blank entry.
401 # Returns a dictionary containing all the details of the application. There are
402 # two major kinds of information in the dictionary. Keys beginning with capital
403 # letters correspond directory to identically named keys in the metadata file.
404 # Keys beginning with lower case letters are generated in one way or another,
405 # and are not found verbatim in the metadata.
407 # Known keys not originating from the metadata are:
409 # 'id' - the application's package ID
410 # 'builds' - a list of dictionaries containing build information
411 # for each defined build
412 # 'comments' - a list of comments from the metadata file. Each is
413 # a tuple of the form (field, comment) where field is
414 # the name of the field it preceded in the metadata
415 # file. Where field is None, the comment goes at the
416 # end of the file. Alternatively, 'build:version' is
417 # for a comment before a particular build version.
418 # 'descriptionlines' - original lines of description as formatted in the
421 def parse_metadata(metafile):
423 def parse_buildline(lines):
424 value = "".join(lines)
425 parts = [p.replace("\\,", ",")
426 for p in re.split(r"(?<!\\),", value)]
428 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
430 thisbuild['origlines'] = lines
431 thisbuild['version'] = parts[0]
432 thisbuild['vercode'] = parts[1]
433 if parts[2].startswith('!'):
434 # For backwards compatibility, handle old-style disabling,
435 # including attempting to extract the commit from the message
436 thisbuild['disable'] = parts[2][1:]
437 commit = 'unknown - see disabled'
438 index = parts[2].rfind('at ')
440 commit = parts[2][index+3:]
441 if commit.endswith(')'):
443 thisbuild['commit'] = commit
445 thisbuild['commit'] = parts[2]
447 pk, pv = p.split('=', 1)
448 thisbuild[pk.strip()] = pv
452 def add_comments(key):
455 for comment in curcomments:
456 thisinfo['comments'].append((key, comment))
462 if not isinstance(metafile, file):
463 metafile = open(metafile, "r")
464 thisinfo['id'] = metafile.name[9:-4]
466 thisinfo['id'] = None
468 # Defaults for fields that come from metadata...
469 thisinfo['Name'] = None
470 thisinfo['Auto Name'] = ''
471 thisinfo['Categories'] = 'None'
472 thisinfo['Description'] = []
473 thisinfo['Summary'] = ''
474 thisinfo['License'] = 'Unknown'
475 thisinfo['Web Site'] = ''
476 thisinfo['Source Code'] = ''
477 thisinfo['Issue Tracker'] = ''
478 thisinfo['Donate'] = None
479 thisinfo['FlattrID'] = None
480 thisinfo['Bitcoin'] = None
481 thisinfo['Litecoin'] = None
482 thisinfo['Disabled'] = None
483 thisinfo['AntiFeatures'] = None
484 thisinfo['Archive Policy'] = None
485 thisinfo['Update Check Mode'] = 'None'
486 thisinfo['Vercode Operation'] = None
487 thisinfo['Auto Update Mode'] = 'None'
488 thisinfo['Current Version'] = ''
489 thisinfo['Current Version Code'] = '0'
490 thisinfo['Repo Type'] = ''
491 thisinfo['Repo'] = ''
492 thisinfo['Requires Root'] = False
493 thisinfo['No Source Since'] = ''
495 # General defaults...
496 thisinfo['builds'] = []
497 thisinfo['comments'] = []
507 for line in metafile:
508 line = line.rstrip('\r\n')
510 if not any(line.startswith(s) for s in (' ', '\t')):
511 if 'commit' not in curbuild and 'disable' not in curbuild:
512 raise MetaDataException("No commit specified for {0} in {1}".format(
513 curbuild['version'], metafile.name))
514 thisinfo['builds'].append(curbuild)
515 add_comments('build:' + curbuild['version'])
518 if line.endswith('\\'):
519 buildlines.append(line[:-1].lstrip())
521 buildlines.append(line.lstrip())
522 bl = ''.join(buildlines)
523 bv = bl.split('=', 1)
525 raise MetaDataException("Invalid build flag at {0} in {1}".
526 format(buildlines[0], metafile.name))
529 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
530 format(name, curbuild['version'], metafile.name))
531 curbuild[name] = val.lstrip()
537 if line.startswith("#"):
538 curcomments.append(line)
540 index = line.find(':')
542 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
544 value = line[index+1:]
546 # Translate obsolete fields...
547 if field == 'Market Version':
548 field = 'Current Version'
549 if field == 'Market Version Code':
550 field = 'Current Version Code'
552 fieldtype = metafieldtype(field)
553 if fieldtype not in ['build', 'buildv2']:
555 if fieldtype == 'multiline':
559 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
560 elif fieldtype == 'string':
561 if field == 'Category' and thisinfo['Categories'] == 'None':
562 thisinfo['Categories'] = value.replace(';',',')
563 thisinfo[field] = value
564 elif fieldtype == 'build':
565 if value.endswith("\\"):
567 buildlines = [value[:-1]]
569 thisinfo['builds'].append(parse_buildline([value]))
570 add_comments('build:' + thisinfo['builds'][-1]['version'])
571 elif fieldtype == 'buildv2':
573 vv = value.split(',')
575 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
576 format(value, metafile.name))
577 curbuild['version'] = vv[0]
578 curbuild['vercode'] = vv[1]
581 elif fieldtype == 'obsolete':
582 pass # Just throw it away!
584 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
585 elif mode == 1: # Multiline field
589 thisinfo[field].append(line)
590 elif mode == 2: # Line continuation mode in Build Version
591 if line.endswith("\\"):
592 buildlines.append(line[:-1])
594 buildlines.append(line)
595 thisinfo['builds'].append(
596 parse_buildline(buildlines))
597 add_comments('build:' + thisinfo['builds'][-1]['version'])
601 # Mode at end of file should always be 0...
603 raise MetaDataException(field + " not terminated in " + metafile.name)
605 raise MetaDataException("Unterminated continuation in " + metafile.name)
607 raise MetaDataException("Unterminated build in " + metafile.name)
609 if not thisinfo['Description']:
610 thisinfo['Description'].append('No description available')
614 # Write a metadata file.
616 # 'dest' - The path to the output file
617 # 'app' - The app data
618 def write_metadata(dest, app):
620 def writecomments(key):
622 for pf, comment in app['comments']:
624 mf.write("%s\n" % comment)
626 #if options.verbose and written > 0:
627 #print "...writing comments for " + (key if key else 'EOF')
629 def writefield(field, value=None):
633 mf.write("%s:%s\n" % (field, value))
637 writefield('Disabled')
638 if app['AntiFeatures']:
639 writefield('AntiFeatures')
640 writefield('Categories')
641 writefield('License')
642 writefield('Web Site')
643 writefield('Source Code')
644 writefield('Issue Tracker')
648 writefield('FlattrID')
650 writefield('Bitcoin')
652 writefield('Litecoin')
657 writefield('Auto Name')
658 writefield('Summary')
659 writefield('Description', '')
660 for line in app['Description']:
661 mf.write("%s\n" % line)
664 if app['Requires Root']:
665 writefield('Requires Root', 'Yes')
668 writefield('Repo Type')
671 for build in app['builds']:
672 writecomments('build:' + build['version'])
673 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
675 # This defines the preferred order for the build items - as in the
676 # manual, they're roughly in order of application.
677 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
678 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
679 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
680 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
681 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
682 'preassemble', 'bindir', 'antcommand', 'novcheck']
684 def write_builditem(key, value):
685 if key in ['version', 'vercode', 'origlines']:
687 if key in valuetypes['bool'].attrs:
692 #print "...writing {0} : {1}".format(key, value)
693 outline = ' %s=' % key
694 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
700 write_builditem(key, build[key])
701 for key, value in build.iteritems():
702 if not key in keyorder:
703 write_builditem(key, value)
706 if 'Maintainer Notes' in app:
707 writefield('Maintainer Notes', '')
708 for line in app['Maintainer Notes']:
709 mf.write("%s\n" % line)
714 if app['Archive Policy']:
715 writefield('Archive Policy')
716 writefield('Auto Update Mode')
717 writefield('Update Check Mode')
718 if app['Vercode Operation']:
719 writefield('Vercode Operation')
720 if 'Update Check Data' in app:
721 writefield('Update Check Data')
722 if app['Current Version']:
723 writefield('Current Version')
724 writefield('Current Version Code')
726 if app['No Source Since']:
727 writefield('No Source Since')