1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 self.matching = matching
42 if type(matching) is str:
43 self.compiled = re.compile(matching)
48 def _assert_regex(self, values, appid):
50 if not self.compiled.match(v):
51 raise MetaDataException("'%s' is not a valid %s in %s. "
52 % (v, self.name, appid) +
53 "Regex pattern: %s" % (self.matching))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s. "
59 % (v, self.name, appid) +
60 "Possible values: %s" % (", ".join(self.matching)))
62 def check(self, value, appid):
63 if type(value) is not str or not value:
65 if self.sep is not None:
66 values = value.split(self.sep)
69 if type(self.matching) is list:
70 self._assert_list(values, appid)
72 self._assert_regex(values, appid)
77 'int' : FieldType("Integer",
82 'http' : FieldType("HTTP link",
83 r'^http[s]?://', None,
84 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
86 'bitcoin' : FieldType("Bitcoin address",
87 r'^[a-zA-Z0-9]{27,34}$', None,
91 'litecoin' : FieldType("Litecoin address",
92 r'^L[a-zA-Z0-9]{33}$', None,
96 'Bool' : FieldType("Boolean",
101 'bool' : FieldType("Boolean",
104 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
105 'fixtrans', 'fixapos', 'novcheck' ]),
107 'Repo Type' : FieldType("Repo Type",
108 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
112 'archive' : FieldType("Archive Policy",
113 r'^[0-9]+ versions$', None,
114 [ "Archive Policy" ],
117 'antifeatures' : FieldType("Anti-Feature",
118 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
122 'autoupdatemodes' : FieldType("Auto Update Mode",
123 r"^(Version .+|None)$", None,
124 [ "Auto Update Mode" ],
127 'updatecheckmodes' : FieldType("Update Check Mode",
128 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
129 [ "Update Check Mode" ],
133 # Check an app's metadata information for integrity errors
134 def check_metadata(info):
135 for k, t in valuetypes.iteritems():
136 for field in t.fields:
138 t.check(info[field], info['id'])
140 info[field] = info[field] == "Yes"
141 for build in info['builds']:
144 t.check(build[attr], info['id'])
146 build[attr] = build[attr] == "yes"
150 # Formatter for descriptions. Create an instance, and call parseline() with
151 # each line of the description source from the metadata. At the end, call
152 # end() and then text_plain, text_wiki and text_html will contain the result.
153 class DescriptionFormatter:
165 def __init__(self, linkres):
166 self.linkResolver = linkres
167 def endcur(self, notstates=None):
168 if notstates and self.state in notstates:
170 if self.state == self.stPARA:
172 elif self.state == self.stUL:
174 elif self.state == self.stOL:
177 self.text_plain += '\n'
178 self.text_html += '</p>'
179 self.state = self.stNONE
181 self.text_html += '</ul>'
182 self.state = self.stNONE
184 self.text_html += '</ol>'
185 self.state = self.stNONE
187 def formatted(self, txt, html):
190 txt = cgi.escape(txt)
192 index = txt.find("''")
194 return formatted + txt
195 formatted += txt[:index]
197 if txt.startswith("'''"):
203 self.bold = not self.bold
211 self.ital = not self.ital
215 def linkify(self, txt):
219 index = txt.find("[")
221 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
222 linkified_plain += self.formatted(txt[:index], False)
223 linkified_html += self.formatted(txt[:index], True)
225 if txt.startswith("[["):
226 index = txt.find("]]")
228 raise MetaDataException("Unterminated ]]")
230 if self.linkResolver:
231 url, urltext = self.linkResolver(url)
234 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
235 linkified_plain += urltext
238 index = txt.find("]")
240 raise MetaDataException("Unterminated ]")
242 index2 = url.find(' ')
246 urltxt = url[index2 + 1:]
248 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
249 linkified_plain += urltxt
251 linkified_plain += ' (' + url + ')'
254 def addtext(self, txt):
255 p, h = self.linkify(txt)
259 def parseline(self, line):
260 self.text_wiki += "%s\n" % line
263 elif line.startswith('*'):
264 self.endcur([self.stUL])
265 if self.state != self.stUL:
266 self.text_html += '<ul>'
267 self.state = self.stUL
268 self.text_html += '<li>'
269 self.text_plain += '*'
270 self.addtext(line[1:])
271 self.text_html += '</li>'
272 elif line.startswith('#'):
273 self.endcur([self.stOL])
274 if self.state != self.stOL:
275 self.text_html += '<ol>'
276 self.state = self.stOL
277 self.text_html += '<li>'
278 self.text_plain += '*' #TODO: lazy - put the numbers in!
279 self.addtext(line[1:])
280 self.text_html += '</li>'
282 self.endcur([self.stPARA])
283 if self.state == self.stNONE:
284 self.text_html += '<p>'
285 self.state = self.stPARA
286 elif self.state == self.stPARA:
287 self.text_html += ' '
288 self.text_plain += ' '
294 # Parse multiple lines of description as written in a metadata file, returning
295 # a single string in plain text format.
296 def description_plain(lines, linkres):
297 ps = DescriptionFormatter(linkres)
303 # Parse multiple lines of description as written in a metadata file, returning
304 # a single string in wiki format. Used for the Maintainer Notes field as well,
305 # because it's the same format.
306 def description_wiki(lines):
307 ps = DescriptionFormatter(None)
313 # Parse multiple lines of description as written in a metadata file, returning
314 # a single string in HTML format.
315 def description_html(lines,linkres):
316 ps = DescriptionFormatter(linkres)
322 def parse_srclib(metafile, **kw):
325 if metafile and not isinstance(metafile, file):
326 metafile = open(metafile, "r")
328 # Defaults for fields that come from metadata
329 thisinfo['Repo Type'] = ''
330 thisinfo['Repo'] = ''
331 thisinfo['Subdir'] = None
332 thisinfo['Prepare'] = None
333 thisinfo['Srclibs'] = None
334 thisinfo['Update Project'] = None
339 for line in metafile:
340 line = line.rstrip('\r\n')
341 if not line or line.startswith("#"):
345 field, value = line.split(':',1)
347 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
349 if field == "Subdir":
350 thisinfo[field] = value.split(',')
352 thisinfo[field] = value
356 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
357 # returned by the parse_metadata function.
358 def read_metadata(xref=True, package=None):
360 for basedir in ('metadata', 'tmp'):
361 if not os.path.exists(basedir):
363 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
364 if package is None or metafile == os.path.join('metadata', package + '.txt'):
366 appinfo = parse_metadata(metafile)
368 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
369 check_metadata(appinfo)
373 # Parse all descriptions at load time, just to ensure cross-referencing
374 # errors are caught early rather than when they hit the build server.
377 if app['id'] == link:
378 return ("fdroid.app:" + link, "Dummy name - don't know yet")
379 raise MetaDataException("Cannot resolve app id " + link)
382 description_html(app['Description'], linkres)
384 raise MetaDataException("Problem with description of " + app['id'] +
389 # Get the type expected for a given metadata field.
390 def metafieldtype(name):
391 if name in ['Description', 'Maintainer Notes']:
393 if name == 'Build Version':
397 if name == 'Use Built':
401 # Parse metadata for a single application.
403 # 'metafile' - the filename to read. The package id for the application comes
404 # from this filename. Pass None to get a blank entry.
406 # Returns a dictionary containing all the details of the application. There are
407 # two major kinds of information in the dictionary. Keys beginning with capital
408 # letters correspond directory to identically named keys in the metadata file.
409 # Keys beginning with lower case letters are generated in one way or another,
410 # and are not found verbatim in the metadata.
412 # Known keys not originating from the metadata are:
414 # 'id' - the application's package ID
415 # 'builds' - a list of dictionaries containing build information
416 # for each defined build
417 # 'comments' - a list of comments from the metadata file. Each is
418 # a tuple of the form (field, comment) where field is
419 # the name of the field it preceded in the metadata
420 # file. Where field is None, the comment goes at the
421 # end of the file. Alternatively, 'build:version' is
422 # for a comment before a particular build version.
423 # 'descriptionlines' - original lines of description as formatted in the
426 def parse_metadata(metafile):
428 def parse_buildline(lines):
429 value = "".join(lines)
430 parts = [p.replace("\\,", ",")
431 for p in re.split(r"(?<!\\),", value)]
433 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
435 thisbuild['origlines'] = lines
436 thisbuild['version'] = parts[0]
437 thisbuild['vercode'] = parts[1]
438 if parts[2].startswith('!'):
439 # For backwards compatibility, handle old-style disabling,
440 # including attempting to extract the commit from the message
441 thisbuild['disable'] = parts[2][1:]
442 commit = 'unknown - see disabled'
443 index = parts[2].rfind('at ')
445 commit = parts[2][index+3:]
446 if commit.endswith(')'):
448 thisbuild['commit'] = commit
450 thisbuild['commit'] = parts[2]
452 pk, pv = p.split('=', 1)
453 thisbuild[pk.strip()] = pv
457 def add_comments(key):
460 for comment in curcomments:
461 thisinfo['comments'].append((key, comment))
467 if not isinstance(metafile, file):
468 metafile = open(metafile, "r")
469 thisinfo['id'] = metafile.name[9:-4]
471 thisinfo['id'] = None
473 # Defaults for fields that come from metadata...
474 thisinfo['Name'] = None
475 thisinfo['Auto Name'] = ''
476 thisinfo['Categories'] = 'None'
477 thisinfo['Description'] = []
478 thisinfo['Summary'] = ''
479 thisinfo['License'] = 'Unknown'
480 thisinfo['Web Site'] = ''
481 thisinfo['Source Code'] = ''
482 thisinfo['Issue Tracker'] = ''
483 thisinfo['Donate'] = None
484 thisinfo['FlattrID'] = None
485 thisinfo['Bitcoin'] = None
486 thisinfo['Litecoin'] = None
487 thisinfo['Disabled'] = None
488 thisinfo['AntiFeatures'] = None
489 thisinfo['Archive Policy'] = None
490 thisinfo['Update Check Mode'] = 'None'
491 thisinfo['Vercode Operation'] = None
492 thisinfo['Auto Update Mode'] = 'None'
493 thisinfo['Current Version'] = ''
494 thisinfo['Current Version Code'] = '0'
495 thisinfo['Repo Type'] = ''
496 thisinfo['Repo'] = ''
497 thisinfo['Requires Root'] = False
498 thisinfo['No Source Since'] = ''
500 # General defaults...
501 thisinfo['builds'] = []
502 thisinfo['comments'] = []
512 for line in metafile:
513 line = line.rstrip('\r\n')
515 if not any(line.startswith(s) for s in (' ', '\t')):
516 if 'commit' not in curbuild and 'disable' not in curbuild:
517 raise MetaDataException("No commit specified for {0} in {1}".format(
518 curbuild['version'], metafile.name))
519 thisinfo['builds'].append(curbuild)
520 add_comments('build:' + curbuild['version'])
523 if line.endswith('\\'):
524 buildlines.append(line[:-1].lstrip())
526 buildlines.append(line.lstrip())
527 bl = ''.join(buildlines)
528 bv = bl.split('=', 1)
530 raise MetaDataException("Invalid build flag at {0} in {1}".
531 format(buildlines[0], metafile.name))
534 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
535 format(name, curbuild['version'], metafile.name))
536 curbuild[name] = val.lstrip()
542 if line.startswith("#"):
543 curcomments.append(line)
546 field, value = line.split(':',1)
548 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
550 # Translate obsolete fields...
551 if field == 'Market Version':
552 field = 'Current Version'
553 if field == 'Market Version Code':
554 field = 'Current Version Code'
556 fieldtype = metafieldtype(field)
557 if fieldtype not in ['build', 'buildv2']:
559 if fieldtype == 'multiline':
563 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
564 elif fieldtype == 'string':
565 if field == 'Category' and thisinfo['Categories'] == 'None':
566 thisinfo['Categories'] = value.replace(';',',')
567 thisinfo[field] = value
568 elif fieldtype == 'build':
569 if value.endswith("\\"):
571 buildlines = [value[:-1]]
573 thisinfo['builds'].append(parse_buildline([value]))
574 add_comments('build:' + thisinfo['builds'][-1]['version'])
575 elif fieldtype == 'buildv2':
577 vv = value.split(',')
579 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
580 format(value, metafile.name))
581 curbuild['version'] = vv[0]
582 curbuild['vercode'] = vv[1]
585 elif fieldtype == 'obsolete':
586 pass # Just throw it away!
588 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
589 elif mode == 1: # Multiline field
593 thisinfo[field].append(line)
594 elif mode == 2: # Line continuation mode in Build Version
595 if line.endswith("\\"):
596 buildlines.append(line[:-1])
598 buildlines.append(line)
599 thisinfo['builds'].append(
600 parse_buildline(buildlines))
601 add_comments('build:' + thisinfo['builds'][-1]['version'])
605 # Mode at end of file should always be 0...
607 raise MetaDataException(field + " not terminated in " + metafile.name)
609 raise MetaDataException("Unterminated continuation in " + metafile.name)
611 raise MetaDataException("Unterminated build in " + metafile.name)
613 if not thisinfo['Description']:
614 thisinfo['Description'].append('No description available')
618 # Write a metadata file.
620 # 'dest' - The path to the output file
621 # 'app' - The app data
622 def write_metadata(dest, app):
624 def writecomments(key):
626 for pf, comment in app['comments']:
628 mf.write("%s\n" % comment)
630 #if options.verbose and written > 0:
631 #print "...writing comments for " + (key if key else 'EOF')
633 def writefield(field, value=None):
637 mf.write("%s:%s\n" % (field, value))
641 writefield('Disabled')
642 if app['AntiFeatures']:
643 writefield('AntiFeatures')
644 writefield('Categories')
645 writefield('License')
646 writefield('Web Site')
647 writefield('Source Code')
648 writefield('Issue Tracker')
652 writefield('FlattrID')
654 writefield('Bitcoin')
656 writefield('Litecoin')
661 writefield('Auto Name')
662 writefield('Summary')
663 writefield('Description', '')
664 for line in app['Description']:
665 mf.write("%s\n" % line)
668 if app['Requires Root']:
669 writefield('Requires Root', 'Yes')
672 writefield('Repo Type')
675 for build in app['builds']:
676 writecomments('build:' + build['version'])
677 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
679 # This defines the preferred order for the build items - as in the
680 # manual, they're roughly in order of application.
681 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
682 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
683 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
684 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
685 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
686 'preassemble', 'bindir', 'antcommand', 'novcheck']
688 def write_builditem(key, value):
689 if key in ['version', 'vercode', 'origlines']:
691 if key in valuetypes['bool'].attrs:
696 #print "...writing {0} : {1}".format(key, value)
697 outline = ' %s=' % key
698 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
704 write_builditem(key, build[key])
705 for key, value in build.iteritems():
706 if not key in keyorder:
707 write_builditem(key, value)
710 if 'Maintainer Notes' in app:
711 writefield('Maintainer Notes', '')
712 for line in app['Maintainer Notes']:
713 mf.write("%s\n" % line)
718 if app['Archive Policy']:
719 writefield('Archive Policy')
720 writefield('Auto Update Mode')
721 writefield('Update Check Mode')
722 if app['Vercode Operation']:
723 writefield('Vercode Operation')
724 if 'Update Check Data' in app:
725 writefield('Update Check Data')
726 if app['Current Version']:
727 writefield('Current Version')
728 writefield('Current Version Code')
730 if app['No Source Since']:
731 writefield('No Source Since')