1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 if type(matching) is str:
42 self.matching = re.compile(matching)
43 elif type(matching) is list:
44 self.matching = matching
49 def _assert_regex(self, values, appid):
51 if not self.matching.match(v):
52 raise MetaDataException("'%s' is not a valid %s in %s"
53 % (v, self.name, appid))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s"
59 % (v, self.name, appid))
61 def check(self, value, appid):
62 if type(value) is not str or not value:
64 if self.sep is not None:
65 values = value.split(self.sep)
68 if type(self.matching) is list:
69 self._assert_list(values, appid)
71 self._assert_regex(values, appid)
76 'int' : FieldType("Integer",
81 'http' : FieldType("HTTP link",
82 r'^http[s]?://', None,
83 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
85 'bitcoin' : FieldType("Bitcoin address",
86 r'^[a-zA-Z0-9]{27,34}$', None,
90 'litecoin' : FieldType("Litecoin address",
91 r'^[a-zA-Z0-9]{27,34}$', None,
95 'archive' : FieldType("Archive Policy",
96 r'^[0-9]+ versions$', None,
100 'bool' : FieldType("Boolean",
103 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
104 'fixtrans', 'fixapos', 'novcheck' ]),
106 'Bool' : FieldType("Boolean",
111 'antifeatures' : FieldType("Anti-Feature",
112 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd" ], ',',
117 # Check an app's metadata information for integrity errors
118 def check_metadata(info):
119 for k, t in valuetypes.iteritems():
120 for field in t.fields:
122 t.check(info[field], info['id'])
124 info[field] = info[field] == "Yes"
125 for build in info['builds']:
128 t.check(build[attr], info['id'])
130 build[attr] = build[attr] == "yes"
134 # Formatter for descriptions. Create an instance, and call parseline() with
135 # each line of the description source from the metadata. At the end, call
136 # end() and then text_plain, text_wiki and text_html will contain the result.
137 class DescriptionFormatter:
149 def __init__(self, linkres):
150 self.linkResolver = linkres
151 def endcur(self, notstates=None):
152 if notstates and self.state in notstates:
154 if self.state == self.stPARA:
156 elif self.state == self.stUL:
158 elif self.state == self.stOL:
161 self.text_plain += '\n'
162 self.text_html += '</p>'
163 self.state = self.stNONE
165 self.text_html += '</ul>'
166 self.state = self.stNONE
168 self.text_html += '</ol>'
169 self.state = self.stNONE
171 def formatted(self, txt, html):
174 txt = cgi.escape(txt)
176 index = txt.find("''")
178 return formatted + txt
179 formatted += txt[:index]
181 if txt.startswith("'''"):
187 self.bold = not self.bold
195 self.ital = not self.ital
199 def linkify(self, txt):
203 index = txt.find("[")
205 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
206 linkified_plain += self.formatted(txt[:index], False)
207 linkified_html += self.formatted(txt[:index], True)
209 if txt.startswith("[["):
210 index = txt.find("]]")
212 raise MetaDataException("Unterminated ]]")
214 if self.linkResolver:
215 url, urltext = self.linkResolver(url)
218 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
219 linkified_plain += urltext
222 index = txt.find("]")
224 raise MetaDataException("Unterminated ]")
226 index2 = url.find(' ')
230 urltxt = url[index2 + 1:]
232 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
233 linkified_plain += urltxt
235 linkified_plain += ' (' + url + ')'
238 def addtext(self, txt):
239 p, h = self.linkify(txt)
243 def parseline(self, line):
244 self.text_wiki += "%s\n" % line
247 elif line.startswith('*'):
248 self.endcur([self.stUL])
249 if self.state != self.stUL:
250 self.text_html += '<ul>'
251 self.state = self.stUL
252 self.text_html += '<li>'
253 self.text_plain += '*'
254 self.addtext(line[1:])
255 self.text_html += '</li>'
256 elif line.startswith('#'):
257 self.endcur([self.stOL])
258 if self.state != self.stOL:
259 self.text_html += '<ol>'
260 self.state = self.stOL
261 self.text_html += '<li>'
262 self.text_plain += '*' #TODO: lazy - put the numbers in!
263 self.addtext(line[1:])
264 self.text_html += '</li>'
266 self.endcur([self.stPARA])
267 if self.state == self.stNONE:
268 self.text_html += '<p>'
269 self.state = self.stPARA
270 elif self.state == self.stPARA:
271 self.text_html += ' '
272 self.text_plain += ' '
278 # Parse multiple lines of description as written in a metadata file, returning
279 # a single string in plain text format.
280 def description_plain(lines, linkres):
281 ps = DescriptionFormatter(linkres)
287 # Parse multiple lines of description as written in a metadata file, returning
288 # a single string in wiki format. Used for the Maintainer Notes field as well,
289 # because it's the same format.
290 def description_wiki(lines):
291 ps = DescriptionFormatter(None)
297 # Parse multiple lines of description as written in a metadata file, returning
298 # a single string in HTML format.
299 def description_html(lines,linkres):
300 ps = DescriptionFormatter(linkres)
306 def parse_srclib(metafile, **kw):
309 if metafile and not isinstance(metafile, file):
310 metafile = open(metafile, "r")
312 # Defaults for fields that come from metadata
313 thisinfo['Repo Type'] = ''
314 thisinfo['Repo'] = ''
315 thisinfo['Subdir'] = None
316 thisinfo['Prepare'] = None
317 thisinfo['Srclibs'] = None
318 thisinfo['Update Project'] = None
323 for line in metafile:
324 line = line.rstrip('\r\n')
325 if not line or line.startswith("#"):
328 index = line.find(':')
330 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
332 value = line[index+1:]
334 if field == "Subdir":
335 thisinfo[field] = value.split(',')
337 thisinfo[field] = value
341 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
342 # returned by the parse_metadata function.
343 def read_metadata(xref=True, package=None):
345 for basedir in ('metadata', 'tmp'):
346 if not os.path.exists(basedir):
348 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
349 if package is None or metafile == os.path.join('metadata', package + '.txt'):
351 appinfo = parse_metadata(metafile)
353 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
354 check_metadata(appinfo)
358 # Parse all descriptions at load time, just to ensure cross-referencing
359 # errors are caught early rather than when they hit the build server.
362 if app['id'] == link:
363 return ("fdroid.app:" + link, "Dummy name - don't know yet")
364 raise MetaDataException("Cannot resolve app id " + link)
367 description_html(app['Description'], linkres)
369 raise MetaDataException("Problem with description of " + app['id'] +
374 # Get the type expected for a given metadata field.
375 def metafieldtype(name):
376 if name in ['Description', 'Maintainer Notes']:
378 if name == 'Build Version':
382 if name == 'Use Built':
386 # Parse metadata for a single application.
388 # 'metafile' - the filename to read. The package id for the application comes
389 # from this filename. Pass None to get a blank entry.
391 # Returns a dictionary containing all the details of the application. There are
392 # two major kinds of information in the dictionary. Keys beginning with capital
393 # letters correspond directory to identically named keys in the metadata file.
394 # Keys beginning with lower case letters are generated in one way or another,
395 # and are not found verbatim in the metadata.
397 # Known keys not originating from the metadata are:
399 # 'id' - the application's package ID
400 # 'builds' - a list of dictionaries containing build information
401 # for each defined build
402 # 'comments' - a list of comments from the metadata file. Each is
403 # a tuple of the form (field, comment) where field is
404 # the name of the field it preceded in the metadata
405 # file. Where field is None, the comment goes at the
406 # end of the file. Alternatively, 'build:version' is
407 # for a comment before a particular build version.
408 # 'descriptionlines' - original lines of description as formatted in the
411 def parse_metadata(metafile):
413 def parse_buildline(lines):
414 value = "".join(lines)
415 parts = [p.replace("\\,", ",")
416 for p in re.split(r"(?<!\\),", value)]
418 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
420 thisbuild['origlines'] = lines
421 thisbuild['version'] = parts[0]
422 thisbuild['vercode'] = parts[1]
423 if parts[2].startswith('!'):
424 # For backwards compatibility, handle old-style disabling,
425 # including attempting to extract the commit from the message
426 thisbuild['disable'] = parts[2][1:]
427 commit = 'unknown - see disabled'
428 index = parts[2].rfind('at ')
430 commit = parts[2][index+3:]
431 if commit.endswith(')'):
433 thisbuild['commit'] = commit
435 thisbuild['commit'] = parts[2]
437 pk, pv = p.split('=', 1)
438 thisbuild[pk.strip()] = pv
442 def add_comments(key):
445 for comment in curcomments:
446 thisinfo['comments'].append((key, comment))
452 if not isinstance(metafile, file):
453 metafile = open(metafile, "r")
454 thisinfo['id'] = metafile.name[9:-4]
456 thisinfo['id'] = None
458 # Defaults for fields that come from metadata...
459 thisinfo['Name'] = None
460 thisinfo['Auto Name'] = ''
461 thisinfo['Categories'] = 'None'
462 thisinfo['Description'] = []
463 thisinfo['Summary'] = ''
464 thisinfo['License'] = 'Unknown'
465 thisinfo['Web Site'] = ''
466 thisinfo['Source Code'] = ''
467 thisinfo['Issue Tracker'] = ''
468 thisinfo['Donate'] = None
469 thisinfo['FlattrID'] = None
470 thisinfo['Bitcoin'] = None
471 thisinfo['Litecoin'] = None
472 thisinfo['Disabled'] = None
473 thisinfo['AntiFeatures'] = None
474 thisinfo['Archive Policy'] = None
475 thisinfo['Update Check Mode'] = 'None'
476 thisinfo['Vercode Operation'] = None
477 thisinfo['Auto Update Mode'] = 'None'
478 thisinfo['Current Version'] = ''
479 thisinfo['Current Version Code'] = '0'
480 thisinfo['Repo Type'] = ''
481 thisinfo['Repo'] = ''
482 thisinfo['Requires Root'] = False
483 thisinfo['No Source Since'] = ''
485 # General defaults...
486 thisinfo['builds'] = []
487 thisinfo['comments'] = []
497 for line in metafile:
498 line = line.rstrip('\r\n')
500 if not any(line.startswith(s) for s in (' ', '\t')):
501 if 'commit' not in curbuild and 'disable' not in curbuild:
502 raise MetaDataException("No commit specified for {0} in {1}".format(
503 curbuild['version'], metafile.name))
504 thisinfo['builds'].append(curbuild)
505 add_comments('build:' + curbuild['version'])
508 if line.endswith('\\'):
509 buildlines.append(line[:-1].lstrip())
511 buildlines.append(line.lstrip())
512 bl = ''.join(buildlines)
513 bv = bl.split('=', 1)
515 raise MetaDataException("Invalid build flag at {0} in {1}".
516 format(buildlines[0], metafile.name))
519 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
520 format(name, curbuild['version'], metafile.name))
521 curbuild[name] = val.lstrip()
527 if line.startswith("#"):
528 curcomments.append(line)
530 index = line.find(':')
532 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
534 value = line[index+1:]
536 # Translate obsolete fields...
537 if field == 'Market Version':
538 field = 'Current Version'
539 if field == 'Market Version Code':
540 field = 'Current Version Code'
542 fieldtype = metafieldtype(field)
543 if fieldtype not in ['build', 'buildv2']:
545 if fieldtype == 'multiline':
549 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
550 elif fieldtype == 'string':
551 if field == 'Category' and thisinfo['Categories'] == 'None':
552 thisinfo['Categories'] = value.replace(';',',')
553 thisinfo[field] = value
554 elif fieldtype == 'build':
555 if value.endswith("\\"):
557 buildlines = [value[:-1]]
559 thisinfo['builds'].append(parse_buildline([value]))
560 add_comments('build:' + thisinfo['builds'][-1]['version'])
561 elif fieldtype == 'buildv2':
563 vv = value.split(',')
565 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
566 format(value, metafile.name))
567 curbuild['version'] = vv[0]
568 curbuild['vercode'] = vv[1]
571 elif fieldtype == 'obsolete':
572 pass # Just throw it away!
574 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
575 elif mode == 1: # Multiline field
579 thisinfo[field].append(line)
580 elif mode == 2: # Line continuation mode in Build Version
581 if line.endswith("\\"):
582 buildlines.append(line[:-1])
584 buildlines.append(line)
585 thisinfo['builds'].append(
586 parse_buildline(buildlines))
587 add_comments('build:' + thisinfo['builds'][-1]['version'])
591 # Mode at end of file should always be 0...
593 raise MetaDataException(field + " not terminated in " + metafile.name)
595 raise MetaDataException("Unterminated continuation in " + metafile.name)
597 raise MetaDataException("Unterminated build in " + metafile.name)
599 if not thisinfo['Description']:
600 thisinfo['Description'].append('No description available')
604 # Write a metadata file.
606 # 'dest' - The path to the output file
607 # 'app' - The app data
608 def write_metadata(dest, app):
610 def writecomments(key):
612 for pf, comment in app['comments']:
614 mf.write("%s\n" % comment)
616 #if options.verbose and written > 0:
617 #print "...writing comments for " + (key if key else 'EOF')
619 def writefield(field, value=None):
623 mf.write("%s:%s\n" % (field, value))
627 writefield('Disabled')
628 if app['AntiFeatures']:
629 writefield('AntiFeatures')
630 writefield('Categories')
631 writefield('License')
632 writefield('Web Site')
633 writefield('Source Code')
634 writefield('Issue Tracker')
638 writefield('FlattrID')
640 writefield('Bitcoin')
642 writefield('Litecoin')
647 writefield('Auto Name')
648 writefield('Summary')
649 writefield('Description', '')
650 for line in app['Description']:
651 mf.write("%s\n" % line)
654 if app['Requires Root']:
655 writefield('Requires Root', 'Yes')
658 writefield('Repo Type')
661 for build in app['builds']:
662 writecomments('build:' + build['version'])
663 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
665 # This defines the preferred order for the build items - as in the
666 # manual, they're roughly in order of application.
667 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
668 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
669 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
670 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
671 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
672 'preassemble', 'bindir', 'antcommand', 'novcheck']
674 def write_builditem(key, value):
675 if key in ['version', 'vercode', 'origlines']:
677 if key in valuetypes['bool'].attrs:
682 #print "...writing {0} : {1}".format(key, value)
683 outline = ' %s=' % key
684 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
690 write_builditem(key, build[key])
691 for key, value in build.iteritems():
692 if not key in keyorder:
693 write_builditem(key, value)
696 if 'Maintainer Notes' in app:
697 writefield('Maintainer Notes', '')
698 for line in app['Maintainer Notes']:
699 mf.write("%s\n" % line)
704 if app['Archive Policy']:
705 writefield('Archive Policy')
706 writefield('Auto Update Mode')
707 writefield('Update Check Mode')
708 if app['Vercode Operation']:
709 writefield('Vercode Operation')
710 if 'Update Check Data' in app:
711 writefield('Update Check Data')
712 if app['Current Version']:
713 writefield('Current Version')
714 writefield('Current Version Code')
716 if app['No Source Since']:
717 writefield('No Source Since')