1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
23 class MetaDataException(Exception):
24 def __init__(self, value):
28 return repr(self.value)
30 # Designates a metadata field type and checks that it matches
32 # 'name' - The long name of the field type
33 # 'matching' - List of possible values or regex expression
34 # 'sep' - Separator to use if value may be a list
35 # 'fields' - Metadata fields (Field:Value) of this type
36 # 'attrs' - Build attributes (attr=value) of this type
39 def __init__(self, name, matching, sep, fields, attrs):
41 self.matching = matching
42 if type(matching) is str:
43 self.compiled = re.compile(matching)
48 def _assert_regex(self, values, appid):
50 if not self.compiled.match(v):
51 raise MetaDataException("'%s' is not a valid %s in %s. "
52 % (v, self.name, appid) +
53 "Regex pattern: %s" % (self.matching))
55 def _assert_list(self, values, appid):
57 if v not in self.matching:
58 raise MetaDataException("'%s' is not a valid %s in %s. "
59 % (v, self.name, appid) +
60 "Possible values: %s" % (", ".join(self.matching)))
62 def check(self, value, appid):
63 if type(value) is not str or not value:
65 if self.sep is not None:
66 values = value.split(self.sep)
69 if type(self.matching) is list:
70 self._assert_list(values, appid)
72 self._assert_regex(values, appid)
77 'int' : FieldType("Integer",
82 'http' : FieldType("HTTP link",
83 r'^http[s]?://', None,
84 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
86 'bitcoin' : FieldType("Bitcoin address",
87 r'^[a-zA-Z0-9]{27,34}$', None,
91 'litecoin' : FieldType("Litecoin address",
92 r'^L[a-zA-Z0-9]{33}$', None,
96 'Bool' : FieldType("Boolean",
101 'bool' : FieldType("Boolean",
104 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
105 'fixtrans', 'fixapos', 'novcheck' ]),
107 'Repo Type' : FieldType("Repo Type",
108 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
112 'archive' : FieldType("Archive Policy",
113 r'^[0-9]+ versions$', None,
114 [ "Archive Policy" ],
117 'antifeatures' : FieldType("Anti-Feature",
118 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
122 'autoupdatemodes' : FieldType("Auto Update Mode",
123 r"^(Version .+|None)$", None,
124 [ "Auto Update Mode" ],
127 'updatecheckmodes' : FieldType("Update Check Mode",
128 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
129 [ "Update Check Mode" ],
133 # Check an app's metadata information for integrity errors
134 def check_metadata(info):
135 for k, t in valuetypes.iteritems():
136 for field in t.fields:
138 t.check(info[field], info['id'])
140 info[field] = info[field] == "Yes"
141 for build in info['builds']:
144 t.check(build[attr], info['id'])
146 build[attr] = build[attr] == "yes"
150 # Formatter for descriptions. Create an instance, and call parseline() with
151 # each line of the description source from the metadata. At the end, call
152 # end() and then text_plain, text_wiki and text_html will contain the result.
153 class DescriptionFormatter:
165 def __init__(self, linkres):
166 self.linkResolver = linkres
167 def endcur(self, notstates=None):
168 if notstates and self.state in notstates:
170 if self.state == self.stPARA:
172 elif self.state == self.stUL:
174 elif self.state == self.stOL:
177 self.text_plain += '\n'
178 self.text_html += '</p>'
179 self.state = self.stNONE
181 self.text_html += '</ul>'
182 self.state = self.stNONE
184 self.text_html += '</ol>'
185 self.state = self.stNONE
187 def formatted(self, txt, html):
190 txt = cgi.escape(txt)
192 index = txt.find("''")
194 return formatted + txt
195 formatted += txt[:index]
197 if txt.startswith("'''"):
203 self.bold = not self.bold
211 self.ital = not self.ital
215 def linkify(self, txt):
219 index = txt.find("[")
221 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
222 linkified_plain += self.formatted(txt[:index], False)
223 linkified_html += self.formatted(txt[:index], True)
225 if txt.startswith("[["):
226 index = txt.find("]]")
228 raise MetaDataException("Unterminated ]]")
230 if self.linkResolver:
231 url, urltext = self.linkResolver(url)
234 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
235 linkified_plain += urltext
238 index = txt.find("]")
240 raise MetaDataException("Unterminated ]")
242 index2 = url.find(' ')
246 urltxt = url[index2 + 1:]
248 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
249 linkified_plain += urltxt
251 linkified_plain += ' (' + url + ')'
254 def addtext(self, txt):
255 p, h = self.linkify(txt)
259 def parseline(self, line):
260 self.text_wiki += "%s\n" % line
263 elif line.startswith('*'):
264 self.endcur([self.stUL])
265 if self.state != self.stUL:
266 self.text_html += '<ul>'
267 self.state = self.stUL
268 self.text_html += '<li>'
269 self.text_plain += '*'
270 self.addtext(line[1:])
271 self.text_html += '</li>'
272 elif line.startswith('#'):
273 self.endcur([self.stOL])
274 if self.state != self.stOL:
275 self.text_html += '<ol>'
276 self.state = self.stOL
277 self.text_html += '<li>'
278 self.text_plain += '*' #TODO: lazy - put the numbers in!
279 self.addtext(line[1:])
280 self.text_html += '</li>'
282 self.endcur([self.stPARA])
283 if self.state == self.stNONE:
284 self.text_html += '<p>'
285 self.state = self.stPARA
286 elif self.state == self.stPARA:
287 self.text_html += ' '
288 self.text_plain += ' '
294 # Parse multiple lines of description as written in a metadata file, returning
295 # a single string in plain text format.
296 def description_plain(lines, linkres):
297 ps = DescriptionFormatter(linkres)
303 # Parse multiple lines of description as written in a metadata file, returning
304 # a single string in wiki format. Used for the Maintainer Notes field as well,
305 # because it's the same format.
306 def description_wiki(lines):
307 ps = DescriptionFormatter(None)
313 # Parse multiple lines of description as written in a metadata file, returning
314 # a single string in HTML format.
315 def description_html(lines,linkres):
316 ps = DescriptionFormatter(linkres)
322 def parse_srclib(metafile, **kw):
325 if metafile and not isinstance(metafile, file):
326 metafile = open(metafile, "r")
328 # Defaults for fields that come from metadata
329 thisinfo['Repo Type'] = ''
330 thisinfo['Repo'] = ''
331 thisinfo['Subdir'] = None
332 thisinfo['Prepare'] = None
333 thisinfo['Srclibs'] = None
334 thisinfo['Update Project'] = None
339 for line in metafile:
340 line = line.rstrip('\r\n')
341 if not line or line.startswith("#"):
344 index = line.find(':')
346 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
348 value = line[index+1:]
350 if field == "Subdir":
351 thisinfo[field] = value.split(',')
353 thisinfo[field] = value
357 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
358 # returned by the parse_metadata function.
359 def read_metadata(xref=True, package=None):
361 for basedir in ('metadata', 'tmp'):
362 if not os.path.exists(basedir):
364 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
365 if package is None or metafile == os.path.join('metadata', package + '.txt'):
367 appinfo = parse_metadata(metafile)
369 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
370 check_metadata(appinfo)
374 # Parse all descriptions at load time, just to ensure cross-referencing
375 # errors are caught early rather than when they hit the build server.
378 if app['id'] == link:
379 return ("fdroid.app:" + link, "Dummy name - don't know yet")
380 raise MetaDataException("Cannot resolve app id " + link)
383 description_html(app['Description'], linkres)
385 raise MetaDataException("Problem with description of " + app['id'] +
390 # Get the type expected for a given metadata field.
391 def metafieldtype(name):
392 if name in ['Description', 'Maintainer Notes']:
394 if name == 'Build Version':
398 if name == 'Use Built':
402 # Parse metadata for a single application.
404 # 'metafile' - the filename to read. The package id for the application comes
405 # from this filename. Pass None to get a blank entry.
407 # Returns a dictionary containing all the details of the application. There are
408 # two major kinds of information in the dictionary. Keys beginning with capital
409 # letters correspond directory to identically named keys in the metadata file.
410 # Keys beginning with lower case letters are generated in one way or another,
411 # and are not found verbatim in the metadata.
413 # Known keys not originating from the metadata are:
415 # 'id' - the application's package ID
416 # 'builds' - a list of dictionaries containing build information
417 # for each defined build
418 # 'comments' - a list of comments from the metadata file. Each is
419 # a tuple of the form (field, comment) where field is
420 # the name of the field it preceded in the metadata
421 # file. Where field is None, the comment goes at the
422 # end of the file. Alternatively, 'build:version' is
423 # for a comment before a particular build version.
424 # 'descriptionlines' - original lines of description as formatted in the
427 def parse_metadata(metafile):
429 def parse_buildline(lines):
430 value = "".join(lines)
431 parts = [p.replace("\\,", ",")
432 for p in re.split(r"(?<!\\),", value)]
434 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
436 thisbuild['origlines'] = lines
437 thisbuild['version'] = parts[0]
438 thisbuild['vercode'] = parts[1]
439 if parts[2].startswith('!'):
440 # For backwards compatibility, handle old-style disabling,
441 # including attempting to extract the commit from the message
442 thisbuild['disable'] = parts[2][1:]
443 commit = 'unknown - see disabled'
444 index = parts[2].rfind('at ')
446 commit = parts[2][index+3:]
447 if commit.endswith(')'):
449 thisbuild['commit'] = commit
451 thisbuild['commit'] = parts[2]
453 pk, pv = p.split('=', 1)
454 thisbuild[pk.strip()] = pv
458 def add_comments(key):
461 for comment in curcomments:
462 thisinfo['comments'].append((key, comment))
468 if not isinstance(metafile, file):
469 metafile = open(metafile, "r")
470 thisinfo['id'] = metafile.name[9:-4]
472 thisinfo['id'] = None
474 # Defaults for fields that come from metadata...
475 thisinfo['Name'] = None
476 thisinfo['Auto Name'] = ''
477 thisinfo['Categories'] = 'None'
478 thisinfo['Description'] = []
479 thisinfo['Summary'] = ''
480 thisinfo['License'] = 'Unknown'
481 thisinfo['Web Site'] = ''
482 thisinfo['Source Code'] = ''
483 thisinfo['Issue Tracker'] = ''
484 thisinfo['Donate'] = None
485 thisinfo['FlattrID'] = None
486 thisinfo['Bitcoin'] = None
487 thisinfo['Litecoin'] = None
488 thisinfo['Disabled'] = None
489 thisinfo['AntiFeatures'] = None
490 thisinfo['Archive Policy'] = None
491 thisinfo['Update Check Mode'] = 'None'
492 thisinfo['Vercode Operation'] = None
493 thisinfo['Auto Update Mode'] = 'None'
494 thisinfo['Current Version'] = ''
495 thisinfo['Current Version Code'] = '0'
496 thisinfo['Repo Type'] = ''
497 thisinfo['Repo'] = ''
498 thisinfo['Requires Root'] = False
499 thisinfo['No Source Since'] = ''
501 # General defaults...
502 thisinfo['builds'] = []
503 thisinfo['comments'] = []
513 for line in metafile:
514 line = line.rstrip('\r\n')
516 if not any(line.startswith(s) for s in (' ', '\t')):
517 if 'commit' not in curbuild and 'disable' not in curbuild:
518 raise MetaDataException("No commit specified for {0} in {1}".format(
519 curbuild['version'], metafile.name))
520 thisinfo['builds'].append(curbuild)
521 add_comments('build:' + curbuild['version'])
524 if line.endswith('\\'):
525 buildlines.append(line[:-1].lstrip())
527 buildlines.append(line.lstrip())
528 bl = ''.join(buildlines)
529 bv = bl.split('=', 1)
531 raise MetaDataException("Invalid build flag at {0} in {1}".
532 format(buildlines[0], metafile.name))
535 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
536 format(name, curbuild['version'], metafile.name))
537 curbuild[name] = val.lstrip()
543 if line.startswith("#"):
544 curcomments.append(line)
546 index = line.find(':')
548 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
550 value = line[index+1:]
552 # Translate obsolete fields...
553 if field == 'Market Version':
554 field = 'Current Version'
555 if field == 'Market Version Code':
556 field = 'Current Version Code'
558 fieldtype = metafieldtype(field)
559 if fieldtype not in ['build', 'buildv2']:
561 if fieldtype == 'multiline':
565 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
566 elif fieldtype == 'string':
567 if field == 'Category' and thisinfo['Categories'] == 'None':
568 thisinfo['Categories'] = value.replace(';',',')
569 thisinfo[field] = value
570 elif fieldtype == 'build':
571 if value.endswith("\\"):
573 buildlines = [value[:-1]]
575 thisinfo['builds'].append(parse_buildline([value]))
576 add_comments('build:' + thisinfo['builds'][-1]['version'])
577 elif fieldtype == 'buildv2':
579 vv = value.split(',')
581 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
582 format(value, metafile.name))
583 curbuild['version'] = vv[0]
584 curbuild['vercode'] = vv[1]
587 elif fieldtype == 'obsolete':
588 pass # Just throw it away!
590 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
591 elif mode == 1: # Multiline field
595 thisinfo[field].append(line)
596 elif mode == 2: # Line continuation mode in Build Version
597 if line.endswith("\\"):
598 buildlines.append(line[:-1])
600 buildlines.append(line)
601 thisinfo['builds'].append(
602 parse_buildline(buildlines))
603 add_comments('build:' + thisinfo['builds'][-1]['version'])
607 # Mode at end of file should always be 0...
609 raise MetaDataException(field + " not terminated in " + metafile.name)
611 raise MetaDataException("Unterminated continuation in " + metafile.name)
613 raise MetaDataException("Unterminated build in " + metafile.name)
615 if not thisinfo['Description']:
616 thisinfo['Description'].append('No description available')
620 # Write a metadata file.
622 # 'dest' - The path to the output file
623 # 'app' - The app data
624 def write_metadata(dest, app):
626 def writecomments(key):
628 for pf, comment in app['comments']:
630 mf.write("%s\n" % comment)
632 #if options.verbose and written > 0:
633 #print "...writing comments for " + (key if key else 'EOF')
635 def writefield(field, value=None):
639 mf.write("%s:%s\n" % (field, value))
643 writefield('Disabled')
644 if app['AntiFeatures']:
645 writefield('AntiFeatures')
646 writefield('Categories')
647 writefield('License')
648 writefield('Web Site')
649 writefield('Source Code')
650 writefield('Issue Tracker')
654 writefield('FlattrID')
656 writefield('Bitcoin')
658 writefield('Litecoin')
663 writefield('Auto Name')
664 writefield('Summary')
665 writefield('Description', '')
666 for line in app['Description']:
667 mf.write("%s\n" % line)
670 if app['Requires Root']:
671 writefield('Requires Root', 'Yes')
674 writefield('Repo Type')
677 for build in app['builds']:
678 writecomments('build:' + build['version'])
679 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
681 # This defines the preferred order for the build items - as in the
682 # manual, they're roughly in order of application.
683 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
684 'gradle', 'maven', 'oldsdkloc', 'target', 'compilesdk',
685 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
686 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
687 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
688 'preassemble', 'bindir', 'antcommand', 'novcheck']
690 def write_builditem(key, value):
691 if key in ['version', 'vercode', 'origlines']:
693 if key in valuetypes['bool'].attrs:
698 #print "...writing {0} : {1}".format(key, value)
699 outline = ' %s=' % key
700 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
706 write_builditem(key, build[key])
707 for key, value in build.iteritems():
708 if not key in keyorder:
709 write_builditem(key, value)
712 if 'Maintainer Notes' in app:
713 writefield('Maintainer Notes', '')
714 for line in app['Maintainer Notes']:
715 mf.write("%s\n" % line)
720 if app['Archive Policy']:
721 writefield('Archive Policy')
722 writefield('Auto Update Mode')
723 writefield('Update Check Mode')
724 if app['Vercode Operation']:
725 writefield('Vercode Operation')
726 if 'Update Check Data' in app:
727 writefield('Update Check Data')
728 if app['Current Version']:
729 writefield('Current Version')
730 writefield('Current Version Code')
732 if app['No Source Since']:
733 writefield('No Source Since')