1 # -*- coding: utf-8 -*-
3 # common.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
24 class MetaDataException(Exception):
25 def __init__(self, value):
29 return repr(self.value)
31 # Designates a metadata field type and checks that it matches
33 # 'name' - The long name of the field type
34 # 'matching' - List of possible values or regex expression
35 # 'sep' - Separator to use if value may be a list
36 # 'fields' - Metadata fields (Field:Value) of this type
37 # 'attrs' - Build attributes (attr=value) of this type
40 def __init__(self, name, matching, sep, fields, attrs):
42 self.matching = matching
43 if type(matching) is str:
44 self.compiled = re.compile(matching)
49 def _assert_regex(self, values, appid):
51 if not self.compiled.match(v):
52 raise MetaDataException("'%s' is not a valid %s in %s. "
53 % (v, self.name, appid) +
54 "Regex pattern: %s" % (self.matching))
56 def _assert_list(self, values, appid):
58 if v not in self.matching:
59 raise MetaDataException("'%s' is not a valid %s in %s. "
60 % (v, self.name, appid) +
61 "Possible values: %s" % (", ".join(self.matching)))
63 def check(self, value, appid):
64 if type(value) is not str or not value:
66 if self.sep is not None:
67 values = value.split(self.sep)
70 if type(self.matching) is list:
71 self._assert_list(values, appid)
73 self._assert_regex(values, appid)
78 'int' : FieldType("Integer",
79 r'^[1-9][0-9]*$', None,
83 'http' : FieldType("HTTP link",
84 r'^http[s]?://', None,
85 [ "Web Site", "Source Code", "Issue Tracker", "Donate" ], []),
87 'bitcoin' : FieldType("Bitcoin address",
88 r'^[a-zA-Z0-9]{27,34}$', None,
92 'litecoin' : FieldType("Litecoin address",
93 r'^L[a-zA-Z0-9]{33}$', None,
97 'dogecoin' : FieldType("Dogecoin address",
98 r'^D[a-zA-Z0-9]{33}$', None,
102 'Bool' : FieldType("Boolean",
107 'bool' : FieldType("Boolean",
110 [ 'submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
111 'fixtrans', 'fixapos', 'novcheck' ]),
113 'Repo Type' : FieldType("Repo Type",
114 [ 'git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib' ], None,
118 'archive' : FieldType("Archive Policy",
119 r'^[0-9]+ versions$', None,
120 [ "Archive Policy" ],
123 'antifeatures' : FieldType("Anti-Feature",
124 [ "Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree" ], ',',
128 'autoupdatemodes' : FieldType("Auto Update Mode",
129 r"^(Version .+|None)$", None,
130 [ "Auto Update Mode" ],
133 'updatecheckmodes' : FieldType("Update Check Mode",
134 r"^(Tags|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
135 [ "Update Check Mode" ],
139 # Check an app's metadata information for integrity errors
140 def check_metadata(info):
141 for k, t in valuetypes.iteritems():
142 for field in t.fields:
144 t.check(info[field], info['id'])
146 info[field] = info[field] == "Yes"
147 for build in info['builds']:
150 t.check(build[attr], info['id'])
152 build[attr] = build[attr] == "yes"
156 # Formatter for descriptions. Create an instance, and call parseline() with
157 # each line of the description source from the metadata. At the end, call
158 # end() and then text_plain, text_wiki and text_html will contain the result.
159 class DescriptionFormatter:
171 def __init__(self, linkres):
172 self.linkResolver = linkres
173 def endcur(self, notstates=None):
174 if notstates and self.state in notstates:
176 if self.state == self.stPARA:
178 elif self.state == self.stUL:
180 elif self.state == self.stOL:
183 self.text_plain += '\n'
184 self.text_html += '</p>'
185 self.state = self.stNONE
187 self.text_html += '</ul>'
188 self.state = self.stNONE
190 self.text_html += '</ol>'
191 self.state = self.stNONE
193 def formatted(self, txt, html):
196 txt = cgi.escape(txt)
198 index = txt.find("''")
200 return formatted + txt
201 formatted += txt[:index]
203 if txt.startswith("'''"):
209 self.bold = not self.bold
217 self.ital = not self.ital
221 def linkify(self, txt):
225 index = txt.find("[")
227 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
228 linkified_plain += self.formatted(txt[:index], False)
229 linkified_html += self.formatted(txt[:index], True)
231 if txt.startswith("[["):
232 index = txt.find("]]")
234 raise MetaDataException("Unterminated ]]")
236 if self.linkResolver:
237 url, urltext = self.linkResolver(url)
240 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
241 linkified_plain += urltext
244 index = txt.find("]")
246 raise MetaDataException("Unterminated ]")
248 index2 = url.find(' ')
252 urltxt = url[index2 + 1:]
254 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
255 linkified_plain += urltxt
257 linkified_plain += ' (' + url + ')'
260 def addtext(self, txt):
261 p, h = self.linkify(txt)
265 def parseline(self, line):
266 self.text_wiki += "%s\n" % line
269 elif line.startswith('*'):
270 self.endcur([self.stUL])
271 if self.state != self.stUL:
272 self.text_html += '<ul>'
273 self.state = self.stUL
274 self.text_html += '<li>'
275 self.text_plain += '*'
276 self.addtext(line[1:])
277 self.text_html += '</li>'
278 elif line.startswith('#'):
279 self.endcur([self.stOL])
280 if self.state != self.stOL:
281 self.text_html += '<ol>'
282 self.state = self.stOL
283 self.text_html += '<li>'
284 self.text_plain += '*' #TODO: lazy - put the numbers in!
285 self.addtext(line[1:])
286 self.text_html += '</li>'
288 self.endcur([self.stPARA])
289 if self.state == self.stNONE:
290 self.text_html += '<p>'
291 self.state = self.stPARA
292 elif self.state == self.stPARA:
293 self.text_html += ' '
294 self.text_plain += ' '
300 # Parse multiple lines of description as written in a metadata file, returning
301 # a single string in plain text format.
302 def description_plain(lines, linkres):
303 ps = DescriptionFormatter(linkres)
309 # Parse multiple lines of description as written in a metadata file, returning
310 # a single string in wiki format. Used for the Maintainer Notes field as well,
311 # because it's the same format.
312 def description_wiki(lines):
313 ps = DescriptionFormatter(None)
319 # Parse multiple lines of description as written in a metadata file, returning
320 # a single string in HTML format.
321 def description_html(lines,linkres):
322 ps = DescriptionFormatter(linkres)
328 def parse_srclib(metafile, **kw):
331 if metafile and not isinstance(metafile, file):
332 metafile = open(metafile, "r")
334 # Defaults for fields that come from metadata
335 thisinfo['Repo Type'] = ''
336 thisinfo['Repo'] = ''
337 thisinfo['Subdir'] = None
338 thisinfo['Prepare'] = None
339 thisinfo['Srclibs'] = None
340 thisinfo['Update Project'] = None
345 for line in metafile:
346 line = line.rstrip('\r\n')
347 if not line or line.startswith("#"):
351 field, value = line.split(':',1)
353 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
355 if field == "Subdir":
356 thisinfo[field] = value.split(',')
358 thisinfo[field] = value
362 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
363 # returned by the parse_metadata function.
364 def read_metadata(xref=True, package=None):
366 for basedir in ('metadata', 'tmp'):
367 if not os.path.exists(basedir):
369 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
370 if package is None or metafile == os.path.join('metadata', package + '.txt'):
372 appinfo = parse_metadata(metafile)
374 raise MetaDataException("Problem reading metadata file %s: - %s" % (metafile, str(e)))
375 check_metadata(appinfo)
379 # Parse all descriptions at load time, just to ensure cross-referencing
380 # errors are caught early rather than when they hit the build server.
383 if app['id'] == link:
384 return ("fdroid.app:" + link, "Dummy name - don't know yet")
385 raise MetaDataException("Cannot resolve app id " + link)
388 description_html(app['Description'], linkres)
390 raise MetaDataException("Problem with description of " + app['id'] +
395 # Get the type expected for a given metadata field.
396 def metafieldtype(name):
397 if name in ['Description', 'Maintainer Notes']:
399 if name == 'Build Version':
403 if name == 'Use Built':
407 # Parse metadata for a single application.
409 # 'metafile' - the filename to read. The package id for the application comes
410 # from this filename. Pass None to get a blank entry.
412 # Returns a dictionary containing all the details of the application. There are
413 # two major kinds of information in the dictionary. Keys beginning with capital
414 # letters correspond directory to identically named keys in the metadata file.
415 # Keys beginning with lower case letters are generated in one way or another,
416 # and are not found verbatim in the metadata.
418 # Known keys not originating from the metadata are:
420 # 'id' - the application's package ID
421 # 'builds' - a list of dictionaries containing build information
422 # for each defined build
423 # 'comments' - a list of comments from the metadata file. Each is
424 # a tuple of the form (field, comment) where field is
425 # the name of the field it preceded in the metadata
426 # file. Where field is None, the comment goes at the
427 # end of the file. Alternatively, 'build:version' is
428 # for a comment before a particular build version.
429 # 'descriptionlines' - original lines of description as formatted in the
432 def parse_metadata(metafile):
434 def parse_buildline(lines):
435 value = "".join(lines)
436 parts = [p.replace("\\,", ",")
437 for p in re.split(r"(?<!\\),", value)]
439 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
441 thisbuild['origlines'] = lines
442 thisbuild['version'] = parts[0]
443 thisbuild['vercode'] = parts[1]
444 if parts[2].startswith('!'):
445 # For backwards compatibility, handle old-style disabling,
446 # including attempting to extract the commit from the message
447 thisbuild['disable'] = parts[2][1:]
448 commit = 'unknown - see disabled'
449 index = parts[2].rfind('at ')
451 commit = parts[2][index+3:]
452 if commit.endswith(')'):
454 thisbuild['commit'] = commit
456 thisbuild['commit'] = parts[2]
458 pk, pv = p.split('=', 1)
459 thisbuild[pk.strip()] = pv
463 def add_comments(key):
466 for comment in curcomments:
467 thisinfo['comments'].append((key, comment))
470 def get_build_type(build):
471 for t in ['maven', 'gradle', 'kivy']:
472 if build.get(t, 'no') != 'no':
474 if 'output' in build:
480 if not isinstance(metafile, file):
481 metafile = open(metafile, "r")
482 thisinfo['id'] = metafile.name[9:-4]
484 thisinfo['id'] = None
486 # Defaults for fields that come from metadata...
487 thisinfo['Name'] = None
488 thisinfo['Provides'] = None
489 thisinfo['Auto Name'] = ''
490 thisinfo['Categories'] = 'None'
491 thisinfo['Description'] = []
492 thisinfo['Summary'] = ''
493 thisinfo['License'] = 'Unknown'
494 thisinfo['Web Site'] = ''
495 thisinfo['Source Code'] = ''
496 thisinfo['Issue Tracker'] = ''
497 thisinfo['Donate'] = None
498 thisinfo['FlattrID'] = None
499 thisinfo['Bitcoin'] = None
500 thisinfo['Litecoin'] = None
501 thisinfo['Dogecoin'] = None
502 thisinfo['Disabled'] = None
503 thisinfo['AntiFeatures'] = None
504 thisinfo['Archive Policy'] = None
505 thisinfo['Update Check Mode'] = 'None'
506 thisinfo['Vercode Operation'] = None
507 thisinfo['Auto Update Mode'] = 'None'
508 thisinfo['Current Version'] = ''
509 thisinfo['Current Version Code'] = '0'
510 thisinfo['Repo Type'] = ''
511 thisinfo['Repo'] = ''
512 thisinfo['Requires Root'] = False
513 thisinfo['No Source Since'] = ''
515 # General defaults...
516 thisinfo['builds'] = []
517 thisinfo['comments'] = []
527 for line in metafile:
528 line = line.rstrip('\r\n')
530 if not any(line.startswith(s) for s in (' ', '\t')):
531 if 'commit' not in curbuild and 'disable' not in curbuild:
532 raise MetaDataException("No commit specified for {0} in {1}".format(
533 curbuild['version'], metafile.name))
534 thisinfo['builds'].append(curbuild)
535 add_comments('build:' + curbuild['version'])
538 if line.endswith('\\'):
539 buildlines.append(line[:-1].lstrip())
541 buildlines.append(line.lstrip())
542 bl = ''.join(buildlines)
543 bv = bl.split('=', 1)
545 raise MetaDataException("Invalid build flag at {0} in {1}".
546 format(buildlines[0], metafile.name))
549 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}".
550 format(name, curbuild['version'], metafile.name))
551 curbuild[name] = val.lstrip()
557 if line.startswith("#"):
558 curcomments.append(line)
561 field, value = line.split(':',1)
563 raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line)
565 # Translate obsolete fields...
566 if field == 'Market Version':
567 field = 'Current Version'
568 if field == 'Market Version Code':
569 field = 'Current Version Code'
571 fieldtype = metafieldtype(field)
572 if fieldtype not in ['build', 'buildv2']:
574 if fieldtype == 'multiline':
578 raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name)
579 elif fieldtype == 'string':
580 if field == 'Category' and thisinfo['Categories'] == 'None':
581 thisinfo['Categories'] = value.replace(';',',')
582 thisinfo[field] = value
583 elif fieldtype == 'build':
584 if value.endswith("\\"):
586 buildlines = [value[:-1]]
588 thisinfo['builds'].append(parse_buildline([value]))
589 add_comments('build:' + thisinfo['builds'][-1]['version'])
590 elif fieldtype == 'buildv2':
592 vv = value.split(',')
594 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'.
595 format(value, metafile.name))
596 curbuild['version'] = vv[0]
597 curbuild['vercode'] = vv[1]
600 elif fieldtype == 'obsolete':
601 pass # Just throw it away!
603 raise MetaDataException("Unrecognised field type for " + field + " in " + metafile.name)
604 elif mode == 1: # Multiline field
608 thisinfo[field].append(line)
609 elif mode == 2: # Line continuation mode in Build Version
610 if line.endswith("\\"):
611 buildlines.append(line[:-1])
613 buildlines.append(line)
614 thisinfo['builds'].append(
615 parse_buildline(buildlines))
616 add_comments('build:' + thisinfo['builds'][-1]['version'])
620 # Mode at end of file should always be 0...
622 raise MetaDataException(field + " not terminated in " + metafile.name)
624 raise MetaDataException("Unterminated continuation in " + metafile.name)
626 raise MetaDataException("Unterminated build in " + metafile.name)
628 if not thisinfo['Description']:
629 thisinfo['Description'].append('No description available')
631 for build in thisinfo['builds']:
632 build['type'] = get_build_type(build)
636 # Write a metadata file.
638 # 'dest' - The path to the output file
639 # 'app' - The app data
640 def write_metadata(dest, app):
642 def writecomments(key):
644 for pf, comment in app['comments']:
646 mf.write("%s\n" % comment)
649 logging.debug("...writing comments for " + (key if key else 'EOF'))
651 def writefield(field, value=None):
655 mf.write("%s:%s\n" % (field, value))
659 writefield('Disabled')
660 if app['AntiFeatures']:
661 writefield('AntiFeatures')
663 writefield('Provides')
664 writefield('Categories')
665 writefield('License')
666 writefield('Web Site')
667 writefield('Source Code')
668 writefield('Issue Tracker')
672 writefield('FlattrID')
674 writefield('Bitcoin')
676 writefield('Litecoin')
678 writefield('Dogecoin')
683 writefield('Auto Name')
684 writefield('Summary')
685 writefield('Description', '')
686 for line in app['Description']:
687 mf.write("%s\n" % line)
690 if app['Requires Root']:
691 writefield('Requires Root', 'Yes')
694 writefield('Repo Type')
697 for build in app['builds']:
698 writecomments('build:' + build['version'])
699 mf.write("Build:%s,%s\n" % ( build['version'], build['vercode']))
701 # This defines the preferred order for the build items - as in the
702 # manual, they're roughly in order of application.
703 keyorder = ['disable', 'commit', 'subdir', 'submodules', 'init',
704 'gradle', 'maven', 'output', 'oldsdkloc', 'target',
705 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
706 'fixtrans', 'fixapos', 'extlibs', 'srclibs', 'patch',
707 'prebuild', 'scanignore', 'scandelete', 'build', 'buildjni',
708 'preassemble', 'bindir', 'antcommand', 'novcheck']
710 def write_builditem(key, value):
711 if key in ['version', 'vercode', 'origlines', 'type']:
713 if key in valuetypes['bool'].attrs:
717 logging.debug("...writing {0} : {1}".format(key, value))
718 outline = ' %s=' % key
719 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
725 write_builditem(key, build[key])
726 for key, value in build.iteritems():
727 if not key in keyorder:
728 write_builditem(key, value)
731 if 'Maintainer Notes' in app:
732 writefield('Maintainer Notes', '')
733 for line in app['Maintainer Notes']:
734 mf.write("%s\n" % line)
739 if app['Archive Policy']:
740 writefield('Archive Policy')
741 writefield('Auto Update Mode')
742 writefield('Update Check Mode')
743 if app['Vercode Operation']:
744 writefield('Vercode Operation')
745 if 'Update Check Data' in app:
746 writefield('Update Check Data')
747 if app['Current Version']:
748 writefield('Current Version')
749 writefield('Current Version Code')
751 if app['No Source Since']:
752 writefield('No Source Since')