1 # -*- coding: utf-8 -*-
3 # metadata.py - part of the FDroid server tools
4 # Copyright (C) 2013, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
28 class MetaDataException(Exception):
29 def __init__(self, value):
39 'Categories': ['None'],
53 'Archive Policy': None,
54 'Update Check Mode': 'None',
55 'Update Check Ignore': None,
56 'Update Check Name': None,
57 'Update Check Data': None,
58 'Vercode Operation': None,
59 'Auto Update Mode': 'None',
60 'Current Version': '',
61 'Current Version Code': '0',
64 'Requires Root': False,
69 # This defines the preferred order for the build items - as in the
70 # manual, they're roughly in order of application.
72 'disable', 'commit', 'subdir', 'submodules', 'init',
73 'gradle', 'maven', 'kivy', 'output', 'oldsdkloc', 'target',
74 'update', 'encoding', 'forceversion', 'forcevercode', 'rm',
75 'extlibs', 'srclibs', 'patch', 'prebuild', 'scanignore',
76 'scandelete', 'build', 'buildjni', 'preassemble', 'bindir',
77 'antcommand', 'novcheck'
81 # Designates a metadata field type and checks that it matches
83 # 'name' - The long name of the field type
84 # 'matching' - List of possible values or regex expression
85 # 'sep' - Separator to use if value may be a list
86 # 'fields' - Metadata fields (Field:Value) of this type
87 # 'attrs' - Build attributes (attr=value) of this type
90 def __init__(self, name, matching, sep, fields, attrs):
92 self.matching = matching
93 if type(matching) is str:
94 self.compiled = re.compile(matching)
99 def _assert_regex(self, values, appid):
101 if not self.compiled.match(v):
102 raise MetaDataException("'%s' is not a valid %s in %s. "
103 % (v, self.name, appid) +
104 "Regex pattern: %s" % (self.matching))
106 def _assert_list(self, values, appid):
108 if v not in self.matching:
109 raise MetaDataException("'%s' is not a valid %s in %s. "
110 % (v, self.name, appid) +
111 "Possible values: %s" % (", ".join(self.matching)))
113 def check(self, value, appid):
114 if type(value) is not str or not value:
116 if self.sep is not None:
117 values = value.split(self.sep)
120 if type(self.matching) is list:
121 self._assert_list(values, appid)
123 self._assert_regex(values, appid)
126 # Generic value types
128 'int': FieldType("Integer",
129 r'^[1-9][0-9]*$', None,
133 'http': FieldType("HTTP link",
134 r'^http[s]?://', None,
135 ["Web Site", "Source Code", "Issue Tracker", "Donate"], []),
137 'bitcoin': FieldType("Bitcoin address",
138 r'^[a-zA-Z0-9]{27,34}$', None,
142 'litecoin': FieldType("Litecoin address",
143 r'^L[a-zA-Z0-9]{33}$', None,
147 'dogecoin': FieldType("Dogecoin address",
148 r'^D[a-zA-Z0-9]{33}$', None,
152 'Bool': FieldType("Boolean",
157 'bool': FieldType("Boolean",
160 ['submodules', 'oldsdkloc', 'forceversion', 'forcevercode',
163 'Repo Type': FieldType("Repo Type",
164 ['git', 'git-svn', 'svn', 'hg', 'bzr', 'srclib'], None,
168 'archive': FieldType("Archive Policy",
169 r'^[0-9]+ versions$', None,
173 'antifeatures': FieldType("Anti-Feature",
174 ["Ads", "Tracking", "NonFreeNet", "NonFreeDep", "NonFreeAdd", "UpstreamNonFree"], ',',
178 'autoupdatemodes': FieldType("Auto Update Mode",
179 r"^(Version .+|None)$", None,
180 ["Auto Update Mode"],
183 'updatecheckmodes': FieldType("Update Check Mode",
184 r"^(Tags|Tags .+|RepoManifest|RepoManifest/.+|RepoTrunk|HTTP|Static|None)$", None,
185 ["Update Check Mode"],
190 # Check an app's metadata information for integrity errors
191 def check_metadata(info):
192 for k, t in valuetypes.iteritems():
193 for field in t.fields:
195 t.check(info[field], info['id'])
197 info[field] = info[field] == "Yes"
198 for build in info['builds']:
201 t.check(build[attr], info['id'])
203 build[attr] = build[attr] == "yes"
208 # Formatter for descriptions. Create an instance, and call parseline() with
209 # each line of the description source from the metadata. At the end, call
210 # end() and then text_plain, text_wiki and text_html will contain the result.
211 class DescriptionFormatter:
224 def __init__(self, linkres):
225 self.linkResolver = linkres
227 def endcur(self, notstates=None):
228 if notstates and self.state in notstates:
230 if self.state == self.stPARA:
232 elif self.state == self.stUL:
234 elif self.state == self.stOL:
238 self.text_plain += '\n'
239 self.text_html += '</p>'
240 self.state = self.stNONE
243 self.text_html += '</ul>'
244 self.state = self.stNONE
247 self.text_html += '</ol>'
248 self.state = self.stNONE
250 def formatted(self, txt, html):
253 txt = cgi.escape(txt)
255 index = txt.find("''")
257 return formatted + txt
258 formatted += txt[:index]
260 if txt.startswith("'''"):
266 self.bold = not self.bold
274 self.ital = not self.ital
277 def linkify(self, txt):
281 index = txt.find("[")
283 return (linkified_plain + self.formatted(txt, False), linkified_html + self.formatted(txt, True))
284 linkified_plain += self.formatted(txt[:index], False)
285 linkified_html += self.formatted(txt[:index], True)
287 if txt.startswith("[["):
288 index = txt.find("]]")
290 raise MetaDataException("Unterminated ]]")
292 if self.linkResolver:
293 url, urltext = self.linkResolver(url)
296 linkified_html += '<a href="' + url + '">' + cgi.escape(urltext) + '</a>'
297 linkified_plain += urltext
298 txt = txt[index + 2:]
300 index = txt.find("]")
302 raise MetaDataException("Unterminated ]")
304 index2 = url.find(' ')
308 urltxt = url[index2 + 1:]
310 linkified_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
311 linkified_plain += urltxt
313 linkified_plain += ' (' + url + ')'
314 txt = txt[index + 1:]
316 def addtext(self, txt):
317 p, h = self.linkify(txt)
321 def parseline(self, line):
322 self.text_wiki += "%s\n" % line
325 elif line.startswith('* '):
326 self.endcur([self.stUL])
327 if self.state != self.stUL:
328 self.text_html += '<ul>'
329 self.state = self.stUL
330 self.text_html += '<li>'
331 self.text_plain += '* '
332 self.addtext(line[1:])
333 self.text_html += '</li>'
334 elif line.startswith('# '):
335 self.endcur([self.stOL])
336 if self.state != self.stOL:
337 self.text_html += '<ol>'
338 self.state = self.stOL
339 self.text_html += '<li>'
340 self.text_plain += '* ' # TODO: lazy - put the numbers in!
341 self.addtext(line[1:])
342 self.text_html += '</li>'
344 self.endcur([self.stPARA])
345 if self.state == self.stNONE:
346 self.text_html += '<p>'
347 self.state = self.stPARA
348 elif self.state == self.stPARA:
349 self.text_html += ' '
350 self.text_plain += ' '
357 # Parse multiple lines of description as written in a metadata file, returning
358 # a single string in plain text format.
359 def description_plain(lines, linkres):
360 ps = DescriptionFormatter(linkres)
367 # Parse multiple lines of description as written in a metadata file, returning
368 # a single string in wiki format. Used for the Maintainer Notes field as well,
369 # because it's the same format.
370 def description_wiki(lines):
371 ps = DescriptionFormatter(None)
378 # Parse multiple lines of description as written in a metadata file, returning
379 # a single string in HTML format.
380 def description_html(lines, linkres):
381 ps = DescriptionFormatter(linkres)
388 def parse_srclib(metafile):
391 if metafile and not isinstance(metafile, file):
392 metafile = open(metafile, "r")
394 # Defaults for fields that come from metadata
395 thisinfo['Repo Type'] = ''
396 thisinfo['Repo'] = ''
397 thisinfo['Subdir'] = None
398 thisinfo['Prepare'] = None
399 thisinfo['Srclibs'] = None
405 for line in metafile:
407 line = line.rstrip('\r\n')
408 if not line or line.startswith("#"):
412 field, value = line.split(':', 1)
414 raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
416 if field == "Subdir":
417 thisinfo[field] = value.split(',')
419 thisinfo[field] = value
425 """Read all srclib metadata.
427 The information read will be accessible as metadata.srclibs, which is a
428 dictionary, keyed on srclib name, with the values each being a dictionary
429 in the same format as that returned by the parse_srclib function.
431 A MetaDataException is raised if there are any problems with the srclib
438 if not os.path.exists(srcdir):
441 for metafile in sorted(glob.glob(os.path.join(srcdir, '*.txt'))):
442 srclibname = os.path.basename(metafile[:-4])
443 srclibs[srclibname] = parse_srclib(metafile)
446 # Read all metadata. Returns a list of 'app' objects (which are dictionaries as
447 # returned by the parse_metadata function.
448 def read_metadata(xref=True):
451 for basedir in ('metadata', 'tmp'):
452 if not os.path.exists(basedir):
455 for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))):
456 appinfo = parse_metadata(metafile)
457 check_metadata(appinfo)
461 # Parse all descriptions at load time, just to ensure cross-referencing
462 # errors are caught early rather than when they hit the build server.
465 if app['id'] == link:
466 return ("fdroid.app:" + link, "Dummy name - don't know yet")
467 raise MetaDataException("Cannot resolve app id " + link)
470 description_html(app['Description'], linkres)
472 raise MetaDataException("Problem with description of " + app['id'] +
478 # Get the type expected for a given metadata field.
479 def metafieldtype(name):
480 if name in ['Description', 'Maintainer Notes']:
482 if name in ['Categories']:
484 if name == 'Build Version':
488 if name == 'Use Built':
490 if name not in app_defaults:
496 if name in ['extlibs', 'srclibs', 'patch', 'rm', 'buildjni',
497 'update', 'scanignore', 'scandelete']:
499 if name in ['init', 'prebuild', 'build']:
504 # Parse metadata for a single application.
506 # 'metafile' - the filename to read. The package id for the application comes
507 # from this filename. Pass None to get a blank entry.
509 # Returns a dictionary containing all the details of the application. There are
510 # two major kinds of information in the dictionary. Keys beginning with capital
511 # letters correspond directory to identically named keys in the metadata file.
512 # Keys beginning with lower case letters are generated in one way or another,
513 # and are not found verbatim in the metadata.
515 # Known keys not originating from the metadata are:
517 # 'id' - the application's package ID
518 # 'builds' - a list of dictionaries containing build information
519 # for each defined build
520 # 'comments' - a list of comments from the metadata file. Each is
521 # a tuple of the form (field, comment) where field is
522 # the name of the field it preceded in the metadata
523 # file. Where field is None, the comment goes at the
524 # end of the file. Alternatively, 'build:version' is
525 # for a comment before a particular build version.
526 # 'descriptionlines' - original lines of description as formatted in the
529 def parse_metadata(metafile):
533 def add_buildflag(p, thisbuild):
536 raise MetaDataException("Invalid build flag at {0} in {1}"
537 .format(buildlines[0], linedesc))
540 raise MetaDataException("Duplicate definition on {0} in version {1} of {2}"
541 .format(pk, thisbuild['version'], linedesc))
544 if pk not in ordered_flags:
545 raise MetaDataException("Unrecognised build flag at {0} in {1}"
546 .format(p, linedesc))
549 # Port legacy ';' separators
550 thisbuild[pk] = [v.strip() for v in pv.replace(';', ',').split(',')]
556 raise MetaDataException("Unrecognised build flag type '%s' at %s in %s"
559 def parse_buildline(lines):
560 value = "".join(lines)
561 parts = [p.replace("\\,", ",")
562 for p in re.split(r"(?<!\\),", value)]
564 raise MetaDataException("Invalid build format: " + value + " in " + metafile.name)
566 thisbuild['origlines'] = lines
567 thisbuild['version'] = parts[0]
568 thisbuild['vercode'] = parts[1]
569 if parts[2].startswith('!'):
570 # For backwards compatibility, handle old-style disabling,
571 # including attempting to extract the commit from the message
572 thisbuild['disable'] = parts[2][1:]
573 commit = 'unknown - see disabled'
574 index = parts[2].rfind('at ')
576 commit = parts[2][index + 3:]
577 if commit.endswith(')'):
579 thisbuild['commit'] = commit
581 thisbuild['commit'] = parts[2]
583 add_buildflag(p, thisbuild)
587 def add_comments(key):
590 for comment in curcomments:
591 thisinfo['comments'].append((key, comment))
594 def get_build_type(build):
595 for t in ['maven', 'gradle', 'kivy']:
596 if build.get(t, 'no') != 'no':
598 if 'output' in build:
604 if not isinstance(metafile, file):
605 metafile = open(metafile, "r")
606 thisinfo['id'] = metafile.name[9:-4]
608 thisinfo['id'] = None
610 thisinfo.update(app_defaults)
612 # General defaults...
613 thisinfo['builds'] = []
614 thisinfo['comments'] = []
625 for line in metafile:
627 linedesc = "%s:%d" % (metafile.name, c)
628 line = line.rstrip('\r\n')
630 if not any(line.startswith(s) for s in (' ', '\t')):
631 if 'commit' not in curbuild and 'disable' not in curbuild:
632 raise MetaDataException("No commit specified for {0} in {1}"
633 .format(curbuild['version'], linedesc))
634 thisinfo['builds'].append(curbuild)
635 add_comments('build:' + curbuild['version'])
638 if line.endswith('\\'):
639 buildlines.append(line[:-1].lstrip())
641 buildlines.append(line.lstrip())
642 bl = ''.join(buildlines)
643 add_buildflag(bl, curbuild)
649 if line.startswith("#"):
650 curcomments.append(line)
653 field, value = line.split(':', 1)
655 raise MetaDataException("Invalid metadata in " + linedesc)
656 if field != field.strip() or value != value.strip():
657 raise MetaDataException("Extra spacing found in " + linedesc)
659 # Translate obsolete fields...
660 if field == 'Market Version':
661 field = 'Current Version'
662 if field == 'Market Version Code':
663 field = 'Current Version Code'
665 fieldtype = metafieldtype(field)
666 if fieldtype not in ['build', 'buildv2']:
668 if fieldtype == 'multiline':
672 raise MetaDataException("Unexpected text on same line as " + field + " in " + linedesc)
673 elif fieldtype == 'string':
674 thisinfo[field] = value
675 elif fieldtype == 'list':
676 thisinfo[field] = [v.strip() for v in value.replace(';', ',').split(',')]
677 elif fieldtype == 'build':
678 if value.endswith("\\"):
680 buildlines = [value[:-1]]
682 thisinfo['builds'].append(parse_buildline([value]))
683 add_comments('build:' + thisinfo['builds'][-1]['version'])
684 elif fieldtype == 'buildv2':
686 vv = value.split(',')
688 raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
689 .format(value, linedesc))
690 curbuild['version'] = vv[0]
691 curbuild['vercode'] = vv[1]
694 elif fieldtype == 'obsolete':
695 pass # Just throw it away!
697 raise MetaDataException("Unrecognised field type for " + field + " in " + linedesc)
698 elif mode == 1: # Multiline field
702 thisinfo[field].append(line)
703 elif mode == 2: # Line continuation mode in Build Version
704 if line.endswith("\\"):
705 buildlines.append(line[:-1])
707 buildlines.append(line)
708 thisinfo['builds'].append(
709 parse_buildline(buildlines))
710 add_comments('build:' + thisinfo['builds'][-1]['version'])
714 # Mode at end of file should always be 0...
716 raise MetaDataException(field + " not terminated in " + metafile.name)
718 raise MetaDataException("Unterminated continuation in " + metafile.name)
720 raise MetaDataException("Unterminated build in " + metafile.name)
722 if not thisinfo['Description']:
723 thisinfo['Description'].append('No description available')
725 for build in thisinfo['builds']:
726 build['type'] = get_build_type(build)
731 # Write a metadata file.
733 # 'dest' - The path to the output file
734 # 'app' - The app data
735 def write_metadata(dest, app):
737 def writecomments(key):
739 for pf, comment in app['comments']:
741 mf.write("%s\n" % comment)
744 logging.debug("...writing comments for " + (key if key else 'EOF'))
746 def writefield(field, value=None):
750 t = metafieldtype(field)
752 value = ','.join(value)
753 mf.write("%s:%s\n" % (field, value))
757 writefield('Disabled')
758 if app['AntiFeatures']:
759 writefield('AntiFeatures')
761 writefield('Provides')
762 writefield('Categories')
763 writefield('License')
764 writefield('Web Site')
765 writefield('Source Code')
766 writefield('Issue Tracker')
770 writefield('FlattrID')
772 writefield('Bitcoin')
774 writefield('Litecoin')
776 writefield('Dogecoin')
781 writefield('Auto Name')
782 writefield('Summary')
783 writefield('Description', '')
784 for line in app['Description']:
785 mf.write("%s\n" % line)
788 if app['Requires Root']:
789 writefield('Requires Root', 'Yes')
792 writefield('Repo Type')
795 for build in app['builds']:
796 writecomments('build:' + build['version'])
797 mf.write("Build:%s,%s\n" % (build['version'], build['vercode']))
799 def write_builditem(key, value):
800 if key in ['version', 'vercode', 'origlines', 'type']:
802 if key in valuetypes['bool'].attrs:
807 logging.debug("...writing {0} : {1}".format(key, value))
808 outline = ' %s=' % key
812 outline += '&& \\\n '.join([s.lstrip() for s in value.split('&& ')])
814 outline += ','.join(value) if type(value) == list else value
818 for key in ordered_flags:
820 write_builditem(key, build[key])
823 if 'Maintainer Notes' in app:
824 writefield('Maintainer Notes', '')
825 for line in app['Maintainer Notes']:
826 mf.write("%s\n" % line)
830 if app['Archive Policy']:
831 writefield('Archive Policy')
832 writefield('Auto Update Mode')
833 writefield('Update Check Mode')
834 if app['Update Check Ignore']:
835 writefield('Update Check Ignore')
836 if app['Vercode Operation']:
837 writefield('Vercode Operation')
838 if app['Update Check Data']:
839 writefield('Update Check Data')
840 if app['Current Version']:
841 writefield('Current Version')
842 writefield('Current Version Code')
844 if app['No Source Since']:
845 writefield('No Source Since')