From ecd3bae855762204d6897e8ed9f6baafe6a2948c Mon Sep 17 00:00:00 2001 From: Ciaran Gultnieks Date: Tue, 10 Jan 2012 23:24:28 +0000 Subject: [PATCH] Metadata is now re-writable --- common.py | 122 +++++++++++++++++++++++++++++++++++++++++-------- rewritemeta.py | 48 +++++++++++++++++++ update.py | 17 +++---- 3 files changed, 159 insertions(+), 28 deletions(-) create mode 100755 rewritemeta.py diff --git a/common.py b/common.py index d1bd6170..cac6a345 100644 --- a/common.py +++ b/common.py @@ -277,10 +277,11 @@ def metafieldtype(name): # 'comments' - a list of comments from the metadata file. Each is # a tuple of the form (field, comment) where field is # the name of the field it preceded in the metadata -# file -# 'name' - the application's name, as displayed. This will -# always be None on return from this parser. The name -# is discovered from built APK files. +# file. Where field is None, the comment goes at the +# end of the file. Alternatively, 'build:version' is +# for a comment before a particular build version. +# 'descriptionlines' - original lines of description as formatted in the +# metadata file. # def parse_metadata(metafile, **kw): @@ -300,6 +301,11 @@ def parse_metadata(metafile, **kw): thisbuild[pk] = pv return thisbuild + def add_comments(key): + for comment in curcomments: + thisinfo['comments'].append((key, comment)) + del curcomments[:] + if not isinstance(metafile, file): metafile = open(metafile, "r") thisinfo = {} @@ -308,7 +314,9 @@ def parse_metadata(metafile, **kw): print "Reading metadata for " + thisinfo['id'] # Defaults for fields that come from metadata... - thisinfo['Description'] = '' + thisinfo['Name'] = None + thisinfo['Category'] = 'None' + thisinfo['Description'] = [] thisinfo['Summary'] = '' thisinfo['License'] = 'Unknown' thisinfo['Web Site'] = '' @@ -325,7 +333,6 @@ def parse_metadata(metafile, **kw): # General defaults... thisinfo['builds'] = [] - thisinfo['name'] = None thisinfo['comments'] = [] mode = 0 @@ -334,23 +341,25 @@ def parse_metadata(metafile, **kw): for line in metafile: line = line.rstrip('\r\n') - if line.startswith("#"): - curcomments.append(line) - elif mode == 0: + if mode == 0: if len(line) == 0: continue + if line.startswith("#"): + curcomments.append(line) + continue index = line.find(':') if index == -1: raise MetaDataException("Invalid metadata in " + metafile.name + " at: " + line) field = line[:index] value = line[index+1:] - for comment in curcomments: - thisinfo['comments'].append((field,comment)) fieldtype = metafieldtype(field) + if fieldtype != 'build': + add_comments(field) if fieldtype == 'multiline': mode = 1 + thisinfo[field] = [] if len(value) > 0: raise MetaDataException("Unexpected text on same line as " + field + " in " + metafile.name) elif fieldtype == 'string': @@ -368,6 +377,7 @@ def parse_metadata(metafile, **kw): buildlines = [value[:-1]] else: thisinfo['builds'].append(parse_buildline([value])) + add_comments('build:' + thisinfo['builds'][-1]['version']) elif fieldtype == 'obsolete': pass # Just throw it away! else: @@ -376,13 +386,7 @@ def parse_metadata(metafile, **kw): if line == '.': mode = 0 else: - if len(line) == 0: - thisinfo[field] += '\n\n' - else: - if (not thisinfo[field].endswith('\n') and - len(thisinfo[field]) > 0): - thisinfo[field] += ' ' - thisinfo[field] += line + thisinfo[field].append(line) elif mode == 2: # Line continuation mode in Build Version if line.endswith("\\"): buildlines.append(line[:-1]) @@ -390,15 +394,18 @@ def parse_metadata(metafile, **kw): buildlines.append(line) thisinfo['builds'].append( parse_buildline(buildlines)) + add_comments('build:' + thisinfo['builds'][-1]['version']) mode = 0 + add_comments(None) + # Mode at end of file should always be 0... if mode == 1: raise MetaDataException(field + " not terminated in " + metafile.name) elif mode == 2: raise MetaDataException("Unterminated continuation in " + metafile.name) if len(thisinfo['Description']) == 0: - thisinfo['Description'] = 'No description available' + thisinfo['Description'].append('No description available') # Ensure all AntiFeatures are recognised... if thisinfo['AntiFeatures']: @@ -414,7 +421,67 @@ def parse_metadata(metafile, **kw): return thisinfo - +# Write a metadata file. +# +# 'dest' - The path to the output file +# 'app' - The app data +def write_metadata(dest, app): + + def writecomments(key): + for pf, comment in app['comments']: + if pf == key: + mf.write(comment + '\n') + + def writefield(field, value=None): + writecomments(field) + if value is None: + value = app[field] + mf.write(field + ':' + value + '\n') + + mf = open(dest, 'w') + if app['Disabled']: + writefield('Disabled') + if app['AntiFeatures']: + writefield('AntiFeatures') + writefield('Category') + writefield('License') + writefield('Web Site') + writefield('Source Code') + writefield('Issue Tracker') + if app['Donate']: + writefield('Donate') + mf.write('\n') + if app['Name']: + writefield('Name') + writefield('Summary') + writefield('Description', '') + for line in app['Description']: + mf.write(line + '\n') + mf.write('.\n') + mf.write('\n') + if app['Requires Root']: + writefield('Requires Root', 'Yes') + mf.write('\n') + if len(app['Repo Type']) > 0: + writefield('Repo Type') + writefield('Repo') + mf.write('\n') + for build in app['builds']: + writecomments('build:' + build['version']) + mf.write('Build Version:') + mf.write('\\\n'.join(build['origlines']) + '\n') + if len(app['builds']) > 0: + mf.write('\n') + if len(app['Market Version']) > 0: + writefield('Market Version') + writefield('Market Version Code') + mf.write('\n') + writecomments(None) + mf.close() + + +# Read all metadata. Returns a list of 'app' objects (which are dictionaries as +# returned by the parse_metadata function. def read_metadata(verbose=False): apps = [] for metafile in sorted(glob.glob(os.path.join('metadata', '*.txt'))): @@ -423,6 +490,21 @@ def read_metadata(verbose=False): apps.append(parse_metadata(metafile, verbose=verbose)) return apps + +# Parse multiple lines of description as written in a metadata file, returning +# a single string. +def parse_description(lines): + text = '' + for line in lines: + if len(line) == 0: + text += '\n\n' + else: + if not text.endswith('\n') and len(text) > 0: + text += ' ' + text += line + return '' + + class BuildException(Exception): def __init__(self, value, stdout = None, stderr = None): self.value = value diff --git a/rewritemeta.py b/rewritemeta.py new file mode 100755 index 00000000..e3535da2 --- /dev/null +++ b/rewritemeta.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# +# rewritemeta.py - part of the FDroid server tools +# Copyright (C) 2010-12, Ciaran Gultnieks, ciaran@ciarang.com +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . + +import sys +import os +import shutil +import re +import urllib +import time +from optparse import OptionParser +import HTMLParser +import common + +#Read configuration... +execfile('config.py') + + +# Parse command line... +parser = OptionParser() +parser.add_option("-v", "--verbose", action="store_true", default=False, + help="Spew out even more information than normal") +(options, args) = parser.parse_args() + +# Get all apps... +apps = common.read_metadata(options.verbose) + +for app in apps: + print "Writing " + app['id'] + common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app) + +print "Finished." + diff --git a/update.py b/update.py index b9740992..738eb528 100755 --- a/update.py +++ b/update.py @@ -193,14 +193,14 @@ for app in apps: bestapk = apk if bestver == 0: - if app['name'] is None: - app['name'] = app['id'] + if app['Name'] is None: + app['Name'] = app['id'] app['icon'] = '' if app['Disabled'] is None: print "WARNING: Application " + app['id'] + " has no packages" else: - if app['name'] is None: - app['name'] = bestapk['name'] + if app['Name'] is None: + app['Name'] = bestapk['name'] app['icon'] = bestapk['icon'] # Generate warnings for apk's with no metadata (or create skeleton @@ -229,7 +229,7 @@ for apk in apks: print " " + apk['name'] + " - " + apk['version'] #Sort the app list by name, then the web site doesn't have to by default: -apps = sorted(apps, key=lambda app: app['name'].upper()) +apps = sorted(apps, key=lambda app: app['Name'].upper()) # Create the index doc = Document() @@ -303,10 +303,11 @@ for app in apps: root.appendChild(apel) addElement('id', app['id'], doc, apel) - addElement('name', app['name'], doc, apel) + addElement('name', app['Name'], doc, apel) addElement('summary', app['Summary'], doc, apel) addElement('icon', app['icon'], doc, apel) - addElement('description', app['Description'], doc, apel) + addElement('description', + common.parse_description(app['Description']), doc, apel) addElement('license', app['License'], doc, apel) if 'Category' in app: addElement('category', app['Category'], doc, apel) @@ -395,7 +396,7 @@ for app in apps: # Output a message of harassment if we don't have the market version: if not gotmarketver and app['Market Version Code'] != '0': addr = app['Source Code'] - print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['name'] + print "WARNING: Don't have market version (" + app['Market Version'] + ") of " + app['Name'] print " (" + app['id'] + ") " + addr warnings += 1 if options.verbose: -- 2.30.2