make metadata exceptions optional based on CLI flag

In many cases, there are times where metadata errors need to be ignored, or
at least not stop the command from running.  For example, there will
inevitably be new metadata fields added, in which case a packaged version
of fdroidserver will throw errors on each one.  This adds a standard -W
flag to customize the response: ignore, default, or error.

* by default, the errors are still errors
* `fdroid readmeta -W` will just print errors
* `fdroid readmeta -Wignore` will not even print errors

https://gitlab.com/fdroid/fdroidserver/issues/150

diff --git a/fdroidserver/build.py b/fdroidserver/build.py
index d02947bdb5aaa4583bfc41ce9cec5c1dc10106ce..daf109dd05182397de52a3f38fe9c94be6789912 100644 (file)
--- a/fdroidserver/build.py
+++ b/fdroidserver/build.py
@@ -981,7 +981,9 @@ def parse_commandline():
                         help="Build all applications available")
     parser.add_argument("-w", "--wiki", default=False, action="store_true",
                         help="Update the wiki")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     # Force --stop with --on-server to get correct exit code
     if options.onserver:

diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py
index e22a6e0f1e1c593343eddcd909f7d4e148921783..b754bf10b6b862804169239051d8e966a16c1812 100644 (file)
--- a/fdroidserver/checkupdates.py
+++ b/fdroidserver/checkupdates.py
@@ -524,7 +524,9 @@ def main():
                         help="Commit changes")
     parser.add_argument("--gplay", action="store_true", default=False,
                         help="Only print differences with the Play Store")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/import.py b/fdroidserver/import.py
index 8c7f5107a5faa9fef7e763b976d5b347207b99a5..97c82f6fd0db5469b91c9a6266c9c626141a0bb6 100644 (file)
--- a/fdroidserver/import.py
+++ b/fdroidserver/import.py
@@ -170,7 +170,9 @@ def main():
                         help="Path to main android project subdirectory, if not in root.")
     parser.add_argument("--rev", default=None,
                         help="Allows a different revision (or git branch) to be specified for the initial import")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/lint.py b/fdroidserver/lint.py
index c805288f2112232ed42371c58e32478314edb75e..41d4b21f43b688955fe59bad5e1b63d2e425bf20 100644 (file)
--- a/fdroidserver/lint.py
+++ b/fdroidserver/lint.py
@@ -377,7 +377,9 @@ def main():
     parser.add_argument("-f", "--format", action="store_true", default=False,
                         help="Also warn about formatting issues, like rewritemeta -l")
     parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/metadata.py b/fdroidserver/metadata.py
index f20b22303ef8291e2bb3db3df073001dc09be614..e03efbeeb4d89d75af9ad5016dc7677ea678897c 100644 (file)
--- a/fdroidserver/metadata.py
+++ b/fdroidserver/metadata.py
@@ -41,6 +41,7 @@ import xml.etree.cElementTree as ElementTree
 import fdroidserver.common
 
 srclibs = None
+warnings_action = None
 
 
 class MetaDataException(Exception):
@@ -51,6 +52,17 @@ class MetaDataException(Exception):
     def __str__(self):
         return self.value
 
+
+def warn_or_exception(value):
+    '''output warning or Exception depending on -W'''
+    if warnings_action == 'ignore':
+        pass
+    elif warnings_action == 'error':
+        raise MetaDataException(value)
+    else:
+        logging.warn(value)
+
+
 # To filter which ones should be written to the metadata files if
 # present
 app_fields = set([
@@ -173,14 +185,14 @@ class App():
     # Gets the value associated to a field name, e.g. 'Auto Name'
     def get_field(self, f):
         if f not in app_fields:
-            raise MetaDataException('Unrecognised app field: ' + f)
+            warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
         return getattr(self, k)
 
     # Sets the value associated to a field name, e.g. 'Auto Name'
     def set_field(self, f, v):
         if f not in app_fields:
-            raise MetaDataException('Unrecognised app field: ' + f)
+            warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
         self.__dict__[k] = v
         self._modified.add(k)
@@ -188,7 +200,7 @@ class App():
     # Appends to the value associated to a field name, e.g. 'Auto Name'
     def append_field(self, f, v):
         if f not in app_fields:
-            raise MetaDataException('Unrecognised app field: ' + f)
+            warn_or_exception('Unrecognised app field: ' + f)
         k = App.field_to_attr(f)
         if k not in self.__dict__:
             self.__dict__[k] = [v]
@@ -307,7 +319,7 @@ class Build():
 
     def get_flag(self, f):
         if f not in build_flags:
-            raise MetaDataException('Unrecognised build flag: ' + f)
+            warn_or_exception('Unrecognised build flag: ' + f)
         return getattr(self, f)
 
     def set_flag(self, f, v):
@@ -316,13 +328,13 @@ class Build():
         if f == 'versionCode':
             f = 'vercode'
         if f not in build_flags:
-            raise MetaDataException('Unrecognised build flag: ' + f)
+            warn_or_exception('Unrecognised build flag: ' + f)
         self.__dict__[f] = v
         self._modified.add(f)
 
     def append_flag(self, f, v):
         if f not in build_flags:
-            raise MetaDataException('Unrecognised build flag: ' + f)
+            warn_or_exception('Unrecognised build flag: ' + f)
         if f not in self.__dict__:
             self.__dict__[f] = [v]
         else:
@@ -414,8 +426,8 @@ class FieldValidator():
             values = [v]
         for v in values:
             if not self.compiled.match(v):
-                raise MetaDataException("'%s' is not a valid %s in %s. Regex pattern: %s"
-                                        % (v, self.name, appid, self.matching))
+                warn_or_exception("'%s' is not a valid %s in %s. Regex pattern: %s"
+                                  % (v, self.name, appid, self.matching))
 
 # Generic value types
 valuetypes = {
@@ -588,7 +600,7 @@ class DescriptionFormatter:
             if txt.startswith("[["):
                 index = txt.find("]]")
                 if index == -1:
-                    raise MetaDataException("Unterminated ]]")
+                    warn_or_exception("Unterminated ]]")
                 url = txt[2:index]
                 if self.linkResolver:
                     url, urltext = self.linkResolver(url)
@@ -600,7 +612,7 @@ class DescriptionFormatter:
             else:
                 index = txt.find("]")
                 if index == -1:
-                    raise MetaDataException("Unterminated ]")
+                    warn_or_exception("Unterminated ]")
                 url = txt[1:index]
                 index2 = url.find(' ')
                 if index2 == -1:
@@ -609,7 +621,7 @@ class DescriptionFormatter:
                     urltxt = url[index2 + 1:]
                     url = url[:index2]
                     if url == urltxt:
-                        raise MetaDataException("Url title is just the URL - use [url]")
+                        warn_or_exception("Url title is just the URL - use [url]")
                 res_html += '<a href="' + url + '">' + cgi.escape(urltxt) + '</a>'
                 res_plain += urltxt
                 if urltxt != url:
@@ -718,7 +730,7 @@ def parse_srclib(metadatapath):
         try:
             f, v = line.split(':', 1)
         except ValueError:
-            raise MetaDataException("Invalid metadata in %s:%d" % (line, n))
+            warn_or_exception("Invalid metadata in %s:%d" % (line, n))
 
         if f == "Subdir":
             thisinfo[f] = v.split(',')
@@ -785,7 +797,7 @@ def read_metadata(xref=True):
                                + glob.glob('.fdroid.yml')):
         packageName, _ = fdroidserver.common.get_extension(os.path.basename(metadatapath))
         if packageName in apps:
-            raise MetaDataException("Found multiple metadata files for " + packageName)
+            warn_or_exception("Found multiple metadata files for " + packageName)
         app = parse_metadata(metadatapath)
         check_metadata(app)
         apps[app.id] = app
@@ -796,14 +808,14 @@ def read_metadata(xref=True):
         def linkres(appid):
             if appid in apps:
                 return ("fdroid.app:" + appid, "Dummy name - don't know yet")
-            raise MetaDataException("Cannot resolve app id " + appid)
+            warn_or_exception("Cannot resolve app id " + appid)
 
         for appid, app in apps.items():
             try:
                 description_html(app.Description, linkres)
             except MetaDataException as e:
-                raise MetaDataException("Problem with description of " + appid +
-                                        " - " + str(e))
+                warn_or_exception("Problem with description of " + appid +
+                                  " - " + str(e))
 
     return apps
 
@@ -845,7 +857,7 @@ def get_default_app_info(metadatapath=None):
                         manifestroot = fdroidserver.common.parse_xml(os.path.join(root, 'AndroidManifest.xml'))
                         break
         if manifestroot is None:
-            raise MetaDataException("Cannot find a packageName for {0}!".format(metadatapath))
+            warn_or_exception("Cannot find a packageName for {0}!".format(metadatapath))
         appid = manifestroot.attrib['package']
 
     app = App()
@@ -930,14 +942,14 @@ def _decode_bool(s):
         return True
     if bool_false.match(s):
         return False
-    raise MetaDataException("Invalid bool '%s'" % s)
+    warn_or_exception("Invalid bool '%s'" % s)
 
 
 def parse_metadata(metadatapath):
     _, ext = fdroidserver.common.get_extension(metadatapath)
     accepted = fdroidserver.common.config['accepted_formats']
     if ext not in accepted:
-        raise MetaDataException('"%s" is not an accepted format, convert to: %s' % (
+        warn_or_exception('"%s" is not an accepted format, convert to: %s' % (
             metadatapath, ', '.join(accepted)))
 
     app = App()
@@ -954,7 +966,7 @@ def parse_metadata(metadatapath):
         elif ext == 'yml':
             parse_yaml_metadata(mf, app)
         else:
-            raise MetaDataException('Unknown metadata format: %s' % metadatapath)
+            warn_or_exception('Unknown metadata format: %s' % metadatapath)
 
     post_metadata_parse(app)
     return app
@@ -979,7 +991,7 @@ def parse_xml_metadata(mf, app):
     root = tree.getroot()
 
     if root.tag != 'resources':
-        raise MetaDataException('resources file does not have root element <resources/>')
+        warn_or_exception('resources file does not have root element <resources/>')
 
     for child in root:
         if child.tag != 'builds':
@@ -1022,12 +1034,12 @@ def parse_txt_metadata(mf, app):
 
     def add_buildflag(p, build):
         if not p.strip():
-            raise MetaDataException("Empty build flag at {1}"
-                                    .format(buildlines[0], linedesc))
+            warn_or_exception("Empty build flag at {1}"
+                              .format(buildlines[0], linedesc))
         bv = p.split('=', 1)
         if len(bv) != 2:
-            raise MetaDataException("Invalid build flag at {0} in {1}"
-                                    .format(buildlines[0], linedesc))
+            warn_or_exception("Invalid build flag at {0} in {1}"
+                              .format(buildlines[0], linedesc))
 
         pk, pv = bv
         pk = pk.lstrip()
@@ -1044,7 +1056,7 @@ def parse_txt_metadata(mf, app):
         v = "".join(lines)
         parts = [p.replace("\\,", ",") for p in re.split(build_line_sep, v)]
         if len(parts) < 3:
-            raise MetaDataException("Invalid build format: " + v + " in " + mf.name)
+            warn_or_exception("Invalid build format: " + v + " in " + mf.name)
         build = Build()
         build.version = parts[0]
         build.vercode = parts[1]
@@ -1095,8 +1107,8 @@ def parse_txt_metadata(mf, app):
                     del buildlines[:]
             else:
                 if not build.commit and not build.disable:
-                    raise MetaDataException("No commit specified for {0} in {1}"
-                                            .format(build.version, linedesc))
+                    warn_or_exception("No commit specified for {0} in {1}"
+                                      .format(build.version, linedesc))
 
                 app.builds.append(build)
                 add_comments('build:' + build.vercode)
@@ -1111,7 +1123,7 @@ def parse_txt_metadata(mf, app):
             try:
                 f, v = line.split(':', 1)
             except ValueError:
-                raise MetaDataException("Invalid metadata in " + linedesc)
+                warn_or_exception("Invalid metadata in " + linedesc)
 
             # Translate obsolete fields...
             if f == 'Market Version':
@@ -1125,7 +1137,8 @@ def parse_txt_metadata(mf, app):
             if ftype == TYPE_MULTILINE:
                 mode = 1
                 if v:
-                    raise MetaDataException("Unexpected text on same line as " + f + " in " + linedesc)
+                    warn_or_exception("Unexpected text on same line as "
+                                      + f + " in " + linedesc)
             elif ftype == TYPE_STRING:
                 app.set_field(f, v)
             elif ftype == TYPE_LIST:
@@ -1142,21 +1155,22 @@ def parse_txt_metadata(mf, app):
             elif ftype == TYPE_BUILD_V2:
                 vv = v.split(',')
                 if len(vv) != 2:
-                    raise MetaDataException('Build should have comma-separated version and vercode, not "{0}", in {1}'
-                                            .format(v, linedesc))
+                    warn_or_exception('Build should have comma-separated',
+                                      'version and vercode,',
+                                      'not "{0}", in {1}'.format(v, linedesc))
                 build = Build()
                 build.version = vv[0]
                 build.vercode = vv[1]
                 if build.vercode in vc_seen:
-                    raise MetaDataException('Duplicate build recipe found for vercode %s in %s' % (
-                                            build.vercode, linedesc))
+                    warn_or_exception('Duplicate build recipe found for vercode %s in %s'
+                                      % (build.vercode, linedesc))
                 vc_seen.add(build.vercode)
                 del buildlines[:]
                 mode = 3
             elif ftype == TYPE_OBSOLETE:
                 pass        # Just throw it away!
             else:
-                raise MetaDataException("Unrecognised field '" + f + "' in " + linedesc)
+                warn_or_exception("Unrecognised field '" + f + "' in " + linedesc)
         elif mode == 1:     # Multiline field
             if line == '.':
                 mode = 0
@@ -1177,11 +1191,11 @@ def parse_txt_metadata(mf, app):
 
     # Mode at end of file should always be 0
     if mode == 1:
-        raise MetaDataException(f + " not terminated in " + mf.name)
+        warn_or_exception(f + " not terminated in " + mf.name)
     if mode == 2:
-        raise MetaDataException("Unterminated continuation in " + mf.name)
+        warn_or_exception("Unterminated continuation in " + mf.name)
     if mode == 3:
-        raise MetaDataException("Unterminated build in " + mf.name)
+        warn_or_exception("Unterminated build in " + mf.name)
 
     return app
 
@@ -1382,12 +1396,18 @@ def write_metadata(metadatapath, app):
     _, ext = fdroidserver.common.get_extension(metadatapath)
     accepted = fdroidserver.common.config['accepted_formats']
     if ext not in accepted:
-        raise MetaDataException('Cannot write "%s", not an accepted format, use: %s' % (
-            metadatapath, ', '.join(accepted)))
+        warn_or_exception('Cannot write "%s", not an accepted format, use: %s'
+                          % (metadatapath, ', '.join(accepted)))
 
     with open(metadatapath, 'w', encoding='utf8') as mf:
         if ext == 'txt':
             return write_txt(mf, app)
         elif ext == 'yml':
             return write_yaml(mf, app)
-    raise MetaDataException('Unknown metadata format: %s' % metadatapath)
+    warn_or_exception('Unknown metadata format: %s' % metadatapath)
+
+
+def add_metadata_arguments(parser):
+    '''add common command line flags related to metadata processing'''
+    parser.add_argument("-W", default='error',
+                        help="force errors to be warnings, or ignore")

diff --git a/fdroidserver/publish.py b/fdroidserver/publish.py
index 668f19a3f9ebc84518779ee8db4a05b911dac0ee..62674480663203db7200213e7866efc4db3d67e6 100644 (file)
--- a/fdroidserver/publish.py
+++ b/fdroidserver/publish.py
@@ -42,7 +42,9 @@ def main():
                             "[APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
     parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/readmeta.py b/fdroidserver/readmeta.py
index e04b5623c7c51cb279a0ceca0f4741984012032e..88ac7be357f9769148846d16b0b991d9a0433489 100644 (file)
--- a/fdroidserver/readmeta.py
+++ b/fdroidserver/readmeta.py
@@ -20,12 +20,16 @@ from argparse import ArgumentParser
 from . import common
 from . import metadata
 
+options = None
+
 
 def main():
 
     parser = ArgumentParser(usage="%(prog)s")
     common.setup_global_opts(parser)
-    parser.parse_args()
+    metadata.add_metadata_arguments(parser)
+    options = parser.parse_args()
+    metadata.warnings_action = options.W
     common.read_config(None)
 
     metadata.read_metadata(xref=True)

diff --git a/fdroidserver/rewritemeta.py b/fdroidserver/rewritemeta.py
index 65a50841fb26e41d1d4a93914a8d6e24058c025e..25ed0875d069584791b724953525e827c2e77251 100644 (file)
--- a/fdroidserver/rewritemeta.py
+++ b/fdroidserver/rewritemeta.py
@@ -53,7 +53,9 @@ def main():
     parser.add_argument("-t", "--to", default=None,
                         help="Rewrite to a specific format")
     parser.add_argument("appid", nargs='*', help="app-id in the form APPID")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/scanner.py b/fdroidserver/scanner.py
index 41ecbb50c286018fced962fb348f882ca4cd4b41..289e041a96433b416f3050e66aa756f89b3550f7 100644 (file)
--- a/fdroidserver/scanner.py
+++ b/fdroidserver/scanner.py
@@ -250,7 +250,9 @@ def main():
     parser = ArgumentParser(usage="%(prog)s [options] [APPID[:VERCODE] [APPID[:VERCODE] ...]]")
     common.setup_global_opts(parser)
     parser.add_argument("appid", nargs='*', help="app-id with optional versioncode in the form APPID[:VERCODE]")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py
index 1a050033a871bca175733ab9a04805e2b0db4483..1f822ba085b500faf9ad5ec32ae2325bc973a6c1 100644 (file)
--- a/fdroidserver/stats.py
+++ b/fdroidserver/stats.py
@@ -66,7 +66,9 @@ def main():
                         "have been made that would invalidate old cached data.")
     parser.add_argument("--nologs", action="store_true", default=False,
                         help="Don't do anything logs-related")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)
 

diff --git a/fdroidserver/update.py b/fdroidserver/update.py
index b1ebfd350c8eae1dde8876376b57a9fd49873e9a..7e1181d1bfa43885785d7d05ab372f89e19f7eeb 100644 (file)
--- a/fdroidserver/update.py
+++ b/fdroidserver/update.py
@@ -1316,7 +1316,9 @@ def main():
                         help="When configured for signed indexes, create only unsigned indexes at this stage")
     parser.add_argument("--use-date-from-apk", action="store_true", default=False,
                         help="Use date from apk instead of current time for newly added apks")
+    metadata.add_metadata_arguments(parser)
     options = parser.parse_args()
+    metadata.warnings_action = options.W
 
     config = common.read_config(options)