build/checkupdates/update: log current fdroiddata commit to wiki

[fdroidserver.git] / fdroidserver / checkupdates.py

diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py
index 02baa061d54f085471a188a5a17bda95cc8585df..72c8b22b4fa3407542e9fc55404e3a0985ffe530 100644 (file)
--- a/fdroidserver/checkupdates.py
+++ b/fdroidserver/checkupdates.py
@@ -23,17 +23,19 @@ import urllib.request
 import urllib.error
 import time
 import subprocess
 import urllib.error
 import time
 import subprocess

+import sys

 from argparse import ArgumentParser
 import traceback
 import html
 from distutils.version import LooseVersion
 import logging
 import copy
 from argparse import ArgumentParser
 import traceback
 import html
 from distutils.version import LooseVersion
 import logging
 import copy

+import urllib.parse

 
 from . import _
 from . import common
 from . import metadata
 
 from . import _
 from . import common
 from . import metadata

-from .exception import VCSException, FDroidException, MetaDataException
+from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException

 
 
 # Check for a new version by looking at a document retrieved via HTTP.
 
 
 # Check for a new version by looking at a document retrieved via HTTP.


@@ -47,6 +49,13 @@ def check_http(app):
             raise FDroidException('Missing Update Check Data')
 
         urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
             raise FDroidException('Missing Update Check Data')
 
         urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')

+        parsed = urllib.parse.urlparse(urlcode)
+        if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
+            raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
+        if urlver != '.':
+            parsed = urllib.parse.urlparse(urlver)
+            if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
+                raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))

 
         vercode = "99999999"
         if len(urlcode) > 0:
 
         vercode = "99999999"
         if len(urlcode) > 0:


@@ -110,8 +119,7 @@ def check_tags(app, pattern):
 
         last_build = app.get_last_build()
 
 
         last_build = app.get_last_build()
 

-        if last_build.submodules:
-            vcs.initsubmodules()
+        try_init_submodules(app, last_build, vcs)

 
         hpak = None
         htag = None
 
         hpak = None
         htag = None


@@ -207,8 +215,7 @@ def check_repomanifest(app, branch=None):
         if len(app.builds) > 0:
             last_build = app.builds[-1]
 
         if len(app.builds) > 0:
             last_build = app.builds[-1]
 

-        if last_build.submodules:
-            vcs.initsubmodules()
+        try_init_submodules(app, last_build, vcs)

 
         hpak = None
         hver = None
 
         hpak = None
         hver = None


@@ -300,6 +307,19 @@ def check_gplay(app):
     return (version.strip(), None)
 
 
     return (version.strip(), None)
 
 

+def try_init_submodules(app, last_build, vcs):
+    """Try to init submodules if the last build entry used them.
+    They might have been removed from the app's repo in the meantime,
+    so if we can't find any submodules we continue with the updates check.
+    If there is any other error in initializing them then we stop the check.
+    """
+    if last_build.submodules:
+        try:
+            vcs.initsubmodules()
+        except NoSubmodulesException:
+            logging.info("No submodules present for {}".format(app.Name))
+
+

 # Return all directories under startdir that contain any of the manifest
 # files, and thus are probably an Android project.
 def dirs_with_manifest(startdir):
 # Return all directories under startdir that contain any of the manifest
 # files, and thus are probably an Android project.
 def dirs_with_manifest(startdir):


@@ -409,6 +429,9 @@ def checkupdates_app(app):
         msg = 'Invalid update check method'
 
     if version and vercode and app.VercodeOperation:
         msg = 'Invalid update check method'
 
     if version and vercode and app.VercodeOperation:

+        if not common.VERCODE_OPERATION_RE.match(app.VercodeOperation):
+            raise MetaDataException(_('Invalid VercodeOperation: {field}')
+                                    .format(field=app.VercodeOperation))

         oldvercode = str(int(vercode))
         op = app.VercodeOperation.replace("%c", oldvercode)
         vercode = str(eval(op))
         oldvercode = str(int(vercode))
         op = app.VercodeOperation.replace("%c", oldvercode)
         vercode = str(eval(op))


@@ -499,8 +522,41 @@ def checkupdates_app(app):
                 raise FDroidException("Git commit failed")
 
 
                 raise FDroidException("Git commit failed")
 
 

+def update_wiki(gplaylog, locallog):
+    if config.get('wiki_server') and config.get('wiki_path'):
+        try:
+            import mwclient
+            site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+                                 path=config['wiki_path'])
+            site.login(config['wiki_user'], config['wiki_password'])
+
+            # Write a page with the last build log for this version code
+            wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
+            newpage = site.Pages[wiki_page_path]
+            txt = ''
+            txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
+            txt += common.get_git_describe_link()
+            txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+            txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+            txt += "\n\n"
+            txt += common.get_android_tools_version_log()
+            txt += "\n\n"
+            if gplaylog:
+                txt += '== --gplay check ==\n\n'
+                txt += gplaylog
+            if locallog:
+                txt += '== local source check ==\n\n'
+                txt += locallog
+            newpage.save(txt, summary='Run log')
+            newpage = site.Pages['checkupdates']
+            newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+        except Exception as e:
+            logging.error(_('Error while attempting to publish log: %s') % e)
+
+

 config = None
 options = None
 config = None
 options = None

+start_timestamp = time.gmtime()

 
 
 def main():
 
 
 def main():


@@ -517,6 +573,8 @@ def main():
                         help=_("Only process apps with auto-updates"))
     parser.add_argument("--commit", action="store_true", default=False,
                         help=_("Commit changes"))
                         help=_("Only process apps with auto-updates"))
     parser.add_argument("--commit", action="store_true", default=False,
                         help=_("Commit changes"))

+    parser.add_argument("--allow-dirty", action="store_true", default=False,
+                        help=_("Run on git repo that has uncommitted changes"))

     parser.add_argument("--gplay", action="store_true", default=False,
                         help=_("Only print differences with the Play Store"))
     metadata.add_metadata_arguments(parser)
     parser.add_argument("--gplay", action="store_true", default=False,
                         help=_("Only print differences with the Play Store"))
     metadata.add_metadata_arguments(parser)


@@ -525,13 +583,21 @@ def main():
 
     config = common.read_config(options)
 
 
     config = common.read_config(options)
 

+    if not options.allow_dirty:
+        status = subprocess.check_output(['git', 'status', '--porcelain'])
+        if status:
+            logging.error(_('Build metadata git repo has uncommited changes!'))
+            sys.exit(1)
+

     # Get all apps...
     allapps = metadata.read_metadata()
 
     apps = common.read_app_args(options.appid, allapps, False)
 
     # Get all apps...
     allapps = metadata.read_metadata()
 
     apps = common.read_app_args(options.appid, allapps, False)
 

+    gplaylog = ''

     if options.gplay:
         for appid, app in apps.items():
     if options.gplay:
         for appid, app in apps.items():

+            gplaylog += '* ' + appid + '\n'

             version, reason = check_gplay(app)
             if version is None:
                 if reason == '404':
             version, reason = check_gplay(app)
             if version is None:
                 if reason == '404':


@@ -553,21 +619,28 @@ def main():
                     else:
                         logging.info("{0} has the same version {1} on the Play Store"
                                      .format(common.getappname(app), version))
                     else:
                         logging.info("{0} has the same version {1} on the Play Store"
                                      .format(common.getappname(app), version))

+        update_wiki(gplaylog, None)

         return
 
         return
 

+    locallog = ''

     for appid, app in apps.items():
 
         if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
             logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
             continue
 
     for appid, app in apps.items():
 
         if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
             logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
             continue
 

-        logging.info(_("Processing {appid}").format(appid=appid))
+        msg = _("Processing {appid}").format(appid=appid)
+        logging.info(msg)
+        locallog += '* ' + msg + '\n'

 
         try:
             checkupdates_app(app)
         except Exception as e:
 
         try:
             checkupdates_app(app)
         except Exception as e:

-            logging.error(_("...checkupdate failed for {appid} : {error}")
-                          .format(appid=appid, error=e))
+            msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
+            logging.error(msg)
+            locallog += msg + '\n'
+
+    update_wiki(None, locallog)

 
     logging.info(_("Finished"))
 
 
     logging.info(_("Finished"))