X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=blobdiff_plain;f=fdroidserver%2Fcheckupdates.py;h=72c8b22b4fa3407542e9fc55404e3a0985ffe530;hb=70d9633555ba07b4bb83dfd7dcda9781cc80cf51;hp=b754bf10b6b862804169239051d8e966a16c1812;hpb=071376b7f4c9b30ca1d3e7ff7d7f670a3c7afaf1;p=fdroidserver.git
diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py
index b754bf10..72c8b22b 100644
--- a/fdroidserver/checkupdates.py
+++ b/fdroidserver/checkupdates.py
@@ -17,24 +17,25 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see .
-import sys
import os
import re
import urllib.request
import urllib.error
import time
import subprocess
+import sys
from argparse import ArgumentParser
import traceback
-from html.parser import HTMLParser
+import html
from distutils.version import LooseVersion
import logging
import copy
+import urllib.parse
+from . import _
from . import common
from . import metadata
-from .common import VCSException, FDroidException
-from .metadata import MetaDataException
+from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
# Check for a new version by looking at a document retrieved via HTTP.
@@ -48,6 +49,13 @@ def check_http(app):
raise FDroidException('Missing Update Check Data')
urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
+ parsed = urllib.parse.urlparse(urlcode)
+ if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
+ raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
+ if urlver != '.':
+ parsed = urllib.parse.urlparse(urlver)
+ if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
+ raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
vercode = "99999999"
if len(urlcode) > 0:
@@ -59,7 +67,7 @@ def check_http(app):
m = re.search(codeex, page)
if not m:
raise FDroidException("No RE match for version code")
- vercode = m.group(1)
+ vercode = m.group(1).strip()
version = "??"
if len(urlver) > 0:
@@ -109,12 +117,9 @@ def check_tags(app, pattern):
vcs.gotorevision(None)
- last_build = metadata.Build()
- if len(app.builds) > 0:
- last_build = app.builds[-1]
+ last_build = app.get_last_build()
- if last_build.submodules:
- vcs.initsubmodules()
+ try_init_submodules(app, last_build, vcs)
hpak = None
htag = None
@@ -210,8 +215,7 @@ def check_repomanifest(app, branch=None):
if len(app.builds) > 0:
last_build = app.builds[-1]
- if last_build.submodules:
- vcs.initsubmodules()
+ try_init_submodules(app, last_build, vcs)
hpak = None
hver = None
@@ -245,7 +249,7 @@ def check_repomanifest(app, branch=None):
return (None, msg)
-def check_repotrunk(app, branch=None):
+def check_repotrunk(app):
try:
if app.RepoType == 'srclib':
@@ -283,7 +287,7 @@ def check_gplay(app):
req = urllib.request.Request(url, None, headers)
try:
resp = urllib.request.urlopen(req, None, 20)
- page = resp.read()
+ page = resp.read().decode()
except urllib.error.HTTPError as e:
return (None, str(e.code))
except Exception as e:
@@ -293,8 +297,7 @@ def check_gplay(app):
m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*', page)
if m:
- html_parser = HTMLParser()
- version = html_parser.unescape(m.group(1))
+ version = html.unescape(m.group(1))
if version == 'Varies with device':
return (None, 'Device-variable version, cannot use this method')
@@ -304,13 +307,26 @@ def check_gplay(app):
return (version.strip(), None)
+def try_init_submodules(app, last_build, vcs):
+ """Try to init submodules if the last build entry used them.
+ They might have been removed from the app's repo in the meantime,
+ so if we can't find any submodules we continue with the updates check.
+ If there is any other error in initializing them then we stop the check.
+ """
+ if last_build.submodules:
+ try:
+ vcs.initsubmodules()
+ except NoSubmodulesException:
+ logging.info("No submodules present for {}".format(app.Name))
+
+
# Return all directories under startdir that contain any of the manifest
# files, and thus are probably an Android project.
def dirs_with_manifest(startdir):
- for r, d, f in os.walk(startdir):
- if any(m in f for m in [
+ for root, dirs, files in os.walk(startdir):
+ if any(m in files for m in [
'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
- yield r
+ yield root
# Tries to find a new subdir starting from the root build_dir. Returns said
@@ -322,9 +338,7 @@ def possible_subdirs(app):
else:
build_dir = os.path.join('build', app.id)
- last_build = metadata.Build()
- if len(app.builds) > 0:
- last_build = app.builds[-1]
+ last_build = app.get_last_build()
for d in dirs_with_manifest(build_dir):
m_paths = common.manifest_paths(d, last_build.gradle)
@@ -351,9 +365,7 @@ def fetch_autoname(app, tag):
except VCSException:
return None
- last_build = metadata.Build()
- if len(app.builds) > 0:
- last_build = app.builds[-1]
+ last_build = app.get_last_build()
logging.debug("...fetch auto name from " + build_dir)
new_name = None
@@ -378,7 +390,7 @@ def fetch_autoname(app, tag):
return commitmsg
-def checkupdates_app(app, first=True):
+def checkupdates_app(app):
# If a change is made, commitmsg should be set to a description of it.
# Only if this is set will changes be written back to the metadata.
@@ -417,6 +429,9 @@ def checkupdates_app(app, first=True):
msg = 'Invalid update check method'
if version and vercode and app.VercodeOperation:
+ if not common.VERCODE_OPERATION_RE.match(app.VercodeOperation):
+ raise MetaDataException(_('Invalid VercodeOperation: {field}')
+ .format(field=app.VercodeOperation))
oldvercode = str(int(vercode))
op = app.VercodeOperation.replace("%c", oldvercode)
vercode = str(eval(op))
@@ -438,6 +453,8 @@ def checkupdates_app(app, first=True):
elif vercode == app.CurrentVersionCode:
logging.info("...up to date")
else:
+ logging.debug("...updating - old vercode={0}, new vercode={1}".format(
+ app.CurrentVersionCode, vercode))
app.CurrentVersion = version
app.CurrentVersionCode = str(int(vercode))
updating = True
@@ -452,7 +469,9 @@ def checkupdates_app(app, first=True):
if options.auto:
mode = app.AutoUpdateMode
- if mode in ('None', 'Static'):
+ if not app.CurrentVersionCode:
+ logging.warn("Can't auto-update app with no current version code: " + app.id)
+ elif mode in ('None', 'Static'):
pass
elif mode.startswith('Version '):
pattern = mode[8:]
@@ -466,22 +485,22 @@ def checkupdates_app(app, first=True):
gotcur = False
latest = None
for build in app.builds:
- if int(build.vercode) >= int(app.CurrentVersionCode):
+ if int(build.versionCode) >= int(app.CurrentVersionCode):
gotcur = True
- if not latest or int(build.vercode) > int(latest.vercode):
+ if not latest or int(build.versionCode) > int(latest.versionCode):
latest = build
- if int(latest.vercode) > int(app.CurrentVersionCode):
+ if int(latest.versionCode) > int(app.CurrentVersionCode):
logging.info("Refusing to auto update, since the latest build is newer")
if not gotcur:
newbuild = copy.deepcopy(latest)
newbuild.disable = False
- newbuild.vercode = app.CurrentVersionCode
- newbuild.version = app.CurrentVersion + suffix
- logging.info("...auto-generating build for " + newbuild.version)
- commit = pattern.replace('%v', newbuild.version)
- commit = commit.replace('%c', newbuild.vercode)
+ newbuild.versionCode = app.CurrentVersionCode
+ newbuild.versionName = app.CurrentVersion + suffix
+ logging.info("...auto-generating build for " + newbuild.versionName)
+ commit = pattern.replace('%v', newbuild.versionName)
+ commit = commit.replace('%c', newbuild.versionCode)
newbuild.commit = commit
app.builds.append(newbuild)
name = common.getappname(app)
@@ -500,12 +519,44 @@ def checkupdates_app(app, first=True):
gitcmd.extend(['--author', config['auto_author']])
gitcmd.extend(["--", metadatapath])
if subprocess.call(gitcmd) != 0:
- logging.error("Git commit failed")
- sys.exit(1)
+ raise FDroidException("Git commit failed")
+
+
+def update_wiki(gplaylog, locallog):
+ if config.get('wiki_server') and config.get('wiki_path'):
+ try:
+ import mwclient
+ site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
+ path=config['wiki_path'])
+ site.login(config['wiki_user'], config['wiki_password'])
+
+ # Write a page with the last build log for this version code
+ wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
+ newpage = site.Pages[wiki_page_path]
+ txt = ''
+ txt += "* command line: " + ' '.join(sys.argv) + "\n"
+ txt += common.get_git_describe_link()
+ txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
+ txt += "* completed at " + common.get_wiki_timestamp() + '\n'
+ txt += "\n\n"
+ txt += common.get_android_tools_version_log()
+ txt += "\n\n"
+ if gplaylog:
+ txt += '== --gplay check ==\n\n'
+ txt += gplaylog
+ if locallog:
+ txt += '== local source check ==\n\n'
+ txt += locallog
+ newpage.save(txt, summary='Run log')
+ newpage = site.Pages['checkupdates']
+ newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
+ except Exception as e:
+ logging.error(_('Error while attempting to publish log: %s') % e)
config = None
options = None
+start_timestamp = time.gmtime()
def main():
@@ -515,28 +566,38 @@ def main():
# Parse command line...
parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
common.setup_global_opts(parser)
- parser.add_argument("appid", nargs='*', help="app-id to check for updates")
+ parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
parser.add_argument("--auto", action="store_true", default=False,
- help="Process auto-updates")
+ help=_("Process auto-updates"))
parser.add_argument("--autoonly", action="store_true", default=False,
- help="Only process apps with auto-updates")
+ help=_("Only process apps with auto-updates"))
parser.add_argument("--commit", action="store_true", default=False,
- help="Commit changes")
+ help=_("Commit changes"))
+ parser.add_argument("--allow-dirty", action="store_true", default=False,
+ help=_("Run on git repo that has uncommitted changes"))
parser.add_argument("--gplay", action="store_true", default=False,
- help="Only print differences with the Play Store")
+ help=_("Only print differences with the Play Store"))
metadata.add_metadata_arguments(parser)
options = parser.parse_args()
metadata.warnings_action = options.W
config = common.read_config(options)
+ if not options.allow_dirty:
+ status = subprocess.check_output(['git', 'status', '--porcelain'])
+ if status:
+ logging.error(_('Build metadata git repo has uncommited changes!'))
+ sys.exit(1)
+
# Get all apps...
allapps = metadata.read_metadata()
apps = common.read_app_args(options.appid, allapps, False)
+ gplaylog = ''
if options.gplay:
- for app in apps:
+ for appid, app in apps.items():
+ gplaylog += '* ' + appid + '\n'
version, reason = check_gplay(app)
if version is None:
if reason == '404':
@@ -558,19 +619,31 @@ def main():
else:
logging.info("{0} has the same version {1} on the Play Store"
.format(common.getappname(app), version))
+ update_wiki(gplaylog, None)
return
+ locallog = ''
for appid, app in apps.items():
if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
- logging.debug("Nothing to do for {0}...".format(appid))
+ logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
continue
- logging.info("Processing " + appid + '...')
+ msg = _("Processing {appid}").format(appid=appid)
+ logging.info(msg)
+ locallog += '* ' + msg + '\n'
+
+ try:
+ checkupdates_app(app)
+ except Exception as e:
+ msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
+ logging.error(msg)
+ locallog += msg + '\n'
+
+ update_wiki(None, locallog)
- checkupdates_app(app)
+ logging.info(_("Finished"))
- logging.info("Finished.")
if __name__ == "__main__":
main()