#
# checkupdates.py - part of the FDroid server tools
# Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
-# Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
+# Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
import traceback
import HTMLParser
from distutils.version import LooseVersion
+import logging
+
import common
+import metadata
from common import BuildException
from common import VCSException
+from metadata import MetaDataException
# Check for a new version by looking at a document retrieved via HTTP.
try:
- if not 'Update Check Data' in app:
+ if 'Update Check Data' not in app:
raise Exception('Missing Update Check Data')
urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
vercode = "99999999"
if len(urlcode) > 0:
- print "...requesting {0}".format(urlcode)
+ logging.debug("...requesting {0}".format(urlcode))
req = urllib2.Request(urlcode, None)
resp = urllib2.urlopen(req, None, 20)
page = resp.read()
version = "??"
if len(urlver) > 0:
if urlver != '.':
- print "...requesting {0}".format(urlver)
+ logging.debug("...requesting {0}".format(urlver))
req = urllib2.Request(urlver, None)
resp = urllib2.urlopen(req, None, 20)
page = resp.read()
return (version, vercode)
except Exception:
- msg = "Could not complete http check for app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
+
# Check for a new version by looking at the tags in the source repo.
# Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
-def check_tags(app, sdk_path):
+def check_tags(app, pattern):
try:
+ appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
+ if repotype == 'git-svn' and ';' not in app['Repo']:
+ return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
+
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
+ vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
flavour = None
if len(app['builds']) > 0:
- if 'subdir' in app['builds'][-1]:
+ if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
- if 'gradle' in app['builds'][-1]:
+ if app['builds'][-1]['gradle']:
flavour = app['builds'][-1]['gradle']
+ if flavour == 'yes':
+ flavour = None
htag = None
hver = None
hcode = "0"
- for tag in vcs.gettags():
+ tags = vcs.gettags()
+ if pattern:
+ pat = re.compile(pattern)
+ tags = [tag for tag in tags if pat.match(tag)]
+
+ if repotype in ('git',):
+ tags = vcs.latesttags(tags, 5)
+
+ for tag in tags:
+ logging.debug("Check tag: '{0}'".format(tag))
vcs.gotorevision(tag)
# Only process tags where the manifest exists...
paths = common.manifest_paths(build_dir, flavour)
- version, vercode, package = common.parse_androidmanifests(paths)
- if package and package == app['id'] and version and vercode:
- print "Manifest exists. Found version %s" % version
- if int(vercode) > int(hcode):
- htag = tag
- hcode = str(int(vercode))
- hver = version
+ version, vercode, package = \
+ common.parse_androidmanifests(paths, app['Update Check Ignore'])
+ if not package or package != appid or not version or not vercode:
+ continue
+
+ logging.debug("Manifest exists. Found version {0} ({1})"
+ .format(version, vercode))
+ if int(vercode) > int(hcode):
+ htag = tag
+ hcode = str(int(vercode))
+ hver = version
if hver:
return (hver, hcode, htag)
return (None, "Couldn't find any version information", None)
except BuildException as be:
- msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+ msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
return (None, msg, None)
except VCSException as vcse:
- msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg, None)
except Exception:
- msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg, None)
+
# Check for a new version by looking at the AndroidManifest.xml at the HEAD
# of the source repo. Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
# caution, because it's inappropriate for many projects.
# Returns (None, "a message") if this didn't work, or (version, vercode) for
# the details of the current version.
-def check_repomanifest(app, sdk_path, branch=None):
+def check_repomanifest(app, branch=None):
try:
+ appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
if app['Repo Type'] == 'srclib':
build_dir = os.path.join('build', 'srclib', app['Repo'])
repotype = common.getsrclibvcs(app['Repo'])
repotype = app['Repo Type']
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
+ vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
if repotype == 'git':
if branch:
- branch = 'origin/'+branch
+ branch = 'origin/' + branch
vcs.gotorevision(branch)
elif repotype == 'git-svn':
vcs.gotorevision(branch)
flavour = None
if len(app['builds']) > 0:
- if 'subdir' in app['builds'][-1]:
+ if app['builds'][-1]['subdir']:
build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
- if 'gradle' in app['builds'][-1]:
+ if app['builds'][-1]['gradle']:
flavour = app['builds'][-1]['gradle']
+ if flavour == 'yes':
+ flavour = None
if not os.path.isdir(build_dir):
return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
paths = common.manifest_paths(build_dir, flavour)
- version, vercode, package = common.parse_androidmanifests(paths)
+ version, vercode, package = \
+ common.parse_androidmanifests(paths, app['Update Check Ignore'])
if not package:
return (None, "Couldn't find package ID")
- if package != app['id']:
+ if package != appid:
return (None, "Package ID mismatch")
if not version:
- return (None,"Couldn't find latest version name")
+ return (None, "Couldn't find latest version name")
if not vercode:
- return (None,"Couldn't find latest version code")
+ if "Ignore" == version:
+ return (None, "Latest version is ignored")
+ return (None, "Couldn't find latest version code")
- return (version, str(int(vercode)))
+ vercode = str(int(vercode))
+
+ logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
+
+ return (version, vercode)
except BuildException as be:
- msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+ msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
return (None, msg)
except VCSException as vcse:
- msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
- msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
-def check_repotrunk(app, sdk_path, branch=None):
+
+def check_repotrunk(app, branch=None):
try:
if app['Repo Type'] == 'srclib':
return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
+ vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except BuildException as be:
- msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+ msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
return (None, msg)
except VCSException as vcse:
- msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
return (None, msg)
except Exception:
- msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
return (None, msg)
+
# Check for a new version by looking at the Google Play Store.
# Returns (None, "a message") if this didn't work, or (version, None) for
# the details of the current version.
def check_gplay(app):
time.sleep(15)
url = 'https://play.google.com/store/apps/details?id=' + app['id']
- headers = {'User-Agent' : 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
+ headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
req = urllib2.Request(url, None, headers)
try:
resp = urllib2.urlopen(req, None, 20)
return (version.strip(), None)
-config = {}
+config = None
+options = None
+
def main():
- # Read configuration...
- common.read_config(config)
+ global config, options
# Parse command line...
- parser = OptionParser()
+ parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
parser.add_option("-v", "--verbose", action="store_true", default=False,
help="Spew out even more information than normal")
- parser.add_option("-p", "--package", default=None,
- help="Check only the specified package")
+ parser.add_option("-q", "--quiet", action="store_true", default=False,
+ help="Restrict output to warnings and errors")
parser.add_option("--auto", action="store_true", default=False,
help="Process auto-updates")
parser.add_option("--autoonly", action="store_true", default=False,
help="Only print differences with the Play Store")
(options, args) = parser.parse_args()
+ config = common.read_config(options)
+
# Get all apps...
- apps = common.read_metadata(options.verbose)
+ allapps = metadata.read_metadata()
+ metadata.read_srclibs()
- # Filter apps according to command-line options
- if options.package:
- apps = [app for app in apps if app['id'] == options.package]
- if len(apps) == 0:
- print "No such package"
- sys.exit(1)
+ apps = common.read_app_args(args, allapps, False)
if options.gplay:
for app in apps:
version, reason = check_gplay(app)
- if version is None and options.verbose:
+ if version is None:
if reason == '404':
- print "%s (%s) is not in the Play Store" % (app['Auto Name'], app['id'])
+ logging.info("{0} is not in the Play Store".format(common.getappname(app)))
else:
- print "%s (%s) encountered a problem: %s" % (app['Auto Name'], app['id'], reason)
+ logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
if version is not None:
stored = app['Current Version']
- if LooseVersion(stored) < LooseVersion(version):
- print "%s (%s) has version %s on the Play Store, which is bigger than %s" % (
- app['Auto Name'], app['id'], version, stored)
- elif options.verbose:
- print "%s (%s) has the same version %s on the Play Store" % (
- app['Auto Name'], app['id'], version)
+ if not stored:
+ logging.info("{0} has no Current Version but has version {1} on the Play Store"
+ .format(common.getappname(app), version))
+ elif LooseVersion(stored) < LooseVersion(version):
+ logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
+ .format(common.getappname(app), version, stored))
+ else:
+ if stored != version:
+ logging.info("{0} has version {1} on the Play Store, which differs from {2}"
+ .format(common.getappname(app), version, stored))
+ else:
+ logging.info("{0} has the same version {1} on the Play Store"
+ .format(common.getappname(app), version))
return
-
for app in apps:
-
- if options.autoonly and app['Auto Update Mode'] == 'None':
- print "Nothing to do for %s..." % app['id']
+ if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
+ logging.debug("Nothing to do for {0}...".format(app['id']))
continue
- print "Processing " + app['id'] + '...'
+ logging.info("Processing " + app['id'] + '...')
- writeit = False
- logmsg = None
+ # If a change is made, commitmsg should be set to a description of it.
+ # Only if this is set will changes be written back to the metadata.
+ commitmsg = None
tag = None
+ msg = None
+ vercode = None
+ noverok = False
mode = app['Update Check Mode']
- if mode == 'Tags':
- (version, vercode, tag) = check_tags(app, config['sdk_path'])
+ if mode.startswith('Tags'):
+ pattern = mode[5:] if len(mode) > 4 else None
+ (version, vercode, tag) = check_tags(app, pattern)
+ msg = vercode
elif mode == 'RepoManifest':
- (version, vercode) = check_repomanifest(app, config['sdk_path'])
+ (version, vercode) = check_repomanifest(app)
+ msg = vercode
elif mode.startswith('RepoManifest/'):
- (version, vercode) = check_repomanifest(app, config['sdk_path'], mode[13:])
+ tag = mode[13:]
+ (version, vercode) = check_repomanifest(app, tag)
+ msg = vercode
elif mode == 'RepoTrunk':
- (version, vercode) = check_repotrunk(app, config['sdk_path'])
+ (version, vercode) = check_repotrunk(app)
+ msg = vercode
elif mode == 'HTTP':
(version, vercode) = check_http(app)
- elif mode == 'Static':
- version = None
- vercode = 'Checking disabled'
- elif mode == 'None':
+ msg = vercode
+ elif mode in ('None', 'Static'):
version = None
- vercode = 'Checking disabled'
+ msg = 'Checking disabled'
+ noverok = True
else:
version = None
- vercode = 'Invalid update check method'
+ msg = 'Invalid update check method'
+
+ if vercode and app['Vercode Operation']:
+ op = app['Vercode Operation'].replace("%c", str(int(vercode)))
+ vercode = str(eval(op))
updating = False
if not version:
- print "..." + vercode
+ logmsg = "...{0} : {1}".format(app['id'], msg)
+ if noverok:
+ logging.info(logmsg)
+ else:
+ logging.warn(logmsg)
elif vercode == app['Current Version Code']:
- print "...up to date"
+ logging.info("...up to date")
else:
app['Current Version'] = version
app['Current Version Code'] = str(int(vercode))
updating = True
- writeit = True
# Do the Auto Name thing as well as finding the CV real name
- if len(app["Repo Type"]) > 0:
+ if len(app["Repo Type"]) > 0 and mode not in ('None', 'Static'):
try:
else:
app_dir = os.path.join('build/', app['id'])
- vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir,
- config['sdk_path'])
+ vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
vcs.gotorevision(tag)
flavour = None
if len(app['builds']) > 0:
- if 'subdir' in app['builds'][-1]:
+ if app['builds'][-1]['subdir']:
app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
- if 'gradle' in app['builds'][-1]:
+ if app['builds'][-1]['gradle']:
flavour = app['builds'][-1]['gradle']
+ if flavour == 'yes':
+ flavour = None
+ logging.debug("...fetch auto name from " + app_dir +
+ ((" (flavour: %s)" % flavour) if flavour else ""))
new_name = common.fetch_real_name(app_dir, flavour)
- if new_name != app['Auto Name']:
- app['Auto Name'] = new_name
+ if new_name:
+ logging.debug("...got autoname '" + new_name + "'")
+ if new_name != app['Auto Name']:
+ app['Auto Name'] = new_name
+ if not commitmsg:
+ commitmsg = "Set autoname of {0}".format(common.getappname(app))
+ else:
+ logging.debug("...couldn't get autoname")
if app['Current Version'].startswith('@string/'):
cv = common.version_name(app['Current Version'], app_dir, flavour)
if app['Current Version'] != cv:
app['Current Version'] = cv
- writeit = True
+ if not commitmsg:
+ commitmsg = "Fix CV of {0}".format(common.getappname(app))
except Exception:
- print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc())
+ logging.error("Auto Name or Current Version failed for {0} due to exception: {1}".format(app['id'], traceback.format_exc()))
if updating:
- print '...updating to version %s (%s)' % (app['Current Version'], app['Current Version Code'])
- name = '%s (%s)' % (app['Auto Name'], app['id']) if app['Auto Name'] else app['id']
- ver = "%s (%s)" % (app['Current Version'], app['Current Version Code'])
- logmsg = 'Update CV of %s to %s' % (name, ver)
+ name = common.getappname(app)
+ ver = common.getcvname(app)
+ logging.info('...updating to version %s' % ver)
+ commitmsg = 'Update CV of %s to %s' % (name, ver)
if options.auto:
mode = app['Auto Update Mode']
- if mode == 'None':
+ if mode in ('None', 'Static'):
pass
elif mode.startswith('Version '):
pattern = mode[8:]
if pattern.startswith('+'):
- o = pattern.find(' ')
- suffix = pattern[1:o]
- pattern = pattern[o + 1:]
+ try:
+ suffix, pattern = pattern.split(' ', 1)
+ except ValueError:
+ raise MetaDataException("Invalid AUM: " + mode)
else:
suffix = ''
gotcur = False
gotcur = True
if not latest or int(build['vercode']) > int(latest['vercode']):
latest = build
+
if not gotcur:
newbuild = latest.copy()
if 'origlines' in newbuild:
del newbuild['origlines']
+ newbuild['disable'] = False
newbuild['vercode'] = app['Current Version Code']
newbuild['version'] = app['Current Version'] + suffix
- print "...auto-generating build for " + newbuild['version']
+ logging.info("...auto-generating build for " + newbuild['version'])
commit = pattern.replace('%v', newbuild['version'])
commit = commit.replace('%c', newbuild['vercode'])
newbuild['commit'] = commit
app['builds'].append(newbuild)
- writeit = True
- name = "%s (%s)" % (app['Auto Name'], app['id']) if app['Auto Name'] else app['id']
- ver = "%s (%s)" % (newbuild['version'], newbuild['vercode'])
- logmsg = "Update %s to %s" % (name, ver)
+ name = common.getappname(app)
+ ver = common.getcvname(app)
+ commitmsg = "Update %s to %s" % (name, ver)
else:
- print 'Invalid auto update mode'
+ logging.warn('Invalid auto update mode "' + mode + '" on ' + app['id'])
- if writeit:
+ if commitmsg:
metafile = os.path.join('metadata', app['id'] + '.txt')
- common.write_metadata(metafile, app)
- if options.commit and logmsg:
- print "Commiting update for " + metafile
- if subprocess.call(["git", "commit", "-m",
- logmsg, "--", metafile]) != 0:
- print "Git commit failed"
+ metadata.write_metadata(metafile, app)
+ if options.commit:
+ logging.info("Commiting update for " + metafile)
+ gitcmd = ["git", "commit", "-m", commitmsg]
+ if 'auto_author' in config:
+ gitcmd.extend(['--author', config['auto_author']])
+ gitcmd.extend(["--", metafile])
+ if subprocess.call(gitcmd) != 0:
+ logging.error("Git commit failed")
sys.exit(1)
- print "Finished."
+ logging.info("Finished.")
if __name__ == "__main__":
main()
-
~ianmdlvl / fdroidserver.git / blobdiff