-#!/usr/bin/env python2
-# -*- coding: utf-8 -*-
+#!/usr/bin/env python3
#
# checkupdates.py - part of the FDroid server tools
# Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
import sys
import os
import re
-import urllib2
+import urllib.request
+import urllib.error
import time
import subprocess
from argparse import ArgumentParser
import traceback
-import HTMLParser
+from html.parser import HTMLParser
from distutils.version import LooseVersion
import logging
+import copy
-import common
-import metadata
-from common import VCSException, FDroidException
-from metadata import MetaDataException
+from . import common
+from . import metadata
+from .common import VCSException, FDroidException
+from .metadata import MetaDataException
# Check for a new version by looking at a document retrieved via HTTP.
try:
- if 'Update Check Data' not in app:
+ if not app.UpdateCheckData:
raise FDroidException('Missing Update Check Data')
- urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
+ urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
vercode = "99999999"
if len(urlcode) > 0:
logging.debug("...requesting {0}".format(urlcode))
- req = urllib2.Request(urlcode, None)
- resp = urllib2.urlopen(req, None, 20)
+ req = urllib.request.Request(urlcode, None)
+ resp = urllib.request.urlopen(req, None, 20)
page = resp.read()
m = re.search(codeex, page)
if len(urlver) > 0:
if urlver != '.':
logging.debug("...requesting {0}".format(urlver))
- req = urllib2.Request(urlver, None)
- resp = urllib2.urlopen(req, None, 20)
+ req = urllib.request.Request(urlver, None)
+ resp = urllib.request.urlopen(req, None, 20)
page = resp.read()
m = re.search(verex, page)
return (version, vercode)
except FDroidException:
- msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
+ msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
-def app_matches_packagename(app, package):
- if not package:
- return False
- appid = app['Update Check Name'] or app['id']
- if appid == "Ignore":
- return True
- return appid == package
-
-
# Check for a new version by looking at the tags in the source repo.
# Whether this can be used reliably or not depends on
# the development procedures used by the project's developers. Use it with
try:
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
- repotype = common.getsrclibvcs(app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
+ repotype = common.getsrclibvcs(app.Repo)
else:
- build_dir = os.path.join('build', app['id'])
- repotype = app['Repo Type']
+ build_dir = os.path.join('build', app.id)
+ repotype = app.RepoType
if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
- if repotype == 'git-svn' and ';' not in app['Repo']:
+ if repotype == 'git-svn' and ';' not in app.Repo:
return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
- flavours = []
- if len(app['builds']) > 0:
- if app['builds'][-1]['gradle']:
- flavours = app['builds'][-1]['gradle']
+ last_build = metadata.Build()
+ if len(app.builds) > 0:
+ last_build = app.builds[-1]
+
+ if last_build.submodules:
+ vcs.initsubmodules()
hpak = None
htag = None
hver = None
hcode = "0"
- tags = vcs.gettags()
+ tags = []
+ if repotype == 'git':
+ tags = vcs.latesttags()
+ else:
+ tags = vcs.gettags()
+ if not tags:
+ return (None, "No tags found", None)
+
logging.debug("All tags: " + ','.join(tags))
if pattern:
pat = re.compile(pattern)
tags = [tag for tag in tags if pat.match(tag)]
+ if not tags:
+ return (None, "No matching tags found", None)
logging.debug("Matching tags: " + ','.join(tags))
- if repotype in ('git',):
- tags = vcs.latesttags(tags, 5)
+ if len(tags) > 5 and repotype == 'git':
+ tags = tags[:5]
logging.debug("Latest tags: " + ','.join(tags))
for tag in tags:
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
- paths = common.manifest_paths(root_dir, flavours)
- version, vercode, package = \
- common.parse_androidmanifests(paths, app['Update Check Ignore'])
+ paths = common.manifest_paths(root_dir, last_build.gradle)
+ version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
return (None, "Couldn't find any version information", None)
except VCSException as vcse:
- msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg, None)
except Exception:
- msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg, None)
try:
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
- repotype = common.getsrclibvcs(app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
+ repotype = common.getsrclibvcs(app.Repo)
else:
- build_dir = os.path.join('build', app['id'])
- repotype = app['Repo Type']
+ build_dir = os.path.join('build', app.id)
+ repotype = app.RepoType
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
if repotype == 'git':
if branch:
elif repotype == 'bzr':
vcs.gotorevision(None)
- flavours = []
- if len(app['builds']) > 0:
- if app['builds'][-1]['gradle']:
- flavours = app['builds'][-1]['gradle']
+ last_build = metadata.Build()
+ if len(app.builds) > 0:
+ last_build = app.builds[-1]
+
+ if last_build.submodules:
+ vcs.initsubmodules()
hpak = None
hver = None
root_dir = build_dir
else:
root_dir = os.path.join(build_dir, subdir)
- paths = common.manifest_paths(root_dir, flavours)
- version, vercode, package = \
- common.parse_androidmanifests(paths, app['Update Check Ignore'])
+ paths = common.manifest_paths(root_dir, last_build.gradle)
+ version, vercode, package = common.parse_androidmanifests(paths, app)
if vercode:
logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
.format(subdir, version, vercode))
return (None, "Couldn't find any version information")
except VCSException as vcse:
- msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
- msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
def check_repotrunk(app, branch=None):
try:
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
- repotype = common.getsrclibvcs(app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
+ repotype = common.getsrclibvcs(app.Repo)
else:
- build_dir = os.path.join('build', app['id'])
- repotype = app['Repo Type']
+ build_dir = os.path.join('build', app.id)
+ repotype = app.RepoType
if repotype not in ('git-svn', ):
return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
# Set up vcs interface and make sure we have the latest code...
- vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(None)
ref = vcs.getref()
return (ref, ref)
except VCSException as vcse:
- msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
+ msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
return (None, msg)
except Exception:
- msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
+ msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
return (None, msg)
# the details of the current version.
def check_gplay(app):
time.sleep(15)
- url = 'https://play.google.com/store/apps/details?id=' + app['id']
+ url = 'https://play.google.com/store/apps/details?id=' + app.id
headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
- req = urllib2.Request(url, None, headers)
+ req = urllib.request.Request(url, None, headers)
try:
- resp = urllib2.urlopen(req, None, 20)
+ resp = urllib.request.urlopen(req, None, 20)
page = resp.read()
- except urllib2.HTTPError, e:
+ except urllib.error.HTTPError as e:
return (None, str(e.code))
- except Exception, e:
+ except Exception as e:
return (None, 'Failed:' + str(e))
version = None
m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
if m:
- html_parser = HTMLParser.HTMLParser()
+ html_parser = HTMLParser()
version = html_parser.unescape(m.group(1))
if version == 'Varies with device':
# subdir relative to the build dir if found, None otherwise.
def possible_subdirs(app):
- if app['Repo Type'] == 'srclib':
- build_dir = os.path.join('build', 'srclib', app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
else:
- build_dir = os.path.join('build', app['id'])
+ build_dir = os.path.join('build', app.id)
- flavours = []
- if len(app['builds']) > 0:
- build = app['builds'][-1]
- if build['gradle']:
- flavours = build['gradle']
+ last_build = metadata.Build()
+ if len(app.builds) > 0:
+ last_build = app.builds[-1]
for d in dirs_with_manifest(build_dir):
- m_paths = common.manifest_paths(d, flavours)
- package = common.parse_androidmanifests(m_paths, app['Update Check Ignore'])[2]
- if app_matches_packagename(app, package):
+ m_paths = common.manifest_paths(d, last_build.gradle)
+ package = common.parse_androidmanifests(m_paths, app)[2]
+ if package is not None:
subdir = os.path.relpath(d, build_dir)
logging.debug("Adding possible subdir %s" % subdir)
yield subdir
def fetch_autoname(app, tag):
- if not app["Repo Type"] or app['Update Check Mode'] in ('None', 'Static'):
+ if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
return None
- if app['Repo Type'] == 'srclib':
- app_dir = os.path.join('build', 'srclib', app['Repo'])
+ if app.RepoType == 'srclib':
+ build_dir = os.path.join('build', 'srclib', app.Repo)
else:
- app_dir = os.path.join('build', app['id'])
+ build_dir = os.path.join('build', app.id)
try:
- vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
+ vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
vcs.gotorevision(tag)
except VCSException:
return None
- flavours = []
- if len(app['builds']) > 0:
- if app['builds'][-1]['subdir']:
- app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
- if app['builds'][-1]['gradle']:
- flavours = app['builds'][-1]['gradle']
+ last_build = metadata.Build()
+ if len(app.builds) > 0:
+ last_build = app.builds[-1]
- logging.debug("...fetch auto name from " + app_dir)
- new_name = common.fetch_real_name(app_dir, flavours)
+ logging.debug("...fetch auto name from " + build_dir)
+ new_name = None
+ for subdir in possible_subdirs(app):
+ if subdir == '.':
+ root_dir = build_dir
+ else:
+ root_dir = os.path.join(build_dir, subdir)
+ new_name = common.fetch_real_name(root_dir, last_build.gradle)
+ if new_name is not None:
+ break
commitmsg = None
if new_name:
logging.debug("...got autoname '" + new_name + "'")
- if new_name != app['Auto Name']:
- app['Auto Name'] = new_name
+ if new_name != app.AutoName:
+ app.AutoName = new_name
if not commitmsg:
commitmsg = "Set autoname of {0}".format(common.getappname(app))
else:
msg = None
vercode = None
noverok = False
- mode = app['Update Check Mode']
+ mode = app.UpdateCheckMode
if mode.startswith('Tags'):
pattern = mode[5:] if len(mode) > 4 else None
(version, vercode, tag) = check_tags(app, pattern)
+ if version == 'Unknown':
+ version = tag
msg = vercode
elif mode == 'RepoManifest':
(version, vercode) = check_repomanifest(app)
version = None
msg = 'Invalid update check method'
- if version and vercode and app['Vercode Operation']:
+ if version and vercode and app.VercodeOperation:
oldvercode = str(int(vercode))
- op = app['Vercode Operation'].replace("%c", oldvercode)
+ op = app.VercodeOperation.replace("%c", oldvercode)
vercode = str(eval(op))
logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
updating = False
if version is None:
- logmsg = "...{0} : {1}".format(app['id'], msg)
+ logmsg = "...{0} : {1}".format(app.id, msg)
if noverok:
logging.info(logmsg)
else:
logging.warn(logmsg)
- elif vercode == app['Current Version Code']:
+ elif vercode == app.CurrentVersionCode:
logging.info("...up to date")
else:
- app['Current Version'] = version
- app['Current Version Code'] = str(int(vercode))
+ app.CurrentVersion = version
+ app.CurrentVersionCode = str(int(vercode))
updating = True
commitmsg = fetch_autoname(app, tag)
commitmsg = 'Update CV of %s to %s' % (name, ver)
if options.auto:
- mode = app['Auto Update Mode']
+ mode = app.AutoUpdateMode
if mode in ('None', 'Static'):
pass
elif mode.startswith('Version '):
suffix = ''
gotcur = False
latest = None
- for build in app['builds']:
- if int(build['vercode']) >= int(app['Current Version Code']):
+ for build in app.builds:
+ if int(build.vercode) >= int(app.CurrentVersionCode):
gotcur = True
- if not latest or int(build['vercode']) > int(latest['vercode']):
+ if not latest or int(build.vercode) > int(latest.vercode):
latest = build
- if int(latest['vercode']) > int(app['Current Version Code']):
+ if int(latest.vercode) > int(app.CurrentVersionCode):
logging.info("Refusing to auto update, since the latest build is newer")
if not gotcur:
- newbuild = latest.copy()
- if 'origlines' in newbuild:
- del newbuild['origlines']
- newbuild['disable'] = False
- newbuild['vercode'] = app['Current Version Code']
- newbuild['version'] = app['Current Version'] + suffix
- logging.info("...auto-generating build for " + newbuild['version'])
- commit = pattern.replace('%v', newbuild['version'])
- commit = commit.replace('%c', newbuild['vercode'])
- newbuild['commit'] = commit
- app['builds'].append(newbuild)
+ newbuild = copy.deepcopy(latest)
+ newbuild.disable = False
+ newbuild.vercode = app.CurrentVersionCode
+ newbuild.version = app.CurrentVersion + suffix
+ logging.info("...auto-generating build for " + newbuild.version)
+ commit = pattern.replace('%v', newbuild.version)
+ commit = commit.replace('%c', newbuild.vercode)
+ newbuild.commit = commit
+ app.builds.append(newbuild)
name = common.getappname(app)
ver = common.getcvname(app)
commitmsg = "Update %s to %s" % (name, ver)
else:
- logging.warn('Invalid auto update mode "' + mode + '" on ' + app['id'])
+ logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
if commitmsg:
- metadatapath = os.path.join('metadata', app['id'] + '.txt')
+ metadatapath = os.path.join('metadata', app.id + '.txt')
with open(metadatapath, 'w') as f:
metadata.write_metadata('txt', f, app)
if options.commit:
else:
logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
if version is not None:
- stored = app['Current Version']
+ stored = app.CurrentVersion
if not stored:
logging.info("{0} has no Current Version but has version {1} on the Play Store"
.format(common.getappname(app), version))
.format(common.getappname(app), version))
return
- for appid, app in apps.iteritems():
+ for appid, app in apps.items():
- if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
+ if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
logging.debug("Nothing to do for {0}...".format(appid))
continue
~ianmdlvl / fdroidserver.git / blobdiff