2 # -*- coding: utf-8 -*-
4 # checkupdates.py - part of the FDroid server tools
5 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from argparse import ArgumentParser
30 from distutils.version import LooseVersion
35 from common import VCSException, FDroidException
36 from metadata import MetaDataException
39 # Check for a new version by looking at a document retrieved via HTTP.
40 # The app's Update Check Data field is used to provide the information
46 if 'Update Check Data' not in app:
47 raise FDroidException('Missing Update Check Data')
49 urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
53 logging.debug("...requesting {0}".format(urlcode))
54 req = urllib2.Request(urlcode, None)
55 resp = urllib2.urlopen(req, None, 20)
58 m = re.search(codeex, page)
60 raise FDroidException("No RE match for version code")
66 logging.debug("...requesting {0}".format(urlver))
67 req = urllib2.Request(urlver, None)
68 resp = urllib2.urlopen(req, None, 20)
71 m = re.search(verex, page)
73 raise FDroidException("No RE match for version")
76 return (version, vercode)
78 except FDroidException:
79 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
83 def app_matches_packagename(app, package):
86 appid = app['Update Check Name'] or app['id']
89 return appid == package
92 # Check for a new version by looking at the tags in the source repo.
93 # Whether this can be used reliably or not depends on
94 # the development procedures used by the project's developers. Use it with
95 # caution, because it's inappropriate for many projects.
96 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
97 # the details of the current version.
98 def check_tags(app, pattern):
102 if app['Repo Type'] == 'srclib':
103 build_dir = os.path.join('build', 'srclib', app['Repo'])
104 repotype = common.getsrclibvcs(app['Repo'])
106 build_dir = os.path.join('build', app['id'])
107 repotype = app['Repo Type']
109 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
110 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
112 if repotype == 'git-svn' and ';' not in app['Repo']:
113 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
115 # Set up vcs interface and make sure we have the latest code...
116 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
118 vcs.gotorevision(None)
121 if len(app['builds']) > 0:
122 if app['builds'][-1]['gradle']:
123 flavours = app['builds'][-1]['gradle']
131 logging.debug("All tags: " + ','.join(tags))
133 pat = re.compile(pattern)
134 tags = [tag for tag in tags if pat.match(tag)]
135 logging.debug("Matching tags: " + ','.join(tags))
137 if repotype in ('git',):
138 tags = vcs.latesttags(tags, 5)
139 logging.debug("Latest tags: " + ','.join(tags))
142 logging.debug("Check tag: '{0}'".format(tag))
143 vcs.gotorevision(tag)
145 for subdir in possible_subdirs(app):
149 root_dir = os.path.join(build_dir, subdir)
150 paths = common.manifest_paths(root_dir, flavours)
151 version, vercode, package = \
152 common.parse_androidmanifests(paths, app['Update Check Ignore'])
154 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
155 .format(subdir, version, vercode))
156 if int(vercode) > int(hcode):
159 hcode = str(int(vercode))
163 return (None, "Couldn't find package ID", None)
165 return (hver, hcode, htag)
166 return (None, "Couldn't find any version information", None)
168 except VCSException as vcse:
169 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
170 return (None, msg, None)
172 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
173 return (None, msg, None)
176 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
177 # of the source repo. Whether this can be used reliably or not depends on
178 # the development procedures used by the project's developers. Use it with
179 # caution, because it's inappropriate for many projects.
180 # Returns (None, "a message") if this didn't work, or (version, vercode) for
181 # the details of the current version.
182 def check_repomanifest(app, branch=None):
186 if app['Repo Type'] == 'srclib':
187 build_dir = os.path.join('build', 'srclib', app['Repo'])
188 repotype = common.getsrclibvcs(app['Repo'])
190 build_dir = os.path.join('build', app['id'])
191 repotype = app['Repo Type']
193 # Set up vcs interface and make sure we have the latest code...
194 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
196 if repotype == 'git':
198 branch = 'origin/' + branch
199 vcs.gotorevision(branch)
200 elif repotype == 'git-svn':
201 vcs.gotorevision(branch)
202 elif repotype == 'hg':
203 vcs.gotorevision(branch)
204 elif repotype == 'bzr':
205 vcs.gotorevision(None)
208 if len(app['builds']) > 0:
209 if app['builds'][-1]['gradle']:
210 flavours = app['builds'][-1]['gradle']
215 for subdir in possible_subdirs(app):
219 root_dir = os.path.join(build_dir, subdir)
220 paths = common.manifest_paths(root_dir, flavours)
221 version, vercode, package = \
222 common.parse_androidmanifests(paths, app['Update Check Ignore'])
224 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
225 .format(subdir, version, vercode))
226 if int(vercode) > int(hcode):
228 hcode = str(int(vercode))
232 return (None, "Couldn't find package ID")
235 return (None, "Couldn't find any version information")
237 except VCSException as vcse:
238 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
241 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
245 def check_repotrunk(app, branch=None):
248 if app['Repo Type'] == 'srclib':
249 build_dir = os.path.join('build', 'srclib', app['Repo'])
250 repotype = common.getsrclibvcs(app['Repo'])
252 build_dir = os.path.join('build', app['id'])
253 repotype = app['Repo Type']
255 if repotype not in ('git-svn', ):
256 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
258 # Set up vcs interface and make sure we have the latest code...
259 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
261 vcs.gotorevision(None)
265 except VCSException as vcse:
266 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
269 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
273 # Check for a new version by looking at the Google Play Store.
274 # Returns (None, "a message") if this didn't work, or (version, None) for
275 # the details of the current version.
276 def check_gplay(app):
278 url = 'https://play.google.com/store/apps/details?id=' + app['id']
279 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
280 req = urllib2.Request(url, None, headers)
282 resp = urllib2.urlopen(req, None, 20)
284 except urllib2.HTTPError, e:
285 return (None, str(e.code))
287 return (None, 'Failed:' + str(e))
291 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
293 html_parser = HTMLParser.HTMLParser()
294 version = html_parser.unescape(m.group(1))
296 if version == 'Varies with device':
297 return (None, 'Device-variable version, cannot use this method')
300 return (None, "Couldn't find version")
301 return (version.strip(), None)
304 # Return all directories under startdir that contain any of the manifest
305 # files, and thus are probably an Android project.
306 def dirs_with_manifest(startdir):
307 for r, d, f in os.walk(startdir):
308 if any(m in f for m in [
309 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
313 # Tries to find a new subdir starting from the root build_dir. Returns said
314 # subdir relative to the build dir if found, None otherwise.
315 def possible_subdirs(app):
317 if app['Repo Type'] == 'srclib':
318 build_dir = os.path.join('build', 'srclib', app['Repo'])
320 build_dir = os.path.join('build', app['id'])
323 if len(app['builds']) > 0:
324 build = app['builds'][-1]
326 flavours = build['gradle']
327 subdir = build['subdir']
328 if subdir and os.path.isdir(os.path.join(build_dir, subdir)):
329 logging.debug("Adding possible subdir %s" % subdir)
332 for d in dirs_with_manifest(build_dir):
333 m_paths = common.manifest_paths(d, flavours)
334 package = common.parse_androidmanifests(m_paths, app['Update Check Ignore'])[2]
335 if app_matches_packagename(app, package):
336 subdir = os.path.relpath(d, build_dir)
337 logging.debug("Adding possible subdir %s" % subdir)
341 def fetch_autoname(app, tag):
343 if not app["Repo Type"] or app['Update Check Mode'] in ('None', 'Static'):
346 if app['Repo Type'] == 'srclib':
347 app_dir = os.path.join('build', 'srclib', app['Repo'])
349 app_dir = os.path.join('build', app['id'])
352 vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
353 vcs.gotorevision(tag)
358 if len(app['builds']) > 0:
359 if app['builds'][-1]['subdir']:
360 app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
361 if app['builds'][-1]['gradle']:
362 flavours = app['builds'][-1]['gradle']
364 logging.debug("...fetch auto name from " + app_dir)
365 new_name = common.fetch_real_name(app_dir, flavours)
368 logging.debug("...got autoname '" + new_name + "'")
369 if new_name != app['Auto Name']:
370 app['Auto Name'] = new_name
372 commitmsg = "Set autoname of {0}".format(common.getappname(app))
374 logging.debug("...couldn't get autoname")
379 def checkupdates_app(app, first=True):
381 # If a change is made, commitmsg should be set to a description of it.
382 # Only if this is set will changes be written back to the metadata.
389 mode = app['Update Check Mode']
390 if mode.startswith('Tags'):
391 pattern = mode[5:] if len(mode) > 4 else None
392 (version, vercode, tag) = check_tags(app, pattern)
394 elif mode == 'RepoManifest':
395 (version, vercode) = check_repomanifest(app)
397 elif mode.startswith('RepoManifest/'):
399 (version, vercode) = check_repomanifest(app, tag)
401 elif mode == 'RepoTrunk':
402 (version, vercode) = check_repotrunk(app)
405 (version, vercode) = check_http(app)
407 elif mode in ('None', 'Static'):
409 msg = 'Checking disabled'
413 msg = 'Invalid update check method'
415 if version and vercode and app['Vercode Operation']:
416 oldvercode = str(int(vercode))
417 op = app['Vercode Operation'].replace("%c", oldvercode)
418 vercode = str(eval(op))
419 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
421 if version and any(version.startswith(s) for s in [
422 '${', # Gradle variable names
423 '@string/', # Strings we could not resolve
429 logmsg = "...{0} : {1}".format(app['id'], msg)
434 elif vercode == app['Current Version Code']:
435 logging.info("...up to date")
437 app['Current Version'] = version
438 app['Current Version Code'] = str(int(vercode))
441 commitmsg = fetch_autoname(app, tag)
444 name = common.getappname(app)
445 ver = common.getcvname(app)
446 logging.info('...updating to version %s' % ver)
447 commitmsg = 'Update CV of %s to %s' % (name, ver)
450 mode = app['Auto Update Mode']
451 if mode in ('None', 'Static'):
453 elif mode.startswith('Version '):
455 if pattern.startswith('+'):
457 suffix, pattern = pattern.split(' ', 1)
459 raise MetaDataException("Invalid AUM: " + mode)
464 for build in app['builds']:
465 if build['vercode'] >= app['Current Version Code']:
467 if not latest or int(build['vercode']) > int(latest['vercode']):
470 if int(latest['vercode']) > int(app['Current Version Code']):
471 logging.info("Refusing to auto update, since the latest build is newer")
474 newbuild = latest.copy()
475 if 'origlines' in newbuild:
476 del newbuild['origlines']
477 newbuild['disable'] = False
478 newbuild['vercode'] = app['Current Version Code']
479 newbuild['version'] = app['Current Version'] + suffix
480 logging.info("...auto-generating build for " + newbuild['version'])
481 commit = pattern.replace('%v', newbuild['version'])
482 commit = commit.replace('%c', newbuild['vercode'])
483 newbuild['commit'] = commit
484 app['builds'].append(newbuild)
485 name = common.getappname(app)
486 ver = common.getcvname(app)
487 commitmsg = "Update %s to %s" % (name, ver)
489 logging.warn('Invalid auto update mode "' + mode + '" on ' + app['id'])
492 metadatapath = os.path.join('metadata', app['id'] + '.txt')
493 with open(metadatapath, 'w') as f:
494 metadata.write_metadata('txt', f, app)
496 logging.info("Commiting update for " + metadatapath)
497 gitcmd = ["git", "commit", "-m", commitmsg]
498 if 'auto_author' in config:
499 gitcmd.extend(['--author', config['auto_author']])
500 gitcmd.extend(["--", metadatapath])
501 if subprocess.call(gitcmd) != 0:
502 logging.error("Git commit failed")
512 global config, options
514 # Parse command line...
515 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
516 common.setup_global_opts(parser)
517 parser.add_argument("appid", nargs='*', help="app-id to check for updates")
518 parser.add_argument("--auto", action="store_true", default=False,
519 help="Process auto-updates")
520 parser.add_argument("--autoonly", action="store_true", default=False,
521 help="Only process apps with auto-updates")
522 parser.add_argument("--commit", action="store_true", default=False,
523 help="Commit changes")
524 parser.add_argument("--gplay", action="store_true", default=False,
525 help="Only print differences with the Play Store")
526 options = parser.parse_args()
528 config = common.read_config(options)
531 allapps = metadata.read_metadata()
533 apps = common.read_app_args(options.appid, allapps, False)
537 version, reason = check_gplay(app)
540 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
542 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
543 if version is not None:
544 stored = app['Current Version']
546 logging.info("{0} has no Current Version but has version {1} on the Play Store"
547 .format(common.getappname(app), version))
548 elif LooseVersion(stored) < LooseVersion(version):
549 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
550 .format(common.getappname(app), version, stored))
552 if stored != version:
553 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
554 .format(common.getappname(app), version, stored))
556 logging.info("{0} has the same version {1} on the Play Store"
557 .format(common.getappname(app), version))
560 for appid, app in apps.iteritems():
562 if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
563 logging.debug("Nothing to do for {0}...".format(appid))
566 logging.info("Processing " + appid + '...')
568 checkupdates_app(app)
570 logging.info("Finished.")
572 if __name__ == "__main__":