2 # -*- coding: utf-8 -*-
4 # checkupdates.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from optparse import OptionParser
30 from distutils.version import LooseVersion
35 from common import BuildException
36 from common import VCSException
37 from metadata import MetaDataException
40 # Check for a new version by looking at a document retrieved via HTTP.
41 # The app's Update Check Data field is used to provide the information
47 if not 'Update Check Data' in app:
48 raise Exception('Missing Update Check Data')
50 urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
54 logging.debug("...requesting {0}".format(urlcode))
55 req = urllib2.Request(urlcode, None)
56 resp = urllib2.urlopen(req, None, 20)
59 m = re.search(codeex, page)
61 raise Exception("No RE match for version code")
67 logging.debug("...requesting {0}".format(urlver))
68 req = urllib2.Request(urlver, None)
69 resp = urllib2.urlopen(req, None, 20)
72 m = re.search(verex, page)
74 raise Exception("No RE match for version")
77 return (version, vercode)
80 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
84 # Check for a new version by looking at the tags in the source repo.
85 # Whether this can be used reliably or not depends on
86 # the development procedures used by the project's developers. Use it with
87 # caution, because it's inappropriate for many projects.
88 # Returns (None, "a message") if this didn't work, or (version, vercode) for
89 # the details of the current version.
90 def check_tags(app, pattern):
94 appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
95 if app['Repo Type'] == 'srclib':
96 build_dir = os.path.join('build', 'srclib', app['Repo'])
97 repotype = common.getsrclibvcs(app['Repo'])
99 build_dir = os.path.join('build/', app['id'])
100 repotype = app['Repo Type']
102 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
103 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
105 # Set up vcs interface and make sure we have the latest code...
106 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
108 vcs.gotorevision(None)
111 if len(app['builds']) > 0:
112 if 'subdir' in app['builds'][-1]:
113 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
114 if 'gradle' in app['builds'][-1]:
115 flavour = app['builds'][-1]['gradle']
125 pat = re.compile(pattern)
126 tags = [tag for tag in tags if pat.match(tag)]
128 if repotype in ('git',):
129 tags = vcs.latesttags(tags, 5)
132 logging.debug("Check tag: '{0}'".format(tag))
133 vcs.gotorevision(tag)
135 # Only process tags where the manifest exists...
136 paths = common.manifest_paths(build_dir, flavour)
137 version, vercode, package = common.parse_androidmanifests(paths)
138 if not package or package != appid or not version or not vercode:
141 logging.debug("Manifest exists. Found version {0} ({1})".format(
143 if int(vercode) > int(hcode):
145 hcode = str(int(vercode))
149 return (hver, hcode, htag)
150 return (None, "Couldn't find any version information", None)
152 except BuildException as be:
153 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
154 return (None, msg, None)
155 except VCSException as vcse:
156 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
157 return (None, msg, None)
159 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
160 return (None, msg, None)
163 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
164 # of the source repo. Whether this can be used reliably or not depends on
165 # the development procedures used by the project's developers. Use it with
166 # caution, because it's inappropriate for many projects.
167 # Returns (None, "a message") if this didn't work, or (version, vercode) for
168 # the details of the current version.
169 def check_repomanifest(app, branch=None):
173 appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
174 if app['Repo Type'] == 'srclib':
175 build_dir = os.path.join('build', 'srclib', app['Repo'])
176 repotype = common.getsrclibvcs(app['Repo'])
178 build_dir = os.path.join('build/', app['id'])
179 repotype = app['Repo Type']
181 # Set up vcs interface and make sure we have the latest code...
182 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
184 if repotype == 'git':
186 branch = 'origin/'+branch
187 vcs.gotorevision(branch)
188 elif repotype == 'git-svn':
189 vcs.gotorevision(branch)
190 elif repotype == 'svn':
191 vcs.gotorevision(None)
192 elif repotype == 'hg':
193 vcs.gotorevision(branch)
194 elif repotype == 'bzr':
195 vcs.gotorevision(None)
199 if len(app['builds']) > 0:
200 if 'subdir' in app['builds'][-1]:
201 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
202 if 'gradle' in app['builds'][-1]:
203 flavour = app['builds'][-1]['gradle']
207 if not os.path.isdir(build_dir):
208 return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
210 paths = common.manifest_paths(build_dir, flavour)
212 version, vercode, package = common.parse_androidmanifests(paths)
214 return (None, "Couldn't find package ID")
216 return (None, "Package ID mismatch")
218 return (None, "Couldn't find latest version name")
220 return (None, "Couldn't find latest version code")
222 vercode = str(int(vercode))
224 logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
226 return (version, vercode)
228 except BuildException as be:
229 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
231 except VCSException as vcse:
232 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
235 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
239 def check_repotrunk(app, branch=None):
242 if app['Repo Type'] == 'srclib':
243 build_dir = os.path.join('build', 'srclib', app['Repo'])
244 repotype = common.getsrclibvcs(app['Repo'])
246 build_dir = os.path.join('build/', app['id'])
247 repotype = app['Repo Type']
249 if repotype not in ('svn', 'git-svn'):
250 return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
252 # Set up vcs interface and make sure we have the latest code...
253 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
255 vcs.gotorevision(None)
259 except BuildException as be:
260 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
262 except VCSException as vcse:
263 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
266 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
270 # Check for a new version by looking at the Google Play Store.
271 # Returns (None, "a message") if this didn't work, or (version, None) for
272 # the details of the current version.
273 def check_gplay(app):
275 url = 'https://play.google.com/store/apps/details?id=' + app['id']
276 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
277 req = urllib2.Request(url, None, headers)
279 resp = urllib2.urlopen(req, None, 20)
281 except urllib2.HTTPError, e:
282 return (None, str(e.code))
284 return (None, 'Failed:' + str(e))
288 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
290 html_parser = HTMLParser.HTMLParser()
291 version = html_parser.unescape(m.group(1))
293 if version == 'Varies with device':
294 return (None, 'Device-variable version, cannot use this method')
297 return (None, "Couldn't find version")
298 return (version.strip(), None)
307 global config, options
309 # Parse command line...
310 parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
311 parser.add_option("-v", "--verbose", action="store_true", default=False,
312 help="Spew out even more information than normal")
313 parser.add_option("-q", "--quiet", action="store_true", default=False,
314 help="Restrict output to warnings and errors")
315 parser.add_option("--auto", action="store_true", default=False,
316 help="Process auto-updates")
317 parser.add_option("--autoonly", action="store_true", default=False,
318 help="Only process apps with auto-updates")
319 parser.add_option("--commit", action="store_true", default=False,
320 help="Commit changes")
321 parser.add_option("--gplay", action="store_true", default=False,
322 help="Only print differences with the Play Store")
323 (options, args) = parser.parse_args()
325 config = common.read_config(options)
328 allapps = metadata.read_metadata(options.verbose)
330 apps = common.read_app_args(args, allapps, False)
334 version, reason = check_gplay(app)
337 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
339 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
340 if version is not None:
341 stored = app['Current Version']
343 logging.info("{0} has no Current Version but has version {1} on the Play Store".format(
344 common.getappname(app), version))
345 elif LooseVersion(stored) < LooseVersion(version):
346 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}".format(
347 common.getappname(app), version, stored))
349 if stored != version:
350 logging.info("{0} has version {1} on the Play Store, which differs from {2}".format(
351 common.getappname(app), version, stored))
353 logging.info("{0} has the same version {1} on the Play Store".format(
354 common.getappname(app), version))
359 if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
360 logging.debug("Nothing to do for {0}...".format(app['id']))
363 logging.info("Processing " + app['id'] + '...')
365 # If a change is made, commitmsg should be set to a description of it.
366 # Only if this is set will changes be written back to the metadata.
373 mode = app['Update Check Mode']
374 if mode.startswith('Tags'):
375 pattern = mode[5:] if len(mode) > 4 else None
376 (version, vercode, tag) = check_tags(app, pattern)
378 elif mode == 'RepoManifest':
379 (version, vercode) = check_repomanifest(app)
381 elif mode.startswith('RepoManifest/'):
383 (version, vercode) = check_repomanifest(app, tag)
385 elif mode == 'RepoTrunk':
386 (version, vercode) = check_repotrunk(app)
389 (version, vercode) = check_http(app)
391 elif mode in ('None', 'Static'):
393 msg = 'Checking disabled'
397 msg = 'Invalid update check method'
399 if vercode and app['Vercode Operation']:
400 op = app['Vercode Operation'].replace("%c", str(int(vercode)))
401 vercode = str(eval(op))
405 logmsg = "...{0} : {1}".format(app['id'], msg)
410 elif vercode == app['Current Version Code']:
411 logging.info("...up to date")
413 app['Current Version'] = version
414 app['Current Version Code'] = str(int(vercode))
417 # Do the Auto Name thing as well as finding the CV real name
418 if len(app["Repo Type"]) > 0 and mode not in ('None', 'Static'):
422 if app['Repo Type'] == 'srclib':
423 app_dir = os.path.join('build', 'srclib', app['Repo'])
425 app_dir = os.path.join('build/', app['id'])
427 vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
428 vcs.gotorevision(tag)
431 if len(app['builds']) > 0:
432 if 'subdir' in app['builds'][-1]:
433 app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
434 if 'gradle' in app['builds'][-1]:
435 flavour = app['builds'][-1]['gradle']
439 logging.debug("...fetch auto name from " + app_dir +
440 ((" (flavour: %s)" % flavour) if flavour else ""))
441 new_name = common.fetch_real_name(app_dir, flavour)
443 logging.debug("...got autoname '" + new_name + "'")
444 if new_name != app['Auto Name']:
445 app['Auto Name'] = new_name
447 commitmsg = "Set autoname of {0}".format(common.getappname(app))
449 logging.debug("...couldn't get autoname")
451 if app['Current Version'].startswith('@string/'):
452 cv = common.version_name(app['Current Version'], app_dir, flavour)
453 if app['Current Version'] != cv:
454 app['Current Version'] = cv
456 commitmsg = "Fix CV of {0}".format(common.getappname(app))
458 logging.error("Auto Name or Current Version failed for {0} due to exception: {1}".format(app['id'], traceback.format_exc()))
461 name = common.getappname(app)
462 ver = common.getcvname(app)
463 logging.info('...updating to version %s' % ver)
464 commitmsg = 'Update CV of %s to %s' % (name, ver)
467 mode = app['Auto Update Mode']
468 if mode in ('None', 'Static'):
470 elif mode.startswith('Version '):
472 if pattern.startswith('+'):
474 suffix, pattern = pattern.split(' ', 1)
476 raise MetaDataException("Invalid AUM: " + mode)
481 for build in app['builds']:
482 if build['vercode'] == app['Current Version Code']:
484 if not latest or int(build['vercode']) > int(latest['vercode']):
488 newbuild = latest.copy()
489 for k in ('origlines', 'disable'):
492 newbuild['vercode'] = app['Current Version Code']
493 newbuild['version'] = app['Current Version'] + suffix
494 logging.info("...auto-generating build for " + newbuild['version'])
495 commit = pattern.replace('%v', newbuild['version'])
496 commit = commit.replace('%c', newbuild['vercode'])
497 newbuild['commit'] = commit
498 app['builds'].append(newbuild)
499 name = common.getappname(app)
500 ver = common.getcvname(app)
501 commitmsg = "Update %s to %s" % (name, ver)
503 logging.warn('Invalid auto update mode "' + mode + '" on ' + app['id'])
506 metafile = os.path.join('metadata', app['id'] + '.txt')
507 metadata.write_metadata(metafile, app)
509 logging.info("Commiting update for " + metafile)
510 gitcmd = ["git", "commit", "-m",
512 if 'auto_author' in config:
513 gitcmd.extend(['--author', config['auto_author']])
514 gitcmd.extend(["--", metafile])
515 if subprocess.call(gitcmd) != 0:
516 logging.error("Git commit failed")
519 logging.info("Finished.")
521 if __name__ == "__main__":