2 # -*- coding: utf-8 -*-
4 # checkupdates.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from optparse import OptionParser
30 from distutils.version import LooseVersion
35 from common import BuildException
36 from common import VCSException
37 from metadata import MetaDataException
40 # Check for a new version by looking at a document retrieved via HTTP.
41 # The app's Update Check Data field is used to provide the information
47 if 'Update Check Data' not in app:
48 raise Exception('Missing Update Check Data')
50 urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
54 logging.debug("...requesting {0}".format(urlcode))
55 req = urllib2.Request(urlcode, None)
56 resp = urllib2.urlopen(req, None, 20)
59 m = re.search(codeex, page)
61 raise Exception("No RE match for version code")
67 logging.debug("...requesting {0}".format(urlver))
68 req = urllib2.Request(urlver, None)
69 resp = urllib2.urlopen(req, None, 20)
72 m = re.search(verex, page)
74 raise Exception("No RE match for version")
77 return (version, vercode)
80 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
84 # Check for a new version by looking at the tags in the source repo.
85 # Whether this can be used reliably or not depends on
86 # the development procedures used by the project's developers. Use it with
87 # caution, because it's inappropriate for many projects.
88 # Returns (None, "a message") if this didn't work, or (version, vercode) for
89 # the details of the current version.
90 def check_tags(app, pattern):
94 appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
95 if app['Repo Type'] == 'srclib':
96 build_dir = os.path.join('build', 'srclib', app['Repo'])
97 repotype = common.getsrclibvcs(app['Repo'])
99 build_dir = os.path.join('build/', app['id'])
100 repotype = app['Repo Type']
102 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
103 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
105 # Set up vcs interface and make sure we have the latest code...
106 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
108 vcs.gotorevision(None)
111 if len(app['builds']) > 0:
112 if app['builds'][-1]['subdir']:
113 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
114 if app['builds'][-1]['gradle']:
115 flavour = app['builds'][-1]['gradle']
125 pat = re.compile(pattern)
126 tags = [tag for tag in tags if pat.match(tag)]
128 if repotype in ('git',):
129 tags = vcs.latesttags(tags, 5)
132 logging.debug("Check tag: '{0}'".format(tag))
133 vcs.gotorevision(tag)
135 # Only process tags where the manifest exists...
136 paths = common.manifest_paths(build_dir, flavour)
137 version, vercode, package = \
138 common.parse_androidmanifests(paths, app['Update Check Ignore'])
139 if not package or package != appid or not version or not vercode:
142 logging.debug("Manifest exists. Found version {0} ({1})"
143 .format(version, vercode))
144 if int(vercode) > int(hcode):
146 hcode = str(int(vercode))
150 return (hver, hcode, htag)
151 return (None, "Couldn't find any version information", None)
153 except BuildException as be:
154 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
155 return (None, msg, None)
156 except VCSException as vcse:
157 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
158 return (None, msg, None)
160 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
161 return (None, msg, None)
164 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
165 # of the source repo. Whether this can be used reliably or not depends on
166 # the development procedures used by the project's developers. Use it with
167 # caution, because it's inappropriate for many projects.
168 # Returns (None, "a message") if this didn't work, or (version, vercode) for
169 # the details of the current version.
170 def check_repomanifest(app, branch=None):
174 appid = app['Update Check Name'] if app['Update Check Name'] else app['id']
175 if app['Repo Type'] == 'srclib':
176 build_dir = os.path.join('build', 'srclib', app['Repo'])
177 repotype = common.getsrclibvcs(app['Repo'])
179 build_dir = os.path.join('build/', app['id'])
180 repotype = app['Repo Type']
182 # Set up vcs interface and make sure we have the latest code...
183 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
185 if repotype == 'git':
187 branch = 'origin/' + branch
188 vcs.gotorevision(branch)
189 elif repotype == 'git-svn':
190 vcs.gotorevision(branch)
191 elif repotype == 'svn':
192 vcs.gotorevision(None)
193 elif repotype == 'hg':
194 vcs.gotorevision(branch)
195 elif repotype == 'bzr':
196 vcs.gotorevision(None)
200 if len(app['builds']) > 0:
201 if app['builds'][-1]['subdir']:
202 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
203 if app['builds'][-1]['gradle']:
204 flavour = app['builds'][-1]['gradle']
208 if not os.path.isdir(build_dir):
209 return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
211 paths = common.manifest_paths(build_dir, flavour)
213 version, vercode, package = \
214 common.parse_androidmanifests(paths, app['Update Check Ignore'])
216 return (None, "Couldn't find package ID")
218 return (None, "Package ID mismatch")
220 return (None, "Couldn't find latest version name")
222 if "Ignore" == version:
223 return (None, "Latest version is ignored")
224 return (None, "Couldn't find latest version code")
226 vercode = str(int(vercode))
228 logging.debug("Manifest exists. Found version {0} ({1})".format(version, vercode))
230 return (version, vercode)
232 except BuildException as be:
233 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
235 except VCSException as vcse:
236 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
239 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
243 def check_repotrunk(app, branch=None):
246 if app['Repo Type'] == 'srclib':
247 build_dir = os.path.join('build', 'srclib', app['Repo'])
248 repotype = common.getsrclibvcs(app['Repo'])
250 build_dir = os.path.join('build/', app['id'])
251 repotype = app['Repo Type']
253 if repotype not in ('svn', 'git-svn'):
254 return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
256 # Set up vcs interface and make sure we have the latest code...
257 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
259 vcs.gotorevision(None)
263 except BuildException as be:
264 msg = "Could not scan app {0} due to BuildException: {1}".format(app['id'], be)
266 except VCSException as vcse:
267 msg = "VCS error while scanning app {0}: {1}".format(app['id'], vcse)
270 msg = "Could not scan app {0} due to unknown error: {1}".format(app['id'], traceback.format_exc())
274 # Check for a new version by looking at the Google Play Store.
275 # Returns (None, "a message") if this didn't work, or (version, None) for
276 # the details of the current version.
277 def check_gplay(app):
279 url = 'https://play.google.com/store/apps/details?id=' + app['id']
280 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
281 req = urllib2.Request(url, None, headers)
283 resp = urllib2.urlopen(req, None, 20)
285 except urllib2.HTTPError, e:
286 return (None, str(e.code))
288 return (None, 'Failed:' + str(e))
292 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
294 html_parser = HTMLParser.HTMLParser()
295 version = html_parser.unescape(m.group(1))
297 if version == 'Varies with device':
298 return (None, 'Device-variable version, cannot use this method')
301 return (None, "Couldn't find version")
302 return (version.strip(), None)
311 global config, options
313 # Parse command line...
314 parser = OptionParser(usage="Usage: %prog [options] [APPID [APPID ...]]")
315 parser.add_option("-v", "--verbose", action="store_true", default=False,
316 help="Spew out even more information than normal")
317 parser.add_option("-q", "--quiet", action="store_true", default=False,
318 help="Restrict output to warnings and errors")
319 parser.add_option("--auto", action="store_true", default=False,
320 help="Process auto-updates")
321 parser.add_option("--autoonly", action="store_true", default=False,
322 help="Only process apps with auto-updates")
323 parser.add_option("--commit", action="store_true", default=False,
324 help="Commit changes")
325 parser.add_option("--gplay", action="store_true", default=False,
326 help="Only print differences with the Play Store")
327 (options, args) = parser.parse_args()
329 config = common.read_config(options)
332 allapps = metadata.read_metadata()
333 metadata.read_srclibs()
335 apps = common.read_app_args(args, allapps, False)
339 version, reason = check_gplay(app)
342 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
344 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
345 if version is not None:
346 stored = app['Current Version']
348 logging.info("{0} has no Current Version but has version {1} on the Play Store"
349 .format(common.getappname(app), version))
350 elif LooseVersion(stored) < LooseVersion(version):
351 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
352 .format(common.getappname(app), version, stored))
354 if stored != version:
355 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
356 .format(common.getappname(app), version, stored))
358 logging.info("{0} has the same version {1} on the Play Store"
359 .format(common.getappname(app), version))
364 if options.autoonly and app['Auto Update Mode'] in ('None', 'Static'):
365 logging.debug("Nothing to do for {0}...".format(app['id']))
368 logging.info("Processing " + app['id'] + '...')
370 # If a change is made, commitmsg should be set to a description of it.
371 # Only if this is set will changes be written back to the metadata.
378 mode = app['Update Check Mode']
379 if mode.startswith('Tags'):
380 pattern = mode[5:] if len(mode) > 4 else None
381 (version, vercode, tag) = check_tags(app, pattern)
383 elif mode == 'RepoManifest':
384 (version, vercode) = check_repomanifest(app)
386 elif mode.startswith('RepoManifest/'):
388 (version, vercode) = check_repomanifest(app, tag)
390 elif mode == 'RepoTrunk':
391 (version, vercode) = check_repotrunk(app)
394 (version, vercode) = check_http(app)
396 elif mode in ('None', 'Static'):
398 msg = 'Checking disabled'
402 msg = 'Invalid update check method'
404 if vercode and app['Vercode Operation']:
405 op = app['Vercode Operation'].replace("%c", str(int(vercode)))
406 vercode = str(eval(op))
410 logmsg = "...{0} : {1}".format(app['id'], msg)
415 elif vercode == app['Current Version Code']:
416 logging.info("...up to date")
418 app['Current Version'] = version
419 app['Current Version Code'] = str(int(vercode))
422 # Do the Auto Name thing as well as finding the CV real name
423 if len(app["Repo Type"]) > 0 and mode not in ('None', 'Static'):
427 if app['Repo Type'] == 'srclib':
428 app_dir = os.path.join('build', 'srclib', app['Repo'])
430 app_dir = os.path.join('build/', app['id'])
432 vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir)
433 vcs.gotorevision(tag)
436 if len(app['builds']) > 0:
437 if app['builds'][-1]['subdir']:
438 app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
439 if app['builds'][-1]['gradle']:
440 flavour = app['builds'][-1]['gradle']
444 logging.debug("...fetch auto name from " + app_dir +
445 ((" (flavour: %s)" % flavour) if flavour else ""))
446 new_name = common.fetch_real_name(app_dir, flavour)
448 logging.debug("...got autoname '" + new_name + "'")
449 if new_name != app['Auto Name']:
450 app['Auto Name'] = new_name
452 commitmsg = "Set autoname of {0}".format(common.getappname(app))
454 logging.debug("...couldn't get autoname")
456 if app['Current Version'].startswith('@string/'):
457 cv = common.version_name(app['Current Version'], app_dir, flavour)
458 if app['Current Version'] != cv:
459 app['Current Version'] = cv
461 commitmsg = "Fix CV of {0}".format(common.getappname(app))
463 logging.error("Auto Name or Current Version failed for {0} due to exception: {1}".format(app['id'], traceback.format_exc()))
466 name = common.getappname(app)
467 ver = common.getcvname(app)
468 logging.info('...updating to version %s' % ver)
469 commitmsg = 'Update CV of %s to %s' % (name, ver)
472 mode = app['Auto Update Mode']
473 if mode in ('None', 'Static'):
475 elif mode.startswith('Version '):
477 if pattern.startswith('+'):
479 suffix, pattern = pattern.split(' ', 1)
481 raise MetaDataException("Invalid AUM: " + mode)
486 for build in app['builds']:
487 if build['vercode'] == app['Current Version Code']:
489 if not latest or int(build['vercode']) > int(latest['vercode']):
493 newbuild = latest.copy()
494 if 'origlines' in newbuild:
495 del newbuild['origlines']
496 newbuild['disable'] = False
497 newbuild['vercode'] = app['Current Version Code']
498 newbuild['version'] = app['Current Version'] + suffix
499 logging.info("...auto-generating build for " + newbuild['version'])
500 commit = pattern.replace('%v', newbuild['version'])
501 commit = commit.replace('%c', newbuild['vercode'])
502 newbuild['commit'] = commit
503 app['builds'].append(newbuild)
504 name = common.getappname(app)
505 ver = common.getcvname(app)
506 commitmsg = "Update %s to %s" % (name, ver)
508 logging.warn('Invalid auto update mode "' + mode + '" on ' + app['id'])
511 metafile = os.path.join('metadata', app['id'] + '.txt')
512 metadata.write_metadata(metafile, app)
514 logging.info("Commiting update for " + metafile)
515 gitcmd = ["git", "commit", "-m", commitmsg]
516 if 'auto_author' in config:
517 gitcmd.extend(['--author', config['auto_author']])
518 gitcmd.extend(["--", metafile])
519 if subprocess.call(gitcmd) != 0:
520 logging.error("Git commit failed")
523 logging.info("Finished.")
525 if __name__ == "__main__":