2 # -*- coding: utf-8 -*-
4 # checkupdates.py - part of the FDroid server tools
5 # Copyright (C) 2010-13, Ciaran Gultnieks, ciaran@ciarang.com
6 # Copyright (C) 2013 Daniel Martà <mvdan@mvdan.cc>
8 # This program is free software: you can redistribute it and/or modify
9 # it under the terms of the GNU Affero General Public License as published by
10 # the Free Software Foundation, either version 3 of the License, or
11 # (at your option) any later version.
13 # This program is distributed in the hope that it will be useful,
14 # but WITHOUT ANY WARRANTY; without even the implied warranty of
15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 # GNU Affero General Public License for more details.
18 # You should have received a copy of the GNU Affero General Public License
19 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from optparse import OptionParser
30 from distutils.version import LooseVersion
32 from common import BuildException
33 from common import VCSException
36 # Check for a new version by looking at a document retrieved via HTTP.
37 # The app's Update Check Data field is used to provide the information
43 if not 'Update Check Data' in app:
44 raise Exception('Missing Update Check Data')
46 urlcode, codeex, urlver, verex = app['Update Check Data'].split('|')
50 print "...requesting {0}".format(urlcode)
51 req = urllib2.Request(urlcode, None)
52 resp = urllib2.urlopen(req, None, 20)
55 m = re.search(codeex, page)
57 raise Exception("No RE match for version code")
63 print "...requesting {0}".format(urlver)
64 req = urllib2.Request(urlver, None)
65 resp = urllib2.urlopen(req, None, 20)
68 m = re.search(verex, page)
70 raise Exception("No RE match for version")
73 return (version, vercode)
76 msg = "Could not complete http check for app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
79 # Check for a new version by looking at the tags in the source repo.
80 # Whether this can be used reliably or not depends on
81 # the development procedures used by the project's developers. Use it with
82 # caution, because it's inappropriate for many projects.
83 # Returns (None, "a message") if this didn't work, or (version, vercode) for
84 # the details of the current version.
85 def check_tags(app, sdk_path):
89 if app['Repo Type'] == 'srclib':
90 build_dir = os.path.join('build', 'srclib', app['Repo'])
91 repotype = common.getsrclibvcs(app['Repo'])
93 build_dir = os.path.join('build/', app['id'])
94 repotype = app['Repo Type']
96 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
97 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
99 # Set up vcs interface and make sure we have the latest code...
100 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
102 vcs.gotorevision(None)
105 if len(app['builds']) > 0:
106 if 'subdir' in app['builds'][-1]:
107 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
108 if 'gradle' in app['builds'][-1]:
109 flavour = app['builds'][-1]['gradle']
115 for tag in vcs.gettags():
116 vcs.gotorevision(tag)
118 # Only process tags where the manifest exists...
119 paths = common.manifest_paths(build_dir, flavour)
120 version, vercode, package = common.parse_androidmanifests(paths)
121 if package and package == app['id'] and version and vercode:
122 print "Manifest exists. Found version %s" % version
123 if int(vercode) > int(hcode):
125 hcode = str(int(vercode))
129 return (hver, hcode, htag)
130 return (None, "Couldn't find any version information", None)
132 except BuildException as be:
133 msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
134 return (None, msg, None)
135 except VCSException as vcse:
136 msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
137 return (None, msg, None)
139 msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
140 return (None, msg, None)
142 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
143 # of the source repo. Whether this can be used reliably or not depends on
144 # the development procedures used by the project's developers. Use it with
145 # caution, because it's inappropriate for many projects.
146 # Returns (None, "a message") if this didn't work, or (version, vercode) for
147 # the details of the current version.
148 def check_repomanifest(app, sdk_path, branch=None):
152 if app['Repo Type'] == 'srclib':
153 build_dir = os.path.join('build', 'srclib', app['Repo'])
154 repotype = common.getsrclibvcs(app['Repo'])
156 build_dir = os.path.join('build/', app['id'])
157 repotype = app['Repo Type']
159 # Set up vcs interface and make sure we have the latest code...
160 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
162 if repotype == 'git':
164 branch = 'origin/'+branch
165 vcs.gotorevision(branch)
166 elif repotype == 'git-svn':
167 vcs.gotorevision(branch)
168 elif repotype == 'svn':
169 vcs.gotorevision(None)
170 elif repotype == 'hg':
171 vcs.gotorevision(branch)
172 elif repotype == 'bzr':
173 vcs.gotorevision(None)
177 if len(app['builds']) > 0:
178 if 'subdir' in app['builds'][-1]:
179 build_dir = os.path.join(build_dir, app['builds'][-1]['subdir'])
180 if 'gradle' in app['builds'][-1]:
181 flavour = app['builds'][-1]['gradle']
183 if not os.path.isdir(build_dir):
184 return (None, "Subdir '" + app['builds'][-1]['subdir'] + "'is not a valid directory")
186 paths = common.manifest_paths(build_dir, flavour)
188 version, vercode, package = common.parse_androidmanifests(paths)
190 return (None, "Couldn't find package ID")
191 if package != app['id']:
192 return (None, "Package ID mismatch")
194 return (None,"Couldn't find latest version name")
196 return (None,"Couldn't find latest version code")
198 return (version, str(int(vercode)))
200 except BuildException as be:
201 msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
203 except VCSException as vcse:
204 msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
207 msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
210 def check_repotrunk(app, sdk_path, branch=None):
213 if app['Repo Type'] == 'srclib':
214 build_dir = os.path.join('build', 'srclib', app['Repo'])
215 repotype = common.getsrclibvcs(app['Repo'])
217 build_dir = os.path.join('build/', app['id'])
218 repotype = app['Repo Type']
220 if repotype not in ('svn', 'git-svn'):
221 return (None, 'RepoTrunk update mode only makes sense in svn and git-svn repositories')
223 # Set up vcs interface and make sure we have the latest code...
224 vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir, sdk_path)
226 vcs.gotorevision(None)
230 except BuildException as be:
231 msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
233 except VCSException as vcse:
234 msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
237 msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
240 # Check for a new version by looking at the Google Play Store.
241 # Returns (None, "a message") if this didn't work, or (version, None) for
242 # the details of the current version.
243 def check_gplay(app):
245 url = 'https://play.google.com/store/apps/details?id=' + app['id']
246 headers = {'User-Agent' : 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
247 req = urllib2.Request(url, None, headers)
249 resp = urllib2.urlopen(req, None, 20)
251 except urllib2.HTTPError, e:
252 return (None, str(e.code))
254 return (None, 'Failed:' + str(e))
258 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
260 html_parser = HTMLParser.HTMLParser()
261 version = html_parser.unescape(m.group(1))
263 if version == 'Varies with device':
264 return (None, 'Device-variable version, cannot use this method')
267 return (None, "Couldn't find version")
268 return (version.strip(), None)
275 # Read configuration...
276 common.read_config(config)
278 # Parse command line...
279 parser = OptionParser()
280 parser.add_option("-v", "--verbose", action="store_true", default=False,
281 help="Spew out even more information than normal")
282 parser.add_option("-p", "--package", default=None,
283 help="Check only the specified package")
284 parser.add_option("--auto", action="store_true", default=False,
285 help="Process auto-updates")
286 parser.add_option("--autoonly", action="store_true", default=False,
287 help="Only process apps with auto-updates")
288 parser.add_option("--commit", action="store_true", default=False,
289 help="Commit changes")
290 parser.add_option("--gplay", action="store_true", default=False,
291 help="Only print differences with the Play Store")
292 (options, args) = parser.parse_args()
295 apps = common.read_metadata(options.verbose)
297 # Filter apps according to command-line options
299 apps = [app for app in apps if app['id'] == options.package]
301 print "No such package"
306 version, reason = check_gplay(app)
307 if version is None and options.verbose:
309 print "%s (%s) is not in the Play Store" % (app['Auto Name'], app['id'])
311 print "%s (%s) encountered a problem: %s" % (app['Auto Name'], app['id'], reason)
312 if version is not None:
313 stored = app['Current Version']
314 if LooseVersion(stored) < LooseVersion(version):
315 print "%s (%s) has version %s on the Play Store, which is bigger than %s" % (
316 app['Auto Name'], app['id'], version, stored)
317 elif options.verbose:
318 print "%s (%s) has the same version %s on the Play Store" % (
319 app['Auto Name'], app['id'], version)
326 if options.autoonly and app['Auto Update Mode'] == 'None':
327 print "Nothing to do for %s..." % app['id']
330 print "Processing " + app['id'] + '...'
336 mode = app['Update Check Mode']
338 (version, vercode, tag) = check_tags(app, config['sdk_path'])
339 elif mode == 'RepoManifest':
340 (version, vercode) = check_repomanifest(app, config['sdk_path'])
341 elif mode.startswith('RepoManifest/'):
342 (version, vercode) = check_repomanifest(app, config['sdk_path'], mode[13:])
343 elif mode == 'RepoTrunk':
344 (version, vercode) = check_repotrunk(app, config['sdk_path'])
346 (version, vercode) = check_http(app)
347 elif mode == 'Static':
349 vercode = 'Checking disabled'
352 vercode = 'Checking disabled'
355 vercode = 'Invalid update check method'
359 print "..." + vercode
360 elif vercode == app['Current Version Code']:
361 print "...up to date"
363 app['Current Version'] = version
364 app['Current Version Code'] = str(int(vercode))
368 # Do the Auto Name thing as well as finding the CV real name
369 if len(app["Repo Type"]) > 0:
373 if app['Repo Type'] == 'srclib':
374 app_dir = os.path.join('build', 'srclib', app['Repo'])
376 app_dir = os.path.join('build/', app['id'])
378 vcs = common.getvcs(app["Repo Type"], app["Repo"], app_dir,
380 vcs.gotorevision(tag)
383 if len(app['builds']) > 0:
384 if 'subdir' in app['builds'][-1]:
385 app_dir = os.path.join(app_dir, app['builds'][-1]['subdir'])
386 if 'gradle' in app['builds'][-1]:
387 flavour = app['builds'][-1]['gradle']
389 new_name = common.fetch_real_name(app_dir, flavour)
390 if new_name != app['Auto Name']:
391 app['Auto Name'] = new_name
393 if app['Current Version'].startswith('@string/'):
394 cv = common.version_name(app['Current Version'], app_dir, flavour)
395 if app['Current Version'] != cv:
396 app['Current Version'] = cv
399 print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc())
402 print '...updating to version %s (%s)' % (app['Current Version'], app['Current Version Code'])
403 name = '%s (%s)' % (app['Auto Name'], app['id']) if app['Auto Name'] else app['id']
404 ver = "%s (%s)" % (app['Current Version'], app['Current Version Code'])
405 logmsg = 'Update CV of %s to %s' % (name, ver)
408 mode = app['Auto Update Mode']
411 elif mode.startswith('Version '):
413 if pattern.startswith('+'):
414 o = pattern.find(' ')
415 suffix = pattern[1:o]
416 pattern = pattern[o + 1:]
421 for build in app['builds']:
422 if build['vercode'] == app['Current Version Code']:
424 if not latest or int(build['vercode']) > int(latest['vercode']):
427 newbuild = latest.copy()
428 if 'origlines' in newbuild:
429 del newbuild['origlines']
430 newbuild['vercode'] = app['Current Version Code']
431 newbuild['version'] = app['Current Version'] + suffix
432 print "...auto-generating build for " + newbuild['version']
433 commit = pattern.replace('%v', newbuild['version'])
434 commit = commit.replace('%c', newbuild['vercode'])
435 newbuild['commit'] = commit
436 app['builds'].append(newbuild)
438 name = "%s (%s)" % (app['Auto Name'], app['id']) if app['Auto Name'] else app['id']
439 ver = "%s (%s)" % (newbuild['version'], newbuild['vercode'])
440 logmsg = "Update %s to %s" % (name, ver)
442 print 'Invalid auto update mode'
445 metafile = os.path.join('metadata', app['id'] + '.txt')
446 common.write_metadata(metafile, app)
447 if options.commit and logmsg:
448 print "Commiting update for " + metafile
449 if subprocess.call(["git", "commit", "-m",
450 "'"+logmsg.replace("'", "\\'")+"'", "--", metafile]) != 0:
451 print "Git commit failed"
456 if __name__ == "__main__":