3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from argparse import ArgumentParser
29 from distutils.version import LooseVersion
35 from . import metadata
36 from .exception import VCSException, FDroidException, MetaDataException
39 # Check for a new version by looking at a document retrieved via HTTP.
40 # The app's Update Check Data field is used to provide the information
46 if not app.UpdateCheckData:
47 raise FDroidException('Missing Update Check Data')
49 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
53 logging.debug("...requesting {0}".format(urlcode))
54 req = urllib.request.Request(urlcode, None)
55 resp = urllib.request.urlopen(req, None, 20)
56 page = resp.read().decode('utf-8')
58 m = re.search(codeex, page)
60 raise FDroidException("No RE match for version code")
61 vercode = m.group(1).strip()
66 logging.debug("...requesting {0}".format(urlver))
67 req = urllib.request.Request(urlver, None)
68 resp = urllib.request.urlopen(req, None, 20)
69 page = resp.read().decode('utf-8')
71 m = re.search(verex, page)
73 raise FDroidException("No RE match for version")
76 return (version, vercode)
78 except FDroidException:
79 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
83 # Check for a new version by looking at the tags in the source repo.
84 # Whether this can be used reliably or not depends on
85 # the development procedures used by the project's developers. Use it with
86 # caution, because it's inappropriate for many projects.
87 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
88 # the details of the current version.
89 def check_tags(app, pattern):
93 if app.RepoType == 'srclib':
94 build_dir = os.path.join('build', 'srclib', app.Repo)
95 repotype = common.getsrclibvcs(app.Repo)
97 build_dir = os.path.join('build', app.id)
98 repotype = app.RepoType
100 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
101 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
103 if repotype == 'git-svn' and ';' not in app.Repo:
104 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
106 # Set up vcs interface and make sure we have the latest code...
107 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
109 vcs.gotorevision(None)
111 last_build = app.get_last_build()
113 if last_build.submodules:
122 if repotype == 'git':
123 tags = vcs.latesttags()
127 return (None, "No tags found", None)
129 logging.debug("All tags: " + ','.join(tags))
131 pat = re.compile(pattern)
132 tags = [tag for tag in tags if pat.match(tag)]
134 return (None, "No matching tags found", None)
135 logging.debug("Matching tags: " + ','.join(tags))
137 if len(tags) > 5 and repotype == 'git':
139 logging.debug("Latest tags: " + ','.join(tags))
142 logging.debug("Check tag: '{0}'".format(tag))
143 vcs.gotorevision(tag)
145 for subdir in possible_subdirs(app):
149 root_dir = os.path.join(build_dir, subdir)
150 paths = common.manifest_paths(root_dir, last_build.gradle)
151 version, vercode, package = common.parse_androidmanifests(paths, app)
153 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
154 .format(subdir, version, vercode))
155 if int(vercode) > int(hcode):
158 hcode = str(int(vercode))
162 return (None, "Couldn't find package ID", None)
164 return (hver, hcode, htag)
165 return (None, "Couldn't find any version information", None)
167 except VCSException as vcse:
168 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
169 return (None, msg, None)
171 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
172 return (None, msg, None)
175 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
176 # of the source repo. Whether this can be used reliably or not depends on
177 # the development procedures used by the project's developers. Use it with
178 # caution, because it's inappropriate for many projects.
179 # Returns (None, "a message") if this didn't work, or (version, vercode) for
180 # the details of the current version.
181 def check_repomanifest(app, branch=None):
185 if app.RepoType == 'srclib':
186 build_dir = os.path.join('build', 'srclib', app.Repo)
187 repotype = common.getsrclibvcs(app.Repo)
189 build_dir = os.path.join('build', app.id)
190 repotype = app.RepoType
192 # Set up vcs interface and make sure we have the latest code...
193 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
195 if repotype == 'git':
197 branch = 'origin/' + branch
198 vcs.gotorevision(branch)
199 elif repotype == 'git-svn':
200 vcs.gotorevision(branch)
201 elif repotype == 'hg':
202 vcs.gotorevision(branch)
203 elif repotype == 'bzr':
204 vcs.gotorevision(None)
206 last_build = metadata.Build()
207 if len(app.builds) > 0:
208 last_build = app.builds[-1]
210 if last_build.submodules:
216 for subdir in possible_subdirs(app):
220 root_dir = os.path.join(build_dir, subdir)
221 paths = common.manifest_paths(root_dir, last_build.gradle)
222 version, vercode, package = common.parse_androidmanifests(paths, app)
224 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
225 .format(subdir, version, vercode))
226 if int(vercode) > int(hcode):
228 hcode = str(int(vercode))
232 return (None, "Couldn't find package ID")
235 return (None, "Couldn't find any version information")
237 except VCSException as vcse:
238 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
241 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
245 def check_repotrunk(app):
248 if app.RepoType == 'srclib':
249 build_dir = os.path.join('build', 'srclib', app.Repo)
250 repotype = common.getsrclibvcs(app.Repo)
252 build_dir = os.path.join('build', app.id)
253 repotype = app.RepoType
255 if repotype not in ('git-svn', ):
256 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
258 # Set up vcs interface and make sure we have the latest code...
259 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
261 vcs.gotorevision(None)
265 except VCSException as vcse:
266 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
269 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
273 # Check for a new version by looking at the Google Play Store.
274 # Returns (None, "a message") if this didn't work, or (version, None) for
275 # the details of the current version.
276 def check_gplay(app):
278 url = 'https://play.google.com/store/apps/details?id=' + app.id
279 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
280 req = urllib.request.Request(url, None, headers)
282 resp = urllib.request.urlopen(req, None, 20)
283 page = resp.read().decode()
284 except urllib.error.HTTPError as e:
285 return (None, str(e.code))
286 except Exception as e:
287 return (None, 'Failed:' + str(e))
291 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
293 version = html.unescape(m.group(1))
295 if version == 'Varies with device':
296 return (None, 'Device-variable version, cannot use this method')
299 return (None, "Couldn't find version")
300 return (version.strip(), None)
303 # Return all directories under startdir that contain any of the manifest
304 # files, and thus are probably an Android project.
305 def dirs_with_manifest(startdir):
306 for root, dirs, files in os.walk(startdir):
307 if any(m in files for m in [
308 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
312 # Tries to find a new subdir starting from the root build_dir. Returns said
313 # subdir relative to the build dir if found, None otherwise.
314 def possible_subdirs(app):
316 if app.RepoType == 'srclib':
317 build_dir = os.path.join('build', 'srclib', app.Repo)
319 build_dir = os.path.join('build', app.id)
321 last_build = app.get_last_build()
323 for d in dirs_with_manifest(build_dir):
324 m_paths = common.manifest_paths(d, last_build.gradle)
325 package = common.parse_androidmanifests(m_paths, app)[2]
326 if package is not None:
327 subdir = os.path.relpath(d, build_dir)
328 logging.debug("Adding possible subdir %s" % subdir)
332 def fetch_autoname(app, tag):
334 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
337 if app.RepoType == 'srclib':
338 build_dir = os.path.join('build', 'srclib', app.Repo)
340 build_dir = os.path.join('build', app.id)
343 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
344 vcs.gotorevision(tag)
348 last_build = app.get_last_build()
350 logging.debug("...fetch auto name from " + build_dir)
352 for subdir in possible_subdirs(app):
356 root_dir = os.path.join(build_dir, subdir)
357 new_name = common.fetch_real_name(root_dir, last_build.gradle)
358 if new_name is not None:
362 logging.debug("...got autoname '" + new_name + "'")
363 if new_name != app.AutoName:
364 app.AutoName = new_name
366 commitmsg = "Set autoname of {0}".format(common.getappname(app))
368 logging.debug("...couldn't get autoname")
373 def checkupdates_app(app):
375 # If a change is made, commitmsg should be set to a description of it.
376 # Only if this is set will changes be written back to the metadata.
383 mode = app.UpdateCheckMode
384 if mode.startswith('Tags'):
385 pattern = mode[5:] if len(mode) > 4 else None
386 (version, vercode, tag) = check_tags(app, pattern)
387 if version == 'Unknown':
390 elif mode == 'RepoManifest':
391 (version, vercode) = check_repomanifest(app)
393 elif mode.startswith('RepoManifest/'):
395 (version, vercode) = check_repomanifest(app, tag)
397 elif mode == 'RepoTrunk':
398 (version, vercode) = check_repotrunk(app)
401 (version, vercode) = check_http(app)
403 elif mode in ('None', 'Static'):
405 msg = 'Checking disabled'
409 msg = 'Invalid update check method'
411 if version and vercode and app.VercodeOperation:
412 oldvercode = str(int(vercode))
413 op = app.VercodeOperation.replace("%c", oldvercode)
414 vercode = str(eval(op))
415 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
417 if version and any(version.startswith(s) for s in [
418 '${', # Gradle variable names
419 '@string/', # Strings we could not resolve
425 logmsg = "...{0} : {1}".format(app.id, msg)
430 elif vercode == app.CurrentVersionCode:
431 logging.info("...up to date")
433 logging.debug("...updating - old vercode={0}, new vercode={1}".format(
434 app.CurrentVersionCode, vercode))
435 app.CurrentVersion = version
436 app.CurrentVersionCode = str(int(vercode))
439 commitmsg = fetch_autoname(app, tag)
442 name = common.getappname(app)
443 ver = common.getcvname(app)
444 logging.info('...updating to version %s' % ver)
445 commitmsg = 'Update CV of %s to %s' % (name, ver)
448 mode = app.AutoUpdateMode
449 if not app.CurrentVersionCode:
450 logging.warn("Can't auto-update app with no current version code: " + app.id)
451 elif mode in ('None', 'Static'):
453 elif mode.startswith('Version '):
455 if pattern.startswith('+'):
457 suffix, pattern = pattern.split(' ', 1)
459 raise MetaDataException("Invalid AUM: " + mode)
464 for build in app.builds:
465 if int(build.versionCode) >= int(app.CurrentVersionCode):
467 if not latest or int(build.versionCode) > int(latest.versionCode):
470 if int(latest.versionCode) > int(app.CurrentVersionCode):
471 logging.info("Refusing to auto update, since the latest build is newer")
474 newbuild = copy.deepcopy(latest)
475 newbuild.disable = False
476 newbuild.versionCode = app.CurrentVersionCode
477 newbuild.versionName = app.CurrentVersion + suffix
478 logging.info("...auto-generating build for " + newbuild.versionName)
479 commit = pattern.replace('%v', newbuild.versionName)
480 commit = commit.replace('%c', newbuild.versionCode)
481 newbuild.commit = commit
482 app.builds.append(newbuild)
483 name = common.getappname(app)
484 ver = common.getcvname(app)
485 commitmsg = "Update %s to %s" % (name, ver)
487 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
490 metadatapath = os.path.join('metadata', app.id + '.txt')
491 metadata.write_metadata(metadatapath, app)
493 logging.info("Commiting update for " + metadatapath)
494 gitcmd = ["git", "commit", "-m", commitmsg]
495 if 'auto_author' in config:
496 gitcmd.extend(['--author', config['auto_author']])
497 gitcmd.extend(["--", metadatapath])
498 if subprocess.call(gitcmd) != 0:
499 raise FDroidException("Git commit failed")
508 global config, options
510 # Parse command line...
511 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
512 common.setup_global_opts(parser)
513 parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
514 parser.add_argument("--auto", action="store_true", default=False,
515 help=_("Process auto-updates"))
516 parser.add_argument("--autoonly", action="store_true", default=False,
517 help=_("Only process apps with auto-updates"))
518 parser.add_argument("--commit", action="store_true", default=False,
519 help=_("Commit changes"))
520 parser.add_argument("--gplay", action="store_true", default=False,
521 help=_("Only print differences with the Play Store"))
522 metadata.add_metadata_arguments(parser)
523 options = parser.parse_args()
524 metadata.warnings_action = options.W
526 config = common.read_config(options)
529 allapps = metadata.read_metadata()
531 apps = common.read_app_args(options.appid, allapps, False)
534 for appid, app in apps.items():
535 version, reason = check_gplay(app)
538 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
540 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
541 if version is not None:
542 stored = app.CurrentVersion
544 logging.info("{0} has no Current Version but has version {1} on the Play Store"
545 .format(common.getappname(app), version))
546 elif LooseVersion(stored) < LooseVersion(version):
547 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
548 .format(common.getappname(app), version, stored))
550 if stored != version:
551 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
552 .format(common.getappname(app), version, stored))
554 logging.info("{0} has the same version {1} on the Play Store"
555 .format(common.getappname(app), version))
558 for appid, app in apps.items():
560 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
561 logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
564 logging.info(_("Processing {appid}").format(appid=appid))
567 checkupdates_app(app)
568 except Exception as e:
569 logging.error(_("...checkupdate failed for {appid} : {error}")
570 .format(appid=appid, error=e))
572 logging.info(_("Finished"))
575 if __name__ == "__main__":