3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
26 from argparse import ArgumentParser
29 from distutils.version import LooseVersion
35 from . import metadata
36 from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
39 # Check for a new version by looking at a document retrieved via HTTP.
40 # The app's Update Check Data field is used to provide the information
46 if not app.UpdateCheckData:
47 raise FDroidException('Missing Update Check Data')
49 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
53 logging.debug("...requesting {0}".format(urlcode))
54 req = urllib.request.Request(urlcode, None)
55 resp = urllib.request.urlopen(req, None, 20)
56 page = resp.read().decode('utf-8')
58 m = re.search(codeex, page)
60 raise FDroidException("No RE match for version code")
61 vercode = m.group(1).strip()
66 logging.debug("...requesting {0}".format(urlver))
67 req = urllib.request.Request(urlver, None)
68 resp = urllib.request.urlopen(req, None, 20)
69 page = resp.read().decode('utf-8')
71 m = re.search(verex, page)
73 raise FDroidException("No RE match for version")
76 return (version, vercode)
78 except FDroidException:
79 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
83 # Check for a new version by looking at the tags in the source repo.
84 # Whether this can be used reliably or not depends on
85 # the development procedures used by the project's developers. Use it with
86 # caution, because it's inappropriate for many projects.
87 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
88 # the details of the current version.
89 def check_tags(app, pattern):
93 if app.RepoType == 'srclib':
94 build_dir = os.path.join('build', 'srclib', app.Repo)
95 repotype = common.getsrclibvcs(app.Repo)
97 build_dir = os.path.join('build', app.id)
98 repotype = app.RepoType
100 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
101 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
103 if repotype == 'git-svn' and ';' not in app.Repo:
104 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
106 # Set up vcs interface and make sure we have the latest code...
107 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
109 vcs.gotorevision(None)
111 last_build = app.get_last_build()
113 try_init_submodules(app, last_build, vcs)
121 if repotype == 'git':
122 tags = vcs.latesttags()
126 return (None, "No tags found", None)
128 logging.debug("All tags: " + ','.join(tags))
130 pat = re.compile(pattern)
131 tags = [tag for tag in tags if pat.match(tag)]
133 return (None, "No matching tags found", None)
134 logging.debug("Matching tags: " + ','.join(tags))
136 if len(tags) > 5 and repotype == 'git':
138 logging.debug("Latest tags: " + ','.join(tags))
141 logging.debug("Check tag: '{0}'".format(tag))
142 vcs.gotorevision(tag)
144 for subdir in possible_subdirs(app):
148 root_dir = os.path.join(build_dir, subdir)
149 paths = common.manifest_paths(root_dir, last_build.gradle)
150 version, vercode, package = common.parse_androidmanifests(paths, app)
152 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
153 .format(subdir, version, vercode))
154 if int(vercode) > int(hcode):
157 hcode = str(int(vercode))
161 return (None, "Couldn't find package ID", None)
163 return (hver, hcode, htag)
164 return (None, "Couldn't find any version information", None)
166 except VCSException as vcse:
167 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
168 return (None, msg, None)
170 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
171 return (None, msg, None)
174 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
175 # of the source repo. Whether this can be used reliably or not depends on
176 # the development procedures used by the project's developers. Use it with
177 # caution, because it's inappropriate for many projects.
178 # Returns (None, "a message") if this didn't work, or (version, vercode) for
179 # the details of the current version.
180 def check_repomanifest(app, branch=None):
184 if app.RepoType == 'srclib':
185 build_dir = os.path.join('build', 'srclib', app.Repo)
186 repotype = common.getsrclibvcs(app.Repo)
188 build_dir = os.path.join('build', app.id)
189 repotype = app.RepoType
191 # Set up vcs interface and make sure we have the latest code...
192 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
194 if repotype == 'git':
196 branch = 'origin/' + branch
197 vcs.gotorevision(branch)
198 elif repotype == 'git-svn':
199 vcs.gotorevision(branch)
200 elif repotype == 'hg':
201 vcs.gotorevision(branch)
202 elif repotype == 'bzr':
203 vcs.gotorevision(None)
205 last_build = metadata.Build()
206 if len(app.builds) > 0:
207 last_build = app.builds[-1]
209 try_init_submodules(app, last_build, vcs)
214 for subdir in possible_subdirs(app):
218 root_dir = os.path.join(build_dir, subdir)
219 paths = common.manifest_paths(root_dir, last_build.gradle)
220 version, vercode, package = common.parse_androidmanifests(paths, app)
222 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
223 .format(subdir, version, vercode))
224 if int(vercode) > int(hcode):
226 hcode = str(int(vercode))
230 return (None, "Couldn't find package ID")
233 return (None, "Couldn't find any version information")
235 except VCSException as vcse:
236 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
239 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
243 def check_repotrunk(app):
246 if app.RepoType == 'srclib':
247 build_dir = os.path.join('build', 'srclib', app.Repo)
248 repotype = common.getsrclibvcs(app.Repo)
250 build_dir = os.path.join('build', app.id)
251 repotype = app.RepoType
253 if repotype not in ('git-svn', ):
254 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
256 # Set up vcs interface and make sure we have the latest code...
257 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
259 vcs.gotorevision(None)
263 except VCSException as vcse:
264 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
267 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
271 # Check for a new version by looking at the Google Play Store.
272 # Returns (None, "a message") if this didn't work, or (version, None) for
273 # the details of the current version.
274 def check_gplay(app):
276 url = 'https://play.google.com/store/apps/details?id=' + app.id
277 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
278 req = urllib.request.Request(url, None, headers)
280 resp = urllib.request.urlopen(req, None, 20)
281 page = resp.read().decode()
282 except urllib.error.HTTPError as e:
283 return (None, str(e.code))
284 except Exception as e:
285 return (None, 'Failed:' + str(e))
289 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
291 version = html.unescape(m.group(1))
293 if version == 'Varies with device':
294 return (None, 'Device-variable version, cannot use this method')
297 return (None, "Couldn't find version")
298 return (version.strip(), None)
301 def try_init_submodules(app, last_build, vcs):
302 """Try to init submodules if the last build entry used them.
303 They might have been removed from the app's repo in the meantime,
304 so if we can't find any submodules we continue with the updates check.
305 If there is any other error in initializing them then we stop the check.
307 if last_build.submodules:
310 except NoSubmodulesException:
311 logging.info("No submodules present for {}".format(app.Name))
314 # Return all directories under startdir that contain any of the manifest
315 # files, and thus are probably an Android project.
316 def dirs_with_manifest(startdir):
317 for root, dirs, files in os.walk(startdir):
318 if any(m in files for m in [
319 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
323 # Tries to find a new subdir starting from the root build_dir. Returns said
324 # subdir relative to the build dir if found, None otherwise.
325 def possible_subdirs(app):
327 if app.RepoType == 'srclib':
328 build_dir = os.path.join('build', 'srclib', app.Repo)
330 build_dir = os.path.join('build', app.id)
332 last_build = app.get_last_build()
334 for d in dirs_with_manifest(build_dir):
335 m_paths = common.manifest_paths(d, last_build.gradle)
336 package = common.parse_androidmanifests(m_paths, app)[2]
337 if package is not None:
338 subdir = os.path.relpath(d, build_dir)
339 logging.debug("Adding possible subdir %s" % subdir)
343 def fetch_autoname(app, tag):
345 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
348 if app.RepoType == 'srclib':
349 build_dir = os.path.join('build', 'srclib', app.Repo)
351 build_dir = os.path.join('build', app.id)
354 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
355 vcs.gotorevision(tag)
359 last_build = app.get_last_build()
361 logging.debug("...fetch auto name from " + build_dir)
363 for subdir in possible_subdirs(app):
367 root_dir = os.path.join(build_dir, subdir)
368 new_name = common.fetch_real_name(root_dir, last_build.gradle)
369 if new_name is not None:
373 logging.debug("...got autoname '" + new_name + "'")
374 if new_name != app.AutoName:
375 app.AutoName = new_name
377 commitmsg = "Set autoname of {0}".format(common.getappname(app))
379 logging.debug("...couldn't get autoname")
384 def checkupdates_app(app):
386 # If a change is made, commitmsg should be set to a description of it.
387 # Only if this is set will changes be written back to the metadata.
394 mode = app.UpdateCheckMode
395 if mode.startswith('Tags'):
396 pattern = mode[5:] if len(mode) > 4 else None
397 (version, vercode, tag) = check_tags(app, pattern)
398 if version == 'Unknown':
401 elif mode == 'RepoManifest':
402 (version, vercode) = check_repomanifest(app)
404 elif mode.startswith('RepoManifest/'):
406 (version, vercode) = check_repomanifest(app, tag)
408 elif mode == 'RepoTrunk':
409 (version, vercode) = check_repotrunk(app)
412 (version, vercode) = check_http(app)
414 elif mode in ('None', 'Static'):
416 msg = 'Checking disabled'
420 msg = 'Invalid update check method'
422 if version and vercode and app.VercodeOperation:
423 oldvercode = str(int(vercode))
424 op = app.VercodeOperation.replace("%c", oldvercode)
425 vercode = str(eval(op))
426 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
428 if version and any(version.startswith(s) for s in [
429 '${', # Gradle variable names
430 '@string/', # Strings we could not resolve
436 logmsg = "...{0} : {1}".format(app.id, msg)
441 elif vercode == app.CurrentVersionCode:
442 logging.info("...up to date")
444 logging.debug("...updating - old vercode={0}, new vercode={1}".format(
445 app.CurrentVersionCode, vercode))
446 app.CurrentVersion = version
447 app.CurrentVersionCode = str(int(vercode))
450 commitmsg = fetch_autoname(app, tag)
453 name = common.getappname(app)
454 ver = common.getcvname(app)
455 logging.info('...updating to version %s' % ver)
456 commitmsg = 'Update CV of %s to %s' % (name, ver)
459 mode = app.AutoUpdateMode
460 if not app.CurrentVersionCode:
461 logging.warn("Can't auto-update app with no current version code: " + app.id)
462 elif mode in ('None', 'Static'):
464 elif mode.startswith('Version '):
466 if pattern.startswith('+'):
468 suffix, pattern = pattern.split(' ', 1)
470 raise MetaDataException("Invalid AUM: " + mode)
475 for build in app.builds:
476 if int(build.versionCode) >= int(app.CurrentVersionCode):
478 if not latest or int(build.versionCode) > int(latest.versionCode):
481 if int(latest.versionCode) > int(app.CurrentVersionCode):
482 logging.info("Refusing to auto update, since the latest build is newer")
485 newbuild = copy.deepcopy(latest)
486 newbuild.disable = False
487 newbuild.versionCode = app.CurrentVersionCode
488 newbuild.versionName = app.CurrentVersion + suffix
489 logging.info("...auto-generating build for " + newbuild.versionName)
490 commit = pattern.replace('%v', newbuild.versionName)
491 commit = commit.replace('%c', newbuild.versionCode)
492 newbuild.commit = commit
493 app.builds.append(newbuild)
494 name = common.getappname(app)
495 ver = common.getcvname(app)
496 commitmsg = "Update %s to %s" % (name, ver)
498 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
501 metadatapath = os.path.join('metadata', app.id + '.txt')
502 metadata.write_metadata(metadatapath, app)
504 logging.info("Commiting update for " + metadatapath)
505 gitcmd = ["git", "commit", "-m", commitmsg]
506 if 'auto_author' in config:
507 gitcmd.extend(['--author', config['auto_author']])
508 gitcmd.extend(["--", metadatapath])
509 if subprocess.call(gitcmd) != 0:
510 raise FDroidException("Git commit failed")
519 global config, options
521 # Parse command line...
522 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
523 common.setup_global_opts(parser)
524 parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
525 parser.add_argument("--auto", action="store_true", default=False,
526 help=_("Process auto-updates"))
527 parser.add_argument("--autoonly", action="store_true", default=False,
528 help=_("Only process apps with auto-updates"))
529 parser.add_argument("--commit", action="store_true", default=False,
530 help=_("Commit changes"))
531 parser.add_argument("--gplay", action="store_true", default=False,
532 help=_("Only print differences with the Play Store"))
533 metadata.add_metadata_arguments(parser)
534 options = parser.parse_args()
535 metadata.warnings_action = options.W
537 config = common.read_config(options)
540 allapps = metadata.read_metadata()
542 apps = common.read_app_args(options.appid, allapps, False)
545 for appid, app in apps.items():
546 version, reason = check_gplay(app)
549 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
551 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
552 if version is not None:
553 stored = app.CurrentVersion
555 logging.info("{0} has no Current Version but has version {1} on the Play Store"
556 .format(common.getappname(app), version))
557 elif LooseVersion(stored) < LooseVersion(version):
558 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
559 .format(common.getappname(app), version, stored))
561 if stored != version:
562 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
563 .format(common.getappname(app), version, stored))
565 logging.info("{0} has the same version {1} on the Play Store"
566 .format(common.getappname(app), version))
569 for appid, app in apps.items():
571 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
572 logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
575 logging.info(_("Processing {appid}").format(appid=appid))
578 checkupdates_app(app)
579 except Exception as e:
580 logging.error(_("...checkupdate failed for {appid} : {error}")
581 .format(appid=appid, error=e))
583 logging.info(_("Finished"))
586 if __name__ == "__main__":