3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from argparse import ArgumentParser
30 from distutils.version import LooseVersion
37 from . import metadata
38 from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
41 # Check for a new version by looking at a document retrieved via HTTP.
42 # The app's Update Check Data field is used to provide the information
48 if not app.UpdateCheckData:
49 raise FDroidException('Missing Update Check Data')
51 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
52 parsed = urllib.parse.urlparse(urlcode)
53 if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
54 raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
56 parsed = urllib.parse.urlparse(urlver)
57 if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
58 raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
62 logging.debug("...requesting {0}".format(urlcode))
63 req = urllib.request.Request(urlcode, None)
64 resp = urllib.request.urlopen(req, None, 20)
65 page = resp.read().decode('utf-8')
67 m = re.search(codeex, page)
69 raise FDroidException("No RE match for version code")
70 vercode = m.group(1).strip()
75 logging.debug("...requesting {0}".format(urlver))
76 req = urllib.request.Request(urlver, None)
77 resp = urllib.request.urlopen(req, None, 20)
78 page = resp.read().decode('utf-8')
80 m = re.search(verex, page)
82 raise FDroidException("No RE match for version")
85 return (version, vercode)
87 except FDroidException:
88 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
92 # Check for a new version by looking at the tags in the source repo.
93 # Whether this can be used reliably or not depends on
94 # the development procedures used by the project's developers. Use it with
95 # caution, because it's inappropriate for many projects.
96 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
97 # the details of the current version.
98 def check_tags(app, pattern):
102 if app.RepoType == 'srclib':
103 build_dir = os.path.join('build', 'srclib', app.Repo)
104 repotype = common.getsrclibvcs(app.Repo)
106 build_dir = os.path.join('build', app.id)
107 repotype = app.RepoType
109 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
110 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
112 if repotype == 'git-svn' and ';' not in app.Repo:
113 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
115 # Set up vcs interface and make sure we have the latest code...
116 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
118 vcs.gotorevision(None)
120 last_build = app.get_last_build()
122 try_init_submodules(app, last_build, vcs)
130 if repotype == 'git':
131 tags = vcs.latesttags()
135 return (None, "No tags found", None)
137 logging.debug("All tags: " + ','.join(tags))
139 pat = re.compile(pattern)
140 tags = [tag for tag in tags if pat.match(tag)]
142 return (None, "No matching tags found", None)
143 logging.debug("Matching tags: " + ','.join(tags))
145 if len(tags) > 5 and repotype == 'git':
147 logging.debug("Latest tags: " + ','.join(tags))
150 logging.debug("Check tag: '{0}'".format(tag))
151 vcs.gotorevision(tag)
153 for subdir in possible_subdirs(app):
157 root_dir = os.path.join(build_dir, subdir)
158 paths = common.manifest_paths(root_dir, last_build.gradle)
159 version, vercode, package = common.parse_androidmanifests(paths, app)
161 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
162 .format(subdir, version, vercode))
163 if int(vercode) > int(hcode):
166 hcode = str(int(vercode))
170 return (None, "Couldn't find package ID", None)
172 return (hver, hcode, htag)
173 return (None, "Couldn't find any version information", None)
175 except VCSException as vcse:
176 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
177 return (None, msg, None)
179 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
180 return (None, msg, None)
183 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
184 # of the source repo. Whether this can be used reliably or not depends on
185 # the development procedures used by the project's developers. Use it with
186 # caution, because it's inappropriate for many projects.
187 # Returns (None, "a message") if this didn't work, or (version, vercode) for
188 # the details of the current version.
189 def check_repomanifest(app, branch=None):
193 if app.RepoType == 'srclib':
194 build_dir = os.path.join('build', 'srclib', app.Repo)
195 repotype = common.getsrclibvcs(app.Repo)
197 build_dir = os.path.join('build', app.id)
198 repotype = app.RepoType
200 # Set up vcs interface and make sure we have the latest code...
201 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
203 if repotype == 'git':
205 branch = 'origin/' + branch
206 vcs.gotorevision(branch)
207 elif repotype == 'git-svn':
208 vcs.gotorevision(branch)
209 elif repotype == 'hg':
210 vcs.gotorevision(branch)
211 elif repotype == 'bzr':
212 vcs.gotorevision(None)
214 last_build = metadata.Build()
215 if len(app.builds) > 0:
216 last_build = app.builds[-1]
218 try_init_submodules(app, last_build, vcs)
223 for subdir in possible_subdirs(app):
227 root_dir = os.path.join(build_dir, subdir)
228 paths = common.manifest_paths(root_dir, last_build.gradle)
229 version, vercode, package = common.parse_androidmanifests(paths, app)
231 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
232 .format(subdir, version, vercode))
233 if int(vercode) > int(hcode):
235 hcode = str(int(vercode))
239 return (None, "Couldn't find package ID")
242 return (None, "Couldn't find any version information")
244 except VCSException as vcse:
245 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
248 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
252 def check_repotrunk(app):
255 if app.RepoType == 'srclib':
256 build_dir = os.path.join('build', 'srclib', app.Repo)
257 repotype = common.getsrclibvcs(app.Repo)
259 build_dir = os.path.join('build', app.id)
260 repotype = app.RepoType
262 if repotype not in ('git-svn', ):
263 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
265 # Set up vcs interface and make sure we have the latest code...
266 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
268 vcs.gotorevision(None)
272 except VCSException as vcse:
273 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
276 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
280 # Check for a new version by looking at the Google Play Store.
281 # Returns (None, "a message") if this didn't work, or (version, None) for
282 # the details of the current version.
283 def check_gplay(app):
285 url = 'https://play.google.com/store/apps/details?id=' + app.id
286 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
287 req = urllib.request.Request(url, None, headers)
289 resp = urllib.request.urlopen(req, None, 20)
290 page = resp.read().decode()
291 except urllib.error.HTTPError as e:
292 return (None, str(e.code))
293 except Exception as e:
294 return (None, 'Failed:' + str(e))
298 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
300 version = html.unescape(m.group(1))
302 if version == 'Varies with device':
303 return (None, 'Device-variable version, cannot use this method')
306 return (None, "Couldn't find version")
307 return (version.strip(), None)
310 def try_init_submodules(app, last_build, vcs):
311 """Try to init submodules if the last build entry used them.
312 They might have been removed from the app's repo in the meantime,
313 so if we can't find any submodules we continue with the updates check.
314 If there is any other error in initializing them then we stop the check.
316 if last_build.submodules:
319 except NoSubmodulesException:
320 logging.info("No submodules present for {}".format(app.Name))
323 # Return all directories under startdir that contain any of the manifest
324 # files, and thus are probably an Android project.
325 def dirs_with_manifest(startdir):
326 for root, dirs, files in os.walk(startdir):
327 if any(m in files for m in [
328 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
332 # Tries to find a new subdir starting from the root build_dir. Returns said
333 # subdir relative to the build dir if found, None otherwise.
334 def possible_subdirs(app):
336 if app.RepoType == 'srclib':
337 build_dir = os.path.join('build', 'srclib', app.Repo)
339 build_dir = os.path.join('build', app.id)
341 last_build = app.get_last_build()
343 for d in dirs_with_manifest(build_dir):
344 m_paths = common.manifest_paths(d, last_build.gradle)
345 package = common.parse_androidmanifests(m_paths, app)[2]
346 if package is not None:
347 subdir = os.path.relpath(d, build_dir)
348 logging.debug("Adding possible subdir %s" % subdir)
352 def fetch_autoname(app, tag):
354 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
357 if app.RepoType == 'srclib':
358 build_dir = os.path.join('build', 'srclib', app.Repo)
360 build_dir = os.path.join('build', app.id)
363 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
364 vcs.gotorevision(tag)
368 last_build = app.get_last_build()
370 logging.debug("...fetch auto name from " + build_dir)
372 for subdir in possible_subdirs(app):
376 root_dir = os.path.join(build_dir, subdir)
377 new_name = common.fetch_real_name(root_dir, last_build.gradle)
378 if new_name is not None:
382 logging.debug("...got autoname '" + new_name + "'")
383 if new_name != app.AutoName:
384 app.AutoName = new_name
386 commitmsg = "Set autoname of {0}".format(common.getappname(app))
388 logging.debug("...couldn't get autoname")
393 def checkupdates_app(app):
395 # If a change is made, commitmsg should be set to a description of it.
396 # Only if this is set will changes be written back to the metadata.
403 mode = app.UpdateCheckMode
404 if mode.startswith('Tags'):
405 pattern = mode[5:] if len(mode) > 4 else None
406 (version, vercode, tag) = check_tags(app, pattern)
407 if version == 'Unknown':
410 elif mode == 'RepoManifest':
411 (version, vercode) = check_repomanifest(app)
413 elif mode.startswith('RepoManifest/'):
415 (version, vercode) = check_repomanifest(app, tag)
417 elif mode == 'RepoTrunk':
418 (version, vercode) = check_repotrunk(app)
421 (version, vercode) = check_http(app)
423 elif mode in ('None', 'Static'):
425 msg = 'Checking disabled'
429 msg = 'Invalid update check method'
431 if version and vercode and app.VercodeOperation:
432 if not common.VERCODE_OPERATION_RE.match(app.VercodeOperation):
433 raise MetaDataException(_('Invalid VercodeOperation: {field}')
434 .format(field=app.VercodeOperation))
435 oldvercode = str(int(vercode))
436 op = app.VercodeOperation.replace("%c", oldvercode)
437 vercode = str(eval(op))
438 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
440 if version and any(version.startswith(s) for s in [
441 '${', # Gradle variable names
442 '@string/', # Strings we could not resolve
448 logmsg = "...{0} : {1}".format(app.id, msg)
453 elif vercode == app.CurrentVersionCode:
454 logging.info("...up to date")
456 logging.debug("...updating - old vercode={0}, new vercode={1}".format(
457 app.CurrentVersionCode, vercode))
458 app.CurrentVersion = version
459 app.CurrentVersionCode = str(int(vercode))
462 commitmsg = fetch_autoname(app, tag)
465 name = common.getappname(app)
466 ver = common.getcvname(app)
467 logging.info('...updating to version %s' % ver)
468 commitmsg = 'Update CV of %s to %s' % (name, ver)
471 mode = app.AutoUpdateMode
472 if not app.CurrentVersionCode:
473 logging.warn("Can't auto-update app with no current version code: " + app.id)
474 elif mode in ('None', 'Static'):
476 elif mode.startswith('Version '):
478 if pattern.startswith('+'):
480 suffix, pattern = pattern.split(' ', 1)
482 raise MetaDataException("Invalid AUM: " + mode)
487 for build in app.builds:
488 if int(build.versionCode) >= int(app.CurrentVersionCode):
490 if not latest or int(build.versionCode) > int(latest.versionCode):
493 if int(latest.versionCode) > int(app.CurrentVersionCode):
494 logging.info("Refusing to auto update, since the latest build is newer")
497 newbuild = copy.deepcopy(latest)
498 newbuild.disable = False
499 newbuild.versionCode = app.CurrentVersionCode
500 newbuild.versionName = app.CurrentVersion + suffix
501 logging.info("...auto-generating build for " + newbuild.versionName)
502 commit = pattern.replace('%v', newbuild.versionName)
503 commit = commit.replace('%c', newbuild.versionCode)
504 newbuild.commit = commit
505 app.builds.append(newbuild)
506 name = common.getappname(app)
507 ver = common.getcvname(app)
508 commitmsg = "Update %s to %s" % (name, ver)
510 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
513 metadata.write_metadata(app.metadatapath, app)
515 logging.info("Commiting update for " + app.metadatapath)
516 gitcmd = ["git", "commit", "-m", commitmsg]
517 if 'auto_author' in config:
518 gitcmd.extend(['--author', config['auto_author']])
519 gitcmd.extend(["--", app.metadatapath])
520 if subprocess.call(gitcmd) != 0:
521 raise FDroidException("Git commit failed")
524 def update_wiki(gplaylog, locallog):
525 if config.get('wiki_server') and config.get('wiki_path'):
528 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
529 path=config['wiki_path'])
530 site.login(config['wiki_user'], config['wiki_password'])
532 # Write a page with the last build log for this version code
533 wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
534 newpage = site.Pages[wiki_page_path]
536 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
537 txt += common.get_git_describe_link()
538 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
539 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
541 txt += common.get_android_tools_version_log()
544 txt += '== --gplay check ==\n\n'
547 txt += '== local source check ==\n\n'
549 newpage.save(txt, summary='Run log')
550 newpage = site.Pages['checkupdates']
551 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
552 except Exception as e:
553 logging.error(_('Error while attempting to publish log: %s') % e)
558 start_timestamp = time.gmtime()
563 global config, options
565 # Parse command line...
566 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
567 common.setup_global_opts(parser)
568 parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
569 parser.add_argument("--auto", action="store_true", default=False,
570 help=_("Process auto-updates"))
571 parser.add_argument("--autoonly", action="store_true", default=False,
572 help=_("Only process apps with auto-updates"))
573 parser.add_argument("--commit", action="store_true", default=False,
574 help=_("Commit changes"))
575 parser.add_argument("--allow-dirty", action="store_true", default=False,
576 help=_("Run on git repo that has uncommitted changes"))
577 parser.add_argument("--gplay", action="store_true", default=False,
578 help=_("Only print differences with the Play Store"))
579 metadata.add_metadata_arguments(parser)
580 options = parser.parse_args()
581 metadata.warnings_action = options.W
583 config = common.read_config(options)
585 if not options.allow_dirty:
586 status = subprocess.check_output(['git', 'status', '--porcelain'])
588 logging.error(_('Build metadata git repo has uncommited changes!'))
592 allapps = metadata.read_metadata()
594 apps = common.read_app_args(options.appid, allapps, False)
598 for appid, app in apps.items():
599 gplaylog += '* ' + appid + '\n'
600 version, reason = check_gplay(app)
603 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
605 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
606 if version is not None:
607 stored = app.CurrentVersion
609 logging.info("{0} has no Current Version but has version {1} on the Play Store"
610 .format(common.getappname(app), version))
611 elif LooseVersion(stored) < LooseVersion(version):
612 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
613 .format(common.getappname(app), version, stored))
615 if stored != version:
616 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
617 .format(common.getappname(app), version, stored))
619 logging.info("{0} has the same version {1} on the Play Store"
620 .format(common.getappname(app), version))
621 update_wiki(gplaylog, None)
625 for appid, app in apps.items():
627 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
628 logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
631 msg = _("Processing {appid}").format(appid=appid)
633 locallog += '* ' + msg + '\n'
636 checkupdates_app(app)
637 except Exception as e:
638 msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
640 locallog += msg + '\n'
642 update_wiki(None, locallog)
644 logging.info(_("Finished"))
647 if __name__ == "__main__":