3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from argparse import ArgumentParser
30 from distutils.version import LooseVersion
37 from . import metadata
38 from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
41 # Check for a new version by looking at a document retrieved via HTTP.
42 # The app's Update Check Data field is used to provide the information
48 if not app.UpdateCheckData:
49 raise FDroidException('Missing Update Check Data')
51 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
52 parsed = urllib.parse.urlparse(urlcode)
53 if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
54 raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
56 parsed = urllib.parse.urlparse(urlver)
57 if not parsed.netloc or not parsed.scheme or parsed.scheme != 'https':
58 raise FDroidException(_('UpdateCheckData has invalid URL: {url}').format(url=urlcode))
62 logging.debug("...requesting {0}".format(urlcode))
63 req = urllib.request.Request(urlcode, None)
64 resp = urllib.request.urlopen(req, None, 20)
65 page = resp.read().decode('utf-8')
67 m = re.search(codeex, page)
69 raise FDroidException("No RE match for version code")
70 vercode = m.group(1).strip()
75 logging.debug("...requesting {0}".format(urlver))
76 req = urllib.request.Request(urlver, None)
77 resp = urllib.request.urlopen(req, None, 20)
78 page = resp.read().decode('utf-8')
80 m = re.search(verex, page)
82 raise FDroidException("No RE match for version")
85 return (version, vercode)
87 except FDroidException:
88 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
92 # Check for a new version by looking at the tags in the source repo.
93 # Whether this can be used reliably or not depends on
94 # the development procedures used by the project's developers. Use it with
95 # caution, because it's inappropriate for many projects.
96 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
97 # the details of the current version.
98 def check_tags(app, pattern):
102 if app.RepoType == 'srclib':
103 build_dir = os.path.join('build', 'srclib', app.Repo)
104 repotype = common.getsrclibvcs(app.Repo)
106 build_dir = os.path.join('build', app.id)
107 repotype = app.RepoType
109 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
110 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
112 if repotype == 'git-svn' and ';' not in app.Repo:
113 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
115 # Set up vcs interface and make sure we have the latest code...
116 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
118 vcs.gotorevision(None)
120 last_build = app.get_last_build()
122 try_init_submodules(app, last_build, vcs)
130 if repotype == 'git':
131 tags = vcs.latesttags()
135 return (None, "No tags found", None)
137 logging.debug("All tags: " + ','.join(tags))
139 pat = re.compile(pattern)
140 tags = [tag for tag in tags if pat.match(tag)]
142 return (None, "No matching tags found", None)
143 logging.debug("Matching tags: " + ','.join(tags))
145 if len(tags) > 5 and repotype == 'git':
147 logging.debug("Latest tags: " + ','.join(tags))
150 logging.debug("Check tag: '{0}'".format(tag))
151 vcs.gotorevision(tag)
153 for subdir in possible_subdirs(app):
157 root_dir = os.path.join(build_dir, subdir)
158 paths = common.manifest_paths(root_dir, last_build.gradle)
159 version, vercode, package = common.parse_androidmanifests(paths, app)
161 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
162 .format(subdir, version, vercode))
163 if int(vercode) > int(hcode):
166 hcode = str(int(vercode))
170 return (None, "Couldn't find package ID", None)
172 return (hver, hcode, htag)
173 return (None, "Couldn't find any version information", None)
175 except VCSException as vcse:
176 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
177 return (None, msg, None)
179 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
180 return (None, msg, None)
183 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
184 # of the source repo. Whether this can be used reliably or not depends on
185 # the development procedures used by the project's developers. Use it with
186 # caution, because it's inappropriate for many projects.
187 # Returns (None, "a message") if this didn't work, or (version, vercode) for
188 # the details of the current version.
189 def check_repomanifest(app, branch=None):
193 if app.RepoType == 'srclib':
194 build_dir = os.path.join('build', 'srclib', app.Repo)
195 repotype = common.getsrclibvcs(app.Repo)
197 build_dir = os.path.join('build', app.id)
198 repotype = app.RepoType
200 # Set up vcs interface and make sure we have the latest code...
201 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
203 if repotype == 'git':
205 branch = 'origin/' + branch
206 vcs.gotorevision(branch)
207 elif repotype == 'git-svn':
208 vcs.gotorevision(branch)
209 elif repotype == 'hg':
210 vcs.gotorevision(branch)
211 elif repotype == 'bzr':
212 vcs.gotorevision(None)
214 last_build = metadata.Build()
215 if len(app.builds) > 0:
216 last_build = app.builds[-1]
218 try_init_submodules(app, last_build, vcs)
223 for subdir in possible_subdirs(app):
227 root_dir = os.path.join(build_dir, subdir)
228 paths = common.manifest_paths(root_dir, last_build.gradle)
229 version, vercode, package = common.parse_androidmanifests(paths, app)
231 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
232 .format(subdir, version, vercode))
233 if int(vercode) > int(hcode):
235 hcode = str(int(vercode))
239 return (None, "Couldn't find package ID")
242 return (None, "Couldn't find any version information")
244 except VCSException as vcse:
245 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
248 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
252 def check_repotrunk(app):
255 if app.RepoType == 'srclib':
256 build_dir = os.path.join('build', 'srclib', app.Repo)
257 repotype = common.getsrclibvcs(app.Repo)
259 build_dir = os.path.join('build', app.id)
260 repotype = app.RepoType
262 if repotype not in ('git-svn', ):
263 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
265 # Set up vcs interface and make sure we have the latest code...
266 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
268 vcs.gotorevision(None)
272 except VCSException as vcse:
273 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
276 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
280 # Check for a new version by looking at the Google Play Store.
281 # Returns (None, "a message") if this didn't work, or (version, None) for
282 # the details of the current version.
283 def check_gplay(app):
285 url = 'https://play.google.com/store/apps/details?id=' + app.id
286 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
287 req = urllib.request.Request(url, None, headers)
289 resp = urllib.request.urlopen(req, None, 20)
290 page = resp.read().decode()
291 except urllib.error.HTTPError as e:
292 return (None, str(e.code))
293 except Exception as e:
294 return (None, 'Failed:' + str(e))
298 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
300 version = html.unescape(m.group(1))
302 if version == 'Varies with device':
303 return (None, 'Device-variable version, cannot use this method')
306 return (None, "Couldn't find version")
307 return (version.strip(), None)
310 def try_init_submodules(app, last_build, vcs):
311 """Try to init submodules if the last build entry used them.
312 They might have been removed from the app's repo in the meantime,
313 so if we can't find any submodules we continue with the updates check.
314 If there is any other error in initializing them then we stop the check.
316 if last_build.submodules:
319 except NoSubmodulesException:
320 logging.info("No submodules present for {}".format(app.Name))
323 # Return all directories under startdir that contain any of the manifest
324 # files, and thus are probably an Android project.
325 def dirs_with_manifest(startdir):
326 for root, dirs, files in os.walk(startdir):
327 if any(m in files for m in [
328 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
332 # Tries to find a new subdir starting from the root build_dir. Returns said
333 # subdir relative to the build dir if found, None otherwise.
334 def possible_subdirs(app):
336 if app.RepoType == 'srclib':
337 build_dir = os.path.join('build', 'srclib', app.Repo)
339 build_dir = os.path.join('build', app.id)
341 last_build = app.get_last_build()
343 for d in dirs_with_manifest(build_dir):
344 m_paths = common.manifest_paths(d, last_build.gradle)
345 package = common.parse_androidmanifests(m_paths, app)[2]
346 if package is not None:
347 subdir = os.path.relpath(d, build_dir)
348 logging.debug("Adding possible subdir %s" % subdir)
352 def fetch_autoname(app, tag):
354 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
357 if app.RepoType == 'srclib':
358 build_dir = os.path.join('build', 'srclib', app.Repo)
360 build_dir = os.path.join('build', app.id)
363 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
364 vcs.gotorevision(tag)
368 last_build = app.get_last_build()
370 logging.debug("...fetch auto name from " + build_dir)
372 for subdir in possible_subdirs(app):
376 root_dir = os.path.join(build_dir, subdir)
377 new_name = common.fetch_real_name(root_dir, last_build.gradle)
378 if new_name is not None:
382 logging.debug("...got autoname '" + new_name + "'")
383 if new_name != app.AutoName:
384 app.AutoName = new_name
386 commitmsg = "Set autoname of {0}".format(common.getappname(app))
388 logging.debug("...couldn't get autoname")
393 def checkupdates_app(app):
395 # If a change is made, commitmsg should be set to a description of it.
396 # Only if this is set will changes be written back to the metadata.
403 mode = app.UpdateCheckMode
404 if mode.startswith('Tags'):
405 pattern = mode[5:] if len(mode) > 4 else None
406 (version, vercode, tag) = check_tags(app, pattern)
407 if version == 'Unknown':
410 elif mode == 'RepoManifest':
411 (version, vercode) = check_repomanifest(app)
413 elif mode.startswith('RepoManifest/'):
415 (version, vercode) = check_repomanifest(app, tag)
417 elif mode == 'RepoTrunk':
418 (version, vercode) = check_repotrunk(app)
421 (version, vercode) = check_http(app)
423 elif mode in ('None', 'Static'):
425 msg = 'Checking disabled'
429 msg = 'Invalid update check method'
431 if version and vercode and app.VercodeOperation:
432 oldvercode = str(int(vercode))
433 op = app.VercodeOperation.replace("%c", oldvercode)
434 vercode = str(eval(op))
435 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
437 if version and any(version.startswith(s) for s in [
438 '${', # Gradle variable names
439 '@string/', # Strings we could not resolve
445 logmsg = "...{0} : {1}".format(app.id, msg)
450 elif vercode == app.CurrentVersionCode:
451 logging.info("...up to date")
453 logging.debug("...updating - old vercode={0}, new vercode={1}".format(
454 app.CurrentVersionCode, vercode))
455 app.CurrentVersion = version
456 app.CurrentVersionCode = str(int(vercode))
459 commitmsg = fetch_autoname(app, tag)
462 name = common.getappname(app)
463 ver = common.getcvname(app)
464 logging.info('...updating to version %s' % ver)
465 commitmsg = 'Update CV of %s to %s' % (name, ver)
468 mode = app.AutoUpdateMode
469 if not app.CurrentVersionCode:
470 logging.warn("Can't auto-update app with no current version code: " + app.id)
471 elif mode in ('None', 'Static'):
473 elif mode.startswith('Version '):
475 if pattern.startswith('+'):
477 suffix, pattern = pattern.split(' ', 1)
479 raise MetaDataException("Invalid AUM: " + mode)
484 for build in app.builds:
485 if int(build.versionCode) >= int(app.CurrentVersionCode):
487 if not latest or int(build.versionCode) > int(latest.versionCode):
490 if int(latest.versionCode) > int(app.CurrentVersionCode):
491 logging.info("Refusing to auto update, since the latest build is newer")
494 newbuild = copy.deepcopy(latest)
495 newbuild.disable = False
496 newbuild.versionCode = app.CurrentVersionCode
497 newbuild.versionName = app.CurrentVersion + suffix
498 logging.info("...auto-generating build for " + newbuild.versionName)
499 commit = pattern.replace('%v', newbuild.versionName)
500 commit = commit.replace('%c', newbuild.versionCode)
501 newbuild.commit = commit
502 app.builds.append(newbuild)
503 name = common.getappname(app)
504 ver = common.getcvname(app)
505 commitmsg = "Update %s to %s" % (name, ver)
507 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
510 metadatapath = os.path.join('metadata', app.id + '.txt')
511 metadata.write_metadata(metadatapath, app)
513 logging.info("Commiting update for " + metadatapath)
514 gitcmd = ["git", "commit", "-m", commitmsg]
515 if 'auto_author' in config:
516 gitcmd.extend(['--author', config['auto_author']])
517 gitcmd.extend(["--", metadatapath])
518 if subprocess.call(gitcmd) != 0:
519 raise FDroidException("Git commit failed")
522 def update_wiki(gplaylog, locallog):
523 if config.get('wiki_server') and config.get('wiki_path'):
526 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
527 path=config['wiki_path'])
528 site.login(config['wiki_user'], config['wiki_password'])
530 # Write a page with the last build log for this version code
531 wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
532 newpage = site.Pages[wiki_page_path]
534 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
535 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
536 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
538 txt += common.get_android_tools_version_log()
541 txt += '== --gplay check ==\n\n'
544 txt += '== local source check ==\n\n'
546 newpage.save(txt, summary='Run log')
547 newpage = site.Pages['checkupdates']
548 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
549 except Exception as e:
550 logging.error(_('Error while attempting to publish log: %s') % e)
555 start_timestamp = time.gmtime()
560 global config, options
562 # Parse command line...
563 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
564 common.setup_global_opts(parser)
565 parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
566 parser.add_argument("--auto", action="store_true", default=False,
567 help=_("Process auto-updates"))
568 parser.add_argument("--autoonly", action="store_true", default=False,
569 help=_("Only process apps with auto-updates"))
570 parser.add_argument("--commit", action="store_true", default=False,
571 help=_("Commit changes"))
572 parser.add_argument("--gplay", action="store_true", default=False,
573 help=_("Only print differences with the Play Store"))
574 metadata.add_metadata_arguments(parser)
575 options = parser.parse_args()
576 metadata.warnings_action = options.W
578 config = common.read_config(options)
581 allapps = metadata.read_metadata()
583 apps = common.read_app_args(options.appid, allapps, False)
587 for appid, app in apps.items():
588 gplaylog += '* ' + appid + '\n'
589 version, reason = check_gplay(app)
592 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
594 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
595 if version is not None:
596 stored = app.CurrentVersion
598 logging.info("{0} has no Current Version but has version {1} on the Play Store"
599 .format(common.getappname(app), version))
600 elif LooseVersion(stored) < LooseVersion(version):
601 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
602 .format(common.getappname(app), version, stored))
604 if stored != version:
605 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
606 .format(common.getappname(app), version, stored))
608 logging.info("{0} has the same version {1} on the Play Store"
609 .format(common.getappname(app), version))
610 update_wiki(gplaylog, None)
614 for appid, app in apps.items():
616 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
617 logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
620 msg = _("Processing {appid}").format(appid=appid)
622 locallog += '* ' + msg + '\n'
625 checkupdates_app(app)
626 except Exception as e:
627 msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
629 locallog += msg + '\n'
631 update_wiki(None, locallog)
633 logging.info(_("Finished"))
636 if __name__ == "__main__":