3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from argparse import ArgumentParser
30 from distutils.version import LooseVersion
36 from . import metadata
37 from .exception import VCSException, NoSubmodulesException, FDroidException, MetaDataException
40 # Check for a new version by looking at a document retrieved via HTTP.
41 # The app's Update Check Data field is used to provide the information
47 if not app.UpdateCheckData:
48 raise FDroidException('Missing Update Check Data')
50 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
54 logging.debug("...requesting {0}".format(urlcode))
55 req = urllib.request.Request(urlcode, None)
56 resp = urllib.request.urlopen(req, None, 20)
57 page = resp.read().decode('utf-8')
59 m = re.search(codeex, page)
61 raise FDroidException("No RE match for version code")
62 vercode = m.group(1).strip()
67 logging.debug("...requesting {0}".format(urlver))
68 req = urllib.request.Request(urlver, None)
69 resp = urllib.request.urlopen(req, None, 20)
70 page = resp.read().decode('utf-8')
72 m = re.search(verex, page)
74 raise FDroidException("No RE match for version")
77 return (version, vercode)
79 except FDroidException:
80 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
84 # Check for a new version by looking at the tags in the source repo.
85 # Whether this can be used reliably or not depends on
86 # the development procedures used by the project's developers. Use it with
87 # caution, because it's inappropriate for many projects.
88 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
89 # the details of the current version.
90 def check_tags(app, pattern):
94 if app.RepoType == 'srclib':
95 build_dir = os.path.join('build', 'srclib', app.Repo)
96 repotype = common.getsrclibvcs(app.Repo)
98 build_dir = os.path.join('build', app.id)
99 repotype = app.RepoType
101 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
102 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
104 if repotype == 'git-svn' and ';' not in app.Repo:
105 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
107 # Set up vcs interface and make sure we have the latest code...
108 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
110 vcs.gotorevision(None)
112 last_build = app.get_last_build()
114 try_init_submodules(app, last_build, vcs)
122 if repotype == 'git':
123 tags = vcs.latesttags()
127 return (None, "No tags found", None)
129 logging.debug("All tags: " + ','.join(tags))
131 pat = re.compile(pattern)
132 tags = [tag for tag in tags if pat.match(tag)]
134 return (None, "No matching tags found", None)
135 logging.debug("Matching tags: " + ','.join(tags))
137 if len(tags) > 5 and repotype == 'git':
139 logging.debug("Latest tags: " + ','.join(tags))
142 logging.debug("Check tag: '{0}'".format(tag))
143 vcs.gotorevision(tag)
145 for subdir in possible_subdirs(app):
149 root_dir = os.path.join(build_dir, subdir)
150 paths = common.manifest_paths(root_dir, last_build.gradle)
151 version, vercode, package = common.parse_androidmanifests(paths, app)
153 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
154 .format(subdir, version, vercode))
155 if int(vercode) > int(hcode):
158 hcode = str(int(vercode))
162 return (None, "Couldn't find package ID", None)
164 return (hver, hcode, htag)
165 return (None, "Couldn't find any version information", None)
167 except VCSException as vcse:
168 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
169 return (None, msg, None)
171 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
172 return (None, msg, None)
175 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
176 # of the source repo. Whether this can be used reliably or not depends on
177 # the development procedures used by the project's developers. Use it with
178 # caution, because it's inappropriate for many projects.
179 # Returns (None, "a message") if this didn't work, or (version, vercode) for
180 # the details of the current version.
181 def check_repomanifest(app, branch=None):
185 if app.RepoType == 'srclib':
186 build_dir = os.path.join('build', 'srclib', app.Repo)
187 repotype = common.getsrclibvcs(app.Repo)
189 build_dir = os.path.join('build', app.id)
190 repotype = app.RepoType
192 # Set up vcs interface and make sure we have the latest code...
193 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
195 if repotype == 'git':
197 branch = 'origin/' + branch
198 vcs.gotorevision(branch)
199 elif repotype == 'git-svn':
200 vcs.gotorevision(branch)
201 elif repotype == 'hg':
202 vcs.gotorevision(branch)
203 elif repotype == 'bzr':
204 vcs.gotorevision(None)
206 last_build = metadata.Build()
207 if len(app.builds) > 0:
208 last_build = app.builds[-1]
210 try_init_submodules(app, last_build, vcs)
215 for subdir in possible_subdirs(app):
219 root_dir = os.path.join(build_dir, subdir)
220 paths = common.manifest_paths(root_dir, last_build.gradle)
221 version, vercode, package = common.parse_androidmanifests(paths, app)
223 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
224 .format(subdir, version, vercode))
225 if int(vercode) > int(hcode):
227 hcode = str(int(vercode))
231 return (None, "Couldn't find package ID")
234 return (None, "Couldn't find any version information")
236 except VCSException as vcse:
237 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
240 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
244 def check_repotrunk(app):
247 if app.RepoType == 'srclib':
248 build_dir = os.path.join('build', 'srclib', app.Repo)
249 repotype = common.getsrclibvcs(app.Repo)
251 build_dir = os.path.join('build', app.id)
252 repotype = app.RepoType
254 if repotype not in ('git-svn', ):
255 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
257 # Set up vcs interface and make sure we have the latest code...
258 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
260 vcs.gotorevision(None)
264 except VCSException as vcse:
265 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
268 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
272 # Check for a new version by looking at the Google Play Store.
273 # Returns (None, "a message") if this didn't work, or (version, None) for
274 # the details of the current version.
275 def check_gplay(app):
277 url = 'https://play.google.com/store/apps/details?id=' + app.id
278 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
279 req = urllib.request.Request(url, None, headers)
281 resp = urllib.request.urlopen(req, None, 20)
282 page = resp.read().decode()
283 except urllib.error.HTTPError as e:
284 return (None, str(e.code))
285 except Exception as e:
286 return (None, 'Failed:' + str(e))
290 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
292 version = html.unescape(m.group(1))
294 if version == 'Varies with device':
295 return (None, 'Device-variable version, cannot use this method')
298 return (None, "Couldn't find version")
299 return (version.strip(), None)
302 def try_init_submodules(app, last_build, vcs):
303 """Try to init submodules if the last build entry used them.
304 They might have been removed from the app's repo in the meantime,
305 so if we can't find any submodules we continue with the updates check.
306 If there is any other error in initializing them then we stop the check.
308 if last_build.submodules:
311 except NoSubmodulesException:
312 logging.info("No submodules present for {}".format(app.Name))
315 # Return all directories under startdir that contain any of the manifest
316 # files, and thus are probably an Android project.
317 def dirs_with_manifest(startdir):
318 for root, dirs, files in os.walk(startdir):
319 if any(m in files for m in [
320 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
324 # Tries to find a new subdir starting from the root build_dir. Returns said
325 # subdir relative to the build dir if found, None otherwise.
326 def possible_subdirs(app):
328 if app.RepoType == 'srclib':
329 build_dir = os.path.join('build', 'srclib', app.Repo)
331 build_dir = os.path.join('build', app.id)
333 last_build = app.get_last_build()
335 for d in dirs_with_manifest(build_dir):
336 m_paths = common.manifest_paths(d, last_build.gradle)
337 package = common.parse_androidmanifests(m_paths, app)[2]
338 if package is not None:
339 subdir = os.path.relpath(d, build_dir)
340 logging.debug("Adding possible subdir %s" % subdir)
344 def fetch_autoname(app, tag):
346 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
349 if app.RepoType == 'srclib':
350 build_dir = os.path.join('build', 'srclib', app.Repo)
352 build_dir = os.path.join('build', app.id)
355 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
356 vcs.gotorevision(tag)
360 last_build = app.get_last_build()
362 logging.debug("...fetch auto name from " + build_dir)
364 for subdir in possible_subdirs(app):
368 root_dir = os.path.join(build_dir, subdir)
369 new_name = common.fetch_real_name(root_dir, last_build.gradle)
370 if new_name is not None:
374 logging.debug("...got autoname '" + new_name + "'")
375 if new_name != app.AutoName:
376 app.AutoName = new_name
378 commitmsg = "Set autoname of {0}".format(common.getappname(app))
380 logging.debug("...couldn't get autoname")
385 def checkupdates_app(app):
387 # If a change is made, commitmsg should be set to a description of it.
388 # Only if this is set will changes be written back to the metadata.
395 mode = app.UpdateCheckMode
396 if mode.startswith('Tags'):
397 pattern = mode[5:] if len(mode) > 4 else None
398 (version, vercode, tag) = check_tags(app, pattern)
399 if version == 'Unknown':
402 elif mode == 'RepoManifest':
403 (version, vercode) = check_repomanifest(app)
405 elif mode.startswith('RepoManifest/'):
407 (version, vercode) = check_repomanifest(app, tag)
409 elif mode == 'RepoTrunk':
410 (version, vercode) = check_repotrunk(app)
413 (version, vercode) = check_http(app)
415 elif mode in ('None', 'Static'):
417 msg = 'Checking disabled'
421 msg = 'Invalid update check method'
423 if version and vercode and app.VercodeOperation:
424 oldvercode = str(int(vercode))
425 op = app.VercodeOperation.replace("%c", oldvercode)
426 vercode = str(eval(op))
427 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
429 if version and any(version.startswith(s) for s in [
430 '${', # Gradle variable names
431 '@string/', # Strings we could not resolve
437 logmsg = "...{0} : {1}".format(app.id, msg)
442 elif vercode == app.CurrentVersionCode:
443 logging.info("...up to date")
445 logging.debug("...updating - old vercode={0}, new vercode={1}".format(
446 app.CurrentVersionCode, vercode))
447 app.CurrentVersion = version
448 app.CurrentVersionCode = str(int(vercode))
451 commitmsg = fetch_autoname(app, tag)
454 name = common.getappname(app)
455 ver = common.getcvname(app)
456 logging.info('...updating to version %s' % ver)
457 commitmsg = 'Update CV of %s to %s' % (name, ver)
460 mode = app.AutoUpdateMode
461 if not app.CurrentVersionCode:
462 logging.warn("Can't auto-update app with no current version code: " + app.id)
463 elif mode in ('None', 'Static'):
465 elif mode.startswith('Version '):
467 if pattern.startswith('+'):
469 suffix, pattern = pattern.split(' ', 1)
471 raise MetaDataException("Invalid AUM: " + mode)
476 for build in app.builds:
477 if int(build.versionCode) >= int(app.CurrentVersionCode):
479 if not latest or int(build.versionCode) > int(latest.versionCode):
482 if int(latest.versionCode) > int(app.CurrentVersionCode):
483 logging.info("Refusing to auto update, since the latest build is newer")
486 newbuild = copy.deepcopy(latest)
487 newbuild.disable = False
488 newbuild.versionCode = app.CurrentVersionCode
489 newbuild.versionName = app.CurrentVersion + suffix
490 logging.info("...auto-generating build for " + newbuild.versionName)
491 commit = pattern.replace('%v', newbuild.versionName)
492 commit = commit.replace('%c', newbuild.versionCode)
493 newbuild.commit = commit
494 app.builds.append(newbuild)
495 name = common.getappname(app)
496 ver = common.getcvname(app)
497 commitmsg = "Update %s to %s" % (name, ver)
499 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
502 metadatapath = os.path.join('metadata', app.id + '.txt')
503 metadata.write_metadata(metadatapath, app)
505 logging.info("Commiting update for " + metadatapath)
506 gitcmd = ["git", "commit", "-m", commitmsg]
507 if 'auto_author' in config:
508 gitcmd.extend(['--author', config['auto_author']])
509 gitcmd.extend(["--", metadatapath])
510 if subprocess.call(gitcmd) != 0:
511 raise FDroidException("Git commit failed")
514 def update_wiki(gplaylog, locallog):
515 if config.get('wiki_server') and config.get('wiki_path'):
518 site = mwclient.Site((config['wiki_protocol'], config['wiki_server']),
519 path=config['wiki_path'])
520 site.login(config['wiki_user'], config['wiki_password'])
522 # Write a page with the last build log for this version code
523 wiki_page_path = 'checkupdates_' + time.strftime('%s', start_timestamp)
524 newpage = site.Pages[wiki_page_path]
526 txt += "* command line: <code>" + ' '.join(sys.argv) + "</code>\n"
527 txt += "* started at " + common.get_wiki_timestamp(start_timestamp) + '\n'
528 txt += "* completed at " + common.get_wiki_timestamp() + '\n'
530 txt += common.get_android_tools_version_log()
533 txt += '== --gplay check ==\n\n'
536 txt += '== local source check ==\n\n'
538 newpage.save(txt, summary='Run log')
539 newpage = site.Pages['checkupdates']
540 newpage.save('#REDIRECT [[' + wiki_page_path + ']]', summary='Update redirect')
541 except Exception as e:
542 logging.error(_('Error while attempting to publish log: %s') % e)
547 start_timestamp = time.gmtime()
552 global config, options
554 # Parse command line...
555 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
556 common.setup_global_opts(parser)
557 parser.add_argument("appid", nargs='*', help=_("applicationId to check for updates"))
558 parser.add_argument("--auto", action="store_true", default=False,
559 help=_("Process auto-updates"))
560 parser.add_argument("--autoonly", action="store_true", default=False,
561 help=_("Only process apps with auto-updates"))
562 parser.add_argument("--commit", action="store_true", default=False,
563 help=_("Commit changes"))
564 parser.add_argument("--gplay", action="store_true", default=False,
565 help=_("Only print differences with the Play Store"))
566 metadata.add_metadata_arguments(parser)
567 options = parser.parse_args()
568 metadata.warnings_action = options.W
570 config = common.read_config(options)
573 allapps = metadata.read_metadata()
575 apps = common.read_app_args(options.appid, allapps, False)
579 for appid, app in apps.items():
580 gplaylog += '* ' + appid + '\n'
581 version, reason = check_gplay(app)
584 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
586 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
587 if version is not None:
588 stored = app.CurrentVersion
590 logging.info("{0} has no Current Version but has version {1} on the Play Store"
591 .format(common.getappname(app), version))
592 elif LooseVersion(stored) < LooseVersion(version):
593 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
594 .format(common.getappname(app), version, stored))
596 if stored != version:
597 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
598 .format(common.getappname(app), version, stored))
600 logging.info("{0} has the same version {1} on the Play Store"
601 .format(common.getappname(app), version))
602 update_wiki(gplaylog, None)
606 for appid, app in apps.items():
608 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
609 logging.debug(_("Nothing to do for {appid}.").format(appid=appid))
612 msg = _("Processing {appid}").format(appid=appid)
614 locallog += '* ' + msg + '\n'
617 checkupdates_app(app)
618 except Exception as e:
619 msg = _("...checkupdate failed for {appid} : {error}").format(appid=appid, error=e)
621 locallog += msg + '\n'
623 update_wiki(None, locallog)
625 logging.info(_("Finished"))
628 if __name__ == "__main__":