3 # checkupdates.py - part of the FDroid server tools
4 # Copyright (C) 2010-2015, Ciaran Gultnieks, ciaran@ciarang.com
5 # Copyright (C) 2013-2014 Daniel Martà <mvdan@mvdan.cc>
7 # This program is free software: you can redistribute it and/or modify
8 # it under the terms of the GNU Affero General Public License as published by
9 # the Free Software Foundation, either version 3 of the License, or
10 # (at your option) any later version.
12 # This program is distributed in the hope that it will be useful,
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 # GNU Affero General Public License for more details.
17 # You should have received a copy of the GNU Affero General Public License
18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
27 from argparse import ArgumentParser
29 from html.parser import HTMLParser
30 from distutils.version import LooseVersion
35 from . import metadata
36 from .common import VCSException, FDroidException
37 from .metadata import MetaDataException
40 # Check for a new version by looking at a document retrieved via HTTP.
41 # The app's Update Check Data field is used to provide the information
47 if not app.UpdateCheckData:
48 raise FDroidException('Missing Update Check Data')
50 urlcode, codeex, urlver, verex = app.UpdateCheckData.split('|')
54 logging.debug("...requesting {0}".format(urlcode))
55 req = urllib.request.Request(urlcode, None)
56 resp = urllib.request.urlopen(req, None, 20)
57 page = resp.read().decode('utf-8')
59 m = re.search(codeex, page)
61 raise FDroidException("No RE match for version code")
67 logging.debug("...requesting {0}".format(urlver))
68 req = urllib.request.Request(urlver, None)
69 resp = urllib.request.urlopen(req, None, 20)
70 page = resp.read().decode('utf-8')
72 m = re.search(verex, page)
74 raise FDroidException("No RE match for version")
77 return (version, vercode)
79 except FDroidException:
80 msg = "Could not complete http check for app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
84 # Check for a new version by looking at the tags in the source repo.
85 # Whether this can be used reliably or not depends on
86 # the development procedures used by the project's developers. Use it with
87 # caution, because it's inappropriate for many projects.
88 # Returns (None, "a message") if this didn't work, or (version, vercode, tag) for
89 # the details of the current version.
90 def check_tags(app, pattern):
94 if app.RepoType == 'srclib':
95 build_dir = os.path.join('build', 'srclib', app.Repo)
96 repotype = common.getsrclibvcs(app.Repo)
98 build_dir = os.path.join('build', app.id)
99 repotype = app.RepoType
101 if repotype not in ('git', 'git-svn', 'hg', 'bzr'):
102 return (None, 'Tags update mode only works for git, hg, bzr and git-svn repositories currently', None)
104 if repotype == 'git-svn' and ';' not in app.Repo:
105 return (None, 'Tags update mode used in git-svn, but the repo was not set up with tags', None)
107 # Set up vcs interface and make sure we have the latest code...
108 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
110 vcs.gotorevision(None)
112 last_build = metadata.Build()
113 if len(app.builds) > 0:
114 last_build = app.builds[-1]
116 if last_build.submodules:
125 if repotype == 'git':
126 tags = vcs.latesttags()
130 return (None, "No tags found", None)
132 logging.debug("All tags: " + ','.join(tags))
134 pat = re.compile(pattern)
135 tags = [tag for tag in tags if pat.match(tag)]
137 return (None, "No matching tags found", None)
138 logging.debug("Matching tags: " + ','.join(tags))
140 if len(tags) > 5 and repotype == 'git':
142 logging.debug("Latest tags: " + ','.join(tags))
145 logging.debug("Check tag: '{0}'".format(tag))
146 vcs.gotorevision(tag)
148 for subdir in possible_subdirs(app):
152 root_dir = os.path.join(build_dir, subdir)
153 paths = common.manifest_paths(root_dir, last_build.gradle)
154 version, vercode, package = common.parse_androidmanifests(paths, app)
156 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
157 .format(subdir, version, vercode))
158 if int(vercode) > int(hcode):
161 hcode = str(int(vercode))
165 return (None, "Couldn't find package ID", None)
167 return (hver, hcode, htag)
168 return (None, "Couldn't find any version information", None)
170 except VCSException as vcse:
171 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
172 return (None, msg, None)
174 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
175 return (None, msg, None)
178 # Check for a new version by looking at the AndroidManifest.xml at the HEAD
179 # of the source repo. Whether this can be used reliably or not depends on
180 # the development procedures used by the project's developers. Use it with
181 # caution, because it's inappropriate for many projects.
182 # Returns (None, "a message") if this didn't work, or (version, vercode) for
183 # the details of the current version.
184 def check_repomanifest(app, branch=None):
188 if app.RepoType == 'srclib':
189 build_dir = os.path.join('build', 'srclib', app.Repo)
190 repotype = common.getsrclibvcs(app.Repo)
192 build_dir = os.path.join('build', app.id)
193 repotype = app.RepoType
195 # Set up vcs interface and make sure we have the latest code...
196 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
198 if repotype == 'git':
200 branch = 'origin/' + branch
201 vcs.gotorevision(branch)
202 elif repotype == 'git-svn':
203 vcs.gotorevision(branch)
204 elif repotype == 'hg':
205 vcs.gotorevision(branch)
206 elif repotype == 'bzr':
207 vcs.gotorevision(None)
209 last_build = metadata.Build()
210 if len(app.builds) > 0:
211 last_build = app.builds[-1]
213 if last_build.submodules:
219 for subdir in possible_subdirs(app):
223 root_dir = os.path.join(build_dir, subdir)
224 paths = common.manifest_paths(root_dir, last_build.gradle)
225 version, vercode, package = common.parse_androidmanifests(paths, app)
227 logging.debug("Manifest exists in subdir '{0}'. Found version {1} ({2})"
228 .format(subdir, version, vercode))
229 if int(vercode) > int(hcode):
231 hcode = str(int(vercode))
235 return (None, "Couldn't find package ID")
238 return (None, "Couldn't find any version information")
240 except VCSException as vcse:
241 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
244 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
248 def check_repotrunk(app, branch=None):
251 if app.RepoType == 'srclib':
252 build_dir = os.path.join('build', 'srclib', app.Repo)
253 repotype = common.getsrclibvcs(app.Repo)
255 build_dir = os.path.join('build', app.id)
256 repotype = app.RepoType
258 if repotype not in ('git-svn', ):
259 return (None, 'RepoTrunk update mode only makes sense in git-svn repositories')
261 # Set up vcs interface and make sure we have the latest code...
262 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
264 vcs.gotorevision(None)
268 except VCSException as vcse:
269 msg = "VCS error while scanning app {0}: {1}".format(app.id, vcse)
272 msg = "Could not scan app {0} due to unknown error: {1}".format(app.id, traceback.format_exc())
276 # Check for a new version by looking at the Google Play Store.
277 # Returns (None, "a message") if this didn't work, or (version, None) for
278 # the details of the current version.
279 def check_gplay(app):
281 url = 'https://play.google.com/store/apps/details?id=' + app.id
282 headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:18.0) Gecko/20100101 Firefox/18.0'}
283 req = urllib.request.Request(url, None, headers)
285 resp = urllib.request.urlopen(req, None, 20)
287 except urllib.error.HTTPError as e:
288 return (None, str(e.code))
289 except Exception as e:
290 return (None, 'Failed:' + str(e))
294 m = re.search('itemprop="softwareVersion">[ ]*([^<]+)[ ]*</div>', page)
296 html_parser = HTMLParser()
297 version = html_parser.unescape(m.group(1))
299 if version == 'Varies with device':
300 return (None, 'Device-variable version, cannot use this method')
303 return (None, "Couldn't find version")
304 return (version.strip(), None)
307 # Return all directories under startdir that contain any of the manifest
308 # files, and thus are probably an Android project.
309 def dirs_with_manifest(startdir):
310 for r, d, f in os.walk(startdir):
311 if any(m in f for m in [
312 'AndroidManifest.xml', 'pom.xml', 'build.gradle']):
316 # Tries to find a new subdir starting from the root build_dir. Returns said
317 # subdir relative to the build dir if found, None otherwise.
318 def possible_subdirs(app):
320 if app.RepoType == 'srclib':
321 build_dir = os.path.join('build', 'srclib', app.Repo)
323 build_dir = os.path.join('build', app.id)
325 last_build = metadata.Build()
326 if len(app.builds) > 0:
327 last_build = app.builds[-1]
329 for d in dirs_with_manifest(build_dir):
330 m_paths = common.manifest_paths(d, last_build.gradle)
331 package = common.parse_androidmanifests(m_paths, app)[2]
332 if package is not None:
333 subdir = os.path.relpath(d, build_dir)
334 logging.debug("Adding possible subdir %s" % subdir)
338 def fetch_autoname(app, tag):
340 if not app.RepoType or app.UpdateCheckMode in ('None', 'Static'):
343 if app.RepoType == 'srclib':
344 build_dir = os.path.join('build', 'srclib', app.Repo)
346 build_dir = os.path.join('build', app.id)
349 vcs = common.getvcs(app.RepoType, app.Repo, build_dir)
350 vcs.gotorevision(tag)
354 last_build = metadata.Build()
355 if len(app.builds) > 0:
356 last_build = app.builds[-1]
358 logging.debug("...fetch auto name from " + build_dir)
360 for subdir in possible_subdirs(app):
364 root_dir = os.path.join(build_dir, subdir)
365 new_name = common.fetch_real_name(root_dir, last_build.gradle)
366 if new_name is not None:
370 logging.debug("...got autoname '" + new_name + "'")
371 if new_name != app.AutoName:
372 app.AutoName = new_name
374 commitmsg = "Set autoname of {0}".format(common.getappname(app))
376 logging.debug("...couldn't get autoname")
381 def checkupdates_app(app, first=True):
383 # If a change is made, commitmsg should be set to a description of it.
384 # Only if this is set will changes be written back to the metadata.
391 mode = app.UpdateCheckMode
392 if mode.startswith('Tags'):
393 pattern = mode[5:] if len(mode) > 4 else None
394 (version, vercode, tag) = check_tags(app, pattern)
395 if version == 'Unknown':
398 elif mode == 'RepoManifest':
399 (version, vercode) = check_repomanifest(app)
401 elif mode.startswith('RepoManifest/'):
403 (version, vercode) = check_repomanifest(app, tag)
405 elif mode == 'RepoTrunk':
406 (version, vercode) = check_repotrunk(app)
409 (version, vercode) = check_http(app)
411 elif mode in ('None', 'Static'):
413 msg = 'Checking disabled'
417 msg = 'Invalid update check method'
419 if version and vercode and app.VercodeOperation:
420 oldvercode = str(int(vercode))
421 op = app.VercodeOperation.replace("%c", oldvercode)
422 vercode = str(eval(op))
423 logging.debug("Applied vercode operation: %s -> %s" % (oldvercode, vercode))
425 if version and any(version.startswith(s) for s in [
426 '${', # Gradle variable names
427 '@string/', # Strings we could not resolve
433 logmsg = "...{0} : {1}".format(app.id, msg)
438 elif vercode == app.CurrentVersionCode:
439 logging.info("...up to date")
441 app.CurrentVersion = version
442 app.CurrentVersionCode = str(int(vercode))
445 commitmsg = fetch_autoname(app, tag)
448 name = common.getappname(app)
449 ver = common.getcvname(app)
450 logging.info('...updating to version %s' % ver)
451 commitmsg = 'Update CV of %s to %s' % (name, ver)
454 mode = app.AutoUpdateMode
455 if mode in ('None', 'Static'):
457 elif mode.startswith('Version '):
459 if pattern.startswith('+'):
461 suffix, pattern = pattern.split(' ', 1)
463 raise MetaDataException("Invalid AUM: " + mode)
468 for build in app.builds:
469 if int(build.vercode) >= int(app.CurrentVersionCode):
471 if not latest or int(build.vercode) > int(latest.vercode):
474 if int(latest.vercode) > int(app.CurrentVersionCode):
475 logging.info("Refusing to auto update, since the latest build is newer")
478 newbuild = copy.deepcopy(latest)
479 newbuild.disable = False
480 newbuild.vercode = app.CurrentVersionCode
481 newbuild.version = app.CurrentVersion + suffix
482 logging.info("...auto-generating build for " + newbuild.version)
483 commit = pattern.replace('%v', newbuild.version)
484 commit = commit.replace('%c', newbuild.vercode)
485 newbuild.commit = commit
486 app.builds.append(newbuild)
487 name = common.getappname(app)
488 ver = common.getcvname(app)
489 commitmsg = "Update %s to %s" % (name, ver)
491 logging.warn('Invalid auto update mode "' + mode + '" on ' + app.id)
494 metadatapath = os.path.join('metadata', app.id + '.txt')
495 metadata.write_metadata(metadatapath, app)
497 logging.info("Commiting update for " + metadatapath)
498 gitcmd = ["git", "commit", "-m", commitmsg]
499 if 'auto_author' in config:
500 gitcmd.extend(['--author', config['auto_author']])
501 gitcmd.extend(["--", metadatapath])
502 if subprocess.call(gitcmd) != 0:
503 logging.error("Git commit failed")
513 global config, options
515 # Parse command line...
516 parser = ArgumentParser(usage="%(prog)s [options] [APPID [APPID ...]]")
517 common.setup_global_opts(parser)
518 parser.add_argument("appid", nargs='*', help="app-id to check for updates")
519 parser.add_argument("--auto", action="store_true", default=False,
520 help="Process auto-updates")
521 parser.add_argument("--autoonly", action="store_true", default=False,
522 help="Only process apps with auto-updates")
523 parser.add_argument("--commit", action="store_true", default=False,
524 help="Commit changes")
525 parser.add_argument("--gplay", action="store_true", default=False,
526 help="Only print differences with the Play Store")
527 options = parser.parse_args()
529 config = common.read_config(options)
532 allapps = metadata.read_metadata()
534 apps = common.read_app_args(options.appid, allapps, False)
538 version, reason = check_gplay(app)
541 logging.info("{0} is not in the Play Store".format(common.getappname(app)))
543 logging.info("{0} encountered a problem: {1}".format(common.getappname(app), reason))
544 if version is not None:
545 stored = app.CurrentVersion
547 logging.info("{0} has no Current Version but has version {1} on the Play Store"
548 .format(common.getappname(app), version))
549 elif LooseVersion(stored) < LooseVersion(version):
550 logging.info("{0} has version {1} on the Play Store, which is bigger than {2}"
551 .format(common.getappname(app), version, stored))
553 if stored != version:
554 logging.info("{0} has version {1} on the Play Store, which differs from {2}"
555 .format(common.getappname(app), version, stored))
557 logging.info("{0} has the same version {1} on the Play Store"
558 .format(common.getappname(app), version))
561 for appid, app in apps.items():
563 if options.autoonly and app.AutoUpdateMode in ('None', 'Static'):
564 logging.debug("Nothing to do for {0}...".format(appid))
567 logging.info("Processing " + appid + '...')
569 checkupdates_app(app)
571 logging.info("Finished.")
573 if __name__ == "__main__":