From: Daniel Martí Date: Mon, 27 Jan 2014 16:04:22 +0000 (+0100) Subject: More logging X-Git-Tag: 0.2~348 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=1b743e8b74726cef0496599ba45bd2b13a78cc47;p=fdroidserver.git More logging --- diff --git a/fdroidserver/build.py b/fdroidserver/build.py index b335e0f5..465c3b9d 100644 --- a/fdroidserver/build.py +++ b/fdroidserver/build.py @@ -462,8 +462,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d jni_components = [c.strip() for c in jni_components.split(';')] ndkbuild = os.path.join(config['ndk_path'], "ndk-build") for d in jni_components: - if options.verbose: - print "Running ndk-build in " + root_dir + '/' + d + logging.info("Building native code in '%s'" % d) manifest = root_dir + '/' + d + '/AndroidManifest.xml' if os.path.exists(manifest): # Read and write the whole AM.xml to fix newlines and avoid @@ -482,7 +481,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d p = None # Build the release... if thisbuild['type'] == 'maven': - print "Building Maven project..." + logging.info("Building Maven project...") if '@' in thisbuild['maven']: maven_dir = os.path.join(root_dir, thisbuild['maven'].split('@',1)[1]) @@ -510,7 +509,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d bindir = os.path.join(root_dir, 'target') elif thisbuild['type'] == 'kivy': - print "Building Kivy project..." + logging.info("Building Kivy project...") spec = os.path.join(root_dir, 'buildozer.spec') if not os.path.exists(spec): @@ -570,7 +569,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d p = FDroidPopen(cmd, cwd=distdir) elif thisbuild['type'] == 'gradle': - print "Building Gradle project..." + logging.info("Building Gradle project...") if '@' in thisbuild['gradle']: flavours = thisbuild['gradle'].split('@')[0].split(',') gradle_dir = thisbuild['gradle'].split('@')[1] @@ -601,7 +600,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d p = FDroidPopen(commands, cwd=gradle_dir) else: - print "Building Ant project..." + logging.info("Building Ant project...") cmd = ['ant'] if 'antcommand' in thisbuild: cmd += [thisbuild['antcommand']] @@ -613,7 +612,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d if p.returncode != 0: raise BuildException("Build failed for %s:%s" % (app['id'], thisbuild['version']), p.stdout) - print "Successfully built version " + thisbuild['version'] + ' of ' + app['id'] + logging.info("Successfully built version " + thisbuild['version'] + ' of ' + app['id']) # Find the apk name in the output... if 'bindir' in thisbuild: @@ -659,7 +658,7 @@ def build_local(app, thisbuild, vcs, build_dir, output_dir, srclib_dir, extlib_d # By way of a sanity check, make sure the version and version # code in our new apk match what we expect... - print "Checking " + src + logging.info("Checking " + src) if not os.path.exists(src): raise BuildException("Unsigned apk is not at expected location of " + src) @@ -758,7 +757,7 @@ def trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, if 'disable' in thisbuild: return False - print "Building version " + thisbuild['version'] + ' of ' + app['id'] + logging.info("Building version " + thisbuild['version'] + ' of ' + app['id']) if server: # When using server mode, still keep a local cache of the repo, by @@ -826,12 +825,12 @@ def main(): log_dir = 'logs' if not os.path.isdir(log_dir): - print "Creating log directory" + logging.info("Creating log directory") os.makedirs(log_dir) tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): - print "Creating temporary directory" + logging.info("Creating temporary directory") os.makedirs(tmp_dir) if options.test: @@ -839,7 +838,7 @@ def main(): else: output_dir = 'unsigned' if not os.path.isdir(output_dir): - print "Creating output directory" + logging.info("Creating output directory") os.makedirs(output_dir) if config['archive_older'] != 0: @@ -851,7 +850,7 @@ def main(): build_dir = 'build' if not os.path.isdir(build_dir): - print "Creating build directory" + logging.info("Creating build directory") os.makedirs(build_dir) srclib_dir = os.path.join(build_dir, 'srclib') extlib_dir = os.path.join(build_dir, 'extlib') @@ -901,15 +900,13 @@ def main(): build_dir = os.path.join('build', app['id']) # Set up vcs interface and make sure we have the latest code... - if options.verbose: - print "Getting {0} vcs interface for {1}".format( - app['Repo Type'], app['Repo']) + logging.info("Getting {0} vcs interface for {1}".format( + app['Repo Type'], app['Repo'])) vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir) first = False - if options.verbose: - print "Checking " + thisbuild['version'] + logging.info("Checking " + thisbuild['version']) if trybuild(app, thisbuild, build_dir, output_dir, also_check_dir, srclib_dir, extlib_dir, tmp_dir, repo_dir, vcs, options.test, options.server, options.force, options.onserver): @@ -919,19 +916,19 @@ def main(): logfile = open(os.path.join(log_dir, app['id'] + '.log'), 'a+') logfile.write(str(be)) logfile.close() - print "Could not build app %s due to BuildException: %s" % (app['id'], be) + logging.info("Could not build app %s due to BuildException: %s" % (app['id'], be)) if options.stop: sys.exit(1) failed_apps[app['id']] = be wikilog = be.get_wikitext() except VCSException as vcse: - print "VCS error while building app %s: %s" % (app['id'], vcse) + logging.info("VCS error while building app %s: %s" % (app['id'], vcse)) if options.stop: sys.exit(1) failed_apps[app['id']] = vcse wikilog = str(vcse) except Exception as e: - print "Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) + logging.info("Could not build app %s due to unknown error: %s" % (app['id'], traceback.format_exc())) if options.stop: sys.exit(1) failed_apps[app['id']] = e @@ -946,20 +943,20 @@ def main(): txt = "Build completed at " + time.strftime("%Y-%m-%d %H:%M:%SZ", time.gmtime()) + "\n\n" + txt newpage.save(txt, summary='Build log') except: - print "Error while attempting to publish build log" + logging.info("Error while attempting to publish build log") for app in build_succeeded: - print "success: %s" % (app['id']) + logging.info("success: %s" % (app['id'])) if not options.verbose: for fa in failed_apps: - print "Build for app %s failed:\n%s" % (fa, failed_apps[fa]) + logging.info("Build for app %s failed:\n%s" % (fa, failed_apps[fa])) - print "Finished." + logging.info("Finished.") if len(build_succeeded) > 0: - print str(len(build_succeeded)) + ' builds succeeded' + logging.info(str(len(build_succeeded)) + ' builds succeeded') if len(failed_apps) > 0: - print str(len(failed_apps)) + ' builds failed' + logging.info(str(len(failed_apps)) + ' builds failed') sys.exit(0) diff --git a/fdroidserver/checkupdates.py b/fdroidserver/checkupdates.py index d93a0646..f37ace28 100644 --- a/fdroidserver/checkupdates.py +++ b/fdroidserver/checkupdates.py @@ -28,6 +28,8 @@ from optparse import OptionParser import traceback import HTMLParser from distutils.version import LooseVersion +import logging + import common, metadata from common import BuildException from common import VCSException @@ -48,7 +50,7 @@ def check_http(app): vercode = "99999999" if len(urlcode) > 0: - print "...requesting {0}".format(urlcode) + logging.info("...requesting {0}".format(urlcode)) req = urllib2.Request(urlcode, None) resp = urllib2.urlopen(req, None, 20) page = resp.read() @@ -61,7 +63,7 @@ def check_http(app): version = "??" if len(urlver) > 0: if urlver != '.': - print "...requesting {0}".format(urlver) + logging.info("...requesting {0}".format(urlver)) req = urllib2.Request(urlver, None) resp = urllib2.urlopen(req, None, 20) page = resp.read() @@ -114,16 +116,15 @@ def check_tags(app): hcode = "0" for tag in vcs.gettags(): - if options.verbose: - print "Check tag: '{0}'".format(tag) + logging.info("Check tag: '{0}'".format(tag)) vcs.gotorevision(tag) # Only process tags where the manifest exists... paths = common.manifest_paths(build_dir, flavour) version, vercode, package = common.parse_androidmanifests(paths) if package and package == app['id'] and version and vercode: - print "Manifest exists. Found version %s (%s)" % ( - version, vercode) + logging.info("Manifest exists. Found version %s (%s)" % ( + version, vercode)) if int(vercode) > int(hcode): htag = tag hcode = str(int(vercode)) @@ -201,7 +202,7 @@ def check_repomanifest(app, branch=None): vercode = str(int(vercode)) - print "Manifest exists. Found version %s (%s)" % (version, vercode) + logging.info("Manifest exists. Found version %s (%s)" % (version, vercode)) return (version, vercode) @@ -307,38 +308,36 @@ def main(): if options.gplay: for app in apps: version, reason = check_gplay(app) - if version is None and options.verbose: + if version is None: if reason == '404': - print "%s is not in the Play Store" % common.getappname(app) + logging.info("%s is not in the Play Store" % common.getappname(app)) else: - print "%s encountered a problem: %s" % (common.getappname(app), reason) + logging.info("%s encountered a problem: %s" % (common.getappname(app), reason)) if version is not None: stored = app['Current Version'] if not stored: - if options.verbose: - print "%s has no Current Version but has version %s on the Play Store" % ( - common.getappname(app), version) + logging.info("%s has no Current Version but has version %s on the Play Store" % ( + common.getappname(app), version)) elif LooseVersion(stored) < LooseVersion(version): - print "%s has version %s on the Play Store, which is bigger than %s" % ( - common.getappname(app), version, stored) - elif options.verbose: + logging.info("%s has version %s on the Play Store, which is bigger than %s" % ( + common.getappname(app), version, stored)) + else: if stored != version: - print "%s has version %s on the Play Store, which differs from %s" % ( - common.getappname(app), version, stored) + logging.info("%s has version %s on the Play Store, which differs from %s" % ( + common.getappname(app), version, stored)) else: - print "%s has the same version %s on the Play Store" % ( - common.getappname(app), version) + logging.info("%s has the same version %s on the Play Store" % ( + common.getappname(app), version)) return for app in apps: if options.autoonly and app['Auto Update Mode'] == 'None': - if options.verbose: - print "Nothing to do for %s..." % app['id'] + logging.info("Nothing to do for %s..." % app['id']) continue - print "Processing " + app['id'] + '...' + logging.info("Processing " + app['id'] + '...') writeit = False logmsg = None @@ -374,9 +373,9 @@ def main(): updating = False if not version: - print "...%s" % msg + logging.info("...%s" % msg) elif vercode == app['Current Version Code']: - print "...up to date" + logging.info("...up to date") else: app['Current Version'] = version app['Current Version Code'] = str(int(vercode)) @@ -413,12 +412,12 @@ def main(): app['Current Version'] = cv writeit = True except Exception: - print "ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc()) + logging.info("ERROR: Auto Name or Current Version failed for %s due to exception: %s" % (app['id'], traceback.format_exc())) if updating: name = common.getappname(app) ver = common.getcvname(app) - print '...updating to version %s' % ver + logging.info('...updating to version %s' % ver) logmsg = 'Update CV of %s to %s' % (name, ver) if options.auto: @@ -447,7 +446,7 @@ def main(): del newbuild['origlines'] newbuild['vercode'] = app['Current Version Code'] newbuild['version'] = app['Current Version'] + suffix - print "...auto-generating build for " + newbuild['version'] + logging.info("...auto-generating build for " + newbuild['version']) commit = pattern.replace('%v', newbuild['version']) commit = commit.replace('%c', newbuild['vercode']) newbuild['commit'] = commit @@ -457,23 +456,23 @@ def main(): ver = common.getcvname(app) logmsg = "Update %s to %s" % (name, ver) else: - print 'Invalid auto update mode "' + mode + '"' + logging.info('Invalid auto update mode "' + mode + '"') if writeit: metafile = os.path.join('metadata', app['id'] + '.txt') metadata.write_metadata(metafile, app) if options.commit and logmsg: - print "Commiting update for " + metafile + logging.info("Commiting update for " + metafile) gitcmd = ["git", "commit", "-m", logmsg] if 'auto_author' in config: gitcmd.extend(['--author', config['auto_author']]) gitcmd.extend(["--", metafile]) if subprocess.call(gitcmd) != 0: - print "Git commit failed" + logging.info("Git commit failed") sys.exit(1) - print "Finished." + logging.info("Finished.") if __name__ == "__main__": main() diff --git a/fdroidserver/import.py b/fdroidserver/import.py index 3a4d3f75..8d066ae7 100644 --- a/fdroidserver/import.py +++ b/fdroidserver/import.py @@ -24,6 +24,7 @@ import shutil import urllib from optparse import OptionParser from ConfigParser import ConfigParser +import logging import common, metadata # Get the repo type and address from the given web page. The page is scanned @@ -106,13 +107,13 @@ def main(): config = common.read_config(options) if not options.url: - print "Specify project url." + logging.info("Specify project url.") sys.exit(1) url = options.url tmp_dir = 'tmp' if not os.path.isdir(tmp_dir): - print "Creating temporary directory" + logging.info("Creating temporary directory") os.makedirs(tmp_dir) # Get all apps... @@ -133,7 +134,7 @@ def main(): if url.endswith('/'): url = url[:-1] if url.endswith('.git'): - print "A github URL should point to the project, not the git repo" + logging.info("A github URL should point to the project, not the git repo") sys.exit(1) projecttype = 'github' repo = url + '.git' @@ -154,7 +155,7 @@ def main(): # Figure out the repo type and adddress... repotype, repo = getrepofrompage(sourcecode) if not repotype: - print "Unable to determine vcs type. " + repo + logging.info("Unable to determine vcs type. " + repo) sys.exit(1) elif url.startswith('http://code.google.com/p/'): if not url.endswith('/'): @@ -168,29 +169,29 @@ def main(): # Figure out the repo type and adddress... repotype, repo = getrepofrompage(sourcecode) if not repotype: - print "Unable to determine vcs type. " + repo + logging.info("Unable to determine vcs type. " + repo) sys.exit(1) # Figure out the license... req = urllib.urlopen(url) if req.getcode() != 200: - print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode()) + logging.info('Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())) sys.exit(1) page = req.read() index = page.find('Code license') if index == -1: - print "Couldn't find license data" + logging.info("Couldn't find license data") sys.exit(1) ltext = page[index:] lprefix = 'rel="nofollow">' index = ltext.find(lprefix) if index == -1: - print "Couldn't find license text" + logging.info("Couldn't find license text") sys.exit(1) ltext = ltext[index + len(lprefix):] index = ltext.find('<') if index == -1: - print "License text not formatted as expected" + logging.info("License text not formatted as expected") sys.exit(1) ltext = ltext[:index] if ltext == 'GNU GPL v3': @@ -208,17 +209,17 @@ def main(): elif ltext == 'New BSD License': license = 'NewBSD' else: - print "License " + ltext + " is not recognised" + logging.info("License " + ltext + " is not recognised") sys.exit(1) if not projecttype: - print "Unable to determine the project type." - print "The URL you supplied was not in one of the supported formats. Please consult" - print "the manual for a list of supported formats, and supply one of those." + logging.info("Unable to determine the project type.") + logging.info("The URL you supplied was not in one of the supported formats. Please consult") + logging.info("the manual for a list of supported formats, and supply one of those.") sys.exit(1) # Get a copy of the source so we can extract some info... - print 'Getting source from ' + repotype + ' repo at ' + repo + logging.info('Getting source from ' + repotype + ' repo at ' + repo) src_dir = os.path.join(tmp_dir, 'importer') if os.path.exists(src_dir): shutil.rmtree(src_dir) @@ -235,12 +236,12 @@ def main(): version, vercode, package = common.parse_androidmanifests(paths) if not package: - print "Couldn't find package ID" + logging.info("Couldn't find package ID") sys.exit(1) if not version: - print "WARNING: Couldn't find latest version name" + logging.info("WARNING: Couldn't find latest version name") if not vercode: - print "WARNING: Couldn't find latest version code" + logging.info("WARNING: Couldn't find latest version code") else: spec = os.path.join(root_dir, 'buildozer.spec') if os.path.exists(spec): @@ -252,13 +253,13 @@ def main(): version = bconfig.get('app', 'version') vercode = None else: - print "No android or kivy project could be found. Specify --subdir?" + logging.info("No android or kivy project could be found. Specify --subdir?") sys.exit(1) # Make sure it's actually new... for app in apps: if app['id'] == package: - print "Package " + package + " already exists" + logging.info("Package " + package + " already exists") sys.exit(1) # Construct the metadata... @@ -295,7 +296,7 @@ def main(): metafile = os.path.join('metadata', package + '.txt') metadata.write_metadata(metafile, app) - print "Wrote " + metafile + logging.info("Wrote " + metafile) if __name__ == "__main__": diff --git a/fdroidserver/stats.py b/fdroidserver/stats.py index f64c6485..6aeb6129 100644 --- a/fdroidserver/stats.py +++ b/fdroidserver/stats.py @@ -26,6 +26,7 @@ import glob from optparse import OptionParser import paramiko import socket +import logging import common, metadata from common import FDroidPopen @@ -57,7 +58,7 @@ def main(): config = common.read_config(options) if not config['update_stats']: - print "Stats are disabled - check your configuration" + logging.info("Stats are disabled - check your configuration") sys.exit(1) # Get all metadata-defined apps... @@ -78,14 +79,14 @@ def main(): ssh = None ftp = None try: - print 'Retrieving logs' + logging.info('Retrieving logs') ssh = paramiko.SSHClient() ssh.load_system_host_keys() ssh.connect('f-droid.org', username='fdroid', timeout=10, key_filename=config['webserver_keyfile']) ftp = ssh.open_sftp() ftp.get_channel().settimeout(60) - print "...connected" + logging.info("...connected") ftp.chdir('logs') files = ftp.listdir() @@ -96,7 +97,7 @@ def main(): destsize = ftp.stat(f).st_size if (not os.path.exists(destpath) or os.path.getsize(destpath) != destsize): - print "...retrieving " + f + logging.info("...retrieving " + f) ftp.get(f, destpath) except Exception: traceback.print_exc() @@ -113,15 +114,13 @@ def main(): if not options.nologs: # Process logs - if options.verbose: - print 'Processing logs...' + logging.info('Processing logs...') apps = {} appsVer = {} logexpr = '(?P[.:0-9a-fA-F]+) - - \[(?P