chiark / gitweb /
All callable scripts now implement main()
authorCiaran Gultnieks <ciaran@ciarang.com>
Sun, 26 Feb 2012 14:18:58 +0000 (14:18 +0000)
committerCiaran Gultnieks <ciaran@ciarang.com>
Sun, 26 Feb 2012 14:18:58 +0000 (14:18 +0000)
checkupdates.py
import.py
publish.py
rewritemeta.py
scanner.py
update.py
updatestats.py

index 2c083044dd7a0888c48bb74570aefab61ee02fb9..b679d6f8958a928fbae21fed9a5c04449addeb8b 100755 (executable)
@@ -27,9 +27,6 @@ from optparse import OptionParser
 import HTMLParser
 import common
 
-#Read configuration...
-execfile('config.py')
-
 
 # Check for a new version by looking at the Google market.
 # Returns (None, "a message") if this didn't work, or (version, vercode) for
@@ -66,48 +63,55 @@ def check_market(app):
 
 
 
+def main():
 
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
-                  help="Build only the specified package")
-(options, args) = parser.parse_args()
-
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+    #Read configuration...
+    execfile('config.py')
 
-html_parser = HTMLParser.HTMLParser()
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Build only the specified package")
+    (options, args) = parser.parse_args()
 
-for app in apps:
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
 
-    if options.package and options.package != app['id']:
-        # Silent skip...
-        pass
-    else:
-        print "Processing " + app['id'] + '...'
+    html_parser = HTMLParser.HTMLParser()
 
-        mode = app['Update Check Mode']
-        if mode == 'Market':
-            (version, vercode) = check_market(app)
-        elif mode == 'None':
-            version = None
-            vercode = 'Checking disabled'
-        else:
-            version = None
-            vercode = 'Invalid update check method'
+    for app in apps:
 
-        if not version:
-            print "..." + vercode
-        elif vercode == app['Current Version Code'] and version == app['Current Version']:
-            print "...up to date"
+        if options.package and options.package != app['id']:
+            # Silent skip...
+            pass
         else:
-            print '...updating to version:' + version + ' vercode:' + vercode
-            app['Current Version'] = version
-            app['Current Version Code'] = vercode
-            metafile = os.path.join('metadata', app['id'] + '.txt')
-            common.write_metadata(metafile, app)
-
-print "Finished."
+            print "Processing " + app['id'] + '...'
+
+            mode = app['Update Check Mode']
+            if mode == 'Market':
+                (version, vercode) = check_market(app)
+            elif mode == 'None':
+                version = None
+                vercode = 'Checking disabled'
+            else:
+                version = None
+                vercode = 'Invalid update check method'
+
+            if not version:
+                print "..." + vercode
+            elif vercode == app['Current Version Code'] and version == app['Current Version']:
+                print "...up to date"
+            else:
+                print '...updating to version:' + version + ' vercode:' + vercode
+                app['Current Version'] = version
+                app['Current Version Code'] = vercode
+                metafile = os.path.join('metadata', app['id'] + '.txt')
+                common.write_metadata(metafile, app)
+
+    print "Finished."
+
+if __name__ == "__main__":
+    main()
 
index e29fbc3a5d90847d74a6c7ad21a113c7fd9b1992..3d794cb8e7a8be8d2600bc3943c6bb9192b8438f 100755 (executable)
--- a/import.py
+++ b/import.py
@@ -25,219 +25,225 @@ import re
 import urllib
 from optparse import OptionParser
 
-#Read configuration...
-repo_name = None
-repo_description = None
-repo_icon = None
-repo_url = None
-execfile('config.py')
-
-import common
-
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-u", "--url", default=None,
-                  help="Project URL to import from.")
-parser.add_option("-s", "--subdir", default=None,
-                  help="Path to main android project subdirectory, if not in root.")
-(options, args) = parser.parse_args()
-
-if not options.url:
-    print "Specify project url."
-    sys.exit(1)
-url = options.url
-
-tmp_dir = 'tmp'
-if not os.path.isdir(tmp_dir):
-    print "Creating temporary directory"
-    os.makedirs(tmp_dir)
-
-# Get all apps...
-apps = common.read_metadata()
-
-# Figure out what kind of project it is...
-projecttype = None
-issuetracker = None
-license = None
-if url.startswith('https://github.com'):
-    projecttype = 'github'
-    repo = url + '.git'
-    repotype = 'git'
-    sourcecode = url
-elif url.startswith('http://code.google.com/p/'):
-    if not url.endswith('/'):
-        print "Expected format for googlecode url is http://code.google.com/p/PROJECT/"
+def main():
+
+    # Read configuration...
+    repo_name = None
+    repo_description = None
+    repo_icon = None
+    repo_url = None
+    execfile('config.py')
+
+    import common
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-u", "--url", default=None,
+                      help="Project URL to import from.")
+    parser.add_option("-s", "--subdir", default=None,
+                      help="Path to main android project subdirectory, if not in root.")
+    (options, args) = parser.parse_args()
+
+    if not options.url:
+        print "Specify project url."
         sys.exit(1)
-    projecttype = 'googlecode'
-    sourcecode = url + 'source/checkout'
-    issuetracker = url + 'issues/list'
-
-    # Figure out the repo type and adddress...
-    req = urllib.urlopen(sourcecode)
-    if req.getcode() != 200:
-        print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode())
-        sys.exit(1)
-    page = req.read()
-    repotype = None
-    index = page.find('hg clone')
-    if index != -1:
-        repotype = 'hg'
-        repo = page[index + 9:]
-        index = repo.find('<')
-        if index == -1:
-            print "Error while getting repo address"
+    url = options.url
+
+    tmp_dir = 'tmp'
+    if not os.path.isdir(tmp_dir):
+        print "Creating temporary directory"
+        os.makedirs(tmp_dir)
+
+    # Get all apps...
+    apps = common.read_metadata()
+
+    # Figure out what kind of project it is...
+    projecttype = None
+    issuetracker = None
+    license = None
+    if url.startswith('https://github.com'):
+        projecttype = 'github'
+        repo = url + '.git'
+        repotype = 'git'
+        sourcecode = url
+    elif url.startswith('http://code.google.com/p/'):
+        if not url.endswith('/'):
+            print "Expected format for googlecode url is http://code.google.com/p/PROJECT/"
             sys.exit(1)
-        repo = repo[:index]
-    if not repotype:
-        index=page.find('git clone')
+        projecttype = 'googlecode'
+        sourcecode = url + 'source/checkout'
+        issuetracker = url + 'issues/list'
+
+        # Figure out the repo type and adddress...
+        req = urllib.urlopen(sourcecode)
+        if req.getcode() != 200:
+            print 'Unable to find source at ' + sourcecode + ' - return code ' + str(req.getcode())
+            sys.exit(1)
+        page = req.read()
+        repotype = None
+        index = page.find('hg clone')
         if index != -1:
-            repotype = 'git'
-            repo = page[index + 10:]
+            repotype = 'hg'
+            repo = page[index + 9:]
             index = repo.find('<')
             if index == -1:
                 print "Error while getting repo address"
                 sys.exit(1)
             repo = repo[:index]
-    if not repotype:
-        index=page.find('svn checkout')
-        if index != -1:
-            repotype = 'git-svn'
-            repo = page[index + 13:]
-            prefix = '<strong><em>http</em></strong>'
-            if not repo.startswith(prefix):
-                print "Unexpected checkout instructions format"
-                sys.exit(1)
-            repo = 'http' + repo[len(prefix):]
-            index = repo.find('<')
-            if index == -1:
-                print "Error while getting repo address - no end tag? '" + repo + "'"
-                sys.exit(1)
-            repo = repo[:index]
-            index = repo.find(' ')
-            if index == -1:
-                print "Error while getting repo address - no space? '" + repo + "'"
-                sys.exit(1)
-            repo = repo[:index]
-    if not repotype:
-        print "Unable to determine vcs type"
-        sys.exit(1)
+        if not repotype:
+            index=page.find('git clone')
+            if index != -1:
+                repotype = 'git'
+                repo = page[index + 10:]
+                index = repo.find('<')
+                if index == -1:
+                    print "Error while getting repo address"
+                    sys.exit(1)
+                repo = repo[:index]
+        if not repotype:
+            index=page.find('svn checkout')
+            if index != -1:
+                repotype = 'git-svn'
+                repo = page[index + 13:]
+                prefix = '<strong><em>http</em></strong>'
+                if not repo.startswith(prefix):
+                    print "Unexpected checkout instructions format"
+                    sys.exit(1)
+                repo = 'http' + repo[len(prefix):]
+                index = repo.find('<')
+                if index == -1:
+                    print "Error while getting repo address - no end tag? '" + repo + "'"
+                    sys.exit(1)
+                repo = repo[:index]
+                index = repo.find(' ')
+                if index == -1:
+                    print "Error while getting repo address - no space? '" + repo + "'"
+                    sys.exit(1)
+                repo = repo[:index]
+        if not repotype:
+            print "Unable to determine vcs type"
+            sys.exit(1)
 
-    # Figure out the license...
-    req = urllib.urlopen(url)
-    if req.getcode() != 200:
-        print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
-        sys.exit(1)
-    page = req.read()
-    index = page.find('Code license')
-    if index == -1:
-        print "Couldn't find license data"
-        sys.exit(1)
-    ltext = page[index:]
-    lprefix = 'rel="nofollow">'
-    index = ltext.find(lprefix)
-    if index == -1:
-        print "Couldn't find license text"
-        sys.exit(1)
-    ltext = ltext[index + len(lprefix):]
-    index = ltext.find('<')
-    if index == -1:
-        print "License text not formatted as expected"
+        # Figure out the license...
+        req = urllib.urlopen(url)
+        if req.getcode() != 200:
+            print 'Unable to find project page at ' + sourcecode + ' - return code ' + str(req.getcode())
+            sys.exit(1)
+        page = req.read()
+        index = page.find('Code license')
+        if index == -1:
+            print "Couldn't find license data"
+            sys.exit(1)
+        ltext = page[index:]
+        lprefix = 'rel="nofollow">'
+        index = ltext.find(lprefix)
+        if index == -1:
+            print "Couldn't find license text"
+            sys.exit(1)
+        ltext = ltext[index + len(lprefix):]
+        index = ltext.find('<')
+        if index == -1:
+            print "License text not formatted as expected"
+            sys.exit(1)
+        ltext = ltext[:index]
+        if ltext == 'GNU GPL v3':
+            license = 'GPLv3'
+        elif ltext == 'GNU GPL v2':
+            license = 'GPLv2'
+        elif ltext == 'Apache License 2.0':
+            license = 'Apache2'
+        else:
+            print "License " + ltext + " is not recognised"
+            sys.exit(1)
+
+    if not projecttype:
+        print "Unable to determine the project type."
         sys.exit(1)
-    ltext = ltext[:index]
-    if ltext == 'GNU GPL v3':
-        license = 'GPLv3'
-    elif ltext == 'GNU GPL v2':
-        license = 'GPLv2'
-    elif ltext == 'Apache License 2.0':
-        license = 'Apache2'
+
+    # Get a copy of the source so we can extract some info...
+    print 'Getting source from ' + repotype + ' repo at ' + repo
+    src_dir = os.path.join(tmp_dir, 'importer')
+    if os.path.exists(tmp_dir):
+        shutil.rmtree(tmp_dir)
+    vcs = common.getvcs(repotype, repo, src_dir)
+    vcs.gotorevision(None)
+    if options.subdir:
+        root_dir = os.path.join(src_dir, options.subdir)
     else:
-        print "License " + ltext + " is not recognised"
+        root_dir = src_dir
+
+    # Check AndroidManiifest.xml exists...
+    manifest = os.path.join(root_dir, 'AndroidManifest.xml')
+    if not os.path.exists(manifest):
+        print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
         sys.exit(1)
 
-if not projecttype:
-    print "Unable to determine the project type."
-    sys.exit(1)
-
-# Get a copy of the source so we can extract some info...
-print 'Getting source from ' + repotype + ' repo at ' + repo
-src_dir = os.path.join(tmp_dir, 'importer')
-if os.path.exists(tmp_dir):
-    shutil.rmtree(tmp_dir)
-vcs = common.getvcs(repotype, repo, src_dir)
-vcs.gotorevision(None)
-if options.subdir:
-    root_dir = os.path.join(src_dir, options.subdir)
-else:
-    root_dir = src_dir
-
-# Check AndroidManiifest.xml exists...
-manifest = os.path.join(root_dir, 'AndroidManifest.xml')
-if not os.path.exists(manifest):
-    print "AndroidManifest.xml did not exist in the expected location. Specify --subdir?"
-    sys.exit(1)
-
-# Extract some information...
-vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
-vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
-psearch = re.compile(r'.*package="([^"]+)".*').search
-version = None
-vercode = None
-package = None
-for line in file(manifest):
+    # Extract some information...
+    vcsearch = re.compile(r'.*android:versionCode="([^"]+)".*').search
+    vnsearch = re.compile(r'.*android:versionName="([^"]+)".*').search
+    psearch = re.compile(r'.*package="([^"]+)".*').search
+    version = None
+    vercode = None
+    package = None
+    for line in file(manifest):
+        if not package:
+            matches = psearch(line)
+            if matches:
+                package = matches.group(1)
+        if not version:
+            matches = vnsearch(line)
+            if matches:
+                version = matches.group(1)
+        if not vercode:
+            matches = vcsearch(line)
+            if matches:
+                vercode = matches.group(1)
     if not package:
-        matches = psearch(line)
-        if matches:
-            package = matches.group(1)
+        print "Couldn't find package ID"
+        sys.exit(1)
     if not version:
-        matches = vnsearch(line)
-        if matches:
-            version = matches.group(1)
+        print "Couldn't find latest version name"
+        sys.exit(1)
     if not vercode:
-        matches = vcsearch(line)
-        if matches:
-            vercode = matches.group(1)
-if not package:
-    print "Couldn't find package ID"
-    sys.exit(1)
-if not version:
-    print "Couldn't find latest version name"
-    sys.exit(1)
-if not vercode:
-    print "Couldn't find latest version code"
-    sys.exit(1)
-
-# Make sure it's actually new...
-for app in apps:
-    if app['id'] == package:
-        print "Package " + package + " already exists"
+        print "Couldn't find latest version code"
         sys.exit(1)
 
-# Construct the metadata...
-app = common.parse_metadata(None)
-app['id'] = package
-app['Web Site'] = url
-app['Source Code'] = sourcecode
-if issuetracker:
-    app['Issue Tracker'] = issuetracker
-if license:
-    app['License'] = license
-app['Repo Type'] = repotype
-app['Repo'] = repo
-
-# Create a build line...
-build = {}
-build['version'] = version
-build['vercode'] = vercode
-build['commit'] = '?'
-if options.subdir:
-    build['subdir'] = options.subdir
-if os.path.exists(os.path.join(root_dir, 'jni')):
-    build['buildjni'] = 'yes'
-app['builds'].append(build)
-app['comments'].append(('build:' + version,
-    "#Generated by import.py - check this is the right version, and find the right commit!"))
-
-metafile = os.path.join('metadata', package + '.txt')
-common.write_metadata(metafile, app)
-print "Wrote " + metafile
+    # Make sure it's actually new...
+    for app in apps:
+        if app['id'] == package:
+            print "Package " + package + " already exists"
+            sys.exit(1)
+
+    # Construct the metadata...
+    app = common.parse_metadata(None)
+    app['id'] = package
+    app['Web Site'] = url
+    app['Source Code'] = sourcecode
+    if issuetracker:
+        app['Issue Tracker'] = issuetracker
+    if license:
+        app['License'] = license
+    app['Repo Type'] = repotype
+    app['Repo'] = repo
+
+    # Create a build line...
+    build = {}
+    build['version'] = version
+    build['vercode'] = vercode
+    build['commit'] = '?'
+    if options.subdir:
+        build['subdir'] = options.subdir
+    if os.path.exists(os.path.join(root_dir, 'jni')):
+        build['buildjni'] = 'yes'
+    app['builds'].append(build)
+    app['comments'].append(('build:' + version,
+        "#Generated by import.py - check this is the right version, and find the right commit!"))
+
+    metafile = os.path.join('metadata', package + '.txt')
+    common.write_metadata(metafile, app)
+    print "Wrote " + metafile
+
+
+if __name__ == "__main__":
+    main()
 
index 7fd9a1fef80d7dfaadcd4527b5bec0302d8a6789..c5610ffe80de795a41177d8375b6ce25eb853490 100755 (executable)
@@ -31,106 +31,112 @@ from optparse import OptionParser
 import common
 from common import BuildException
 
-#Read configuration...
-execfile('config.py')
-
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
-                  help="Publish only the specified package")
-(options, args) = parser.parse_args()
-
-log_dir = 'logs'
-if not os.path.isdir(log_dir):
-    print "Creating log directory"
-    os.makedirs(log_dir)
-
-tmp_dir = 'tmp'
-if not os.path.isdir(tmp_dir):
-    print "Creating temporary directory"
-    os.makedirs(tmp_dir)
-
-output_dir = 'repo'
-if not os.path.isdir(output_dir):
-    print "Creating output directory"
-    os.makedirs(output_dir)
-
-unsigned_dir = 'unsigned'
-if not os.path.isdir(unsigned_dir):
-    print "No unsigned directory - nothing to do"
-    sys.exit(0)
-
-for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
-
-    apkfilename = os.path.basename(apkfile)
-    i = apkfilename.rfind('_')
-    if i == -1:
-        raise BuildException("Invalid apk name")
-    appid = apkfilename[:i]
-    print "Processing " + appid
-
-    if not options.package or options.package == appid:
-
-        # Figure out the key alias name we'll use. Only the first 8
-        # characters are significant, so we'll use the first 8 from
-        # the MD5 of the app's ID and hope there are no collisions.
-        # If a collision does occur later, we're going to have to
-        # come up with a new alogrithm, AND rename all existing keys
-        # in the keystore!
-        if keyaliases.has_key(appid):
-            # For this particular app, the key alias is overridden...
-            keyalias = keyaliases[appid]
-        else:
-            m = md5.new()
-            m.update(appid)
-            keyalias = m.hexdigest()[:8]
-        print "Key alias: " + keyalias
-
-        # See if we already have a key for this application, and
-        # if not generate one...
-        p = subprocess.Popen(['keytool', '-list',
-            '-alias', keyalias, '-keystore', keystore,
-            '-storepass', keystorepass], stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        if p.returncode !=0:
-            print "Key does not exist - generating..."
-            p = subprocess.Popen(['keytool', '-genkey',
-                '-keystore', keystore, '-alias', keyalias,
-                '-keyalg', 'RSA', '-keysize', '2048',
-                '-validity', '10000',
+def main():
+
+    #Read configuration...
+    execfile('config.py')
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Publish only the specified package")
+    (options, args) = parser.parse_args()
+
+    log_dir = 'logs'
+    if not os.path.isdir(log_dir):
+        print "Creating log directory"
+        os.makedirs(log_dir)
+
+    tmp_dir = 'tmp'
+    if not os.path.isdir(tmp_dir):
+        print "Creating temporary directory"
+        os.makedirs(tmp_dir)
+
+    output_dir = 'repo'
+    if not os.path.isdir(output_dir):
+        print "Creating output directory"
+        os.makedirs(output_dir)
+
+    unsigned_dir = 'unsigned'
+    if not os.path.isdir(unsigned_dir):
+        print "No unsigned directory - nothing to do"
+        sys.exit(0)
+
+    for apkfile in sorted(glob.glob(os.path.join(unsigned_dir, '*.apk'))):
+
+        apkfilename = os.path.basename(apkfile)
+        i = apkfilename.rfind('_')
+        if i == -1:
+            raise BuildException("Invalid apk name")
+        appid = apkfilename[:i]
+        print "Processing " + appid
+
+        if not options.package or options.package == appid:
+
+            # Figure out the key alias name we'll use. Only the first 8
+            # characters are significant, so we'll use the first 8 from
+            # the MD5 of the app's ID and hope there are no collisions.
+            # If a collision does occur later, we're going to have to
+            # come up with a new alogrithm, AND rename all existing keys
+            # in the keystore!
+            if keyaliases.has_key(appid):
+                # For this particular app, the key alias is overridden...
+                keyalias = keyaliases[appid]
+            else:
+                m = md5.new()
+                m.update(appid)
+                keyalias = m.hexdigest()[:8]
+            print "Key alias: " + keyalias
+
+            # See if we already have a key for this application, and
+            # if not generate one...
+            p = subprocess.Popen(['keytool', '-list',
+                '-alias', keyalias, '-keystore', keystore,
+                '-storepass', keystorepass], stdout=subprocess.PIPE)
+            output = p.communicate()[0]
+            if p.returncode !=0:
+                print "Key does not exist - generating..."
+                p = subprocess.Popen(['keytool', '-genkey',
+                    '-keystore', keystore, '-alias', keyalias,
+                    '-keyalg', 'RSA', '-keysize', '2048',
+                    '-validity', '10000',
+                    '-storepass', keystorepass, '-keypass', keypass,
+                    '-dname', keydname], stdout=subprocess.PIPE)
+                output = p.communicate()[0]
+                print output
+                if p.returncode != 0:
+                    raise BuildException("Failed to generate key")
+
+            # Sign the application...
+            p = subprocess.Popen(['jarsigner', '-keystore', keystore,
                 '-storepass', keystorepass, '-keypass', keypass,
-                '-dname', keydname], stdout=subprocess.PIPE)
+                    apkfile, keyalias], stdout=subprocess.PIPE)
+            output = p.communicate()[0]
+            print output
+            if p.returncode != 0:
+                raise BuildException("Failed to sign application")
+
+            # Zipalign it...
+            p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
+                                '-v', '4', apkfile,
+                                os.path.join(output_dir, apkfilename)],
+                                stdout=subprocess.PIPE)
             output = p.communicate()[0]
             print output
             if p.returncode != 0:
-                raise BuildException("Failed to generate key")
-
-        # Sign the application...
-        p = subprocess.Popen(['jarsigner', '-keystore', keystore,
-            '-storepass', keystorepass, '-keypass', keypass,
-                apkfile, keyalias], stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        print output
-        if p.returncode != 0:
-            raise BuildException("Failed to sign application")
-
-        # Zipalign it...
-        p = subprocess.Popen([os.path.join(sdk_path,'tools','zipalign'),
-                            '-v', '4', apkfile,
-                            os.path.join(output_dir, apkfilename)],
-                            stdout=subprocess.PIPE)
-        output = p.communicate()[0]
-        print output
-        if p.returncode != 0:
-            raise BuildException("Failed to align application")
-        os.remove(apkfile)
-
-        # Move the source tarball into the output directory...
-        tarfilename = apkfilename[:-4] + '_src.tar.gz'
-        shutil.move(os.path.join(unsigned_dir, tarfilename),
-                os.path.join(output_dir, tarfilename))
-
-        print 'Published ' + apkfilename
+                raise BuildException("Failed to align application")
+            os.remove(apkfile)
+
+            # Move the source tarball into the output directory...
+            tarfilename = apkfilename[:-4] + '_src.tar.gz'
+            shutil.move(os.path.join(unsigned_dir, tarfilename),
+                    os.path.join(output_dir, tarfilename))
+
+            print 'Published ' + apkfilename
+
+
+if __name__ == "__main__":
+    main()
 
index e3535da271bd747149bb4d0f32057b6a38d72f8b..b633f0cce1e55a11b03d1cc6a27e2250b7bb2a17 100755 (executable)
@@ -27,22 +27,26 @@ from optparse import OptionParser
 import HTMLParser
 import common
 
-#Read configuration...
-execfile('config.py')
+def main():
 
+    #Read configuration...
+    execfile('config.py')
 
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-(options, args) = parser.parse_args()
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    (options, args) = parser.parse_args()
 
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
 
-for app in apps:
-    print "Writing " + app['id']
-    common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
+    for app in apps:
+        print "Writing " + app['id']
+        common.write_metadata(os.path.join('metadata', app['id']) + '.txt', app)
 
-print "Finished."
+    print "Finished."
+
+if __name__ == "__main__":
+    main()
 
index ee00a3ca5d7344d7c91cc9e9b412c96fd5209d15..ad4e20fba71bcdb69c0c52662e774cbde4918c00 100755 (executable)
@@ -31,80 +31,85 @@ import common
 from common import BuildException
 from common import VCSException
 
-#Read configuration...
-execfile('config.py')
+def main():
 
+    # Read configuration...
+    execfile('config.py')
 
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-p", "--package", default=None,
-                  help="Scan only the specified package")
-(options, args) = parser.parse_args()
 
-# Get all apps...
-apps = common.read_metadata(options.verbose)
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-p", "--package", default=None,
+                      help="Scan only the specified package")
+    (options, args) = parser.parse_args()
 
-html_parser = HTMLParser.HTMLParser()
+    # Get all apps...
+    apps = common.read_metadata(options.verbose)
 
-problems = []
+    html_parser = HTMLParser.HTMLParser()
 
-extlib_dir = os.path.join('build', 'extlib')
+    problems = []
 
-for app in apps:
+    extlib_dir = os.path.join('build', 'extlib')
 
-    skip = False
-    if options.package and app['id'] != options.package:
-        skip = True
-    elif app['Disabled']:
-        print "Skipping %s: disabled" % app['id']
-        skip = True
-    elif not app['builds']:
-        print "Skipping %s: no builds specified" % app['id']
-        skip = True
+    for app in apps:
 
-    if not skip:
+        skip = False
+        if options.package and app['id'] != options.package:
+            skip = True
+        elif app['Disabled']:
+            print "Skipping %s: disabled" % app['id']
+            skip = True
+        elif not app['builds']:
+            print "Skipping %s: no builds specified" % app['id']
+            skip = True
 
-        print "Processing " + app['id']
+        if not skip:
 
-        try:
+            print "Processing " + app['id']
 
-            build_dir = 'build/' + app['id']
+            try:
 
-            # Set up vcs interface and make sure we have the latest code...
-            vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
+                build_dir = 'build/' + app['id']
 
-            for thisbuild in app['builds']:
+                # Set up vcs interface and make sure we have the latest code...
+                vcs = common.getvcs(app['Repo Type'], app['Repo'], build_dir)
 
-                if thisbuild['commit'].startswith('!'):
-                    print ("..skipping version " + thisbuild['version'] + " - " +
-                            thisbuild['commit'][1:])
-                else:
-                    print "..scanning version " + thisbuild['version']
+                for thisbuild in app['builds']:
 
-                    # Prepare the source code...
-                    root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
+                    if thisbuild['commit'].startswith('!'):
+                        print ("..skipping version " + thisbuild['version'] + " - " +
+                                thisbuild['commit'][1:])
+                    else:
+                        print "..scanning version " + thisbuild['version']
 
-                    # Do the scan...
-                    buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
-                    for problem in buildprobs:
-                        problems.append(problem + 
-                            ' in ' + app['id'] + ' ' + thisbuild['version'])
+                        # Prepare the source code...
+                        root_dir = common.prepare_source(vcs, app, thisbuild,
+                                build_dir, extlib_dir, sdk_path, ndk_path, javacc_path)
 
-        except BuildException as be:
-            msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
-            problems.append(msg)
-        except VCSException as vcse:
-            msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
-            problems.append(msg)
-        except Exception:
-            msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
-            problems.append(msg)
+                        # Do the scan...
+                        buildprobs = common.scan_source(build_dir, root_dir, thisbuild)
+                        for problem in buildprobs:
+                            problems.append(problem + 
+                                ' in ' + app['id'] + ' ' + thisbuild['version'])
 
-print "Finished:"
-for problem in problems:
-    print problem
-print str(len(problems)) + ' problems.'
+            except BuildException as be:
+                msg = "Could not scan app %s due to BuildException: %s" % (app['id'], be)
+                problems.append(msg)
+            except VCSException as vcse:
+                msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
+                problems.append(msg)
+            except Exception:
+                msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
+                problems.append(msg)
+
+    print "Finished:"
+    for problem in problems:
+        print problem
+    print str(len(problems)) + ' problems.'
+
+if __name__ == "__main__":
+    main()
 
index ad0f2377b27c84905e5e7496bbebdbfd89a0e11e..f378db7caf77c7aba3e268b4b77ad1aed8b74945 100755 (executable)
--- a/update.py
+++ b/update.py
@@ -29,491 +29,496 @@ from xml.dom.minidom import Document
 from optparse import OptionParser
 import time
 
-#Read configuration...
-repo_name = None
-repo_description = None
-repo_icon = None
-repo_url = None
-execfile('config.py')
-
-import common
-
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-c", "--createmeta", action="store_true", default=False,
-                  help="Create skeleton metadata files that are missing")
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-q", "--quiet", action="store_true", default=False,
-                  help="No output, except for warnings and errors")
-parser.add_option("-b", "--buildreport", action="store_true", default=False,
-                  help="Report on build data status")
-parser.add_option("-i", "--interactive", default=False, action="store_true",
-                  help="Interactively ask about things that need updating.")
-parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
-                  help="Specify editor to use in interactive mode. Default "+
-                      "is /etc/alternatives/editor")
-parser.add_option("", "--pretty", action="store_true", default=False,
-                  help="Produce human-readable index.xml")
-(options, args) = parser.parse_args()
-
-
-icon_dir=os.path.join('repo','icons')
-
-# Delete and re-create the icon directory...
-if os.path.exists(icon_dir):
-    shutil.rmtree(icon_dir)
-os.mkdir(icon_dir)
-
-warnings = 0
-
-#Make sure we have the repository description...
-if (repo_url is None or repo_name is None or
-        repo_icon is None or repo_description is None):
-    print "Repository description fields are required in config.py"
-    print "See config.sample.py for details"
-    sys.exit(1)
-
-# Get all apps...
-apps = common.read_metadata(verbose=options.verbose)
-
-# Generate a list of categories...
-categories = []
-for app in apps:
-    if app['Category'] not in categories:
-        categories.append(app['Category'])
-
-# Gather information about all the apk files in the repo directory...
-apks = []
-for apkfile in glob.glob(os.path.join('repo','*.apk')):
-
-    apkfilename = apkfile[5:]
-    if apkfilename.find(' ') != -1:
-        print "No spaces in APK filenames!"
+def main():
+
+    # Read configuration...
+    repo_name = None
+    repo_description = None
+    repo_icon = None
+    repo_url = None
+    execfile('config.py')
+
+    import common
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-c", "--createmeta", action="store_true", default=False,
+                      help="Create skeleton metadata files that are missing")
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-q", "--quiet", action="store_true", default=False,
+                      help="No output, except for warnings and errors")
+    parser.add_option("-b", "--buildreport", action="store_true", default=False,
+                      help="Report on build data status")
+    parser.add_option("-i", "--interactive", default=False, action="store_true",
+                      help="Interactively ask about things that need updating.")
+    parser.add_option("-e", "--editor", default="/etc/alternatives/editor",
+                      help="Specify editor to use in interactive mode. Default "+
+                          "is /etc/alternatives/editor")
+    parser.add_option("", "--pretty", action="store_true", default=False,
+                      help="Produce human-readable index.xml")
+    (options, args) = parser.parse_args()
+
+
+    icon_dir=os.path.join('repo','icons')
+
+    # Delete and re-create the icon directory...
+    if os.path.exists(icon_dir):
+        shutil.rmtree(icon_dir)
+    os.mkdir(icon_dir)
+
+    warnings = 0
+
+    # Make sure we have the repository description...
+    if (repo_url is None or repo_name is None or
+            repo_icon is None or repo_description is None):
+        print "Repository description fields are required in config.py"
+        print "See config.sample.py for details"
         sys.exit(1)
-    srcfilename = apkfilename[:-4] + "_src.tar.gz"
-
-    if not options.quiet:
-        print "Processing " + apkfilename
-    thisinfo = {}
-    thisinfo['apkname'] = apkfilename
-    if os.path.exists(os.path.join('repo', srcfilename)):
-        thisinfo['srcname'] = srcfilename
-    thisinfo['size'] = os.path.getsize(apkfile)
-    thisinfo['permissions'] = []
-    thisinfo['features'] = []
-    p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
-                          'dump', 'badging', apkfile],
-                         stdout=subprocess.PIPE)
-    output = p.communicate()[0]
-    if options.verbose:
-        print output
-    if p.returncode != 0:
-        print "ERROR: Failed to get apk information"
-        sys.exit(1)
-    for line in output.splitlines():
-        if line.startswith("package:"):
-            pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
-            thisinfo['id'] = re.match(pat, line).group(1)
-            pat = re.compile(".*versionCode='([0-9]*)'.*")
-            thisinfo['versioncode'] = int(re.match(pat, line).group(1))
-            pat = re.compile(".*versionName='([^']*)'.*")
-            thisinfo['version'] = re.match(pat, line).group(1)
-        if line.startswith("application:"):
-            pat = re.compile(".*label='([^']*)'.*")
-            thisinfo['name'] = re.match(pat, line).group(1)
-            pat = re.compile(".*icon='([^']*)'.*")
-            thisinfo['iconsrc'] = re.match(pat, line).group(1)
-        if line.startswith("sdkVersion:"):
-            pat = re.compile(".*'([0-9]*)'.*")
-            thisinfo['sdkversion'] = re.match(pat, line).group(1)
-        if line.startswith("native-code:"):
-            pat = re.compile(".*'([^']*)'.*")
-            thisinfo['nativecode'] = re.match(pat, line).group(1)
-        if line.startswith("uses-permission:"):
-            pat = re.compile(".*'([^']*)'.*")
-            perm = re.match(pat, line).group(1)
-            if perm.startswith("android.permission."):
-                perm = perm[19:]
-            thisinfo['permissions'].append(perm)
-        if line.startswith("uses-feature:"):
-            pat = re.compile(".*'([^']*)'.*")
-            perm = re.match(pat, line).group(1)
-            #Filter out this, it's only added with the latest SDK tools and
-            #causes problems for lots of apps.
-            if (perm != "android.hardware.screen.portrait" and
-                perm != "android.hardware.screen.landscape"):
-                if perm.startswith("android.feature."):
-                    perm = perm[16:]
-                thisinfo['features'].append(perm)
-
-    if not thisinfo.has_key('sdkversion'):
-        print "  WARNING: no SDK version information found"
-        thisinfo['sdkversion'] = 0
-
-    # Calculate the md5 and sha256...
-    m = hashlib.md5()
-    sha = hashlib.sha256()
-    f = open(apkfile, 'rb')
-    while True:
-        t = f.read(1024)
-        if len(t) == 0:
-            break
-        m.update(t)
-        sha.update(t)
-    thisinfo['md5'] = m.hexdigest()
-    thisinfo['sha256'] = sha.hexdigest()
-    f.close()
 
-    # Get the signature (or md5 of, to be precise)...
-    p = subprocess.Popen(['java', 'getsig',
-                          os.path.join(os.getcwd(), apkfile)],
-                         cwd=os.path.join(sys.path[0], 'getsig'),
-                         stdout=subprocess.PIPE)
-    output = p.communicate()[0]
-    if options.verbose:
-        print output
-    if p.returncode != 0 or not output.startswith('Result:'):
-        print "ERROR: Failed to get apk signature"
-        sys.exit(1)
-    thisinfo['sig'] = output[7:].strip()
-
-    # Extract the icon file...
-    apk = zipfile.ZipFile(apkfile, 'r')
-    thisinfo['icon'] = (thisinfo['id'] + '.' +
-        str(thisinfo['versioncode']) + '.png')
-    iconfilename = os.path.join(icon_dir, thisinfo['icon'])
-    try:
-        iconfile = open(iconfilename, 'wb')
-        iconfile.write(apk.read(thisinfo['iconsrc']))
-        iconfile.close()
-    except:
-        print "WARNING: Error retrieving icon file"
-        warnings += 1
-    apk.close()
-
-    apks.append(thisinfo)
-
-# Some information from the apks needs to be applied up to the application
-# level. When doing this, we use the info from the most recent version's apk.
-for app in apps:
-    bestver = 0 
-    for apk in apks:
-        if apk['id'] == app['id']:
-            if apk['versioncode'] > bestver:
-                bestver = apk['versioncode']
-                bestapk = apk
-
-    if bestver == 0:
-        if app['Name'] is None:
-            app['Name'] = app['id']
-        app['icon'] = ''
-        if app['Disabled'] is None:
-            print "WARNING: Application " + app['id'] + " has no packages"
-    else:
-        if app['Name'] is None:
-            app['Name'] = bestapk['name']
-        app['icon'] = bestapk['icon']
-
-# Generate warnings for apk's with no metadata (or create skeleton
-# metadata files, if requested on the command line)
-for apk in apks:
-    found = False
+    # Get all apps...
+    apps = common.read_metadata(verbose=options.verbose)
+
+    # Generate a list of categories...
+    categories = []
     for app in apps:
-        if app['id'] == apk['id']:
-            found = True
-            break
-    if not found:
-        if options.createmeta:
-            f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
-            f.write("License:Unknown\n")
-            f.write("Web Site:\n")
-            f.write("Source Code:\n")
-            f.write("Issue Tracker:\n")
-            f.write("Summary:" + apk['name'] + "\n")
-            f.write("Description:\n")
-            f.write(apk['name'] + "\n")
-            f.write(".\n")
-            f.close()
-            print "Generated skeleton metadata for " + apk['id']
-        else:
-            print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
-            print "       " + apk['name'] + " - " + apk['version']  
-
-#Sort the app list by name, then the web site doesn't have to by default:
-apps = sorted(apps, key=lambda app: app['Name'].upper())
-
-# Create the index
-doc = Document()
-
-def addElement(name, value, doc, parent):
-    el = doc.createElement(name)
-    el.appendChild(doc.createTextNode(value))
-    parent.appendChild(el)
-
-root = doc.createElement("fdroid")
-doc.appendChild(root)
-
-repoel = doc.createElement("repo")
-repoel.setAttribute("name", repo_name)
-repoel.setAttribute("icon", os.path.basename(repo_icon))
-repoel.setAttribute("url", repo_url)
-
-if repo_keyalias != None:
-
-    # Generate a certificate fingerprint the same way keytool does it
-    # (but with slightly different formatting)
-    def cert_fingerprint(data):
-        digest = hashlib.sha1(data).digest()
-        ret = []
-        for i in range(4):
-            ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
-        return " ".join(ret)
-
-    def extract_pubkey():
-        p = subprocess.Popen(['keytool', '-exportcert',
-                              '-alias', repo_keyalias,
-                              '-keystore', keystore,
-                              '-storepass', keystorepass],
+        if app['Category'] not in categories:
+            categories.append(app['Category'])
+
+    # Gather information about all the apk files in the repo directory...
+    apks = []
+    for apkfile in glob.glob(os.path.join('repo','*.apk')):
+
+        apkfilename = apkfile[5:]
+        if apkfilename.find(' ') != -1:
+            print "No spaces in APK filenames!"
+            sys.exit(1)
+        srcfilename = apkfilename[:-4] + "_src.tar.gz"
+
+        if not options.quiet:
+            print "Processing " + apkfilename
+        thisinfo = {}
+        thisinfo['apkname'] = apkfilename
+        if os.path.exists(os.path.join('repo', srcfilename)):
+            thisinfo['srcname'] = srcfilename
+        thisinfo['size'] = os.path.getsize(apkfile)
+        thisinfo['permissions'] = []
+        thisinfo['features'] = []
+        p = subprocess.Popen([os.path.join(sdk_path, 'platform-tools', 'aapt'),
+                              'dump', 'badging', apkfile],
                              stdout=subprocess.PIPE)
-        cert = p.communicate()[0]
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
         if p.returncode != 0:
-            print "ERROR: Failed to get repo pubkey"
+            print "ERROR: Failed to get apk information"
             sys.exit(1)
-        global repo_pubkey_fingerprint
-        repo_pubkey_fingerprint = cert_fingerprint(cert)
-        return "".join("%02x" % ord(b) for b in cert)
-
-    repoel.setAttribute("pubkey", extract_pubkey())
+        for line in output.splitlines():
+            if line.startswith("package:"):
+                pat = re.compile(".*name='([a-zA-Z0-9._]*)'.*")
+                thisinfo['id'] = re.match(pat, line).group(1)
+                pat = re.compile(".*versionCode='([0-9]*)'.*")
+                thisinfo['versioncode'] = int(re.match(pat, line).group(1))
+                pat = re.compile(".*versionName='([^']*)'.*")
+                thisinfo['version'] = re.match(pat, line).group(1)
+            if line.startswith("application:"):
+                pat = re.compile(".*label='([^']*)'.*")
+                thisinfo['name'] = re.match(pat, line).group(1)
+                pat = re.compile(".*icon='([^']*)'.*")
+                thisinfo['iconsrc'] = re.match(pat, line).group(1)
+            if line.startswith("sdkVersion:"):
+                pat = re.compile(".*'([0-9]*)'.*")
+                thisinfo['sdkversion'] = re.match(pat, line).group(1)
+            if line.startswith("native-code:"):
+                pat = re.compile(".*'([^']*)'.*")
+                thisinfo['nativecode'] = re.match(pat, line).group(1)
+            if line.startswith("uses-permission:"):
+                pat = re.compile(".*'([^']*)'.*")
+                perm = re.match(pat, line).group(1)
+                if perm.startswith("android.permission."):
+                    perm = perm[19:]
+                thisinfo['permissions'].append(perm)
+            if line.startswith("uses-feature:"):
+                pat = re.compile(".*'([^']*)'.*")
+                perm = re.match(pat, line).group(1)
+                #Filter out this, it's only added with the latest SDK tools and
+                #causes problems for lots of apps.
+                if (perm != "android.hardware.screen.portrait" and
+                    perm != "android.hardware.screen.landscape"):
+                    if perm.startswith("android.feature."):
+                        perm = perm[16:]
+                    thisinfo['features'].append(perm)
+
+        if not thisinfo.has_key('sdkversion'):
+            print "  WARNING: no SDK version information found"
+            thisinfo['sdkversion'] = 0
+
+        # Calculate the md5 and sha256...
+        m = hashlib.md5()
+        sha = hashlib.sha256()
+        f = open(apkfile, 'rb')
+        while True:
+            t = f.read(1024)
+            if len(t) == 0:
+                break
+            m.update(t)
+            sha.update(t)
+        thisinfo['md5'] = m.hexdigest()
+        thisinfo['sha256'] = sha.hexdigest()
+        f.close()
+
+        # Get the signature (or md5 of, to be precise)...
+        p = subprocess.Popen(['java', 'getsig',
+                              os.path.join(os.getcwd(), apkfile)],
+                             cwd=os.path.join(sys.path[0], 'getsig'),
+                             stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
+        if p.returncode != 0 or not output.startswith('Result:'):
+            print "ERROR: Failed to get apk signature"
+            sys.exit(1)
+        thisinfo['sig'] = output[7:].strip()
+
+        # Extract the icon file...
+        apk = zipfile.ZipFile(apkfile, 'r')
+        thisinfo['icon'] = (thisinfo['id'] + '.' +
+            str(thisinfo['versioncode']) + '.png')
+        iconfilename = os.path.join(icon_dir, thisinfo['icon'])
+        try:
+            iconfile = open(iconfilename, 'wb')
+            iconfile.write(apk.read(thisinfo['iconsrc']))
+            iconfile.close()
+        except:
+            print "WARNING: Error retrieving icon file"
+            warnings += 1
+        apk.close()
 
-addElement('description', repo_description, doc, repoel)
-root.appendChild(repoel)
+        apks.append(thisinfo)
 
-apps_inrepo = 0
-apps_disabled = 0
-apps_nopkg = 0
+    # Some information from the apks needs to be applied up to the application
+    # level. When doing this, we use the info from the most recent version's apk.
+    for app in apps:
+        bestver = 0 
+        for apk in apks:
+            if apk['id'] == app['id']:
+                if apk['versioncode'] > bestver:
+                    bestver = apk['versioncode']
+                    bestapk = apk
+
+        if bestver == 0:
+            if app['Name'] is None:
+                app['Name'] = app['id']
+            app['icon'] = ''
+            if app['Disabled'] is None:
+                print "WARNING: Application " + app['id'] + " has no packages"
+        else:
+            if app['Name'] is None:
+                app['Name'] = bestapk['name']
+            app['icon'] = bestapk['icon']
 
-for app in apps:
+    # Generate warnings for apk's with no metadata (or create skeleton
+    # metadata files, if requested on the command line)
+    for apk in apks:
+        found = False
+        for app in apps:
+            if app['id'] == apk['id']:
+                found = True
+                break
+        if not found:
+            if options.createmeta:
+                f = open(os.path.join('metadata', apk['id'] + '.txt'), 'w')
+                f.write("License:Unknown\n")
+                f.write("Web Site:\n")
+                f.write("Source Code:\n")
+                f.write("Issue Tracker:\n")
+                f.write("Summary:" + apk['name'] + "\n")
+                f.write("Description:\n")
+                f.write(apk['name'] + "\n")
+                f.write(".\n")
+                f.close()
+                print "Generated skeleton metadata for " + apk['id']
+            else:
+                print "WARNING: " + apk['apkname'] + " (" + apk['id'] + ") has no metadata"
+                print "       " + apk['name'] + " - " + apk['version']  
+
+    #Sort the app list by name, then the web site doesn't have to by default:
+    apps = sorted(apps, key=lambda app: app['Name'].upper())
+
+    # Create the index
+    doc = Document()
+
+    def addElement(name, value, doc, parent):
+        el = doc.createElement(name)
+        el.appendChild(doc.createTextNode(value))
+        parent.appendChild(el)
+
+    root = doc.createElement("fdroid")
+    doc.appendChild(root)
+
+    repoel = doc.createElement("repo")
+    repoel.setAttribute("name", repo_name)
+    repoel.setAttribute("icon", os.path.basename(repo_icon))
+    repoel.setAttribute("url", repo_url)
+
+    if repo_keyalias != None:
+
+        # Generate a certificate fingerprint the same way keytool does it
+        # (but with slightly different formatting)
+        def cert_fingerprint(data):
+            digest = hashlib.sha1(data).digest()
+            ret = []
+            for i in range(4):
+                ret.append(":".join("%02X" % ord(b) for b in digest[i*5:i*5+5]))
+            return " ".join(ret)
+
+        def extract_pubkey():
+            p = subprocess.Popen(['keytool', '-exportcert',
+                                  '-alias', repo_keyalias,
+                                  '-keystore', keystore,
+                                  '-storepass', keystorepass],
+                                 stdout=subprocess.PIPE)
+            cert = p.communicate()[0]
+            if p.returncode != 0:
+                print "ERROR: Failed to get repo pubkey"
+                sys.exit(1)
+            global repo_pubkey_fingerprint
+            repo_pubkey_fingerprint = cert_fingerprint(cert)
+            return "".join("%02x" % ord(b) for b in cert)
+
+        repoel.setAttribute("pubkey", extract_pubkey())
+
+    addElement('description', repo_description, doc, repoel)
+    root.appendChild(repoel)
+
+    apps_inrepo = 0
+    apps_disabled = 0
+    apps_nopkg = 0
 
-    if app['Disabled'] is None:
+    for app in apps:
 
-        # Get a list of the apks for this app...
-        gotcurrentver = False
-        apklist = []
-        for apk in apks:
-            if apk['id'] == app['id']:
-                if str(apk['versioncode']) == app['Current Version Code']:
-                    gotcurrentver = True
-                apklist.append(apk)
+        if app['Disabled'] is None:
 
-        if len(apklist) == 0:
-            apps_nopkg += 1
-        else:
-            apps_inrepo += 1
-            apel = doc.createElement("application")
-            apel.setAttribute("id", app['id'])
-            root.appendChild(apel)
-
-            addElement('id', app['id'], doc, apel)
-            addElement('name', app['Name'], doc, apel)
-            addElement('summary', app['Summary'], doc, apel)
-            addElement('icon', app['icon'], doc, apel)
-            addElement('description',
-                    common.parse_description(app['Description']), doc, apel)
-            addElement('license', app['License'], doc, apel)
-            if 'Category' in app:
-                addElement('category', app['Category'], doc, apel)
-            addElement('web', app['Web Site'], doc, apel)
-            addElement('source', app['Source Code'], doc, apel)
-            addElement('tracker', app['Issue Tracker'], doc, apel)
-            if app['Donate'] != None:
-                addElement('donate', app['Donate'], doc, apel)
-
-            # These elements actually refer to the current version (i.e. which
-            # one is recommended. They are historically mis-named, and need
-            # changing, but stay like this for now to support existing clients.
-            addElement('marketversion', app['Current Version'], doc, apel)
-            addElement('marketvercode', app['Current Version Code'], doc, apel)
-
-            if not (app['AntiFeatures'] is None):
-                addElement('antifeatures', app['AntiFeatures'], doc, apel)
-            if app['Requires Root']:
-                addElement('requirements', 'root', doc, apel)
-
-            # Sort the apk list into version order, just so the web site
-            # doesn't have to do any work by default...
-            apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
-
-            # Check for duplicates - they will make the client unhappy...
-            for i in range(len(apklist) - 1):
-                if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
-                    print "ERROR - duplicate versions"
-                    print apklist[i]['apkname']
-                    print apklist[i+1]['apkname']
-                    sys.exit(1)
-
-            for apk in apklist:
-                apkel = doc.createElement("package")
-                apel.appendChild(apkel)
-                addElement('version', apk['version'], doc, apkel)
-                addElement('versioncode', str(apk['versioncode']), doc, apkel)
-                addElement('apkname', apk['apkname'], doc, apkel)
-                if apk.has_key('srcname'):
-                    addElement('srcname', apk['srcname'], doc, apkel)
-                for hash_type in ('sha256', 'md5'):
-                    if not hash_type in apk:
-                        continue
-                    hashel = doc.createElement("hash")
-                    hashel.setAttribute("type", hash_type)
-                    hashel.appendChild(doc.createTextNode(apk[hash_type]))
-                    apkel.appendChild(hashel)
-                addElement('sig', apk['sig'], doc, apkel)
-                addElement('size', str(apk['size']), doc, apkel)
-                addElement('sdkver', str(apk['sdkversion']), doc, apkel)
-                perms = ""
-                for p in apk['permissions']:
+            # Get a list of the apks for this app...
+            gotcurrentver = False
+            apklist = []
+            for apk in apks:
+                if apk['id'] == app['id']:
+                    if str(apk['versioncode']) == app['Current Version Code']:
+                        gotcurrentver = True
+                    apklist.append(apk)
+
+            if len(apklist) == 0:
+                apps_nopkg += 1
+            else:
+                apps_inrepo += 1
+                apel = doc.createElement("application")
+                apel.setAttribute("id", app['id'])
+                root.appendChild(apel)
+
+                addElement('id', app['id'], doc, apel)
+                addElement('name', app['Name'], doc, apel)
+                addElement('summary', app['Summary'], doc, apel)
+                addElement('icon', app['icon'], doc, apel)
+                addElement('description',
+                        common.parse_description(app['Description']), doc, apel)
+                addElement('license', app['License'], doc, apel)
+                if 'Category' in app:
+                    addElement('category', app['Category'], doc, apel)
+                addElement('web', app['Web Site'], doc, apel)
+                addElement('source', app['Source Code'], doc, apel)
+                addElement('tracker', app['Issue Tracker'], doc, apel)
+                if app['Donate'] != None:
+                    addElement('donate', app['Donate'], doc, apel)
+
+                # These elements actually refer to the current version (i.e. which
+                # one is recommended. They are historically mis-named, and need
+                # changing, but stay like this for now to support existing clients.
+                addElement('marketversion', app['Current Version'], doc, apel)
+                addElement('marketvercode', app['Current Version Code'], doc, apel)
+
+                if not (app['AntiFeatures'] is None):
+                    addElement('antifeatures', app['AntiFeatures'], doc, apel)
+                if app['Requires Root']:
+                    addElement('requirements', 'root', doc, apel)
+
+                # Sort the apk list into version order, just so the web site
+                # doesn't have to do any work by default...
+                apklist = sorted(apklist, key=lambda apk: apk['versioncode'], reverse=True)
+
+                # Check for duplicates - they will make the client unhappy...
+                for i in range(len(apklist) - 1):
+                    if apklist[i]['versioncode'] == apklist[i+1]['versioncode']:
+                        print "ERROR - duplicate versions"
+                        print apklist[i]['apkname']
+                        print apklist[i+1]['apkname']
+                        sys.exit(1)
+
+                for apk in apklist:
+                    apkel = doc.createElement("package")
+                    apel.appendChild(apkel)
+                    addElement('version', apk['version'], doc, apkel)
+                    addElement('versioncode', str(apk['versioncode']), doc, apkel)
+                    addElement('apkname', apk['apkname'], doc, apkel)
+                    if apk.has_key('srcname'):
+                        addElement('srcname', apk['srcname'], doc, apkel)
+                    for hash_type in ('sha256', 'md5'):
+                        if not hash_type in apk:
+                            continue
+                        hashel = doc.createElement("hash")
+                        hashel.setAttribute("type", hash_type)
+                        hashel.appendChild(doc.createTextNode(apk[hash_type]))
+                        apkel.appendChild(hashel)
+                    addElement('sig', apk['sig'], doc, apkel)
+                    addElement('size', str(apk['size']), doc, apkel)
+                    addElement('sdkver', str(apk['sdkversion']), doc, apkel)
+                    perms = ""
+                    for p in apk['permissions']:
+                        if len(perms) > 0:
+                            perms += ","
+                        perms += p
                     if len(perms) > 0:
-                        perms += ","
-                    perms += p
-                if len(perms) > 0:
-                    addElement('permissions', perms, doc, apkel)
-                features = ""
-                for f in apk['features']:
+                        addElement('permissions', perms, doc, apkel)
+                    features = ""
+                    for f in apk['features']:
+                        if len(features) > 0:
+                            features += ","
+                        features += f
                     if len(features) > 0:
-                        features += ","
-                    features += f
-                if len(features) > 0:
-                    addElement('features', features, doc, apkel)
-
-        if options.buildreport:
-            if len(app['builds']) == 0:
-                print ("WARNING: No builds defined for " + app['id'] +
-                        " Source: " + app['Source Code'])
+                        addElement('features', features, doc, apkel)
+
+            if options.buildreport:
+                if len(app['builds']) == 0:
+                    print ("WARNING: No builds defined for " + app['id'] +
+                            " Source: " + app['Source Code'])
+                    warnings += 1
+                else:
+                    if app['Current Version Code'] != '0':
+                        gotbuild = False
+                        for build in app['builds']:
+                            if build['vercode'] == app['Current Version Code']:
+                                gotbuild = True
+                        if not gotbuild:
+                            print ("WARNING: No build data for current version of "
+                                    + app['id'] + " (" + app['Current Version']
+                                    + ") " + app['Source Code'])
+                            warnings += 1
+
+            # If we don't have the current version, check if there is a build
+            # with a commit ID starting with '!' - this means we can't build it
+            # for some reason, and don't want hassling about it...
+            if not gotcurrentver and app['Current Version Code'] != '0':
+                for build in app['builds']:
+                    if build['vercode'] == app['Current Version Code']:
+                        gotcurrentver = True
+
+            # Output a message of harassment if we don't have the current version:
+            if not gotcurrentver and app['Current Version Code'] != '0':
+                addr = app['Source Code']
+                print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
+                print "         (" + app['id'] + ") " + addr
                 warnings += 1
-            else:
-                if app['Current Version Code'] != '0':
-                    gotbuild = False
-                    for build in app['builds']:
-                        if build['vercode'] == app['Current Version Code']:
-                            gotbuild = True
-                    if not gotbuild:
-                        print ("WARNING: No build data for current version of "
-                                + app['id'] + " (" + app['Current Version']
-                                + ") " + app['Source Code'])
-                        warnings += 1
-
-        # If we don't have the current version, check if there is a build
-        # with a commit ID starting with '!' - this means we can't build it
-        # for some reason, and don't want hassling about it...
-        if not gotcurrentver and app['Current Version Code'] != '0':
-            for build in app['builds']:
-                if build['vercode'] == app['Current Version Code']:
-                    gotcurrentver = True
-
-        # Output a message of harassment if we don't have the current version:
-        if not gotcurrentver and app['Current Version Code'] != '0':
-            addr = app['Source Code']
-            print "WARNING: Don't have current version (" + app['Current Version'] + ") of " + app['Name']
-            print "         (" + app['id'] + ") " + addr
-            warnings += 1
-            if options.verbose:
-                # A bit of extra debug info, basically for diagnosing
-                # app developer mistakes:
-                print "         Current vercode:" + app['Current Version Code']
-                print "         Got:"
-                for apk in apks:
-                    if apk['id'] == app['id']:
-                        print "           " + str(apk['versioncode']) + " - " + apk['version']
-            if options.interactive:
-                print "Build data out of date for " + app['id']
-                while True:
-                    answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
-                    if answer == 'i':
-                        break
-                    elif answer == 'e':
-                        subprocess.call([options.editor,
-                            os.path.join('metadata',
-                            app['id'] + '.txt')])
-                        break
-                    elif answer == 'q':
-                        sys.exit(0)
+                if options.verbose:
+                    # A bit of extra debug info, basically for diagnosing
+                    # app developer mistakes:
+                    print "         Current vercode:" + app['Current Version Code']
+                    print "         Got:"
+                    for apk in apks:
+                        if apk['id'] == app['id']:
+                            print "           " + str(apk['versioncode']) + " - " + apk['version']
+                if options.interactive:
+                    print "Build data out of date for " + app['id']
+                    while True:
+                        answer = raw_input("[I]gnore, [E]dit or [Q]uit?").lower()
+                        if answer == 'i':
+                            break
+                        elif answer == 'e':
+                            subprocess.call([options.editor,
+                                os.path.join('metadata',
+                                app['id'] + '.txt')])
+                            break
+                        elif answer == 'q':
+                            sys.exit(0)
+        else:
+            apps_disabled += 1
+
+    of = open(os.path.join('repo','index.xml'), 'wb')
+    if options.pretty:
+        output = doc.toprettyxml()
     else:
-        apps_disabled += 1
-
-of = open(os.path.join('repo','index.xml'), 'wb')
-if options.pretty:
-    output = doc.toprettyxml()
-else:
-    output = doc.toxml()
-of.write(output)
-of.close()
-
-if repo_keyalias != None:
-
-    if not options.quiet:
-        print "Creating signed index."
-        print "Key fingerprint:", repo_pubkey_fingerprint
-    
-    #Create a jar of the index...
-    p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
-        cwd='repo', stdout=subprocess.PIPE)
-    output = p.communicate()[0]
-    if options.verbose:
-        print output
-    if p.returncode != 0:
-        print "ERROR: Failed to create jar file"
-        sys.exit(1)
+        output = doc.toxml()
+    of.write(output)
+    of.close()
+
+    if repo_keyalias != None:
+
+        if not options.quiet:
+            print "Creating signed index."
+            print "Key fingerprint:", repo_pubkey_fingerprint
+        
+        #Create a jar of the index...
+        p = subprocess.Popen(['jar', 'cf', 'index.jar', 'index.xml'],
+            cwd='repo', stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if options.verbose:
+            print output
+        if p.returncode != 0:
+            print "ERROR: Failed to create jar file"
+            sys.exit(1)
 
-    # Sign the index...
-    p = subprocess.Popen(['jarsigner', '-keystore', keystore,
-        '-storepass', keystorepass, '-keypass', keypass,
-        os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
-    output = p.communicate()[0]
-    if p.returncode != 0:
-        print "Failed to sign index"
-        print output
-        sys.exit(1)
-    if options.verbose:
-        print output
-
-# Copy the repo icon into the repo directory...
-iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
-shutil.copyfile(repo_icon, iconfilename)
-
-# Write a category list in the repo to allow quick access...
-catdata = ''
-for cat in categories:
-    catdata += cat + '\n'
-f = open('repo/categories.txt', 'w')
-f.write(catdata)
-f.close()
-
-# Update known apks info...
-knownapks = common.KnownApks()
-for apk in apks:
-    knownapks.recordapk(apk['apkname'], apk['id'])
-knownapks.writeifchanged()
-
-# Generate latest apps data for widget
-data = ''
-for line in file(os.path.join('stats', 'latestapps.txt')):
-    appid = line.rstrip()
-    data += appid + "\t"
-    for app in apps:
-        if app['id'] == appid:
-            data += app['Name'] + "\t"
-            data += app['icon'] + "\t"
-            data += app['License'] + "\n"
-            break
-f = open('repo/latestapps.dat', 'w')
-f.write(data)
-f.close()
-
-
-
-print "Finished."
-print str(apps_inrepo) + " apps in repo"
-print str(apps_disabled) + " disabled"
-print str(apps_nopkg) + " with no packages"
-print str(warnings) + " warnings"
+        # Sign the index...
+        p = subprocess.Popen(['jarsigner', '-keystore', keystore,
+            '-storepass', keystorepass, '-keypass', keypass,
+            os.path.join('repo', 'index.jar') , repo_keyalias], stdout=subprocess.PIPE)
+        output = p.communicate()[0]
+        if p.returncode != 0:
+            print "Failed to sign index"
+            print output
+            sys.exit(1)
+        if options.verbose:
+            print output
+
+    # Copy the repo icon into the repo directory...
+    iconfilename = os.path.join(icon_dir, os.path.basename(repo_icon))
+    shutil.copyfile(repo_icon, iconfilename)
+
+    # Write a category list in the repo to allow quick access...
+    catdata = ''
+    for cat in categories:
+        catdata += cat + '\n'
+    f = open('repo/categories.txt', 'w')
+    f.write(catdata)
+    f.close()
+
+    # Update known apks info...
+    knownapks = common.KnownApks()
+    for apk in apks:
+        knownapks.recordapk(apk['apkname'], apk['id'])
+    knownapks.writeifchanged()
+
+    # Generate latest apps data for widget
+    data = ''
+    for line in file(os.path.join('stats', 'latestapps.txt')):
+        appid = line.rstrip()
+        data += appid + "\t"
+        for app in apps:
+            if app['id'] == appid:
+                data += app['Name'] + "\t"
+                data += app['icon'] + "\t"
+                data += app['License'] + "\n"
+                break
+    f = open('repo/latestapps.dat', 'w')
+    f.write(data)
+    f.close()
+
+
+
+    print "Finished."
+    print str(apps_inrepo) + " apps in repo"
+    print str(apps_disabled) + " disabled"
+    print str(apps_nopkg) + " with no packages"
+    print str(warnings) + " warnings"
+
+if __name__ == "__main__":
+    main()
 
index 892c20bce41221d0860fe24de7b3074e47342f75..a6f9816929dd53b3c71e8fcb9485016721cf8c99 100755 (executable)
@@ -30,121 +30,125 @@ import HTMLParser
 import paramiko
 import common
 
-#Read configuration...
-execfile('config.py')
-
-
-# Parse command line...
-parser = OptionParser()
-parser.add_option("-v", "--verbose", action="store_true", default=False,
-                  help="Spew out even more information than normal")
-parser.add_option("-d", "--download", action="store_true", default=False,
-                  help="Download logs we don't have")
-(options, args) = parser.parse_args()
-
-
-statsdir = 'stats'
-logsdir = os.path.join(statsdir, 'logs')
-logsarchivedir = os.path.join(logsdir, 'archive')
-datadir = os.path.join(statsdir, 'data')
-if not os.path.exists(statsdir):
-    os.mkdir(statsdir)
-if not os.path.exists(logsdir):
-    os.mkdir(logsdir)
-if not os.path.exists(datadir):
-    os.mkdir(datadir)
-
-if options.download:
-    # Get any access logs we don't have...
-    ssh = None
-    ftp = None
-    try:
-        print 'Retrieving logs'
-        ssh = paramiko.SSHClient()
-        ssh.load_system_host_keys()
-        ssh.connect('f-droid.org', username='fdroid', timeout=10,
-                key_filename=webserver_keyfile)
-        ftp = ssh.open_sftp()
-        ftp.get_channel().settimeout(15)
-        print "...connected"
-
-        ftp.chdir('logs')
-        files = ftp.listdir()
-        for f in files:
-            if f.startswith('access-') and f.endswith('.log'):
-
-                destpath = os.path.join(logsdir, f)
-                archivepath = os.path.join(logsarchivedir, f + '.gz')
-                if os.path.exists(archivepath):
-                    if os.path.exists(destpath):
-                        # Just in case we have it archived but failed to remove
-                        # the original...
-                        os.remove(destpath)
-                else:
-                    destsize = ftp.stat(f).st_size
-                    if (not os.path.exists(destpath) or
-                            os.path.getsize(destpath) != destsize):
-                        print "...retrieving " + f
-                        ftp.get(f, destpath)
-    except Exception as e:
-        traceback.print_exc()
-        sys.exit(1)
-    finally:
-        #Disconnect
-        if ftp != None:
-            ftp.close()
-        if ssh != None:
-            ssh.close()
-
-# Process logs
-logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
-logsearch = re.compile(logexpr).search
-apps = {}
-unknownapks = []
-knownapks = common.KnownApks()
-for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
-    logdate = logfile[len(logsdir) + 1 + len('access-'):-4]
-    matches = (logsearch(line) for line in file(logfile))
-    for match in matches:
-        if match and match.group('statuscode') == '200':
-            uri = match.group('uri')
-            if uri.endswith('.apk'):
-                _, apkname = os.path.split(uri)
-                app = knownapks.getapp(apkname)
-                if app:
-                    appid, _ = app
-                    if appid in apps:
-                        apps[appid] += 1
+def main():
+
+    # Read configuration...
+    execfile('config.py')
+
+    # Parse command line...
+    parser = OptionParser()
+    parser.add_option("-v", "--verbose", action="store_true", default=False,
+                      help="Spew out even more information than normal")
+    parser.add_option("-d", "--download", action="store_true", default=False,
+                      help="Download logs we don't have")
+    (options, args) = parser.parse_args()
+
+
+    statsdir = 'stats'
+    logsdir = os.path.join(statsdir, 'logs')
+    logsarchivedir = os.path.join(logsdir, 'archive')
+    datadir = os.path.join(statsdir, 'data')
+    if not os.path.exists(statsdir):
+        os.mkdir(statsdir)
+    if not os.path.exists(logsdir):
+        os.mkdir(logsdir)
+    if not os.path.exists(datadir):
+        os.mkdir(datadir)
+
+    if options.download:
+        # Get any access logs we don't have...
+        ssh = None
+        ftp = None
+        try:
+            print 'Retrieving logs'
+            ssh = paramiko.SSHClient()
+            ssh.load_system_host_keys()
+            ssh.connect('f-droid.org', username='fdroid', timeout=10,
+                    key_filename=webserver_keyfile)
+            ftp = ssh.open_sftp()
+            ftp.get_channel().settimeout(15)
+            print "...connected"
+
+            ftp.chdir('logs')
+            files = ftp.listdir()
+            for f in files:
+                if f.startswith('access-') and f.endswith('.log'):
+
+                    destpath = os.path.join(logsdir, f)
+                    archivepath = os.path.join(logsarchivedir, f + '.gz')
+                    if os.path.exists(archivepath):
+                        if os.path.exists(destpath):
+                            # Just in case we have it archived but failed to remove
+                            # the original...
+                            os.remove(destpath)
                     else:
-                        apps[appid] = 1
-                else:
-                    if not apkname in unknownapks:
-                        unknownapks.append(apkname)
-
-# Calculate and write stats for total downloads...
-f = open('stats/total_downloads_app.txt', 'w')
-lst = []
-alldownloads = 0
-for app, count in apps.iteritems():
-    lst.append(app + " " + str(count))
-    alldownloads += count
-lst.append("ALL " + str(alldownloads))
-f.write('# Total downloads by application, since October 2011\n')
-for line in sorted(lst):
-    f.write(line + '\n')
-f.close()
-
-# Write list of latest apps added to the repo...
-latest = knownapks.getlatest(10)
-f = open('stats/latestapps.txt', 'w')
-for app in latest:
-    f.write(app + '\n')
-f.close()
-
-if len(unknownapks) > 0:
-    print '\nUnknown apks:'
-    for apk in unknownapks:
-        print apk
-
-print "Finished."
+                        destsize = ftp.stat(f).st_size
+                        if (not os.path.exists(destpath) or
+                                os.path.getsize(destpath) != destsize):
+                            print "...retrieving " + f
+                            ftp.get(f, destpath)
+        except Exception as e:
+            traceback.print_exc()
+            sys.exit(1)
+        finally:
+            #Disconnect
+            if ftp != None:
+                ftp.close()
+            if ssh != None:
+                ssh.close()
+
+    # Process logs
+    logexpr = '(?P<ip>[.:0-9a-fA-F]+) - - \[(?P<time>.*?)\] "GET (?P<uri>.*?) HTTP/1.\d" (?P<statuscode>\d+) \d+ "(?P<referral>.*?)" "(?P<useragent>.*?)"'
+    logsearch = re.compile(logexpr).search
+    apps = {}
+    unknownapks = []
+    knownapks = common.KnownApks()
+    for logfile in glob.glob(os.path.join(logsdir,'access-*.log')):
+        logdate = logfile[len(logsdir) + 1 + len('access-'):-4]
+        matches = (logsearch(line) for line in file(logfile))
+        for match in matches:
+            if match and match.group('statuscode') == '200':
+                uri = match.group('uri')
+                if uri.endswith('.apk'):
+                    _, apkname = os.path.split(uri)
+                    app = knownapks.getapp(apkname)
+                    if app:
+                        appid, _ = app
+                        if appid in apps:
+                            apps[appid] += 1
+                        else:
+                            apps[appid] = 1
+                    else:
+                        if not apkname in unknownapks:
+                            unknownapks.append(apkname)
+
+    # Calculate and write stats for total downloads...
+    f = open('stats/total_downloads_app.txt', 'w')
+    lst = []
+    alldownloads = 0
+    for app, count in apps.iteritems():
+        lst.append(app + " " + str(count))
+        alldownloads += count
+    lst.append("ALL " + str(alldownloads))
+    f.write('# Total downloads by application, since October 2011\n')
+    for line in sorted(lst):
+        f.write(line + '\n')
+    f.close()
+
+    # Write list of latest apps added to the repo...
+    latest = knownapks.getlatest(10)
+    f = open('stats/latestapps.txt', 'w')
+    for app in latest:
+        f.write(app + '\n')
+    f.close()
+
+    if len(unknownapks) > 0:
+        print '\nUnknown apks:'
+        for apk in unknownapks:
+            print apk
+
+    print "Finished."
+
+if __name__ == "__main__":
+    main()