From: Ciaran Gultnieks Date: Wed, 4 Jan 2012 21:37:11 +0000 (+0000) Subject: Various fixes and improvements, and git-svn support X-Git-Tag: 0.1~1172 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ianmdlvl/git?a=commitdiff_plain;h=14bc39010b1c5631197e68fcc4f806ae05d468ec;p=fdroidserver.git Various fixes and improvements, and git-svn support --- diff --git a/README b/README index f10a9a3c..a76a34ad 100644 --- a/README +++ b/README @@ -91,7 +91,11 @@ The type of repository - for automatic building from source. If this is not specified, automatic building is disabled for this application. Possible values are: - git, svn, hg, bzr + git, git-svn, svn, hg, bzr + +The git-svn option connects to an SVN repository, and you specify the URL in +exactly the same way, but git is used as a back-end. This is preferable for +performance reasons. ==Repo== @@ -99,7 +103,7 @@ The repository location. Usually a git: or svn: URL. For a Subversion repo that requires authentication, you can precede the repo URL with username:password@ and those parameters will be passed as --username -and --password to the SVN checkout command. +and --password to the SVN checkout command. (Doesn't work for git-svn). ==Build Version== @@ -125,9 +129,7 @@ configuration to the build. These are: subdir= - Specifies to build from a subdirectory of the checked out source code. Normally this directory is changed to before - building, but there is a special case for SVN repositories - where the URL is specified with a * at the end. See the - documentation for the Repo field for more information. + building. bindir= - Normally the build output (apk) is expected to be in the bin subdirectory below the ant build files. If the project is configured to put it elsewhere, that can be specified diff --git a/build.py b/build.py index d4ba7004..be4fc3d0 100755 --- a/build.py +++ b/build.py @@ -104,7 +104,7 @@ for app in apps: # Prepare the source code... root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, sdk_path, ndk_path, + build_dir, sdk_path, ndk_path, javacc_path, not refreshed_source) refreshed_source = True @@ -112,7 +112,11 @@ for app in apps: tarname = app['id'] + '_' + thisbuild['vercode'] + '_src' tarball = tarfile.open(os.path.join(output_dir, tarname + '.tar.gz'), "w:gz") - tarball.add(build_dir, tarname) + def tarexc(f): + if f in ['.svn', '.git', '.hg', '.bzr']: + return True + return False + tarball.add(build_dir, tarname, exclude=tarexc) tarball.close() # Build native stuff if required... diff --git a/common.py b/common.py index a1abfeca..052122bf 100644 --- a/common.py +++ b/common.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import glob, os, sys, re +import shutil import subprocess @@ -25,6 +26,8 @@ def getvcs(vcstype, remote, local): return vcs_git(remote, local) elif vcstype == 'svn': return vcs_svn(remote, local) + elif vcstype == 'git-svn': + return vcs_gitsvn(remote, local) elif vcstype == 'hg': return vcs_hg(remote,local) elif vcstype == 'bzr': @@ -114,6 +117,33 @@ class vcs_git(vcs): raise VCSException("Git submodule update failed") +class vcs_gitsvn(vcs): + + def clone(self): + if subprocess.call(['git', 'svn', 'clone', self.remote, self.local]) != 0: + raise VCSException("Git clone failed") + + def reset(self, rev=None): + if rev is None: + rev = 'HEAD' + else: + p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev], + cwd=self.local, stdout=subprocess.PIPE) + rev = p.communicate()[0].rstrip() + if p.returncode != 0: + raise VCSException("Failed to get git treeish from svn rev") + if subprocess.call(['git', 'reset', '--hard', rev], + cwd=self.local) != 0: + raise VCSException("Git reset failed") + if subprocess.call(['git', 'clean', '-dfx'], + cwd=self.local) != 0: + raise VCSException("Git clean failed") + + def pull(self): + if subprocess.call(['git', 'svn', 'rebase'], + cwd=self.local) != 0: + raise VCSException("Git svn rebase failed") + class vcs_svn(vcs): @@ -357,16 +387,17 @@ class MetaDataException(Exception): # Prepare the source code for a particular build -# 'vcs' - the appropriate vcs object for the application -# 'app' - the application details from the metadata -# 'build' - the build details from the metadata -# 'build_dir' - the path to the build directory -# 'sdk_path' - the path to the Android SDK -# 'ndk_path' - the path to the Android NDK -# 'refresh' - True to refresh from the remote repo +# 'vcs' - the appropriate vcs object for the application +# 'app' - the application details from the metadata +# 'build' - the build details from the metadata +# 'build_dir' - the path to the build directory +# 'sdk_path' - the path to the Android SDK +# 'ndk_path' - the path to the Android NDK +# 'javacc_path' - the path to javacc +# 'refresh' - True to refresh from the remote repo # Returns the root directory, which may be the same as 'build_dir' or may # be a subdirectory of it. -def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh): +def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, javacc_path, refresh): if refresh: vcs.refreshlocal() @@ -374,12 +405,13 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh): # Optionally, the actual app source can be in a subdirectory... if build.has_key('subdir'): root_dir = os.path.join(build_dir, build['subdir']) + if not os.path.exists(root_dir): + raise BuildException('Missing subdir ' + root_dir) else: root_dir = build_dir # Get a working copy of the right revision... - if options.verbose: - print "Resetting repository to " + build['commit'] + print "Resetting repository to " + build['commit'] vcs.reset(build['commit']) # Initialise submodules if requred... @@ -445,7 +477,7 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh): # Fix apostrophes translation files if necessary... if build.get('fixapos', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for root, dirs, files in os.walk(os.path.join(root_dir, 'res')): for filename in files: if filename.endswith('.xml'): if subprocess.call(['sed','-i','s@' + @@ -456,7 +488,7 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh): # Fix translation files if necessary... if build.get('fixtrans', 'no') == 'yes': - for root, dirs, files in os.walk(os.path.join(root_dir,'res')): + for root, dirs, files in os.walk(os.path.join(root_dir, 'res')): for filename in files: if filename.endswith('.xml'): f = open(os.path.join(root, filename)) diff --git a/metadata/An.stop.txt b/metadata/An.stop.txt index e8c2e730..b2b7c43b 100644 --- a/metadata/An.stop.txt +++ b/metadata/An.stop.txt @@ -9,7 +9,7 @@ Description:A simple stopwatch, that also supports lap timing and a countdown timer. . -Repo Type:svn +Repo Type:git-svn Repo:http://anstop.googlecode.com/svn/trunk Build Version:1.4,9,34 diff --git a/metadata/android.androidVNC.txt b/metadata/android.androidVNC.txt index 9dd1cd7e..bf843730 100644 --- a/metadata/android.androidVNC.txt +++ b/metadata/android.androidVNC.txt @@ -8,7 +8,7 @@ Summary:VNC viewer Description: A VNC ('remote desktop') client. . -Repo Type:svn +Repo Type:git-svn Repo:http://android-vnc-viewer.googlecode.com/svn/branches/antlersoft Build Version:0.5.0,13,197,subdir=androidVNC diff --git a/metadata/com.agiro.scanner.android.txt b/metadata/com.agiro.scanner.android.txt index 6dc28ba4..c5ae6c9f 100644 --- a/metadata/com.agiro.scanner.android.txt +++ b/metadata/com.agiro.scanner.android.txt @@ -8,7 +8,7 @@ server. The server component is available separately: https://github.com/johannilsson/agiro-server . -Repo Type: +Repo Type:git Repo:https://github.com/pakerfeldt/aGiro.git Build Version:alpha 2,2,!repo moved and renamed 20bd0f021dd852afcc9aa9008ee713419ae8e05c diff --git a/metadata/com.android.inputmethod.norwegian.txt b/metadata/com.android.inputmethod.norwegian.txt index f702a0ac..a5d5489e 100644 --- a/metadata/com.android.inputmethod.norwegian.txt +++ b/metadata/com.android.inputmethod.norwegian.txt @@ -9,7 +9,7 @@ A modified version of the standard onscreen keyboard in Android with support for Norwegian, Swedish, Danish, Faroese, German, Icelandic and Northern Sámi keyboard layouts. . -Repo Type:svn +Repo Type:git-svn Repo:http://scandinavian-keyboard.googlecode.com/svn/trunk Build Version:1.4.4,13,15,target=android-4 Build Version:1.4.6,15,17,target=android-4 diff --git a/metadata/com.funambol.androidsync.txt b/metadata/com.funambol.androidsync.txt index f9d82274..7a31a9e8 100644 --- a/metadata/com.funambol.androidsync.txt +++ b/metadata/com.funambol.androidsync.txt @@ -13,7 +13,7 @@ Repo:guest:x@https://android-client.forge.funambol.org/svn/android-client/ Build Version:8.7.3,8,1032,subdir=tags/8.7.3,update=no,initfun=yes Build Version:9.0.1,9,1437,subdir=tags/9.0.1,update=no,initfun=yes -Build Version:9.0.3,10,1546,subdir=tags/9.0.3,update=no,initfun=yes +Build Version:9.0.3,10,1547,subdir=tags/9.0.3,update=no,initfun=yes Build Version:10.0.4,14,2162,subdir=tags/10.0.4,update=no,initfun=yes Build Version:10.0.5,15,2211,subdir=tags/10.0.5,update=no,initfun=yes Build Version:10.0.6,16,2337,subdir=tags/10.0.6,update=no,initfun=yes diff --git a/metadata/org.jessies.mathdroid.txt b/metadata/org.jessies.mathdroid.txt index e4e96053..631a1733 100644 --- a/metadata/org.jessies.mathdroid.txt +++ b/metadata/org.jessies.mathdroid.txt @@ -10,10 +10,10 @@ Description: A calculator with full on-screen history and many functions. . -Repo Type:svn +Repo Type:git-svn Repo:http://enh.googlecode.com/svn/trunk -Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java +Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm -rf src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java Market Version:2.5 Market Version Code:25 diff --git a/scanner.py b/scanner.py index 32fd0a38..475cecdf 100755 --- a/scanner.py +++ b/scanner.py @@ -24,6 +24,7 @@ import re import urllib import time import subprocess +import traceback from optparse import OptionParser import HTMLParser import common @@ -38,6 +39,8 @@ execfile('config.py') parser = OptionParser() parser.add_option("-v", "--verbose", action="store_true", default=False, help="Spew out even more information than normal") +parser.add_option("-p", "--package", default=None, + help="Scan only the specified package") (options, args) = parser.parse_args() # Get all apps... @@ -49,13 +52,17 @@ problems = [] for app in apps: - if app['disabled']: + skip = False + if options.package and app['id'] != options.package: + skip = True + elif app['disabled']: print "Skipping %s: disabled" % app['id'] + skip = True elif not app['builds']: print "Skipping %s: no builds specified" % app['id'] + skip = True - if (app['disabled'] is None and app['repo'] != '' - and app['repotype'] != '' and len(app['builds']) > 0): + if not skip: print "Processing " + app['id'] @@ -79,16 +86,18 @@ for app in apps: # Prepare the source code... root_dir = common.prepare_source(vcs, app, thisbuild, - build_dir, sdk_path, ndk_path, + build_dir, sdk_path, ndk_path, javacc_path, not refreshed_source) refreshed_source = True # Scan for common known non-free blobs: - usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar'] + usual_suspects = ['flurryagent.jar', + 'paypal_mpl.jar', + 'admob-sdk-android.jar'] for r,d,f in os.walk(build_dir): for curfile in f: if curfile.lower() in usual_suspects: - msg = 'Found probable non-free blob ' + os.path.join(r,file) + msg = 'Found probable non-free blob ' + os.path.join(r, curfile) msg += ' in ' + app['id'] + ' ' + thisbuild['version'] problems.append(msg) @@ -98,8 +107,8 @@ for app in apps: except VCSException as vcse: msg = "VCS error while scanning app %s: %s" % (app['id'], vcse) problems.append(msg) - except Exception as e: - msg = "Could not scan app %s due to unknown error: %s" % (app['id'], e) + except Exception: + msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc()) problems.append(msg) print "Finished:"