chiark / gitweb /
Various fixes and improvements, and git-svn support
authorCiaran Gultnieks <ciaran@ciarang.com>
Wed, 4 Jan 2012 21:37:11 +0000 (21:37 +0000)
committerCiaran Gultnieks <ciaran@ciarang.com>
Wed, 4 Jan 2012 21:37:11 +0000 (21:37 +0000)
README
build.py
common.py
metadata/An.stop.txt
metadata/android.androidVNC.txt
metadata/com.agiro.scanner.android.txt
metadata/com.android.inputmethod.norwegian.txt
metadata/com.funambol.androidsync.txt
metadata/org.jessies.mathdroid.txt
scanner.py

diff --git a/README b/README
index f10a9a3c39e1c30ff3077d1f3023a4dac779bf23..a76a34ad236f395422c2b2092b687d2e379ebe21 100644 (file)
--- a/README
+++ b/README
@@ -91,7 +91,11 @@ The type of repository - for automatic building from source. If this is not
 specified, automatic building is disabled for this application. Possible
 values are:
 
-  git, svn, hg, bzr
+  git, git-svn, svn, hg, bzr
+The git-svn option connects to an SVN repository, and you specify the URL in
+exactly the same way, but git is used as a back-end. This is preferable for
+performance reasons.
 
 ==Repo==
 
@@ -99,7 +103,7 @@ The repository location. Usually a git: or svn: URL.
 
 For a Subversion repo that requires authentication, you can precede the repo
 URL with username:password@ and those parameters will be passed as --username
-and --password to the SVN checkout command.
+and --password to the SVN checkout command. (Doesn't work for git-svn).
 
 ==Build Version==
 
@@ -125,9 +129,7 @@ configuration to the build. These are:
 
  subdir=<path>   - Specifies to build from a subdirectory of the checked out
                    source code. Normally this directory is changed to before
-                   building, but there is a special case for SVN repositories
-                   where the URL is specified with a * at the end. See the
-                   documentation for the Repo field for more information.
+                   building.
  bindir=<path>   - Normally the build output (apk) is expected to be in the
                    bin subdirectory below the ant build files. If the project
                    is configured to put it elsewhere, that can be specified
index d4ba700401435a13e391a53ac42fdd54f19bd0e9..be4fc3d0cde5e275ef519248e1f538b83967460f 100755 (executable)
--- a/build.py
+++ b/build.py
@@ -104,7 +104,7 @@ for app in apps:
 
                     # Prepare the source code...
                     root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, sdk_path, ndk_path,
+                            build_dir, sdk_path, ndk_path, javacc_path,
                             not refreshed_source)
                     refreshed_source = True
 
@@ -112,7 +112,11 @@ for app in apps:
                     tarname = app['id'] + '_' + thisbuild['vercode'] + '_src'
                     tarball = tarfile.open(os.path.join(output_dir,
                         tarname + '.tar.gz'), "w:gz")
-                    tarball.add(build_dir, tarname)
+                    def tarexc(f):
+                        if f in ['.svn', '.git', '.hg', '.bzr']:
+                            return True
+                        return False
+                    tarball.add(build_dir, tarname, exclude=tarexc)
                     tarball.close()
 
                     # Build native stuff if required...
index a1abfeca8264ab6ca2db8bd01cbbf415885dfded..052122bf37c2bc1d6a038bc11a8037c67a02c0c7 100644 (file)
--- a/common.py
+++ b/common.py
@@ -17,6 +17,7 @@
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import glob, os, sys, re
+import shutil
 import subprocess
 
 
@@ -25,6 +26,8 @@ def getvcs(vcstype, remote, local):
         return vcs_git(remote, local)
     elif vcstype == 'svn':
         return vcs_svn(remote, local)
+    elif vcstype == 'git-svn':
+        return vcs_gitsvn(remote, local)
     elif vcstype == 'hg':
         return vcs_hg(remote,local)
     elif vcstype == 'bzr':
@@ -114,6 +117,33 @@ class vcs_git(vcs):
             raise VCSException("Git submodule update failed")
 
 
+class vcs_gitsvn(vcs):
+
+    def clone(self):
+        if subprocess.call(['git', 'svn', 'clone', self.remote, self.local]) != 0:
+            raise VCSException("Git clone failed")
+
+    def reset(self, rev=None):
+        if rev is None:
+            rev = 'HEAD'
+        else:
+            p = subprocess.Popen(['git', 'svn', 'find-rev', 'r' + rev],
+                cwd=self.local, stdout=subprocess.PIPE)
+            rev = p.communicate()[0].rstrip()
+            if p.returncode != 0:
+                raise VCSException("Failed to get git treeish from svn rev")
+        if subprocess.call(['git', 'reset', '--hard', rev],
+                cwd=self.local) != 0:
+            raise VCSException("Git reset failed")
+        if subprocess.call(['git', 'clean', '-dfx'],
+                cwd=self.local) != 0:
+            raise VCSException("Git clean failed")
+
+    def pull(self):
+        if subprocess.call(['git', 'svn', 'rebase'],
+                cwd=self.local) != 0:
+            raise VCSException("Git svn rebase failed")
+
 
 class vcs_svn(vcs):
 
@@ -357,16 +387,17 @@ class MetaDataException(Exception):
 
 
 # Prepare the source code for a particular build
-#  'vcs'       - the appropriate vcs object for the application
-#  'app'       - the application details from the metadata
-#  'build'     - the build details from the metadata
-#  'build_dir' - the path to the build directory
-#  'sdk_path'  - the path to the Android SDK
-#  'ndk_path'  - the path to the Android NDK
-#  'refresh'   - True to refresh from the remote repo
+#  'vcs'         - the appropriate vcs object for the application
+#  'app'         - the application details from the metadata
+#  'build'       - the build details from the metadata
+#  'build_dir'   - the path to the build directory
+#  'sdk_path'    - the path to the Android SDK
+#  'ndk_path'    - the path to the Android NDK
+#  'javacc_path' - the path to javacc
+#  'refresh'     - True to refresh from the remote repo
 # Returns the root directory, which may be the same as 'build_dir' or may
 # be a subdirectory of it.
-def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
+def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, javacc_path, refresh):
 
     if refresh:
         vcs.refreshlocal()
@@ -374,12 +405,13 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
     # Optionally, the actual app source can be in a subdirectory...
     if build.has_key('subdir'):
         root_dir = os.path.join(build_dir, build['subdir'])
+        if not os.path.exists(root_dir):
+            raise BuildException('Missing subdir ' + root_dir)
     else:
         root_dir = build_dir
 
     # Get a working copy of the right revision...
-    if options.verbose:
-        print "Resetting repository to " + build['commit']
+    print "Resetting repository to " + build['commit']
     vcs.reset(build['commit'])
 
     # Initialise submodules if requred...
@@ -445,7 +477,7 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
 
     # Fix apostrophes translation files if necessary...
     if build.get('fixapos', 'no') == 'yes':
-        for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
+        for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
             for filename in files:
                 if filename.endswith('.xml'):
                     if subprocess.call(['sed','-i','s@' +
@@ -456,7 +488,7 @@ def prepare_source(vcs, app, build, build_dir, sdk_path, ndk_path, refresh):
 
     # Fix translation files if necessary...
     if build.get('fixtrans', 'no') == 'yes':
-        for root, dirs, files in os.walk(os.path.join(root_dir,'res')):
+        for root, dirs, files in os.walk(os.path.join(root_dir, 'res')):
             for filename in files:
                 if filename.endswith('.xml'):
                     f = open(os.path.join(root, filename))
index e8c2e7301fcecb5fd39f96d89f5ca96cb015fcb1..b2b7c43b440c3d22c24261a683a6ae77731623bc 100644 (file)
@@ -9,7 +9,7 @@ Description:A simple stopwatch, that also supports lap timing and a countdown
 timer.
 .
 
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://anstop.googlecode.com/svn/trunk
 
 Build Version:1.4,9,34
index 9dd1cd7eefda1aa706ad122b2097eab88c21ddec..bf843730f2a9f29ee1f08226e46e9da1069de737 100644 (file)
@@ -8,7 +8,7 @@ Summary:VNC viewer
 Description:
 A VNC ('remote desktop') client.
 .
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://android-vnc-viewer.googlecode.com/svn/branches/antlersoft
 
 Build Version:0.5.0,13,197,subdir=androidVNC
index 6dc28ba46ca0cd4c5b15f3b9a5bddbe706d94961..c5ae6c9f809f5ce9facfb8db68da88cf1a205b59 100644 (file)
@@ -8,7 +8,7 @@ server. The server component is available separately:
 https://github.com/johannilsson/agiro-server
 .
 
-Repo Type:
+Repo Type:git
 Repo:https://github.com/pakerfeldt/aGiro.git
 
 Build Version:alpha 2,2,!repo moved and renamed 20bd0f021dd852afcc9aa9008ee713419ae8e05c
index f702a0acffb35527dc7a21816631b7bf814e6433..a5d5489e179dee47f752de60159aa0d1d083e1d0 100644 (file)
@@ -9,7 +9,7 @@ A modified version of the standard onscreen keyboard in Android
 with support for Norwegian, Swedish, Danish, Faroese, German,
 Icelandic and Northern Sámi keyboard layouts.
 .
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://scandinavian-keyboard.googlecode.com/svn/trunk
 Build Version:1.4.4,13,15,target=android-4
 Build Version:1.4.6,15,17,target=android-4
index f9d8227418844a3aebb77646882119de4df46476..7a31a9e8b94f39365bf39b62170a3ae4703fbdf6 100644 (file)
@@ -13,7 +13,7 @@ Repo:guest:x@https://android-client.forge.funambol.org/svn/android-client/
 
 Build Version:8.7.3,8,1032,subdir=tags/8.7.3,update=no,initfun=yes
 Build Version:9.0.1,9,1437,subdir=tags/9.0.1,update=no,initfun=yes
-Build Version:9.0.3,10,1546,subdir=tags/9.0.3,update=no,initfun=yes
+Build Version:9.0.3,10,1547,subdir=tags/9.0.3,update=no,initfun=yes
 Build Version:10.0.4,14,2162,subdir=tags/10.0.4,update=no,initfun=yes
 Build Version:10.0.5,15,2211,subdir=tags/10.0.5,update=no,initfun=yes
 Build Version:10.0.6,16,2337,subdir=tags/10.0.6,update=no,initfun=yes
index e4e960538d84db40231e050b81242b3dd2ad6773..631a1733e21cd9209bdda6138025c66de6fe72b4 100644 (file)
@@ -10,10 +10,10 @@ Description:
 A calculator with full on-screen history and many functions.
 .
 
-Repo Type:svn
+Repo Type:git-svn
 Repo:http://enh.googlecode.com/svn/trunk
 
-Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java
+Build Version:2.5,25,525,oldsdkloc=yes,target=android-9,subdir=mathdroid,prebuild=rm -rf src/org/jessies/test && mkdir src/org/jessies/test && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Assert.java -O src/org/jessies/test/Assert.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/Test.java -O src/org/jessies/test/Test.java && wget http://software.jessies.org/svn/salma-hayek/trunk/src/org/jessies/test/TestHelper.java -O src/org/jessies/test/TestHelper.java
 
 Market Version:2.5
 Market Version Code:25
index 32fd0a381bf3ed082add23bfc9346f1252f9d6b6..475cecdf6db80231d490ef2ef885d10b684e206b 100755 (executable)
@@ -24,6 +24,7 @@ import re
 import urllib
 import time
 import subprocess
+import traceback
 from optparse import OptionParser
 import HTMLParser
 import common
@@ -38,6 +39,8 @@ execfile('config.py')
 parser = OptionParser()
 parser.add_option("-v", "--verbose", action="store_true", default=False,
                   help="Spew out even more information than normal")
+parser.add_option("-p", "--package", default=None,
+                  help="Scan only the specified package")
 (options, args) = parser.parse_args()
 
 # Get all apps...
@@ -49,13 +52,17 @@ problems = []
 
 for app in apps:
 
-    if app['disabled']:
+    skip = False
+    if options.package and app['id'] != options.package:
+        skip = True
+    elif app['disabled']:
         print "Skipping %s: disabled" % app['id']
+        skip = True
     elif not app['builds']:
         print "Skipping %s: no builds specified" % app['id']
+        skip = True
 
-    if (app['disabled'] is None and app['repo'] != '' 
-            and app['repotype'] != '' and len(app['builds']) > 0):
+    if not skip:
 
         print "Processing " + app['id']
 
@@ -79,16 +86,18 @@ for app in apps:
 
                     # Prepare the source code...
                     root_dir = common.prepare_source(vcs, app, thisbuild,
-                            build_dir, sdk_path, ndk_path,
+                            build_dir, sdk_path, ndk_path, javacc_path,
                             not refreshed_source)
                     refreshed_source = True
 
                     # Scan for common known non-free blobs:
-                    usual_suspects = ['flurryagent.jar', 'paypal_mpl.jar']
+                    usual_suspects = ['flurryagent.jar',
+                                      'paypal_mpl.jar',
+                                      'admob-sdk-android.jar']
                     for r,d,f in os.walk(build_dir):
                         for curfile in f:
                             if curfile.lower() in usual_suspects:
-                                msg = 'Found probable non-free blob ' + os.path.join(r,file)
+                                msg = 'Found probable non-free blob ' + os.path.join(r, curfile)
                                 msg += ' in ' + app['id'] + ' ' + thisbuild['version']
                                 problems.append(msg)
 
@@ -98,8 +107,8 @@ for app in apps:
         except VCSException as vcse:
             msg = "VCS error while scanning app %s: %s" % (app['id'], vcse)
             problems.append(msg)
-        except Exception as e:
-            msg = "Could not scan app %s due to unknown error: %s" % (app['id'], e)
+        except Exception:
+            msg = "Could not scan app %s due to unknown error: %s" % (app['id'], traceback.format_exc())
             problems.append(msg)
 
 print "Finished:"