Merge branch 'rsync' into 'master'

build: rsync instead of sftp dirs to the buildserver

Closes #227

See merge request fdroid/fdroidserver!379

diff --git a/buildserver/config.buildserver.py b/buildserver/config.buildserver.py
index 2ac78979310807b1543f1b2013c769bc08087913..1ecf43ced7ceab264d415adae21e9406c86e0e29 100644 (file)
--- a/buildserver/config.buildserver.py
+++ b/buildserver/config.buildserver.py
@@ -7,6 +7,7 @@ ndk_paths = {
     'r13b': "/home/vagrant/android-ndk/r13b",
     'r14b': "/home/vagrant/android-ndk/r14b",
     'r15c': "/home/vagrant/android-ndk/r15c",
+    'r16': "/home/vagrant/android-ndk/r16",
 }
 qt_sdk_path = "/home/vagrant/qt-sdk/5.7.0/5.7"
 java_paths = {

diff --git a/buildserver/provision-apt-get-install b/buildserver/provision-apt-get-install
index a1b0f4b1e705eaa5a37c3c3da777dce914dc2a27..70f9acd8ca0169cfa89a8aa69233911abf16a736 100644 (file)
--- a/buildserver/provision-apt-get-install
+++ b/buildserver/provision-apt-get-install
@@ -50,8 +50,8 @@ packages="
  flex
  gettext/jessie-backports
  gettext-base/jessie-backports
- git-core
- git-svn
+ git-core/jessie-backports
+ git-svn/jessie-backports
  gperf
  graphviz
  imagemagick

diff --git a/buildserver/setup-env-vars b/buildserver/setup-env-vars
index 00220a234ccca6f5aa00b7bcf45966266d178ae2..66e3d7e6452220d23a00acad2657d1e0719665e5 100644 (file)
--- a/buildserver/setup-env-vars
+++ b/buildserver/setup-env-vars
@@ -13,3 +13,7 @@ echo export ANDROID_HOME=$1 >> $bsenv
 echo 'export PATH=$PATH:${ANDROID_HOME}/tools:${ANDROID_HOME}/platform-tools:/opt/gradle/bin' >> $bsenv
 
 chmod 0644 $bsenv
+
+# make sure that SSH never hangs at a password or key prompt
+printf '    StrictHostKeyChecking yes' >> /etc/ssh/ssh_config
+printf '    BatchMode yes' >> /etc/ssh/config

diff --git a/examples/config.py b/examples/config.py
index 82f77431997ef2ccc220fdbc771cde78ee23b61b..0551e1c0b6568646d752a89dc3507e70a0e91ebd 100644 (file)
--- a/examples/config.py
+++ b/examples/config.py
@@ -18,6 +18,7 @@
 #     'r13b': None,
 #     'r14b': None,
 #     'r15c': None,
+#     'r16': None,
 # }
 
 # Path to the Qt SDK. It is of the form "/path/to/Qt5.7.0/5.7"

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 98bc2cb2aacd8e3601b9aba2d146768bfa2fcab8..89c1a1d5ac84414f353f62f53d61a0a8eb04af4f 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -80,6 +80,7 @@ default_config = {
         'r13b': None,
         'r14b': None,
         'r15c': None,
+        'r16': None,
     },
     'qt_sdk_path': None,
     'build_tools': "25.0.2",
@@ -782,6 +783,37 @@ class vcs_git(vcs):
     def clientversioncmd(self):
         return ['git', '--version']
 
+    def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
+        '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
+
+        While fetch/pull/clone respect the command line option flags,
+        it seems that submodule commands do not.  They do seem to
+        follow whatever is in env vars, if the version of git is new
+        enough.  So we just throw the kitchen sink at it to see what
+        sticks.
+
+        '''
+        if cwd is None:
+            cwd = self.local
+        git_config = []
+        for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+        # add helpful tricks supported in git >= 2.3
+        ssh_command = 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes'
+        git_config.append('-c')
+        git_config.append('core.sshCommand="' + ssh_command + '"')  # git >= 2.10
+        envs.update({
+            'GIT_TERMINAL_PROMPT': '0',
+            'GIT_SSH_COMMAND': ssh_command,  # git >= 2.3
+        })
+        return FDroidPopen(['git', ] + git_config + gitargs,
+                           envs=envs, cwd=cwd, output=output)
+
     def checkrepo(self):
         """If the local directory exists, but is somehow not a git repository,
         git will traverse up the directory tree until it finds one
@@ -798,7 +830,7 @@ class vcs_git(vcs):
     def gotorevisionx(self, rev):
         if not os.path.exists(self.local):
             # Brand new checkout
-            p = FDroidPopen(['git', 'clone', self.remote, self.local])
+            p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
             if p.returncode != 0:
                 self.clone_failed = True
                 raise VCSException("Git clone failed", p.output)
@@ -818,10 +850,10 @@ class vcs_git(vcs):
                 raise VCSException(_("Git clean failed"), p.output)
             if not self.refreshed:
                 # Get latest commits and tags from remote
-                p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
+                p = self.GitFetchFDroidPopen(['fetch', 'origin'])
                 if p.returncode != 0:
                     raise VCSException(_("Git fetch failed"), p.output)
-                p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
+                p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
                 if p.returncode != 0:
                     raise VCSException(_("Git fetch failed"), p.output)
                 # Recreate origin/HEAD as git clone would do it, in case it disappeared
@@ -857,16 +889,14 @@ class vcs_git(vcs):
             lines = f.readlines()
         with open(submfile, 'w') as f:
             for line in lines:
-                if 'git@github.com' in line:
-                    line = line.replace('git@github.com:', 'https://github.com/')
-                if 'git@gitlab.com' in line:
-                    line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
+                for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+                    line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
                 f.write(line)
 
         p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
         if p.returncode != 0:
             raise VCSException(_("Git submodule sync failed"), p.output)
-        p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
+        p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
         if p.returncode != 0:
             raise VCSException(_("Git submodule update failed"), p.output)
 
@@ -1583,10 +1613,11 @@ def prepare_source(vcs, app, build, build_dir, srclib_dir, extlib_dir, onserver=
             dest = os.path.join(build_dir, part)
             logging.info("Removing {0}".format(part))
             if os.path.lexists(dest):
-                if os.path.islink(dest):
-                    FDroidPopen(['unlink', dest], output=False)
+                # rmtree can only handle directories that are not symlinks, so catch anything else
+                if not os.path.isdir(dest) or os.path.islink(dest):
+                    os.remove(dest)
                 else:
-                    FDroidPopen(['rm', '-rf', dest], output=False)
+                    shutil.rmtree(dest)
             else:
                 logging.info("...but it didn't exist")
 

diff --git a/fdroidserver/vmtools.py b/fdroidserver/vmtools.py
index 97e1c52ce6b20a9d8558fb1393f3d1d54307ae04..aae46c75cc8cfe44a093529a84d655c5e8163b60 100644 (file)
--- a/fdroidserver/vmtools.py
+++ b/fdroidserver/vmtools.py
@@ -146,7 +146,7 @@ def get_build_vm(srvdir, provider=None):
     has_libvirt_machine = isdir(joinpath(abssrvdir, '.vagrant',
                                          'machines', 'default', 'libvirt'))
     has_vbox_machine = isdir(joinpath(abssrvdir, '.vagrant',
-                                      'machines', 'default', 'libvirt'))
+                                      'machines', 'default', 'virtualbox'))
     if has_libvirt_machine and has_vbox_machine:
         logger.info('build vm provider lookup found virtualbox and libvirt, defaulting to \'virtualbox\'')
         return VirtualboxBuildVm(abssrvdir)

diff --git a/jenkins-build-all b/jenkins-build-all
index d33f33b4f2331aad8f6a85da39f2287c84f48ba8..a1ee7c11c8f336c5d7fe76dde39d6fd57e0d1472 100755 (executable)
--- a/jenkins-build-all
+++ b/jenkins-build-all
@@ -59,11 +59,6 @@ cd $WORKSPACE
 # set up Android SDK to use the Debian packages in stretch
 export ANDROID_HOME=/usr/lib/android-sdk
 
-# ignore username/password prompt for non-existant repos
-git config --global url."https://fakeusername:fakepassword@github.com".insteadOf https://github.com
-git config --global url."https://fakeusername:fakepassword@gitlab.com".insteadOf https://gitlab.com
-git config --global url."https://fakeusername:fakepassword@bitbucket.org".insteadOf https://bitbucket.org
-
 # now build the whole archive
 cd $WORKSPACE
 

diff --git a/makebuildserver b/makebuildserver
index ac0383deb2b6c295899dbb14521d8bd9ae0359fa..7ce2e62d1a4842a29fb3c2dae642a111e20de92d 100755 (executable)
--- a/makebuildserver
+++ b/makebuildserver
@@ -352,6 +352,8 @@ cachefiles = [
      '0ecc2017802924cf81fffc0f51d342e3e69de6343da892ac9fa1cd79bc106024'),
     ('https://dl.google.com/android/repository/android-ndk-r15c-linux-x86_64.zip',
      'f01788946733bf6294a36727b99366a18369904eb068a599dde8cca2c1d2ba3c'),
+    ('https://dl.google.com/android/repository/android-ndk-r16-linux-x86_64.zip',
+     'a8550b81771c67cc6ab7b479a6918d29aa78de3482901762b4f9e0132cd9672e'),
     ('https://download.qt.io/official_releases/qt/5.7/5.7.0/qt-opensource-linux-x64-android-5.7.0.run',
      'f7e55b7970e59bdaabb88cb7afc12e9061e933992bda2f076f52600358644586'),
 ]