build: stop git from waiting forever at username/password prompts

If a git fetch/clone/submodule URL points to gitlab, github, bitbucket, etc
and that repo does not exist any more, those services will prompt the user
for a username/password so that the service can check if its a private
repo.  Private repos show up the same as non-existent repos.  This employs
two techniques for making sure that git never waits at those prompts.  It
instead should just fail immediately.  The buildserver has been hanging on
these prompts forever, until manually killed.  This change will apply to
updates both on the buildserver host, and the buildserver guest vm.

This uses the "insteadOf" git config option to rewrite URLs to always use
HTTPS and then include a fake username/password so that git will use those
in the prompts and fail immediately.  This trick has been in use on the
verification server for a long while and has been working well.  It has
also been used on jenkins.debian.net in the host.
https://f-droid.org/en/docs/Verification_Server/

It also includes GIT_TERMINAL_PROMPT, which also prevents the bad behavior,
which was added in git 2.3.
https://github.com/blog/1957-git-2-3-has-been-released

diff --git a/fdroidserver/common.py b/fdroidserver/common.py
index 98bc2cb2aacd8e3601b9aba2d146768bfa2fcab8..a6afc8b67422557b750be47c96acaa06a342eb28 100644 (file)
--- a/fdroidserver/common.py
+++ b/fdroidserver/common.py
@@ -782,6 +782,30 @@ class vcs_git(vcs):
     def clientversioncmd(self):
         return ['git', '--version']
 
+    def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True):
+        '''Prevent git fetch/clone/submodule from hanging at the username/password prompt
+
+        While fetch/pull/clone respect the command line option flags,
+        it seems that submodule commands do not.  They do seem to
+        follow whatever is in env vars, if the version of git is new
+        enough.  So we just throw the kitchen sink at it to see what
+        sticks.
+
+        '''
+        if cwd is None:
+            cwd = self.local
+        git_config = []
+        for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+            git_config.append('-c')
+            git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+        envs.update({'GIT_TERMINAL_PROMPT': '0'})  # supported in git >= 2.3
+        return FDroidPopen(['git', ] + git_config + gitargs,
+                           envs=envs, cwd=cwd, output=output)
+
     def checkrepo(self):
         """If the local directory exists, but is somehow not a git repository,
         git will traverse up the directory tree until it finds one
@@ -798,7 +822,7 @@ class vcs_git(vcs):
     def gotorevisionx(self, rev):
         if not os.path.exists(self.local):
             # Brand new checkout
-            p = FDroidPopen(['git', 'clone', self.remote, self.local])
+            p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None)
             if p.returncode != 0:
                 self.clone_failed = True
                 raise VCSException("Git clone failed", p.output)
@@ -818,10 +842,10 @@ class vcs_git(vcs):
                 raise VCSException(_("Git clean failed"), p.output)
             if not self.refreshed:
                 # Get latest commits and tags from remote
-                p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local)
+                p = self.GitFetchFDroidPopen(['fetch', 'origin'])
                 if p.returncode != 0:
                     raise VCSException(_("Git fetch failed"), p.output)
-                p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False)
+                p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False)
                 if p.returncode != 0:
                     raise VCSException(_("Git fetch failed"), p.output)
                 # Recreate origin/HEAD as git clone would do it, in case it disappeared
@@ -857,16 +881,14 @@ class vcs_git(vcs):
             lines = f.readlines()
         with open(submfile, 'w') as f:
             for line in lines:
-                if 'git@github.com' in line:
-                    line = line.replace('git@github.com:', 'https://github.com/')
-                if 'git@gitlab.com' in line:
-                    line = line.replace('git@gitlab.com:', 'https://gitlab.com/')
+                for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+                    line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line)
                 f.write(line)
 
         p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False)
         if p.returncode != 0:
             raise VCSException(_("Git submodule sync failed"), p.output)
-        p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local)
+        p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive'])
         if p.returncode != 0:
             raise VCSException(_("Git submodule update failed"), p.output)
 

diff --git a/jenkins-build-all b/jenkins-build-all
index d33f33b4f2331aad8f6a85da39f2287c84f48ba8..a1ee7c11c8f336c5d7fe76dde39d6fd57e0d1472 100755 (executable)
--- a/jenkins-build-all
+++ b/jenkins-build-all
@@ -59,11 +59,6 @@ cd $WORKSPACE
 # set up Android SDK to use the Debian packages in stretch
 export ANDROID_HOME=/usr/lib/android-sdk
 
-# ignore username/password prompt for non-existant repos
-git config --global url."https://fakeusername:fakepassword@github.com".insteadOf https://github.com
-git config --global url."https://fakeusername:fakepassword@gitlab.com".insteadOf https://gitlab.com
-git config --global url."https://fakeusername:fakepassword@bitbucket.org".insteadOf https://bitbucket.org
-
 # now build the whole archive
 cd $WORKSPACE