From fdbfb4d1a2b4f97ff0e1b93739fee2f0c5652e63 Mon Sep 17 00:00:00 2001 From: Hans-Christoph Steiner Date: Thu, 23 Nov 2017 17:08:57 +0100 Subject: [PATCH] build: stop git from waiting forever at username/password prompts If a git fetch/clone/submodule URL points to gitlab, github, bitbucket, etc and that repo does not exist any more, those services will prompt the user for a username/password so that the service can check if its a private repo. Private repos show up the same as non-existent repos. This employs two techniques for making sure that git never waits at those prompts. It instead should just fail immediately. The buildserver has been hanging on these prompts forever, until manually killed. This change will apply to updates both on the buildserver host, and the buildserver guest vm. This uses the "insteadOf" git config option to rewrite URLs to always use HTTPS and then include a fake username/password so that git will use those in the prompts and fail immediately. This trick has been in use on the verification server for a long while and has been working well. It has also been used on jenkins.debian.net in the host. https://f-droid.org/en/docs/Verification_Server/ It also includes GIT_TERMINAL_PROMPT, which also prevents the bad behavior, which was added in git 2.3. https://github.com/blog/1957-git-2-3-has-been-released --- fdroidserver/common.py | 38 ++++++++++++++++++++++++++++++-------- jenkins-build-all | 5 ----- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/fdroidserver/common.py b/fdroidserver/common.py index 98bc2cb2..a6afc8b6 100644 --- a/fdroidserver/common.py +++ b/fdroidserver/common.py @@ -782,6 +782,30 @@ class vcs_git(vcs): def clientversioncmd(self): return ['git', '--version'] + def GitFetchFDroidPopen(self, gitargs, envs=dict(), cwd=None, output=True): + '''Prevent git fetch/clone/submodule from hanging at the username/password prompt + + While fetch/pull/clone respect the command line option flags, + it seems that submodule commands do not. They do seem to + follow whatever is in env vars, if the version of git is new + enough. So we just throw the kitchen sink at it to see what + sticks. + + ''' + if cwd is None: + cwd = self.local + git_config = [] + for domain in ('bitbucket.org', 'github.com', 'gitlab.com'): + git_config.append('-c') + git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':') + git_config.append('-c') + git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain) + git_config.append('-c') + git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain) + envs.update({'GIT_TERMINAL_PROMPT': '0'}) # supported in git >= 2.3 + return FDroidPopen(['git', ] + git_config + gitargs, + envs=envs, cwd=cwd, output=output) + def checkrepo(self): """If the local directory exists, but is somehow not a git repository, git will traverse up the directory tree until it finds one @@ -798,7 +822,7 @@ class vcs_git(vcs): def gotorevisionx(self, rev): if not os.path.exists(self.local): # Brand new checkout - p = FDroidPopen(['git', 'clone', self.remote, self.local]) + p = FDroidPopen(['git', 'clone', self.remote, self.local], cwd=None) if p.returncode != 0: self.clone_failed = True raise VCSException("Git clone failed", p.output) @@ -818,10 +842,10 @@ class vcs_git(vcs): raise VCSException(_("Git clean failed"), p.output) if not self.refreshed: # Get latest commits and tags from remote - p = FDroidPopen(['git', 'fetch', 'origin'], cwd=self.local) + p = self.GitFetchFDroidPopen(['fetch', 'origin']) if p.returncode != 0: raise VCSException(_("Git fetch failed"), p.output) - p = FDroidPopen(['git', 'fetch', '--prune', '--tags', 'origin'], cwd=self.local, output=False) + p = self.GitFetchFDroidPopen(['fetch', '--prune', '--tags', 'origin'], output=False) if p.returncode != 0: raise VCSException(_("Git fetch failed"), p.output) # Recreate origin/HEAD as git clone would do it, in case it disappeared @@ -857,16 +881,14 @@ class vcs_git(vcs): lines = f.readlines() with open(submfile, 'w') as f: for line in lines: - if 'git@github.com' in line: - line = line.replace('git@github.com:', 'https://github.com/') - if 'git@gitlab.com' in line: - line = line.replace('git@gitlab.com:', 'https://gitlab.com/') + for domain in ('bitbucket.org', 'github.com', 'gitlab.com'): + line = re.sub('git@' + domain + ':', 'https://u:p@' + domain + '/', line) f.write(line) p = FDroidPopen(['git', 'submodule', 'sync'], cwd=self.local, output=False) if p.returncode != 0: raise VCSException(_("Git submodule sync failed"), p.output) - p = FDroidPopen(['git', 'submodule', 'update', '--init', '--force', '--recursive'], cwd=self.local) + p = self.GitFetchFDroidPopen(['submodule', 'update', '--init', '--force', '--recursive']) if p.returncode != 0: raise VCSException(_("Git submodule update failed"), p.output) diff --git a/jenkins-build-all b/jenkins-build-all index d33f33b4..a1ee7c11 100755 --- a/jenkins-build-all +++ b/jenkins-build-all @@ -59,11 +59,6 @@ cd $WORKSPACE # set up Android SDK to use the Debian packages in stretch export ANDROID_HOME=/usr/lib/android-sdk -# ignore username/password prompt for non-existant repos -git config --global url."https://fakeusername:fakepassword@github.com".insteadOf https://github.com -git config --global url."https://fakeusername:fakepassword@gitlab.com".insteadOf https://gitlab.com -git config --global url."https://fakeusername:fakepassword@bitbucket.org".insteadOf https://bitbucket.org - # now build the whole archive cd $WORKSPACE -- 2.30.2