test suite: In ad-hoc runs, honour DGIT_TEST_RETRY_COUNT

This lets us retry each failed test.

This is beneficial because the gnupg race is sort-of independent.
When we want to test every commit, the success probability before
is
  ( 1 - P(gnupg causes test fail))        ) ^ (number of tests)
for each commit.  With this change it's
  ( 1 - P(gnupg causes test fail)^retries ) ^ (number of tests)
which is much much better.

Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>

diff --git a/tests/Makefile b/tests/Makefile
index 5bd3eee2c10d5274d2dc1eca91863393b439c8b9..e6ce153bae615795e3dacc51ceafd1b324ab1353 100644 (file)
--- a/tests/Makefile
+++ b/tests/Makefile
@@ -1,5 +1,7 @@
 # usage: tests/using-intree make -f tests/Makefile
 # usage: tests/using-intree make -f tests/Makefile

-# (optionally setting TESTSCRIPTS='tests/tests/foo tests/tests/bar')
+# optionally setting
+#   TESTSCRIPTS='tests/tests/foo tests/tests/bar')
+#   DGIT_TEST_RETRY_COUNT=<a smallish integer>

 
 TESTSCRIPTS ?= $(shell tests/enumerate-tests)
 TESTNAMES := $(notdir $(TESTSCRIPTS))
 
 TESTSCRIPTS ?= $(shell tests/enumerate-tests)
 TESTNAMES := $(notdir $(TESTSCRIPTS))


@@ -10,5 +12,20 @@ all: $(foreach t,$(TESTNAMES),tests/tmp/$t.ok)
 tests/tmp:
        mkdir -p $@
 
 tests/tmp:
        mkdir -p $@
 

+ifeq ($(DGIT_TEST_RETRY_COUNT),)
+

 tests/tmp/%.ok: tests/tmp
        tests/tests/$* >tests/tmp/$*.log 2>&1
 tests/tmp/%.ok: tests/tmp
        tests/tests/$* >tests/tmp/$*.log 2>&1

+
+else
+
+tests/tmp/%.ok: tests/tmp
+       @for retry in $$( seq 1 $(DGIT_TEST_RETRY_COUNT) ); do          \
+               echo "[$$retry] $*";                                    \
+               tests/tests/$* >tests/tmp/$*.$$retry.log 2>&1;          \
+               rc=$$?;                                                 \
+               if [ $$rc = 0 ]; then exit 0; fi;                       \
+               echo >&2 "[$$retry] $* TEST FAILED $$rc";               \
+       done; exit $$rc
+
+endif