chiark / gitweb /
test suite: In ad-hoc runs, honour DGIT_TEST_RETRY_COUNT
authorIan Jackson <ijackson@chiark.greenend.org.uk>
Sun, 7 Jan 2018 15:50:36 +0000 (15:50 +0000)
committerIan Jackson <ijackson@chiark.greenend.org.uk>
Sun, 7 Jan 2018 16:18:53 +0000 (16:18 +0000)
This lets us retry each failed test.

This is beneficial because the gnupg race is sort-of independent.
When we want to test every commit, the success probability before
is
  ( 1 - P(gnupg causes test fail))        ) ^ (number of tests)
for each commit.  With this change it's
  ( 1 - P(gnupg causes test fail)^retries ) ^ (number of tests)
which is much much better.

Signed-off-by: Ian Jackson <ijackson@chiark.greenend.org.uk>
tests/Makefile

index 5bd3eee..e6ce153 100644 (file)
@@ -1,5 +1,7 @@
 # usage: tests/using-intree make -f tests/Makefile
-# (optionally setting TESTSCRIPTS='tests/tests/foo tests/tests/bar')
+# optionally setting
+#   TESTSCRIPTS='tests/tests/foo tests/tests/bar')
+#   DGIT_TEST_RETRY_COUNT=<a smallish integer>
 
 TESTSCRIPTS ?= $(shell tests/enumerate-tests)
 TESTNAMES := $(notdir $(TESTSCRIPTS))
@@ -10,5 +12,20 @@ all: $(foreach t,$(TESTNAMES),tests/tmp/$t.ok)
 tests/tmp:
        mkdir -p $@
 
+ifeq ($(DGIT_TEST_RETRY_COUNT),)
+
 tests/tmp/%.ok: tests/tmp
        tests/tests/$* >tests/tmp/$*.log 2>&1
+
+else
+
+tests/tmp/%.ok: tests/tmp
+       @for retry in $$( seq 1 $(DGIT_TEST_RETRY_COUNT) ); do          \
+               echo "[$$retry] $*";                                    \
+               tests/tests/$* >tests/tmp/$*.$$retry.log 2>&1;          \
+               rc=$$?;                                                 \
+               if [ $$rc = 0 ]; then exit 0; fi;                       \
+               echo >&2 "[$$retry] $* TEST FAILED $$rc";               \
+       done; exit $$rc
+
+endif