From 88c8440dc1ac4e6de35615ac18fdb3ec281e1d96 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Sun, 7 Jan 2018 15:50:36 +0000 Subject: [PATCH] test suite: In ad-hoc runs, honour DGIT_TEST_RETRY_COUNT This lets us retry each failed test. This is beneficial because the gnupg race is sort-of independent. When we want to test every commit, the success probability before is ( 1 - P(gnupg causes test fail)) ) ^ (number of tests) for each commit. With this change it's ( 1 - P(gnupg causes test fail)^retries ) ^ (number of tests) which is much much better. Signed-off-by: Ian Jackson --- tests/Makefile | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/tests/Makefile b/tests/Makefile index 5bd3eee2..e6ce153b 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,5 +1,7 @@ # usage: tests/using-intree make -f tests/Makefile -# (optionally setting TESTSCRIPTS='tests/tests/foo tests/tests/bar') +# optionally setting +# TESTSCRIPTS='tests/tests/foo tests/tests/bar') +# DGIT_TEST_RETRY_COUNT= TESTSCRIPTS ?= $(shell tests/enumerate-tests) TESTNAMES := $(notdir $(TESTSCRIPTS)) @@ -10,5 +12,20 @@ all: $(foreach t,$(TESTNAMES),tests/tmp/$t.ok) tests/tmp: mkdir -p $@ +ifeq ($(DGIT_TEST_RETRY_COUNT),) + tests/tmp/%.ok: tests/tmp tests/tests/$* >tests/tmp/$*.log 2>&1 + +else + +tests/tmp/%.ok: tests/tmp + @for retry in $$( seq 1 $(DGIT_TEST_RETRY_COUNT) ); do \ + echo "[$$retry] $*"; \ + tests/tests/$* >tests/tmp/$*.$$retry.log 2>&1; \ + rc=$$?; \ + if [ $$rc = 0 ]; then exit 0; fi; \ + echo >&2 "[$$retry] $* TEST FAILED $$rc"; \ + done; exit $$rc + +endif -- 2.30.2