1 /* $Id: runtests.c 7578 2006-09-11 23:03:12Z eagle $
3 Run a set of tests, reporting results.
5 Copyright 2000, 2001 Russ Allbery <rra@stanford.edu>
7 Please note that this file is maintained separately from INN by the above
8 author (which is why the coding style is slightly different). Any fixes
9 added to the INN tree should also be reported to the above author if
12 Permission is hereby granted, free of charge, to any person obtaining a
13 copy of this software and associated documentation files (the
14 "Software"), to deal in the Software without restriction, including
15 without limitation the rights to use, copy, modify, merge, publish,
16 distribute, sublicense, and/or sell copies of the Software, and to
17 permit persons to whom the Software is furnished to do so, subject to
18 the following conditions:
20 The above copyright notice and this permission notice shall be included
21 in all copies or substantial portions of the Software.
23 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
24 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
26 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
27 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
28 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
29 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
35 Expects a list of executables located in the given file, one line per
36 executable. For each one, runs it as part of a test suite, reporting
37 results. Test output should start with a line containing the number of
38 tests (numbered from 1 to this number), and then each line should be in
45 where <number> is the number of the test. ok indicates success, not ok
46 indicates failure, and "# skip" indicates the test was skipped for some
47 reason (maybe because it doesn't apply to this platform).
49 This file is completely stand-alone by intention. As stated more
50 formally in the license above, you are welcome to include it in your
51 packages as a test suite driver. It requires ANSI C (__FILE__, __LINE__,
52 void, const, stdarg.h, string.h) and POSIX (fcntl.h, unistd.h, pid_t) and
53 won't compile out of the box on SunOS without adjustments to include
54 strings.h instead. This is intentionally not fixed using autoconf so
55 that this file will not have a dependency on autoconf (although you're
56 welcome to fix it for your project if you want). Since it doesn't matter
57 as much that the test suite for the software package be utterly portable
58 to older systems, this file should be portable enough for most purposes.
60 Any bug reports, bug fixes, and improvements are very much welcome and
61 should be sent to the e-mail address above. */
65 #include "portable/wait.h"
66 #include "portable/time.h"
73 /* sys/time.h must be included before sys/resource.h on some platforms. */
74 #include <sys/resource.h>
76 /* Test status codes. */
84 /* Error exit statuses for test processes. */
85 #define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
86 #define CHILDERR_EXEC 101 /* Couldn't exec child process. */
87 #define CHILDERR_STDERR 102 /* Couldn't open stderr file. */
89 /* Structure to hold data for a set of tests. */
91 const char *file; /* The file name of the test. */
92 int count; /* Expected count of tests. */
93 int current; /* The last seen test number. */
94 int passed; /* Count of passing tests. */
95 int failed; /* Count of failing lists. */
96 int skipped; /* Count of skipped tests (passed). */
97 enum test_status *results; /* Table of results by test number. */
98 int aborted; /* Whether the set as aborted. */
99 int reported; /* Whether the results were reported. */
100 int status; /* The exit status of the test. */
103 /* Structure to hold a linked list of test sets. */
106 struct testlist *next;
109 /* Header used for test output. %s is replaced by the file name of the list
111 static const char banner[] = "\n\
112 Running all tests listed in %s. If any tests fail, run the failing\n\
113 test program by hand to see more details. The test program will have the\n\
114 same name as the test set but with \".t\" appended.\n\n";
116 /* Header for reports of failed tests. */
117 static const char header[] = "\n\
118 Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
119 -------------------------- -------------- ---- ---- ------------------------";
121 /* Include the file name and line number in malloc failures. */
122 #define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
123 #define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
125 /* Internal prototypes. */
126 static void sysdie(const char *format, ...);
127 static void *x_malloc(size_t, const char *file, int line);
128 static char *x_strdup(const char *, const char *file, int line);
129 static int test_analyze(struct testset *);
130 static int test_batch(const char *testlist);
131 static void test_checkline(const char *line, struct testset *);
132 static void test_fail_summary(const struct testlist *);
133 static int test_init(const char *line, struct testset *);
134 static int test_print_range(int first, int last, int chars, int limit);
135 static void test_summarize(struct testset *, int status);
136 static pid_t test_start(const char *path, int *fd);
137 static double tv_diff(const struct timeval *, const struct timeval *);
138 static double tv_seconds(const struct timeval *);
139 static double tv_sum(const struct timeval *, const struct timeval *);
142 /* Report a fatal error, including the results of strerror, and exit. */
144 sysdie(const char *format, ...)
151 fprintf(stderr, "runtests: ");
152 va_start(args, format);
153 vfprintf(stderr, format, args);
155 fprintf(stderr, ": %s\n", strerror(oerrno));
160 /* Allocate memory, reporting a fatal error and exiting on failure. */
162 x_malloc(size_t size, const char *file, int line)
168 sysdie("failed to malloc %lu bytes at %s line %d",
169 (unsigned long) size, file, line);
174 /* Copy a string, reporting a fatal error and exiting on failure. */
176 x_strdup(const char *s, const char *file, int line)
184 sysdie("failed to strdup %lu bytes at %s line %d",
185 (unsigned long) len, file, line);
191 /* Given a struct timeval, return the number of seconds it represents as a
192 double. Use difftime() to convert a time_t to a double. */
194 tv_seconds(const struct timeval *tv)
196 return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
199 /* Given two struct timevals, return the difference in seconds. */
201 tv_diff(const struct timeval *tv1, const struct timeval *tv0)
203 return tv_seconds(tv1) - tv_seconds(tv0);
206 /* Given two struct timevals, return the sum in seconds as a double. */
208 tv_sum(const struct timeval *tv1, const struct timeval *tv2)
210 return tv_seconds(tv1) + tv_seconds(tv2);
214 /* Read the first line of test output, which should contain the range of
215 test numbers, and initialize the testset structure. Assume it was zeroed
216 before being passed in. Return true if initialization succeeds, false
219 test_init(const char *line, struct testset *ts)
223 /* Prefer a simple number of tests, but if the count is given as a range
224 such as 1..10, accept that too for compatibility with Perl's
226 while (isspace((unsigned char)(*line))) line++;
227 if (!strncmp(line, "1..", 3)) line += 3;
229 /* Get the count, check it for validity, and initialize the struct. */
232 puts("invalid test count");
238 ts->results = xmalloc(ts->count * sizeof(enum test_status));
239 for (i = 0; i < ts->count; i++) ts->results[i] = TEST_INVALID;
244 /* Start a program, connecting its stdout to a pipe on our end and its
245 stderr to /dev/null, and storing the file descriptor to read from in the
246 two argument. Returns the PID of the new process. Errors are fatal. */
248 test_start(const char *path, int *fd)
253 if (pipe(fds) == -1) sysdie("can't create pipe");
255 if (child == (pid_t) -1) {
256 sysdie("can't fork");
257 } else if (child == 0) {
258 /* In child. Set up our stdout and stderr. */
259 errfd = open("/dev/null", O_WRONLY);
260 if (errfd < 0) _exit(CHILDERR_STDERR);
261 if (dup2(errfd, 2) == -1) _exit(CHILDERR_DUP);
263 if (dup2(fds[1], 1) == -1) _exit(CHILDERR_DUP);
265 /* Now, exec our process. */
266 if (execl(path, path, (char *) 0) == -1) _exit(CHILDERR_EXEC);
268 /* In parent. Close the extra file descriptor. */
276 /* Given a single line of output from a test, parse it and return the
277 success status of that test. Anything printed to stdout not matching the
278 form /^(not )?ok \d+/ is ignored. Sets ts->current to the test number
279 that just reported status. */
281 test_checkline(const char *line, struct testset *ts)
283 enum test_status status = TEST_PASS;
286 /* If the given line isn't newline-terminated, it was too big for an
287 fgets(), which means ignore it. */
288 if (line[strlen(line) - 1] != '\n') return;
290 /* Parse the line, ignoring something we can't parse. */
291 if (!strncmp(line, "not ", 4)) {
295 if (strncmp(line, "ok ", 3)) return;
297 current = atoi(line);
298 if (current == 0) return;
299 if (current < 0 || current > ts->count) {
300 printf("invalid test number %d\n", current);
305 while (isspace((unsigned char)(*line))) line++;
306 while (isdigit((unsigned char)(*line))) line++;
307 while (isspace((unsigned char)(*line))) line++;
310 while (isspace((unsigned char)(*line))) line++;
311 if (!strncmp(line, "skip", 4)) status = TEST_SKIP;
314 /* Make sure that the test number is in range and not a duplicate. */
315 if (ts->results[current - 1] != TEST_INVALID) {
316 printf("duplicate test number %d\n", current);
322 /* Good results. Increment our various counters. */
324 case TEST_PASS: ts->passed++; break;
325 case TEST_FAIL: ts->failed++; break;
326 case TEST_SKIP: ts->skipped++; break;
329 ts->current = current;
330 ts->results[current - 1] = status;
334 /* Print out a range of test numbers, returning the number of characters it
335 took up. Add a comma and a space before the range if chars indicates
336 that something has already been printed on the line, and print
337 ... instead if chars plus the space needed would go over the limit (use a
338 limit of 0 to disable this. */
340 test_print_range(int first, int last, int chars, int limit)
348 if (!limit || chars <= limit) out += printf(", ");
350 for (n = first; n > 0; n /= 10)
353 for (n = last; n > 0; n /= 10)
357 if (limit && chars + needed > limit) {
358 if (chars <= limit) out += printf("...");
360 if (last > first) out += printf("%d-", first);
361 out += printf("%d", last);
367 /* Summarize a single test set. The second argument is 0 if the set exited
368 cleanly, a positive integer representing the exit status if it exited
369 with a non-zero status, and a negative integer representing the signal
370 that terminated it if it was killed by a signal. */
372 test_summarize(struct testset *ts, int status)
381 fputs("aborted", stdout);
383 printf(", passed %d/%d", ts->passed, ts->count - ts->skipped);
385 for (i = 0; i < ts->count; i++) {
386 if (ts->results[i] == TEST_INVALID) {
387 if (missing == 0) fputs("MISSED ", stdout);
388 if (first && i == last) {
392 test_print_range(first, last, missing - 1, 0);
400 if (first) test_print_range(first, last, missing - 1, 0);
403 for (i = 0; i < ts->count; i++) {
404 if (ts->results[i] == TEST_FAIL) {
405 if (missing && !failed) fputs("; ", stdout);
406 if (failed == 0) fputs("FAILED ", stdout);
407 if (first && i == last) {
411 test_print_range(first, last, failed - 1, 0);
419 if (first) test_print_range(first, last, failed - 1, 0);
420 if (!missing && !failed) {
421 fputs(!status ? "ok" : "dubious", stdout);
422 if (ts->skipped > 0) printf(" (skipped %d tests)", ts->skipped);
426 printf(" (exit status %d)", status);
427 } else if (status < 0) {
428 printf(" (killed by signal %d%s)", -status,
429 WCOREDUMP(ts->status) ? ", core dumped" : "");
435 /* Given a test set, analyze the results, classify the exit status, handle a
436 few special error messages, and then pass it along to test_summarize()
437 for the regular output. */
439 test_analyze(struct testset *ts)
441 if (ts->reported) return 0;
442 if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
443 switch (WEXITSTATUS(ts->status)) {
445 if (!ts->reported) puts("can't dup file descriptors");
448 if (!ts->reported) puts("execution failed (not found?)");
450 case CHILDERR_STDERR:
451 if (!ts->reported) puts("can't open /dev/null");
454 test_summarize(ts, WEXITSTATUS(ts->status));
458 } else if (WIFSIGNALED(ts->status)) {
459 test_summarize(ts, -WTERMSIG(ts->status));
462 test_summarize(ts, 0);
463 return (ts->failed == 0);
468 /* Runs a single test set, accumulating and then reporting the results.
469 Returns true if the test set was successfully run and all tests passed,
472 test_run(struct testset *ts)
474 pid_t testpid, child;
475 int outfd, i, status;
480 /* Initialize the test and our data structures, flagging this set in
481 error if the initialization fails. */
482 file = xmalloc(strlen(ts->file) + 3);
483 strcpy(file, ts->file);
485 testpid = test_start(file, &outfd);
487 output = fdopen(outfd, "r");
488 if (!output) sysdie("fdopen failed");
489 if (!fgets(buffer, sizeof(buffer), output)) ts->aborted = 1;
490 if (!ts->aborted && !test_init(buffer, ts)) {
491 while (fgets(buffer, sizeof(buffer), output))
496 /* Pass each line of output to test_checkline(). */
497 while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
498 test_checkline(buffer, ts);
499 if (ferror(output)) ts->aborted = 1;
501 /* Close the output descriptor, retrieve the exit status, and pass that
502 information to test_analyze() for eventual output. */
504 child = waitpid(testpid, &ts->status, 0);
505 if (child == (pid_t) -1)
506 sysdie("waitpid for %u failed", (unsigned int) testpid);
507 status = test_analyze(ts);
509 /* Convert missing tests to failed tests. */
510 for (i = 0; i < ts->count; i++) {
511 if (ts->results[i] == TEST_INVALID) {
513 ts->results[i] = TEST_FAIL;
521 /* Summarize a list of test failures. */
523 test_fail_summary(const struct testlist *fails)
526 int i, chars, total, first, last;
530 /* Failed Set Fail/Total (%) Skip Stat Failing (25)
531 -------------------------- -------------- ---- ---- -------------- */
532 for (; fails; fails = fails->next) {
534 total = ts->count - ts->skipped;
535 printf("%-26.26s %4d/%-4d %3.0f%% %4d ", ts->file, ts->failed,
536 total, total ? (ts->failed * 100.0) / total : 0,
538 if (WIFEXITED(ts->status)) {
539 printf("%4d ", WEXITSTATUS(ts->status));
550 for (i = 0; i < ts->count; i++) {
551 if (ts->results[i] == TEST_FAIL) {
552 if (first && i == last) {
556 chars += test_print_range(first, last, chars, 20);
562 if (first) test_print_range(first, last, chars, 20);
568 /* Run a batch of tests from a given file listing each test on a line by
569 itself. The file must be rewindable. Returns true iff all tests
572 test_batch(const char *testlist)
579 struct testset ts, *tmp;
580 struct timeval start, end;
582 struct testlist *failhead = 0;
583 struct testlist *failtail = 0;
590 /* Open our file of tests to run and scan it, checking for lines that
591 are too long and searching for the longest line. */
592 tests = fopen(testlist, "r");
593 if (!tests) sysdie("can't open %s", testlist);
595 while (fgets(buffer, sizeof(buffer), tests)) {
597 length = strlen(buffer) - 1;
598 if (buffer[length] != '\n') {
599 fprintf(stderr, "%s:%d: line too long\n", testlist, line);
602 if (length > longest) longest = length;
604 if (fseek(tests, 0, SEEK_SET) == -1)
605 sysdie("can't rewind %s", testlist);
607 /* Add two to longest and round up to the nearest tab stop. This is how
608 wide the column for printing the current test name will be. */
610 if (longest % 8) longest += 8 - (longest % 8);
612 /* Start the wall clock timer. */
613 gettimeofday(&start, NULL);
615 /* Now, plow through our tests again, running each one. Check line
616 length again out of paranoia. */
618 while (fgets(buffer, sizeof(buffer), tests)) {
620 length = strlen(buffer) - 1;
621 if (buffer[length] != '\n') {
622 fprintf(stderr, "%s:%d: line too long\n", testlist, line);
625 buffer[length] = '\0';
626 fputs(buffer, stdout);
627 for (i = length; i < longest; i++) putchar('.');
628 memset(&ts, 0, sizeof(ts));
629 ts.file = xstrdup(buffer);
630 if (!test_run(&ts)) {
631 tmp = xmalloc(sizeof(struct testset));
632 memcpy(tmp, &ts, sizeof(struct testset));
634 failhead = xmalloc(sizeof(struct testset));
639 failtail->next = xmalloc(sizeof(struct testset));
640 failtail = failtail->next;
645 aborted += ts.aborted;
648 skipped += ts.skipped;
653 /* Stop the timer and get our child resource statistics. */
654 gettimeofday(&end, NULL);
655 getrusage(RUSAGE_CHILDREN, &stats);
657 /* Print out our final results. */
658 if (failhead) test_fail_summary(failhead);
661 printf("Aborted %d test sets, passed %d/%d tests.\n", aborted,
663 } else if (failed == 0) {
664 fputs("All tests successful", stdout);
665 if (skipped) printf(", %d tests skipped", skipped);
668 printf("Failed %d/%d tests, %.2f%% okay.\n", failed, total,
669 (total - failed) * 100.0 / total);
671 printf("Files=%d, Tests=%d", line, total);
672 printf(", %.2f seconds", tv_diff(&end, &start));
673 printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
674 tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
675 tv_sum(&stats.ru_utime, &stats.ru_stime));
676 return !(failed || aborted);
680 /* Main routine. Given a file listing tests, run each test listed. */
682 main(int argc, char *argv[])
685 fprintf(stderr, "Usage: runtests <test-list>\n");
688 printf(banner, argv[1]);
689 exit(test_batch(argv[1]) ? 0 : 1);