+++ /dev/null
-/* $Id: runtests.c 7578 2006-09-11 23:03:12Z eagle $
-
- Run a set of tests, reporting results.
-
- Copyright 2000, 2001 Russ Allbery <rra@stanford.edu>
-
- Please note that this file is maintained separately from INN by the above
- author (which is why the coding style is slightly different). Any fixes
- added to the INN tree should also be reported to the above author if
- necessary.
-
- Permission is hereby granted, free of charge, to any person obtaining a
- copy of this software and associated documentation files (the
- "Software"), to deal in the Software without restriction, including
- without limitation the rights to use, copy, modify, merge, publish,
- distribute, sublicense, and/or sell copies of the Software, and to
- permit persons to whom the Software is furnished to do so, subject to
- the following conditions:
-
- The above copyright notice and this permission notice shall be included
- in all copies or substantial portions of the Software.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
- CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
- Usage:
-
- runtests <test-list>
-
- Expects a list of executables located in the given file, one line per
- executable. For each one, runs it as part of a test suite, reporting
- results. Test output should start with a line containing the number of
- tests (numbered from 1 to this number), and then each line should be in
- the following format:
-
- ok <number>
- not ok <number>
- ok <number> # skip
-
- where <number> is the number of the test. ok indicates success, not ok
- indicates failure, and "# skip" indicates the test was skipped for some
- reason (maybe because it doesn't apply to this platform).
-
- This file is completely stand-alone by intention. As stated more
- formally in the license above, you are welcome to include it in your
- packages as a test suite driver. It requires ANSI C (__FILE__, __LINE__,
- void, const, stdarg.h, string.h) and POSIX (fcntl.h, unistd.h, pid_t) and
- won't compile out of the box on SunOS without adjustments to include
- strings.h instead. This is intentionally not fixed using autoconf so
- that this file will not have a dependency on autoconf (although you're
- welcome to fix it for your project if you want). Since it doesn't matter
- as much that the test suite for the software package be utterly portable
- to older systems, this file should be portable enough for most purposes.
-
- Any bug reports, bug fixes, and improvements are very much welcome and
- should be sent to the e-mail address above. */
-
-#include "config.h"
-#include "clibrary.h"
-#include "portable/wait.h"
-#include "portable/time.h"
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <stdarg.h>
-#include <sys/stat.h>
-
-/* sys/time.h must be included before sys/resource.h on some platforms. */
-#include <sys/resource.h>
-
-/* Test status codes. */
-enum test_status {
- TEST_FAIL,
- TEST_PASS,
- TEST_SKIP,
- TEST_INVALID
-};
-
-/* Error exit statuses for test processes. */
-#define CHILDERR_DUP 100 /* Couldn't redirect stderr or stdout. */
-#define CHILDERR_EXEC 101 /* Couldn't exec child process. */
-#define CHILDERR_STDERR 102 /* Couldn't open stderr file. */
-
-/* Structure to hold data for a set of tests. */
-struct testset {
- const char *file; /* The file name of the test. */
- int count; /* Expected count of tests. */
- int current; /* The last seen test number. */
- int passed; /* Count of passing tests. */
- int failed; /* Count of failing lists. */
- int skipped; /* Count of skipped tests (passed). */
- enum test_status *results; /* Table of results by test number. */
- int aborted; /* Whether the set as aborted. */
- int reported; /* Whether the results were reported. */
- int status; /* The exit status of the test. */
-};
-
-/* Structure to hold a linked list of test sets. */
-struct testlist {
- struct testset *ts;
- struct testlist *next;
-};
-
-/* Header used for test output. %s is replaced by the file name of the list
- of tests. */
-static const char banner[] = "\n\
-Running all tests listed in %s. If any tests fail, run the failing\n\
-test program by hand to see more details. The test program will have the\n\
-same name as the test set but with \".t\" appended.\n\n";
-
-/* Header for reports of failed tests. */
-static const char header[] = "\n\
-Failed Set Fail/Total (%) Skip Stat Failing Tests\n\
--------------------------- -------------- ---- ---- ------------------------";
-
-/* Include the file name and line number in malloc failures. */
-#define xmalloc(size) x_malloc((size), __FILE__, __LINE__)
-#define xstrdup(p) x_strdup((p), __FILE__, __LINE__)
-
-/* Internal prototypes. */
-static void sysdie(const char *format, ...);
-static void *x_malloc(size_t, const char *file, int line);
-static char *x_strdup(const char *, const char *file, int line);
-static int test_analyze(struct testset *);
-static int test_batch(const char *testlist);
-static void test_checkline(const char *line, struct testset *);
-static void test_fail_summary(const struct testlist *);
-static int test_init(const char *line, struct testset *);
-static int test_print_range(int first, int last, int chars, int limit);
-static void test_summarize(struct testset *, int status);
-static pid_t test_start(const char *path, int *fd);
-static double tv_diff(const struct timeval *, const struct timeval *);
-static double tv_seconds(const struct timeval *);
-static double tv_sum(const struct timeval *, const struct timeval *);
-
-
-/* Report a fatal error, including the results of strerror, and exit. */
-static void
-sysdie(const char *format, ...)
-{
- int oerrno;
- va_list args;
-
- oerrno = errno;
- fflush(stdout);
- fprintf(stderr, "runtests: ");
- va_start(args, format);
- vfprintf(stderr, format, args);
- va_end(args);
- fprintf(stderr, ": %s\n", strerror(oerrno));
- exit(1);
-}
-
-
-/* Allocate memory, reporting a fatal error and exiting on failure. */
-static void *
-x_malloc(size_t size, const char *file, int line)
-{
- void *p;
-
- p = malloc(size);
- if (!p)
- sysdie("failed to malloc %lu bytes at %s line %d",
- (unsigned long) size, file, line);
- return p;
-}
-
-
-/* Copy a string, reporting a fatal error and exiting on failure. */
-static char *
-x_strdup(const char *s, const char *file, int line)
-{
- char *p;
- size_t len;
-
- len = strlen(s) + 1;
- p = malloc(len);
- if (!p)
- sysdie("failed to strdup %lu bytes at %s line %d",
- (unsigned long) len, file, line);
- memcpy(p, s, len);
- return p;
-}
-
-
-/* Given a struct timeval, return the number of seconds it represents as a
- double. Use difftime() to convert a time_t to a double. */
-static double
-tv_seconds(const struct timeval *tv)
-{
- return difftime(tv->tv_sec, 0) + tv->tv_usec * 1e-6;
-}
-
-/* Given two struct timevals, return the difference in seconds. */
-static double
-tv_diff(const struct timeval *tv1, const struct timeval *tv0)
-{
- return tv_seconds(tv1) - tv_seconds(tv0);
-}
-
-/* Given two struct timevals, return the sum in seconds as a double. */
-static double
-tv_sum(const struct timeval *tv1, const struct timeval *tv2)
-{
- return tv_seconds(tv1) + tv_seconds(tv2);
-}
-
-
-/* Read the first line of test output, which should contain the range of
- test numbers, and initialize the testset structure. Assume it was zeroed
- before being passed in. Return true if initialization succeeds, false
- otherwise. */
-static int
-test_init(const char *line, struct testset *ts)
-{
- int i;
-
- /* Prefer a simple number of tests, but if the count is given as a range
- such as 1..10, accept that too for compatibility with Perl's
- Test::Harness. */
- while (isspace((unsigned char)(*line))) line++;
- if (!strncmp(line, "1..", 3)) line += 3;
-
- /* Get the count, check it for validity, and initialize the struct. */
- i = atoi(line);
- if (i <= 0) {
- puts("invalid test count");
- ts->aborted = 1;
- ts->reported = 1;
- return 0;
- }
- ts->count = i;
- ts->results = xmalloc(ts->count * sizeof(enum test_status));
- for (i = 0; i < ts->count; i++) ts->results[i] = TEST_INVALID;
- return 1;
-}
-
-
-/* Start a program, connecting its stdout to a pipe on our end and its
- stderr to /dev/null, and storing the file descriptor to read from in the
- two argument. Returns the PID of the new process. Errors are fatal. */
-static pid_t
-test_start(const char *path, int *fd)
-{
- int fds[2], errfd;
- pid_t child;
-
- if (pipe(fds) == -1) sysdie("can't create pipe");
- child = fork();
- if (child == (pid_t) -1) {
- sysdie("can't fork");
- } else if (child == 0) {
- /* In child. Set up our stdout and stderr. */
- errfd = open("/dev/null", O_WRONLY);
- if (errfd < 0) _exit(CHILDERR_STDERR);
- if (dup2(errfd, 2) == -1) _exit(CHILDERR_DUP);
- close(fds[0]);
- if (dup2(fds[1], 1) == -1) _exit(CHILDERR_DUP);
-
- /* Now, exec our process. */
- if (execl(path, path, (char *) 0) == -1) _exit(CHILDERR_EXEC);
- } else {
- /* In parent. Close the extra file descriptor. */
- close(fds[1]);
- }
- *fd = fds[0];
- return child;
-}
-
-
-/* Given a single line of output from a test, parse it and return the
- success status of that test. Anything printed to stdout not matching the
- form /^(not )?ok \d+/ is ignored. Sets ts->current to the test number
- that just reported status. */
-static void
-test_checkline(const char *line, struct testset *ts)
-{
- enum test_status status = TEST_PASS;
- int current;
-
- /* If the given line isn't newline-terminated, it was too big for an
- fgets(), which means ignore it. */
- if (line[strlen(line) - 1] != '\n') return;
-
- /* Parse the line, ignoring something we can't parse. */
- if (!strncmp(line, "not ", 4)) {
- status = TEST_FAIL;
- line += 4;
- }
- if (strncmp(line, "ok ", 3)) return;
- line += 3;
- current = atoi(line);
- if (current == 0) return;
- if (current < 0 || current > ts->count) {
- printf("invalid test number %d\n", current);
- ts->aborted = 1;
- ts->reported = 1;
- return;
- }
- while (isspace((unsigned char)(*line))) line++;
- while (isdigit((unsigned char)(*line))) line++;
- while (isspace((unsigned char)(*line))) line++;
- if (*line == '#') {
- line++;
- while (isspace((unsigned char)(*line))) line++;
- if (!strncmp(line, "skip", 4)) status = TEST_SKIP;
- }
-
- /* Make sure that the test number is in range and not a duplicate. */
- if (ts->results[current - 1] != TEST_INVALID) {
- printf("duplicate test number %d\n", current);
- ts->aborted = 1;
- ts->reported = 1;
- return;
- }
-
- /* Good results. Increment our various counters. */
- switch (status) {
- case TEST_PASS: ts->passed++; break;
- case TEST_FAIL: ts->failed++; break;
- case TEST_SKIP: ts->skipped++; break;
- default: break;
- }
- ts->current = current;
- ts->results[current - 1] = status;
-}
-
-
-/* Print out a range of test numbers, returning the number of characters it
- took up. Add a comma and a space before the range if chars indicates
- that something has already been printed on the line, and print
- ... instead if chars plus the space needed would go over the limit (use a
- limit of 0 to disable this. */
-static int
-test_print_range(int first, int last, int chars, int limit)
-{
- int needed = 0;
- int out = 0;
- int n;
-
- if (chars > 0) {
- needed += 2;
- if (!limit || chars <= limit) out += printf(", ");
- }
- for (n = first; n > 0; n /= 10)
- needed++;
- if (last > first) {
- for (n = last; n > 0; n /= 10)
- needed++;
- needed++;
- }
- if (limit && chars + needed > limit) {
- if (chars <= limit) out += printf("...");
- } else {
- if (last > first) out += printf("%d-", first);
- out += printf("%d", last);
- }
- return out;
-}
-
-
-/* Summarize a single test set. The second argument is 0 if the set exited
- cleanly, a positive integer representing the exit status if it exited
- with a non-zero status, and a negative integer representing the signal
- that terminated it if it was killed by a signal. */
-static void
-test_summarize(struct testset *ts, int status)
-{
- int i;
- int missing = 0;
- int failed = 0;
- int first = 0;
- int last = 0;
-
- if (ts->aborted) {
- fputs("aborted", stdout);
- if (ts->count > 0)
- printf(", passed %d/%d", ts->passed, ts->count - ts->skipped);
- } else {
- for (i = 0; i < ts->count; i++) {
- if (ts->results[i] == TEST_INVALID) {
- if (missing == 0) fputs("MISSED ", stdout);
- if (first && i == last) {
- last = i + 1;
- } else {
- if (first) {
- test_print_range(first, last, missing - 1, 0);
- }
- missing++;
- first = i + 1;
- last = i + 1;
- }
- }
- }
- if (first) test_print_range(first, last, missing - 1, 0);
- first = 0;
- last = 0;
- for (i = 0; i < ts->count; i++) {
- if (ts->results[i] == TEST_FAIL) {
- if (missing && !failed) fputs("; ", stdout);
- if (failed == 0) fputs("FAILED ", stdout);
- if (first && i == last) {
- last = i + 1;
- } else {
- if (first) {
- test_print_range(first, last, failed - 1, 0);
- }
- failed++;
- first = i + 1;
- last = i + 1;
- }
- }
- }
- if (first) test_print_range(first, last, failed - 1, 0);
- if (!missing && !failed) {
- fputs(!status ? "ok" : "dubious", stdout);
- if (ts->skipped > 0) printf(" (skipped %d tests)", ts->skipped);
- }
- }
- if (status > 0) {
- printf(" (exit status %d)", status);
- } else if (status < 0) {
- printf(" (killed by signal %d%s)", -status,
- WCOREDUMP(ts->status) ? ", core dumped" : "");
- }
- putchar('\n');
-}
-
-
-/* Given a test set, analyze the results, classify the exit status, handle a
- few special error messages, and then pass it along to test_summarize()
- for the regular output. */
-static int
-test_analyze(struct testset *ts)
-{
- if (ts->reported) return 0;
- if (WIFEXITED(ts->status) && WEXITSTATUS(ts->status) != 0) {
- switch (WEXITSTATUS(ts->status)) {
- case CHILDERR_DUP:
- if (!ts->reported) puts("can't dup file descriptors");
- break;
- case CHILDERR_EXEC:
- if (!ts->reported) puts("execution failed (not found?)");
- break;
- case CHILDERR_STDERR:
- if (!ts->reported) puts("can't open /dev/null");
- break;
- default:
- test_summarize(ts, WEXITSTATUS(ts->status));
- break;
- }
- return 0;
- } else if (WIFSIGNALED(ts->status)) {
- test_summarize(ts, -WTERMSIG(ts->status));
- return 0;
- } else {
- test_summarize(ts, 0);
- return (ts->failed == 0);
- }
-}
-
-
-/* Runs a single test set, accumulating and then reporting the results.
- Returns true if the test set was successfully run and all tests passed,
- false otherwise. */
-static int
-test_run(struct testset *ts)
-{
- pid_t testpid, child;
- int outfd, i, status;
- FILE *output;
- char buffer[BUFSIZ];
- char *file;
-
- /* Initialize the test and our data structures, flagging this set in
- error if the initialization fails. */
- file = xmalloc(strlen(ts->file) + 3);
- strcpy(file, ts->file);
- strcat(file, ".t");
- testpid = test_start(file, &outfd);
- free(file);
- output = fdopen(outfd, "r");
- if (!output) sysdie("fdopen failed");
- if (!fgets(buffer, sizeof(buffer), output)) ts->aborted = 1;
- if (!ts->aborted && !test_init(buffer, ts)) {
- while (fgets(buffer, sizeof(buffer), output))
- ;
- ts->aborted = 1;
- }
-
- /* Pass each line of output to test_checkline(). */
- while (!ts->aborted && fgets(buffer, sizeof(buffer), output))
- test_checkline(buffer, ts);
- if (ferror(output)) ts->aborted = 1;
-
- /* Close the output descriptor, retrieve the exit status, and pass that
- information to test_analyze() for eventual output. */
- fclose(output);
- child = waitpid(testpid, &ts->status, 0);
- if (child == (pid_t) -1)
- sysdie("waitpid for %u failed", (unsigned int) testpid);
- status = test_analyze(ts);
-
- /* Convert missing tests to failed tests. */
- for (i = 0; i < ts->count; i++) {
- if (ts->results[i] == TEST_INVALID) {
- ts->failed++;
- ts->results[i] = TEST_FAIL;
- status = 0;
- }
- }
- return status;
-}
-
-
-/* Summarize a list of test failures. */
-static void
-test_fail_summary(const struct testlist *fails)
-{
- struct testset *ts;
- int i, chars, total, first, last;
-
- puts(header);
-
- /* Failed Set Fail/Total (%) Skip Stat Failing (25)
- -------------------------- -------------- ---- ---- -------------- */
- for (; fails; fails = fails->next) {
- ts = fails->ts;
- total = ts->count - ts->skipped;
- printf("%-26.26s %4d/%-4d %3.0f%% %4d ", ts->file, ts->failed,
- total, total ? (ts->failed * 100.0) / total : 0,
- ts->skipped);
- if (WIFEXITED(ts->status)) {
- printf("%4d ", WEXITSTATUS(ts->status));
- } else {
- printf(" -- ");
- }
- if (ts->aborted) {
- puts("aborted");
- continue;
- }
- chars = 0;
- first = 0;
- last = 0;
- for (i = 0; i < ts->count; i++) {
- if (ts->results[i] == TEST_FAIL) {
- if (first && i == last) {
- last = i + 1;
- } else {
- if (first)
- chars += test_print_range(first, last, chars, 20);
- first = i + 1;
- last = i + 1;
- }
- }
- }
- if (first) test_print_range(first, last, chars, 20);
- putchar('\n');
- }
-}
-
-
-/* Run a batch of tests from a given file listing each test on a line by
- itself. The file must be rewindable. Returns true iff all tests
- passed. */
-static int
-test_batch(const char *testlist)
-{
- FILE *tests;
- size_t length, i;
- size_t longest = 0;
- char buffer[BUFSIZ];
- int line;
- struct testset ts, *tmp;
- struct timeval start, end;
- struct rusage stats;
- struct testlist *failhead = 0;
- struct testlist *failtail = 0;
- int total = 0;
- int passed = 0;
- int skipped = 0;
- int failed = 0;
- int aborted = 0;
-
- /* Open our file of tests to run and scan it, checking for lines that
- are too long and searching for the longest line. */
- tests = fopen(testlist, "r");
- if (!tests) sysdie("can't open %s", testlist);
- line = 0;
- while (fgets(buffer, sizeof(buffer), tests)) {
- line++;
- length = strlen(buffer) - 1;
- if (buffer[length] != '\n') {
- fprintf(stderr, "%s:%d: line too long\n", testlist, line);
- exit(1);
- }
- if (length > longest) longest = length;
- }
- if (fseek(tests, 0, SEEK_SET) == -1)
- sysdie("can't rewind %s", testlist);
-
- /* Add two to longest and round up to the nearest tab stop. This is how
- wide the column for printing the current test name will be. */
- longest += 2;
- if (longest % 8) longest += 8 - (longest % 8);
-
- /* Start the wall clock timer. */
- gettimeofday(&start, NULL);
-
- /* Now, plow through our tests again, running each one. Check line
- length again out of paranoia. */
- line = 0;
- while (fgets(buffer, sizeof(buffer), tests)) {
- line++;
- length = strlen(buffer) - 1;
- if (buffer[length] != '\n') {
- fprintf(stderr, "%s:%d: line too long\n", testlist, line);
- exit(1);
- }
- buffer[length] = '\0';
- fputs(buffer, stdout);
- for (i = length; i < longest; i++) putchar('.');
- memset(&ts, 0, sizeof(ts));
- ts.file = xstrdup(buffer);
- if (!test_run(&ts)) {
- tmp = xmalloc(sizeof(struct testset));
- memcpy(tmp, &ts, sizeof(struct testset));
- if (!failhead) {
- failhead = xmalloc(sizeof(struct testset));
- failhead->ts = tmp;
- failhead->next = 0;
- failtail = failhead;
- } else {
- failtail->next = xmalloc(sizeof(struct testset));
- failtail = failtail->next;
- failtail->ts = tmp;
- failtail->next = 0;
- }
- }
- aborted += ts.aborted;
- total += ts.count;
- passed += ts.passed;
- skipped += ts.skipped;
- failed += ts.failed;
- }
- total -= skipped;
-
- /* Stop the timer and get our child resource statistics. */
- gettimeofday(&end, NULL);
- getrusage(RUSAGE_CHILDREN, &stats);
-
- /* Print out our final results. */
- if (failhead) test_fail_summary(failhead);
- putchar('\n');
- if (aborted) {
- printf("Aborted %d test sets, passed %d/%d tests.\n", aborted,
- passed, total);
- } else if (failed == 0) {
- fputs("All tests successful", stdout);
- if (skipped) printf(", %d tests skipped", skipped);
- puts(".");
- } else {
- printf("Failed %d/%d tests, %.2f%% okay.\n", failed, total,
- (total - failed) * 100.0 / total);
- }
- printf("Files=%d, Tests=%d", line, total);
- printf(", %.2f seconds", tv_diff(&end, &start));
- printf(" (%.2f usr + %.2f sys = %.2f CPU)\n",
- tv_seconds(&stats.ru_utime), tv_seconds(&stats.ru_stime),
- tv_sum(&stats.ru_utime, &stats.ru_stime));
- return !(failed || aborted);
-}
-
-
-/* Main routine. Given a file listing tests, run each test listed. */
-int
-main(int argc, char *argv[])
-{
- if (argc != 2) {
- fprintf(stderr, "Usage: runtests <test-list>\n");
- exit(1);
- }
- printf(banner, argv[1]);
- exit(test_batch(argv[1]) ? 0 : 1);
-}
~ian / innduct.git / blobdiff