From 9fcce036ef8b14c12fc4482efc7c8cf75da3b31f Mon Sep 17 00:00:00 2001 Message-Id: <9fcce036ef8b14c12fc4482efc7c8cf75da3b31f.1715891957.git.mdw@distorted.org.uk> From: Mark Wooding Date: Sat, 9 Feb 2008 19:54:08 +0000 Subject: [PATCH] testrig: Provide useful interface for more complicated test rigs. Organization: Straylight/Edgeware From: Mark Wooding Provide a lower-level interface to the test-vector processing machinery, and introduce `suites' over the top of the existing chunks. --- man/testrig.3 | 75 +++++++++++++--- testrig.c | 245 +++++++++++++++++++++++++++++++++----------------- testrig.h | 26 ++++++ 3 files changed, 254 insertions(+), 92 deletions(-) diff --git a/man/testrig.3 b/man/testrig.3 index c010cb0..b996617 100644 --- a/man/testrig.3 +++ b/man/testrig.3 @@ -19,17 +19,50 @@ testrig \- generic test rig .nf .B "#include " +.BI "void test_do(const test_suite " suite [], +.BI " FILE *" fp , +.BI " test_results *" results ); .BI "void test_run(int " argc ", char *" argv [], .BI " const test_chunk " chunk [], .BI " const char *" def ); .fi .SH DESCRIPTION +.SS Structure +Test vectors are gathered together into +.I chunks +which should be processed in the same way. Chunks, in turn, are grouped +into +.IR suites . +.SS Functions +The +.B test_do +function runs a collection of tests, as defined by +.IR suite , +given the test vectors in the file +.IR fp . +It returns results in the +.B test_results +structure +.IR results , +which has two members: +.TP +.B "unsigned tests" +counts the number of tests carried out, and +.TP +.B "unsigned failed" +counts the number of tests which failed. +.PP +The function returns negative if there was a system error or the test +vector file was corrupt in some way, zero if all the tests were +successful, or positive if some tests failed. +.PP The .B test_run -function is intended to be called from the +provides a simple command-line interface to the test system. It is +intended to be called from the .B main function of a test rig program to check that a particular function or -suite of functions are running properly. The arguments +suite of functions are running properly. It does not return. The arguments .I argc and .I argv @@ -55,13 +88,18 @@ Test vector files have the following syntax: .PP .I file ::= -.RI [ chunk ...] +.RI [ suite-header | chunk " ...]" +.br +.I suite-header +::= +.B suite +.I name .br .I chunk ::= .I name .B { -.RI [ test-vector ...] +.RI [ test-vector " ...]" .B } .br .I test-vector @@ -80,13 +118,30 @@ A value is either a sequence of or a string enclosed in quote marks (double or single). Quoted strings may contain newlines. In either type of value, a backslash escapes the following character. +.SS "Suite definitions" +A +.I suite definition +is described by the structure +.VS +typedef struct test_suite { + const char *name; /* Name of this suite */ + const test_chunk *chunks; /* Pointer to chunks */ +} test_suite; +.VE +The +.I suite +argument to +.B test_do +is a pointer to an array of these structures, terminated by one with a +null +.BR name . .SS "Chunk definitions" -The caller must supply an array of one or more -.IR "chunk definitions" . -Each one describes the format of a named chunk: the number and type of -the values required and the function to call in order to test the system -against that test vector. The array is terminated by a chunk definition -whose name field is a null pointer. +A +.I "chunk definition" +describes the format of a named chunk: the number and type of the values +required and the function to call in order to test the system against +that test vector. The array is terminated by a chunk definition whose +name field is a null pointer. .PP A chunk definition is described by the following structure: .VS diff --git a/testrig.c b/testrig.c index 05fef07..b24ab33 100644 --- a/testrig.c +++ b/testrig.c @@ -283,81 +283,37 @@ static void dump_uint32(dstr *d, FILE *fp) const test_type type_uint32 = { cvt_uint32, dump_uint32 }; -/* --- @test_run@ --- * +/* --- @test_do@ --- * * - * Arguments: @int argc@ = number of command line arguments - * @char *argv[]@ = pointer to command line arguments - * @const test_chunk chunk[]@ = pointer to chunk definitions - * @const char *vec@ = name of default test vector file + * Arguments: @const test_suite suites[]@ = pointer to suite definitions + * @FILE *fp@ = test vector file, ready opened + * @test_results *results@ = where to put results * - * Returns: Doesn't. + * Returns: Negative if something bad happened, or the number of + * failures. * - * Use: Runs a set of test vectors to ensure that a component is - * working properly. + * Use: Runs a collection of tests against a file of test vectors and + * reports the results. */ -void test_run(int argc, char *argv[], - const test_chunk chunk[], - const char *vec) +int test_do(const test_suite suites[], FILE *fp, test_results *results) { - FILE *fp; - int i; - const test_chunk *cch; + test_results dummy; dstr dv[TEST_FIELDMAX]; - int fail = 0, ok = 1; - int sofar = 0; - - /* --- Silly bits of initialization --- */ - - ego(argv[0]); + const test_suite *ss; + const test_chunk *chunks = suites[0].chunks; + const test_chunk *cch; + int rc = -1; + int ok; + int i; for (i = 0; i < TEST_FIELDMAX; i++) DCREATE(&dv[i]); - /* --- Parse command line arguments --- */ - - { - const char *p = 0; - - i = 0; - for (;;) { - if (!p || !*p) { - if (i >= argc - 1) - break; - p = argv[++i]; - if (strcmp(p, "--") == 0) { - i++; - break; - } - if (p[0] != '-' || p[1] == 0) - break; - p++; - } - switch (*p++) { - case 'h': - printf("%s test driver\n" - "Usage: %s [-f FILENAME]\n", QUIS, QUIS); - exit(0); - case 'f': - if (!*p) { - if (i >= argc - 1) - die(1, "option `-f' expects an argument"); - p = argv[++i]; - } - vec = p; - p = 0; - break; - default: - die(1, "option `-%c' unknown", p[-1]); - break; - } - } - } - - /* --- Start parsing from the file --- */ - - if ((fp = fopen(vec, "r")) == 0) - die(1, "couldn't open test vector file `%s': %s", vec, strerror(errno)); + if (!results) + results = &dummy; + results->tests = 0; + results->failed = 0; for (;;) { int t = gettok(fp); @@ -369,12 +325,35 @@ void test_run(int argc, char *argv[], /* --- Pick out the chunk name --- */ - if (t != TOK_WORD) - die(1, "expected ; found `%s'", decode(t)); + if (t != TOK_WORD) { + moan("expected ; found `%s'", decode(t)); + goto done; + } + + if (strcmp(tok.buf, "SUITE") == 0) { + t = gettok(fp); + if (t != TOK_WORD) { + moan("expected ; found `%s'", decode(t)); + goto done; + } + for (ss = suites; ; ss++) { + if (!ss->name) { + chunks = 0; + break; + } + if (strcmp(tok.buf, ss->name) == 0) { + chunks = ss->chunks; + break; + } + } + continue; + } /* --- Find the right chunk block --- */ - for (cch = chunk; ; cch++) { + if (!chunks) + goto skip_chunk; + for (cch = chunks; ; cch++) { if (!cch->name) goto skip_chunk; if (strcmp(tok.buf, cch->name) == 0) @@ -383,14 +362,15 @@ void test_run(int argc, char *argv[], /* --- Past the open brace to the first chunk --- */ - if ((t = gettok(fp)) != '{') - die(1, "expected `{'; found `%s'", decode(t)); + if ((t = gettok(fp)) != '{') { + moan("expected `{'; found `%s'", decode(t)); + goto done; + } /* --- Start on the test data now --- */ printf("%s: ", cch->name); fflush(stdout); - sofar = 0; ok = 1; for (;;) { @@ -405,26 +385,31 @@ void test_run(int argc, char *argv[], for (i = 0; cch->f[i]; i++) { DRESET(&dv[i]); - if (t != TOK_WORD) - die(1, "expected ; found `%s'", decode(t)); + if (t != TOK_WORD) { + moan("expected ; found `%s'", decode(t)); + goto done; + } cch->f[i]->cvt(tok.buf, &dv[i]); t = gettok(fp); } /* --- And a terminating semicolon --- */ - if (t != ';') - die(1, "expected `;'; found `%s'", decode(t)); + if (t != ';') { + moan("expected `;'; found `%s'", decode(t)); + goto done; + } /* --- Run the test code --- */ if (!cch->test(dv)) { + ok = 0; printf("%s: ", cch->name); - for (i = 0; i < sofar; i++) putchar('.'); - fail = 1; ok = 0; + for (i = 0; i < results->tests; i++) putchar('.'); + results->failed++; } - sofar++; putchar('.'); + results->tests++; fflush(stdout); } @@ -433,20 +418,116 @@ void test_run(int argc, char *argv[], continue; skip_chunk: - if ((t = gettok(fp)) != '{') - die(1, "expected '{'; found `%s'", decode(t)); + if ((t = gettok(fp)) != '{') { + moan("expected '{'; found `%s'", decode(t)); + goto done; + } for (;;) { t = gettok(fp); if (t == '}') break; while (t == TOK_WORD) t = gettok(fp); - if (t != ';') - die(1, "expected `;'; found `%s'", decode(t)); + if (t != ';') { + moan("expected `;'; found `%s'", decode(t)); + goto done; + } + } + } + rc = results->failed; + + /* --- All done --- */ + +done: + for (i = 0; i < TEST_FIELDMAX; i++) + dstr_destroy(&dv[i]); + return (rc); +} + +/* --- @test_run@ --- * + * + * Arguments: @int argc@ = number of command line arguments + * @char *argv[]@ = pointer to command line arguments + * @const test_chunk chunk[]@ = pointer to chunk definitions + * @const char *vec@ = name of default test vector file + * + * Returns: Doesn't. + * + * Use: Runs a set of test vectors to ensure that a component is + * working properly. + */ + +void test_run(int argc, char *argv[], + const test_chunk chunk[], + const char *vec) +{ + FILE *fp; + test_results res; + int rc; + test_suite suite[2]; + + /* --- Silly bits of initialization --- */ + + ego(argv[0]); + + /* --- Parse command line arguments --- */ + + { + const char *p = 0; + int i = 0; + + for (;;) { + if (!p || !*p) { + if (i >= argc - 1) + break; + p = argv[++i]; + if (strcmp(p, "--") == 0) { + i++; + break; + } + if (p[0] != '-' || p[1] == 0) + break; + p++; + } + switch (*p++) { + case 'h': + printf("%s test driver\n" + "Usage: %s [-f FILENAME]\n", QUIS, QUIS); + exit(0); + case 'f': + if (!*p) { + if (i >= argc - 1) + die(1, "option `-f' expects an argument"); + p = argv[++i]; + } + vec = p; + p = 0; + break; + default: + die(1, "option `-%c' unknown", p[-1]); + break; + } } } - exit(fail); + /* --- Start parsing from the file --- */ + + if ((fp = fopen(vec, "r")) == 0) + die(1, "couldn't open test vector file `%s': %s", vec, strerror(errno)); + suite[0].name = "simple"; + suite[0].chunks = chunk; + suite[1].name = 0; + rc = test_do(suite, fp, &res); + if (rc < 0) + exit(127); + if (res.failed) { + fprintf(stderr, "FAILED %u of %u test%s\n", + res.failed, res.tests, res.tests == 1 ? "" : "s"); + } else { + fprintf(stderr, "PASSED all %u test%s\n", + res.tests, res.tests == 1 ? "" : "s"); + } + exit(!!res.failed); } /*----- That's all, folks -------------------------------------------------*/ diff --git a/testrig.h b/testrig.h index 8588287..c63edb1 100644 --- a/testrig.h +++ b/testrig.h @@ -47,6 +47,10 @@ /*----- Data structures ---------------------------------------------------*/ +typedef struct test_results { + unsigned tests, failed; +} test_results; + /* --- Test field definition --- */ typedef struct test_type { @@ -62,6 +66,11 @@ typedef struct test_chunk { const test_type *f[TEST_FIELDMAX]; /* Field definitions */ } test_chunk; +typedef struct test_suite { + const char *name; /* Name of this suite */ + const test_chunk *chunks; /* Chunks contained in this suite */ +} test_suite; + /*----- Predefined data types ---------------------------------------------*/ extern const test_type type_hex; @@ -73,6 +82,23 @@ extern const test_type type_uint32; /*----- Functions provided ------------------------------------------------*/ +/* --- @test_do@ --- * + * + * Arguments: @const test_suite suites[]@ = pointer to suite definitions + * @FILE *fp@ = test vector file, ready opened + * @test_results *results@ = where to put results + * + * Returns: Negative if something bad happened, or the number of + * failures. + * + * Use: Runs a collection of tests against a file of test vectors and + * reports the results. + */ + +extern int test_do(const test_suite /*suite*/[], + FILE */*fp*/, + test_results */*results*/); + /* --- @test_run@ --- * * * Arguments: @int argc@ = number of command line arguments -- [mdw]