From bca75e8d0ad7ce51192de3ef053ec63d0c79cfaa Mon Sep 17 00:00:00 2001 Message-Id: From: Mark Wooding Date: Thu, 22 Feb 2024 19:49:54 +0000 Subject: [PATCH] @@@ output doc Organization: Straylight/Edgeware From: Mark Wooding --- test/tvec-output.c | 492 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 484 insertions(+), 8 deletions(-) diff --git a/test/tvec-output.c b/test/tvec-output.c index add820d..c0573f1 100644 --- a/test/tvec-output.c +++ b/test/tvec-output.c @@ -710,7 +710,23 @@ static void report_location(struct human_output *h, FILE *fp, #undef f_flush } -/* Output layout. Pass everything along to the layout machinery above. */ +/* --- @human_writech@, @human_write@, @human_writef@ --- * + * + * Arguments: @void *go@ = output sink, secretly a @struct human_output@ + * @int ch@ = character to write + * @const char *@p@, @size_t sz@ = string (with explicit length) + * to write + * @const char *p, ...@ = format control string and arguments to + * write + * + * Returns: --- + * + * Use: Write characters, strings, or formatted strings to the + * output, applying appropriate layout. + * + * For the human output driver, the layout machinery just strips + * trailing spaces. + */ static int human_writech(void *go, int ch) { struct human_output *h = go; return (layout_char(&h->lyt, ch)); } @@ -733,11 +749,35 @@ static int human_nwritef(void *go, size_t maxsz, const char *p, ...) static const struct gprintf_ops human_printops = { human_writech, human_writem, human_nwritef }; -/* Output methods. */ +/* --- @human_bsession@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @struct tvec_state *tv@ = the test state producing output + * + * Returns: --- + * + * Use: Begin a test session. + * + * The human driver just records the test state for later + * reference. + */ static void human_bsession(struct tvec_output *o, struct tvec_state *tv) { struct human_output *h = (struct human_output *)o; h->tv = tv; } +/* --- @report_unusual@ --- * + * + * Arguments: @struct human_output *h@ = output sink + * @unsigned nxfail, nskip@ = number of expected failures and + * skipped tests + * + * Returns: --- + * + * Use: Write (directly on the output stream) a note about expected + * failures and/or skipped tests, if there were any. + */ + static void report_unusual(struct human_output *h, unsigned nxfail, unsigned nskip) { @@ -764,6 +804,19 @@ static void report_unusual(struct human_output *h, #undef f_any } +/* --- @human_esession@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * + * Returns: Suggested exit status. + * + * Use: End a test session. + * + * The human driver prints a final summary of the rest results + * and returns a suitable exit code. + */ + static int human_esession(struct tvec_output *o) { struct human_output *h = (struct human_output *)o; @@ -804,6 +857,20 @@ static int human_esession(struct tvec_output *o) h->tv = 0; return (tv->f&TVSF_ERROR ? 2 : all_lose ? 1 : 0); } +/* --- @human_bgroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * + * Returns: --- + * + * Use: Begin a test group. + * + * The human driver determines the length of the longest + * register name, resets the group progress scoreboard, and + * activates the progress display. + */ + static void human_bgroup(struct tvec_output *o) { struct human_output *h = (struct human_output *)o; @@ -812,6 +879,21 @@ static void human_bgroup(struct tvec_output *o) dstr_reset(&h->scoreboard); show_progress(h); } +/* --- @human_skipgroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @const char *excuse@, @va_list *ap@ = reason for skipping the + * group, or null + * + * Returns: --- + * + * Use: Report that a test group is being skipped. + * + * The human driver just reports the situation to its output + * stream. + */ + static void human_skipgroup(struct tvec_output *o, const char *excuse, va_list *ap) { @@ -828,6 +910,18 @@ static void human_skipgroup(struct tvec_output *o, fputc('\n', h->lyt.fp); } +/* --- @human_egroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * + * Returns: --- + * + * Use: Report that a test group has finished. + * + * The human driver reports a summary of the group's tests. + */ + static void human_egroup(struct tvec_output *o) { struct human_output *h = (struct human_output *)o; @@ -851,9 +945,41 @@ static void human_egroup(struct tvec_output *o) fputc('\n', h->lyt.fp); } +/* --- @human_btest@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * + * Returns: --- + * + * Use: Report that a test is starting. + * + * The human driver makes sure the progress display is active. + */ + static void human_btest(struct tvec_output *o) { struct human_output *h = (struct human_output *)o; show_progress(h); } +/* --- @human_outcome@, @human_skip@, @human_fail@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @unsigned attr@ = attribute to apply to the outcome + * @const char *outcome@ = outcome string to report + * @const char *detail@, @va_list *ap@ = a detail message + * @const char *excuse@, @va_list *ap@ = reason for skipping the + * test + * + * Returns: --- + * + * Use: Report that a test has been skipped or failed. + * + * The human driver reports the situation on its output stream. + */ + +static void human_outcome(struct tvec_output *o, + unsigned attr, const char *outcome, + const char *detail, va_list *ap) { struct human_output *h = (struct human_output *)o; struct tvec_state *tv = h->tv; @@ -873,6 +999,22 @@ static void human_fail(struct tvec_output *o, const char *detail, va_list *ap) { human_outcome(o, HA_LOSE, "FAILED", detail, ap); } +/* --- @human_dumpreg@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @unsigned disp@ = register disposition + * @const union tvec_regval *rv@ = register value + * @const struct tvec_regdef *rd@ = register definition + * + * Returns: --- + * + * Use: Dump a register. + * + * The human driver applies highlighting to mismatching output + * registers, but otherwise delegates to the register type + * handler and the layout machinery. + */ static void human_dumpreg(struct tvec_output *o, unsigned disp, const union tvec_regval *rv, @@ -894,6 +1036,20 @@ static void human_dumpreg(struct tvec_output *o, setattr(h, HA_PLAIN); layout_char(&h->lyt, '\n'); } +/* --- @human_etest@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @unsigned outcome@ = the test outcome + * + * Returns: --- + * + * Use: Report that a test has finished. + * + * The human driver reactivates the progress display, if + * necessary, and adds a new character for the completed test. + */ + static void human_etest(struct tvec_output *o, unsigned outcome) { struct human_output *h = (struct human_output *)o; @@ -913,6 +1069,21 @@ static void human_etest(struct tvec_output *o, unsigned outcome) } } +/* --- @human_bbench@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @const char *ident@ = identifying register values + * @unsigned unit@ = measurement unit (@TVBU_...@) + * + * Returns: --- + * + * Use: Report that a benchmark has started. + * + * The human driver just prints the start of the benchmark + * report. + */ + static void human_bbench(struct tvec_output *o, const char *ident, unsigned unit) { @@ -923,6 +1094,22 @@ static void human_bbench(struct tvec_output *o, fprintf(h->lyt.fp, "%s: %s: ", tv->test->name, ident); fflush(h->lyt.fp); } +/* --- @human_ebench@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @const char *ident@ = identifying register values + * @unsigned unit@ = measurement unit (@TVBU_...@) + * @const struct bench_timing *tm@ = measurement + * + * Returns: --- + * + * Use: Report a benchmark's results + * + * The human driver just delegates to the default benchmark + * reporting, via the layout machinery. + */ + static void human_ebench(struct tvec_output *o, const char *ident, unsigned unit, const struct bench_timing *tm) @@ -933,6 +1120,23 @@ static void human_ebench(struct tvec_output *o, fputc('\n', h->lyt.fp); } +/* --- @human_report@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * @unsigned level@ = message level (@TVLEV_...@) + * @const char *msg@, @va_list *ap@ = format string and + * arguments + * + * Returns: --- + * + * Use: Report a message to the user. + * + * The human driver arranges to show the message on @stderr@ as + * well as the usual output, with a certain amount of + * intelligence in case they're both actually the same device. + */ + static void human_report(struct tvec_output *o, unsigned level, const char *msg, va_list *ap) { @@ -954,6 +1158,16 @@ static void human_report(struct tvec_output *o, unsigned level, show_progress(h); } +/* --- @human_destroy@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * human_output@ + * + * Returns: --- + * + * Use: Release the resources held by the output driver. + */ + static void human_destroy(struct tvec_output *o) { struct human_output *h = (struct human_output *)o; @@ -973,6 +1187,20 @@ static const struct tvec_outops human_ops = { human_destroy }; +/* --- @tvec_humanoutput@ --- * + * + * Arguments: @FILE *fp@ = output file to write on + * + * Returns: An output formatter. + * + * Use: Return an output formatter which writes on @fp@ with the + * expectation that a human will be watching and interpreting + * the output. If @fp@ denotes a terminal, the display shows a + * `scoreboard' indicating the outcome of each test case + * attempted, and may in addition use colour and other + * highlighting. + */ + struct tvec_output *tvec_humanoutput(FILE *fp) { struct human_output *h; @@ -1010,14 +1238,33 @@ struct tvec_output *tvec_humanoutput(FILE *fp) /*----- Perl's `Test Anything Protocol' -----------------------------------*/ struct tap_output { - struct tvec_output _o; - struct tvec_state *tv; - struct layout lyt; - unsigned last; - char *outbuf; size_t outsz; - int maxlen; + struct tvec_output _o; /* output base class */ + struct tvec_state *tv; /* stashed testing state */ + struct layout lyt; /* output layout */ + char *outbuf; size_t outsz; /* buffer for formatted output */ + unsigned grpix, testix; /* group and test indices */ + unsigned previx; /* previously reported test index */ + int maxlen; /* longest register name */ }; +/* --- @tap_writech@, @tap_write@, @tap_writef@ --- * + * + * Arguments: @void *go@ = output sink, secretly a @struct tap_output@ + * @int ch@ = character to write + * @const char *@p@, @size_t sz@ = string (with explicit length) + * to write + * @const char *p, ...@ = format control string and arguments to + * write + * + * Returns: --- + * + * Use: Write characters, strings, or formatted strings to the + * output, applying appropriate layout. + * + * For the TAP output driver, the layout machinery prefixes each + * line with ` ## ' and strips trailing spaces. + */ + static int tap_writech(void *go, int ch) { struct tap_output *t = go; return (layout_char(&t->lyt, ch)); } @@ -1039,6 +1286,21 @@ static int tap_nwritef(void *go, size_t maxsz, const char *p, ...) static const struct gprintf_ops tap_printops = { tap_writech, tap_writem, tap_nwritef }; +/* --- @tap_bsession@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @struct tvec_state *tv@ = the test state producing output + * + * Returns: --- + * + * Use: Begin a test session. + * + * The TAP driver records the test state for later reference, + * initializes the group index counter, and prints the version + * number. + */ + static void tap_bsession(struct tvec_output *o, struct tvec_state *tv) { struct tap_output *t = (struct tap_output *)o; @@ -1047,6 +1309,18 @@ static void tap_bsession(struct tvec_output *o, struct tvec_state *tv) fputs("TAP version 13\n", t->lyt.fp); /* but secretly 14 really */ } +/* --- @tap_esession@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * + * Returns: Suggested exit status. + * + * Use: End a test session. + * + * The TAP driver prints a final summary of the rest results + * and returns a suitable exit code. + */ static int tap_esession(struct tvec_output *o) { @@ -1064,6 +1338,20 @@ static int tap_esession(struct tvec_output *o) t->tv = 0; return (tv->all[TVOUT_LOSE] ? 1 : 0); } +/* --- @tap_bgroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * + * Returns: --- + * + * Use: Begin a test group. + * + * The TAP driver determines the length of the longest + * register name, resets the group progress scoreboard, and + * activates the progress display. + */ + static void tap_bgroup(struct tvec_output *o) { struct tap_output *t = (struct tap_output *)o; @@ -1074,6 +1362,21 @@ static void tap_bgroup(struct tvec_output *o) fprintf(t->lyt.fp, "# Subtest: %s\n", tv->test->name); } +/* --- @tap_skipgroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @const char *excuse@, @va_list *ap@ = reason for skipping the + * group, or null + * + * Returns: --- + * + * Use: Report that a test group is being skipped. + * + * The TAP driver just reports the situation to its output + * stream. + */ + static void tap_skipgroup(struct tvec_output *o, const char *excuse, va_list *ap) { @@ -1085,6 +1388,18 @@ static void tap_skipgroup(struct tvec_output *o, fputc('\n', t->lyt.fp); } +/* --- @tap_egroup@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * + * Returns: --- + * + * Use: Report that a test group has finished. + * + * The TAP driver reports a summary of the group's tests. + */ + static void tap_egroup(struct tvec_output *o) { struct tap_output *t = (struct tap_output *)o; @@ -1096,9 +1411,45 @@ static void tap_egroup(struct tvec_output *o) t->grpix, tv->test->name); } +/* --- @tap_btest@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * + * Returns: --- + * + * Use: Report that a test is starting. + * + * The TAP driver advances its test counter. (We could do this + * by adding up up the counters in @tv->curr@, and add on the + * current test, but it's easier this way.) + */ + static void tap_btest(struct tvec_output *o) { struct tap_output *t = (struct tap_output *)o; t->testix++; } +/* --- @tap_outcome@, @tap_skip@, @tap_fail@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @unsigned attr@ = attribute to apply to the outcome + * @const char *outcome@ = outcome string to report + * @const char *detail@, @va_list *ap@ = a detail message + * @const char *excuse@, @va_list *ap@ = reason for skipping the + * test + * + * Returns: --- + * + * Use: Report that a test has been skipped or failed. + * + * The TAP driver reports the situation on its output stream. + * TAP only allows us to report a single status for each + * subtest, so we notice when we've already reported a status + * for the current test and convert the second report as a + * comment. This should only happen in the case of multiple + * failures. + */ + static void tap_outcome(struct tvec_output *o, const char *head, const char *tail, const char *detail, va_list *ap) @@ -1120,6 +1471,24 @@ static void tap_skip(struct tvec_output *o, const char *excuse, va_list *ap) static void tap_fail(struct tvec_output *o, const char *detail, va_list *ap) { tap_outcome(o, "not ok", "", detail, ap); } +/* --- @tap_dumpreg@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @unsigned disp@ = register disposition + * @const union tvec_regval *rv@ = register value + * @const struct tvec_regdef *rd@ = register definition + * + * Returns: --- + * + * Use: Dump a register. + * + * The TAP driver applies highlighting to mismatching output + * registers, but otherwise delegates to the register type + * handler and the layout machinery. The result is that the + * register dump is marked as a comment and indented. + */ + static void tap_dumpreg(struct tvec_output *o, unsigned disp, const union tvec_regval *rv, const struct tvec_regdef *rd) @@ -1134,6 +1503,20 @@ static void tap_dumpreg(struct tvec_output *o, layout_char(&t->lyt, '\n'); } +/* --- @tap_etest@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @unsigned outcome@ = the test outcome + * + * Returns: --- + * + * Use: Report that a test has finished. + * + * The TAP driver reports the outcome of the test, if that's not + * already decided. + */ + static void tap_etest(struct tvec_output *o, unsigned outcome) { switch (outcome) { @@ -1146,10 +1529,42 @@ static void tap_etest(struct tvec_output *o, unsigned outcome) } } +/* --- @tap_bbench@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @const char *ident@ = identifying register values + * @unsigned unit@ = measurement unit (@TVBU_...@) + * + * Returns: --- + * + * Use: Report that a benchmark has started. + * + * The TAP driver does nothing here. All of the reporting + * happens in @tap_ebench@. + */ + static void tap_bbench(struct tvec_output *o, const char *ident, unsigned unit) { ; } +/* --- @tap_ebench@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @const char *ident@ = identifying register values + * @unsigned unit@ = measurement unit (@TVBU_...@) + * @const struct bench_timing *tm@ = measurement + * + * Returns: --- + * + * Use: Report a benchmark's results + * + * The TAP driver just delegates to the default benchmark + * reporting, via the layout machinery so that the result is + * printed as a comment. + */ + static void tap_ebench(struct tvec_output *o, const char *ident, unsigned unit, const struct bench_timing *tm) @@ -1162,6 +1577,22 @@ static void tap_ebench(struct tvec_output *o, layout_char(&t->lyt, '\n'); } +/* --- @tap_report@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * @unsigned level@ = message level (@TVLEV_...@) + * @const char *msg@, @va_list *ap@ = format string and + * arguments + * + * Returns: --- + * + * Use: Report a message to the user. + * + * The TAP driver converts error reports into TAP `Bail out!' + * errors. Less critical notices end up as comments. + */ + static void tap_report(struct tvec_output *o, unsigned level, const char *msg, va_list *ap) { @@ -1179,6 +1610,16 @@ static void tap_report(struct tvec_output *o, unsigned level, gprintf(gops, go, msg, ap); gops->putch(go, '\n'); } +/* --- @tap_destroy@ --- * + * + * Arguments: @struct tvec_output *o@ = output sink, secretly a @struct + * tap_output@ + * + * Returns: --- + * + * Use: Release the resources held by the output driver. + */ + static void tap_destroy(struct tvec_output *o) { struct tap_output *t = (struct tap_output *)o; @@ -1197,6 +1638,28 @@ static const struct tvec_outops tap_ops = { tap_destroy }; +/* --- @tvec_tapoutput@ --- * + * + * Arguments: @FILE *fp@ = output file to write on + * + * Returns: An output formatter. + * + * Use: Return an output formatter which writes on @fp@ in `TAP' + * (`Test Anything Protocol') format. + * + * TAP comes from the Perl community, but has spread rather + * further. This driver produces TAP version 14, but pretends + * to be version 13. The driver produces a TAP `test point' -- + * i.e., a result reported as `ok' or `not ok' -- for each input + * test group. Failure reports and register dumps are produced + * as diagnostic messages before the final group result. (TAP + * permits structuerd YAML data after the test-point result, + * which could be used to report details, but (a) postponing the + * details until after the report is inconvenient, and (b) there + * is no standardization for the YAML anyway, so in practice + * it's no more useful than the unstructured diagnostics. + */ + struct tvec_output *tvec_tapoutput(FILE *fp) { struct tap_output *t; @@ -1209,6 +1672,19 @@ struct tvec_output *tvec_tapoutput(FILE *fp) /*----- Default output ----------------------------------------------------*/ +/* --- @tvec_dfltoutput@ --- * + * + * Arguments: @FILE *fp@ = output file to write on + * + * Returns: An output formatter. + * + * Use: Selects and instantiates an output formatter suitable for + * writing on @fp@. The policy is subject to change, but + * currently the `human' output format is selected if @fp@ is + * interactive (i.e., if @isatty(fileno(fp))@ is true), and + * otherwise the `tap' format is used. + */ + struct tvec_output *tvec_dfltout(FILE *fp) { int ttyp = getenv_boolean("TVEC_TTY", -1); -- [mdw]