.\" -*-nroff-*-
.\"
.\" Manual for the test vector framework
.\"
.\" (c) 2024 Straylight/Edgeware
.\"
.
.\"----- Licensing notice ---------------------------------------------------
.\"
.\" This file is part of the mLib utilities library.
.\"
.\" mLib is free software: you can redistribute it and/or modify it under
.\" the terms of the GNU Library General Public License as published by
.\" the Free Software Foundation; either version 2 of the License, or (at
.\" your option) any later version.
.\"
.\" mLib is distributed in the hope that it will be useful, but WITHOUT
.\" ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
.\" FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
.\" License for more details.
.\"
.\" You should have received a copy of the GNU Library General Public
.\" License along with mLib.  If not, write to the Free Software
.\" Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
.\" USA.
.
.\"--------------------------------------------------------------------------
.so ../defs.man \" @@@PRE@@@
.
.\"--------------------------------------------------------------------------
.TH tvec 3mLib "11 March 2024" "Straylight/Edgeware" "mLib utilities library"
.\" @tvec_begin
.\" @tvec_end
.\" @tvec_read
.\" @tvec_humanoutput
.\" @tvec_tapoutput
.\" @tvec_dfltoutput
.
.\"--------------------------------------------------------------------------
.SH NAME
tvec \- test vector framework
.
.\"--------------------------------------------------------------------------
.SH SYNOPSIS
.nf
.B "#include <mLib/tvec.h>"
.PP
.ta 2n
.B "union tvec_misc {"
.B "	const void *p;"
.B "	long i;"
.B "	unsigned long u;"
.B "	double f;"
.B "};"
.B "enum {"
.B "	TVMISC_PTR,"
.B "	TVMISC_INT,"
.B "	TVMISC_UINT,"
.B "	TVMISC_FLT,"
.B "	...,"
.B "	TVMISC_LIMIT,"
.B "};"
.PP
.ta 2n +2n
.B "union tvec_regval {"
.B "	long i;"
.B "	unsigned long u;"
.B "	void *p;"
.B "	double f;"
.B "	struct { char *p; size_t sz; } text;"
.B "	struct { unsigned char *p; size_t sz; } bytes;"
.B "	struct {"
.B "		unsigned char *p; size_t sz;"
.B "		size_t a, m;"
.B "		size_t off;"
.B "	} buf;"
.B "	TVEC_REGSLOTS"
.B "};"
.B "struct tvec_reg {"
.B "	unsigned f;"
.B "	union tvec_regval v;"
.B "};"
.B "#define TVRF_LIVE ..."
.PP
.ta 2n
.B "struct tvec_regdef {"
.B "	const char *name;"
.B "	const struct tvec_regty *ty;"
.B "	unsigned i;"
.B "	unsigned f;"
.B "	union tvec_misc arg;"
.B "};"
.B "#define TVRF_UNSET ..."
.B "#define TVRF_OPT ..."
.B "#define TVRF_ID ..."
.B "#define TVEC_ENDREGS ..."
.PP
.B "struct tvec_state;"
.PP
.B "struct tvec_env;"
.ta \w'\fBtypedef void tvec_testfn('u
.BI "typedef void tvec_testfn(const struct tvec_reg *" in ,
.BI "	struct tvec_reg *" out ,
.BI "	void *" ctx );
.ta 2n
.B "struct tvec_test {"
.B "	const char *name;"
.B "	const struct tvec_regdef *regs;"
.B "	const struct tvec_env *env;"
.B "	tvec_testfn *fn;"
.B "};"
.B "#define TVEC_ENDTESTS ..."
.PP
.ta 2n
.B "struct tvec_config {"
.B "	const struct tvec_test *tests;"
.B "	unsigned nrout, nreg;"
.B "	size_t regsz;"
.B "};"
.B "struct tvec_output;"
.PP
.ta \w'\fBvoid tvec_begin('u
.BI "void tvec_begin(struct tvec_state *" tv_out ,
.BI "	const struct tvec_config *" config ,
.BI "	struct tvec_output *" o );
.BI "int tvec_end(struct tvec_state *" tv );
.BI "int tvec_read(struct tvec_state *" tv ", const char *" infile ", FILE *" fp );
.PP
.BI "extern struct tvec_output *tvec_humanoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_machineoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_tapoutput(FILE *" fp );
.BI "extern struct tvec_output *tvec_dfltoutput(FILE *" fp );
.fi
.
.\"--------------------------------------------------------------------------
.SH DESCRIPTION
.
The
.B <mLib/tvec.h>
header file provides definitions and declarations
for the core of mLib's
.IR "test vector framework" .
.PP
The test vector framework is rather large and complicated,
so the documentation for it is split into multiple manual pages.
This one provides a conceptual overview
and describes the essentials for using it to build simple tests.
.
.SS Conceptual overview
A
.I "test session"
runs a collection of tests
and reports on the outcome.
.PP
A
.I test
involves exercising some functionality
and checking that it behaves properly.
A test can have four
.IR outcomes .
It can
.IR pass :
the functionality behaved properly.
It can
.IR fail :
the functionality did not behave properly.
It can experience an
.IR "expected failure" :
the functionality behaved as expected,
but the expected behaviour is known to be incorrect.
Or it can be
.IR skipped :
for some reason, the test couldn't be performed.
.PP
Tests are gathered together into
.IR "test groups" .
Each test group has a name.
Like a individual tests, test groups also have outcomes:
they can pass, fail, or be skipped.
A test group cannot experience expected failure.
.PP
A session may also encounter
.IR errors ,
e.g., as a result of malformed input
or failures reported by system facilities.
.PP
A test session can either
be driven from data provided by an input file,
or it can be driven by the program alone.
The latter case is called
.I "ad-hoc testing",
and is described in
.BR tvec-adhoc (3).
This manual page describes file-driven testing.
.PP
When it begins a session for file-directed testing,
the program provides a table of
.IR "test definitions" .
A test definition has a
.IR name ,
and also specifies a
.IR "test function" ,
a
.IR "test environment" ,
and a table of
.IR "register definitions" .
Test environments are explained further below.
.PP
A
.I register
is a place which can store a single item of test data;
registers are the means
by which input test data is provided to a test function,
and by which a test function returns its results.
A test definition's register definitions
establish a collection of
.I active
registers.
Each active register has a
.IR name ,
an
.IR index ,
and a
.IR type ,
which are established by its register definition.
The register's name is used to refer to the register in the test data file,
and its index is used to refer to it
in the test function and test environments.
The register's type describes the acceptable values for the register,
and how they are to be compared,
read from the input file,
and dumped in diagnostic output.
New register types can be defined fairly easily: see
.BR tvec_tyimpl (3)
for the details.
A register definition may describe an
.I input
register or an
.I output
register:
input registers provide input data to the test function, while
output registers collect output data from the test function.
The data file provides values for both input and output registers:
the values for the input registers are passed to the test function;
the values for the output registers are
.I "reference outputs"
against which the test function's outputs are to be checked.
.PP
The test function is called with two vectors of registers,
one containing input values for the test function to read
and also reference output values,
and another for output values that the test function should write;
and a
.I context
provided by the test environment.
The test function's task is to exercise the functionality to be tested,
providing it the input data from its input registers,
and collecting the output in its output registers.
It is the responsibility of the test environment or the framework
to compare the output register values against reference values
provided in the input data.
.PP
The input file syntax is described in full below.
In overview, it is a
.BR .ini -style
file.
Comments begin with a semicolon character
.RB ` ; ',
and extend to the end of the line.
It is divided into
.I sections
by headings in square brackets:
.IP
.BR [ test ]
.PP
Each section contains a number of
.I paragraphs
separated by blank lines.
Each paragraph consists of one or more
.I assignments
of the form
.IP
.IB reg " = " value
.PP
or
.IP
.IB reg ": " value
.PP
When the framework encounters a section heading,
it finishes any test group currently in progress,
and searches for a test definition whose name matches the
.I test
name in the section heading.
If it finds a match,
it begins a new test group with the same name.
Each paragraph of assignments is used to provide
input and reference output values
for a single test.
The
.I reg
name in an assignment must match the name
of an active register or a
.I "special variable" ;
the corresponding
.I value
is stored in the named register or variable.
.PP
A register which has been assigned a value is said to be
.IR live ;
otherwise, it is
.IR dead .
By default, every active register must be live for a test to proceed;
but a register definition can mark its register as
.I optional
or
.IR may-be-unset .
An optional register need not be assigned a value explicitly;
instead, the register is left dead.
A may-be-unset register must be mentioned,
but a distinctive syntax
.IP
.IB reg " *"
.PP
(with no colon or equals sign)
says that the register should be left dead.
Optional registers are suitable for cases where
there is an obvious default value
which would otherwise be mentioned frequently in input files.
May-be-unset registers are mostly useful as outputs,
where the output is not always set,
e.g., in error cases,
but where omitting a value in the usual case is likely a mistake.
.PP
A test environment fits in between
the framework and the test function.
It can establish hook functions which are called
at various points throughout a test group
(at the start and and, and before and after each test).
It can define special variables
which can be set from the input file using assignments.
And, finally, it can take on the responsibility
of running the test function.
The registers will have been set up already,
based on the assignments in the input file,
but the environment can modify them.
It must also check the test function's output
against the reference values,
though there are functions provided for doing this.
The environment can choose to run the test function once,
multiple times, or, indeed, not at all.
When it calls the test function, it can provide a context pointer,
with whatever additional information might be useful:
this usually involves coordination
between the environment and the test function.
It is the test environment's responsibility
to check the outputs returned by the test function
and to report on mismatches,
but there are functions provided by the framework
to do the heavy lifting.
.PP
The following are examples of what test environments can do.
.hP \*o
It can fill in default values for optional dead registers.
For example, if a function returnns error codes,
then you can save reptition in the input file
by marking the error-code output register optional
and letting it default to the `success' value in a test environment.
.hP \*o
It can run the test function mulitple times.
For example, a test of functions like
.BR strcmp (3)
might run the test twice,
first with its operands as supplied by the input file,
and then again with the operands swapped
and the opposite expected result.
A test for bignum addition could verify commutativity
by checking both that
.IR x "\ + \ " y "\ =\ " z
and that
.IR y "\ + \ " x "\ =\ " z \fR.
Similarly, a subtraction test could check both that
.IR x "\ \- \ " y "\ =\ " z
and that
.IR y "\ \- \ " x "\ =\ \-" z \fR.
.hP \*o
The
.BR tvec-remote (3),
.BR tvec-timeout (3),
and
.BR tvec-bench (3)
extensions all slot in as test environments.
Sadly,
.BR tvec-bench (3)
requires support from the output protocol
in order to format benchmark results properly;
apart from that,
these three facilities are pure extensions