From e32a2ebcbf999ccd39c73317c3ea87f3cbdf3b49 Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Thu, 18 Oct 2007 16:29:23 +0100 Subject: [PATCH] * adt-testreport-runloop: manpage * adt-testreport-runloop: better command line parsing * adt-testreport-runloop: removed old lalonde junk * adt-testreport-runloop: replaced `..._ok' vars with sentinel files * hosts/chinstrap/update-suppressions: big comment * hosts/cadmium/onepackage-config: set PATH (was in lalonde-nightly) * adt-testreport-onepackage: many manpage improvements --- debian/changelog | 12 +++ hosts/cadmium/cronjob | 3 +- hosts/cadmium/lalonde-nightly | 72 ++++++------- hosts/cadmium/onepackage-config | 3 + hosts/chinstrap/update-suppressions | 32 ++++++ runner/adt-testreport-onepackage.1 | 125 ++++++++++++++++++++--- runner/adt-testreport-runloop.1 | 150 ++++++++++++++++++++++++++++ 7 files changed, 348 insertions(+), 49 deletions(-) create mode 100644 runner/adt-testreport-runloop.1 diff --git a/debian/changelog b/debian/changelog index a180712..5a9c0ef 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +autopkgtest (1.1.0) unstable; urgency=low + + * adt-testreport-runloop: manpage + * adt-testreport-runloop: better command line parsing + * adt-testreport-runloop: removed old lalonde junk + * adt-testreport-runloop: replaced `..._ok' vars with sentinel files + * hosts/chinstrap/update-suppressions: big comment + * hosts/cadmium/onepackage-config: set PATH (was in lalonde-nightly) + * adt-testreport-onepackage: many manpage improvements + + + autopkgtest (1.0.9) unstable; urgency=low * Separate email_sourcepackage_header and email_binarypackage_header diff --git a/hosts/cadmium/cronjob b/hosts/cadmium/cronjob index 28d45ce..3db7ded 100755 --- a/hosts/cadmium/cronjob +++ b/hosts/cadmium/cronjob @@ -3,4 +3,5 @@ set -e cd adt-play exec >>overall.log 2>&1 test -f go -with-lock-ex -q nightly-lock ./lalonde-nightly max_test_count=100 +with-lock-ex -q nightly-lock \ + ./lalonde-nightly @onepackage-config max_test_count=100 diff --git a/hosts/cadmium/lalonde-nightly b/hosts/cadmium/lalonde-nightly index 70ce91a..7f1f9de 100755 --- a/hosts/cadmium/lalonde-nightly +++ b/hosts/cadmium/lalonde-nightly @@ -2,72 +2,74 @@ set -e -PATH=/home/iwj/autopkgtest/runner:/usr/local/sbin:/usr/sbin:$PATH -export PATH - -#any_tests_ok=false -any_tests_ok=true -source_tests_ok=true -binary_tests_ok=true -gutsy_tests_ok=true max_test_count=10 go_anyway=false -for x in "$@"; do eval "$x"; done +for arg in "$@"; do + case "$arg" in + !*) arg="${arg#!}" ;; + *) onepackage_args[${#onepackage_args[*]}]="${arg#@}" ;; + esac + + case "$arg" in + @*) ;; + *=*) eval "$arg" ;; + *) . "$arg" ;; + esac +done + +if [ "x$distro" = x ]; then + : ${testbed_check_path:=/var/lib/autopkgtest/xenlvm/adt_${distro}/good} +else + : ${testbed_check_path:=/dev/null} +fi -: ${testbed_check_path:=/var/lib/autopkgtest/xenlvm/adt_gutsy/good} +if [ "x$test_classes" = x]; then + : ${architecture:=`dpkg --print-architecture`} + test_classes="source $distro, target=source : binary $distro, target=binary-$architecture" +fi fail () { printf >&2 "%s\n" "$*"; exit 127; } progress () { printf "========== %s ==========\n" "$1"; } x () { printf "+ %s\n" "$*"; "$@"; } x2 () { printf >&2 "+ %s\n" "$*"; "$@"; } -wait_for () { - remain=$1; interval=$2; on_not=$3; shift;shift;shift - progress "await ($interval/$remain) $*" - while true; do - date - if "$@"; then return; fi - remain=$(( $remain - $interval )) - if [ $remain -lt 0 ]; then fail "wait timed out for $*"; fi - eval "$on_not" - sleep $interval - done -} - progress testing -xoptslist="source gutsy, target=source:binary gutsy, target=binary-amd64" - test_count=0 -while $any_tests_ok && ($go_anyway || test -f go) && test $test_count -lt $max_test_count; do +while ($go_anyway || test -f go) && test $test_count -lt $max_test_count; do now=`date` printf "%s" "$now: " - if ! test -f $testbed_check_path; then + if ! test -e $testbed_check_path; then printf " testbed broken, bailing\n" break fi - xopts="${xoptslist%%:*}" + xopts="${test_classes%%:*}" xopts_opts="${xopts#*,}" - xoptslist="${xoptslist#*:}:$xopts" + test_classes="${test_classes#*:}:$xopts" printf "considering (%s) " "$xopts_opts" anybroken=false for x in ${xopts%%,*}; do - if eval \$${x}_tests_ok; then :; else - printf " skipping because of %s\n" "$x" + if test -f "stop-$x"; then + printf " skipping because of %s" "$x" anybroken=true break fi done - if $anybroken; then continue; fi + if $anybroken; then + printf "\n" + continue + fi - adt-testreport-onepackage /home/iwj/adt-play/onepackage-config \ - adtrun_extra_opts=--built-binaries-filter=__ \ - interactive=false $xopts_opts + adt-testreport-onepackage \ + adtrun_extra_opts=--built-binaries-filter=__ \ + interactive=false \ + "${onepackage_args[@]}" \ + $xopts_opts test_count=$(( $test_count + 1 )) done diff --git a/hosts/cadmium/onepackage-config b/hosts/cadmium/onepackage-config index 616f068..08f012c 100644 --- a/hosts/cadmium/onepackage-config +++ b/hosts/cadmium/onepackage-config @@ -1,4 +1,7 @@ cd /home/iwj/adt-play +PATH=/home/iwj/autopkgtest/runner:/usr/local/sbin:/usr/sbin:$PATH +export PATH + mirror=http://archive.ubuntu.com/ubuntu distro=gutsy suite=main diff --git a/hosts/chinstrap/update-suppressions b/hosts/chinstrap/update-suppressions index 35f813c..cbd61a5 100755 --- a/hosts/chinstrap/update-suppressions +++ b/hosts/chinstrap/update-suppressions @@ -1,4 +1,36 @@ #!/bin/sh +# +# This is an absolutely hideous hack. +# +# We scrape a suitable launchpad advanced search to find bugs we've +# previously submitted. +# +# We have to run this on chinstrap instead of +# cadmium because cadmium (being treated as a buildd from a security +# point of view) is not permitted to make external requests other than +# to the archive. +# +# +# For the reasons described below this should be replaced ASAP! +# +# BUGS +# +# This will break if the list gets too long for LP to display. +# +# Also, there is a problem with Duplicate bugs. See LP #147754. A +# tedious workaround is needed for this case. When an autopkgtest +# bug, which was marked a duplicate and not the master, is fixed, the +# duplicate target will be marked fixed. To work around the LP bug, +# it is necessary to adjust the status of the autopkgtest-filed bug: +# 1. unduplicate the bug; 2. set it to Fix Released; +# 3. duplicate it again. +# +# Occasionally LP seems to burp and give us an empty page. +# This means that a cronmail is generated which reports +# NO SUPPRESSIONS - PROBABLY WENT WRONG +# It is not clear whether the lack of any error message or nonzero +# exit status from curl is a bug in curl or LP. + cd ${0%/*} diff --git a/runner/adt-testreport-onepackage.1 b/runner/adt-testreport-onepackage.1 index 2d8286f..234b4cf 100644 --- a/runner/adt-testreport-onepackage.1 +++ b/runner/adt-testreport-onepackage.1 @@ -8,13 +8,16 @@ adt\-testreport\-onepackage \- choose one package and test it with adt-run .SH DESCRIPTION .B adt\-testreport\-onepackage is a rather ad-hoc script which chooses a package to test, calls -adt-run, and reports the results. a-t-o has the following -functionality: +adt-run, and reports the results. In a service installation +a-t-o is usually called from +.BR adt\-testreport\-runloop . -a-t-o downloads the Sources file, chooses a package to test (see -below), downloads its source, invokes adt-run, writes a report -describing the outcome, and emails the report to some appropriate -address. +a-t-o has the following functionality: + +a-t-o downloads the Sources or Packages file, chooses a package to +test (see below), downloads its source, invokes \fBadt\-run\fR, writes a +report describing the outcome, and emails the report to some +appropriate address. \fBWARNING\fR - it is essential to reconfigure this program before use, as the supplied examples will \fBsend emails\fR to people who @@ -29,7 +32,11 @@ Various variables can be set for useful effect. A selection of the most usual ones, with default values, can be seen at the top of the a-t-o script itself. Certain others are mentioned in this manpage. There is no exhaustive list and no documentation of the semantics. -.SH PACKAGE SELECTION +.SH PACKAGE AND TEST SELECTION +The configuration variable \fBtarget\fR specifies whether a source +package, or a binary package, is to be tested. It should be set to +\fBsource\fR or \fBbinary\-\fR\fIarch\fR. + a-t-o maintains a record of when each package was last tested, what version was tested, and whether the package had any tests. @@ -37,8 +44,8 @@ This record is used to choose which package should be tested next: packages which have changed since the last test are preferred, as are packages which didn't lack tests last time they were examined. -If the configuration sets \fBpkg\fR to a package name (for example by -passing \fBpkg=\fR\fIpackage-name\fR), this overrides +If the configuration sets \fBpkg\fR to a package name (for example if +\fBpkg=\fR\fIpackage\fR is passed as an argument), this overrides a-t-o's package selection algorithm. .SH REPORTING If \fBinteractive\fR is set to \fBtrue\fR (the default), a-t-o will @@ -59,6 +66,43 @@ specified with the \fBdest\fR* options (see the \fBonepackage-config\fR example configuration file). Some vague progress information is printed to a-t-o's stdout. +.SH AUTOMATIC BUG REPORTING +With suitable configuration, a-t-o can be made to automatically file +bugs when it detects a problem with a package. + +It is necessary to avoid the submission of multiple reports for the +same problem. This is done by fetching somehow a list of the open bug +reports from the relevant bug system and extracting a list of the +source package names. These are to be listed in the file named by the +.B suppresspackages +configuration variable. Testing frequency of these source packages is +reduced, and when a problem is found the email is not sent. + +When the system does decide to send an email about a package it will +normally email the maintainer as determined from the package's control +file. To submit bugs instead, set +.B maintainer_email_override +to the submission email address for the bug system. You must also set +.BR email_sourcepackage_header and email_binarypackage_header +to appropriate templates. These templates specify what information +should be prepended at the top of the body (not the RFC822 header) of +the mail to be sent. In these templates +.B @p +stands for the package name being tested (the source package name for +source packages or the binary package name for binary package tests), +.B @v +stands for the version being tested, +.B @s +stands for the source package name (even if a binary package generated +by that source is being tested) and +.B @a +stands for a single at sign \fB@\fR. + +If you an appropriately useable GPG signing key is available, you may +specify +.B email_signing_key +which is passed to gpg's \fB-u\fR option. Otherwise the email is not +signed. .SH FILES The \fBtmp=\fR\fItmp\fR and \fBvar=\fR\fIvar\fR configuration variables name two directories which must belong solely to this @@ -66,7 +110,9 @@ instance of a-t-o. \fItmp\fR will be erased by a-t-o at startup and then used for temporary files. It is not cleaned up afterwards; it is assumed that -the next run of a-t-o will do that. +the next run of a-t-o will do that. \fItmp\fR should not be made to +refer to a symlink or mountpoint as a-t-o would break the link or +attempt to remove the mountpoint; specify a subdirectory instead. \fIvar\fR is used to record the state, including the information about each package used for package selection. \fIvar\fR\fB/log\fR contains @@ -83,6 +129,41 @@ other other unexpected failures including bad usage, disk full, etc. When a-t-o is run in a loop, the loop should stop pending administrator intervention if a-t-o exits nonzero. +.SH FILES +.TP +.IR tmp " (default: \fB./tmp\fR)" +Temporary directory for working files. Do not specify /tmp. +.TP +.IR var " (default: \fB./var\fR)" +Storage directory for package testing history, score reports, +logfiles, etc. +.TP +.IB var /lastinfo- target +Information about the last test and outcome for each package. This is +used for package scoring and selection. +.TP +.IB var /scores- target +Report giving the score for each package, and the factors which +influenced that score. +.TP +.IB var /emailed/last- pkg , emailaddress +The summary of the last message sent to emailaddress about pkg. +If the configuration variable \fBsuppressrepeatedemails\fR is +\fBtrue\fR then this is used to ensure that a maintainer (or the +test installation operator) is only emailed when the situation for a +particular package changes. +.TP +.IB var /emailed/diff- pkg +Changes to the summary for the last email sent regarding pkg. +This can be used to find out why an email was sent. +.TP +.IB var /log +Logfile from a-t-o. If a-t-runloop is being used its stdout output +often makes a better logfile. +.TP +.IB destrsynchead / destdirfin / pkg +Full logfile and other information is rsync'd here. It is assumed +that this area will be published via a webserver. .SH SEE ALSO \fBadt-run\fR(1). .SH BUGS @@ -97,13 +178,31 @@ adjusted. This algorithm is not hugely sophisticated. There should be a facility for explicitly requesting retests (other than setting \fBpkg\fR); there should be some kind of manual prioritisation queue. -a-t-o cannot choose a distrorelease (eg, dapper, feisty, etch, sid) or -suite (eg main, non-free, universe); the caller must do that. +Distributions whose management systems permit archive skew (that is, +distributions whose contents sometimes contains unsatisfiable +dependencies) cause pathological behaviour. a-t-o will prefer to test +the newly-changed packages and immediately detect and report the +archive skew as test failures. It would be better if it would detect +this situation and not report a failure unless the problem persisted. +And there ought to be an option to have it not try to test a package +until its version has been stable for a nominated period. -Configuration by the use of bash fragments is rather unsophisticated. +If multiple distributions (multiple values of \fBdistro\fR) are used, +a-t-o will conflate their last-tested versions. It should use +separate files. + +Cross-architecture testing (where host and target have differing +Debian architectures) has not been tested and is likely to have +bugs both in interfaces and implementation. + +a-t-o should be capable of automatically closing bug reports if it +retests a package and finds that the bug is fixed. The script lacks flexibility in many respects; many additional configuration settings and hooks are needed. + +There is no facility for sharing test queues across multiple +machines. This means that it is difficult to share the testing load. .SH AUTHORS AND COPYRIGHT This manpage is part of autopkgtest, a tool for testing Debian binary packages. autopkgtest is Copyright (C) 2006-2007 Canonical Ltd and diff --git a/runner/adt-testreport-runloop.1 b/runner/adt-testreport-runloop.1 new file mode 100644 index 0000000..9536c7f --- /dev/null +++ b/runner/adt-testreport-runloop.1 @@ -0,0 +1,150 @@ +.TH adt\-testreport\-runloop 1 2007 autopkgtest "Linux Programmer's Manual" +.SH NAME +adt\-testreport\-runloop \- repeatedly choose and test packages +.SH SYNOPSYS +.B adt\-testreport\-runloop +.RI [[\fB!@\fR] variable \fB=\fR value |[\fB!@\fR] settings\-file ]... +.br +.SH DESCRIPTION +.B adt\-testreport\-runloop +is a rather ad-hoc script which repeatedly invokes +.BR adt\-testreport\-onepackage ; +it is designed to be invoked from cron. + +\fBWARNING\fR - it is essential to reconfigure this program before +use, as the supplied examples will \fBsend emails\fR to people who +will become \fBvery upset\fR with you! +.SH CONFIGURATION +Command-line arguments containing \fB=\fR are interpreted +with \fBeval\fR; other arguments are sourced using \fB.\fR, +in each case interpreted as bash script. + +Supplied arguments are both executed by a-t-r and passed through to +the invocation of a-t-o. You may prefix an argument with \fB@\fR to +indicate that it is for a-t-onepackage only, or with \fB!\fR to +indicate that it is for a-t-runloop only. + +Thus the set of variables which can be set is the union of those for +a-t-runloop itself and those for a-t-onepackage. See the a-t-o +manpage for such information about its configuration as is available. +Configuration fragment authors should bear in mind that both programs +use \fBset -e\fR. + +It can be a good idea to \fBcd\fR, and set PATH, in the configuration +file. + +The following variables may be set: +.TP +.B distro +distribution (`suite') to be tested. There is no default. This +variable is used only for the default values of +.B testbed_check_path +and +.BR test_classes . +However, it is probably a convenient way of specifying the intended +effect particularly as attempting to do cross-distribution testing in a +single installation does not work properly right now (see a-t-o's BUGS). +.TP +.B max_test_count +the maximum number of times to invoke a-t-o. This counts both passed +and failed tests, but it doesn't count possible runs which were +suppressed. Default: 10. +.TP +.B go_anway +Run even though the file \fBgo\fR does not exist. See SUPPRESSION, +below. Default: \fBfalse\fR. +.TP +.B architecture +Used in combination with \fBdistro\fR when calculating the right +default for test_classes. The default is the result from +.BR "dpkg --print-architecture" . +.TP +.B testbed_check_path +If the file named by this variable doesn't exist, a-t-r will exit. +This avoids it spinning and reporting many immediate failures due to a +broken testbed. The default is +.BI /var/lib/autopkgtest/xenlvm/adt_ distro /good +if distro is set, which assumes that the Xen-based virtualisation +system is being used; if distro is not set then default is /dev/null +which nullifies this feature. +.TP +.B test_classes +Specifies the set of test classes to try, with the details of each +test class separated by a colon \fB:\fR from the next. a-t-r will run +a-t-o once for each listed test class and then return to the beginning +of the list. + +Tests in each class are thus selected approximately equally often. +(If max_test_count is not a multiple of the number of actual test +classes then early classes are used slightly more.) + +Each test class consists of a set of suppression checks, a comma +\fB,\fR, and some configuration arguments for a-t-o. The suppression +checks are whitespace-separated words; see SUPPRESSION. The +configuration arguments are likewise whitespace-separated. There is +no provision for passing configuration arguments containing spaces to +a-t-o. a-t-o arguments in test classes override other configuration +settings. + +The default is +.BI "source " distro ", target=source : binary " distro , +.BI target=binary- architecture +which tests binary packages and source packages from the same +distribution in alternation +.SH SUPPRESSION +a-t-r is designed to be enabled and disabled by sentinel files. + +Firstly, if the file \fBgo\fR does not exist in its working directory, +it will stop at the first sensible opportunity. (See also the +go_anyway configuration variable.) + +Secondly, test classes may specify suppression checks. For each such +check \fIclasscheck\fR, a-t-r will test for the existence of +.BI stop- classcheck +in its current directory. If that file exists, it will suppress that +class of test. Do not suppress all of the tests without removing +\fBgo\fR. +.SH FILES +.TP +.BI /var/lib/autopkgtest/xenlvm/adt_ distro /good +default sentinel file for testbed_check_path. +.TP +.B ./go +Sentinel file indicating that operation should be enabled. +.TP +.BI ./stop- classcheck +Sentinel file indicating that test classes mentioning \fIclasscheck\fR +should be suppressed. +.LP +adt-testreport-runloop does not use any other files directly, but it +will probably need a configuration file as an argument, and it will +invoke adt-testreport-onepackage which uses many files in the current +directory and which invokes an autopkgtest virtualisation server. +.SH BUGS AND WARNINGS +\fBIt is essential not to use the example configuration!\fR +This configuration is likely to send inappropriate emails! + +There is no locking. Whatever invokes a-t-r should take out a lock of +some kind. See the example cronjobs provided in +.B hosts/*/cronjob +in the source package. + +If all of the test classes are suppressed, a-t-r will spin in a tight +loop looking for something to do. + +The +.B testbed_check_path +feature is rather ad-hoc. It would be better to query the +virtualisation server to be used directly and then suppress the test +class if relevant. (But the infinite loop bug would need to be fixed +too.) + +The design precludes detecting attempts to set incorrectly +named configuration options. +.SH AUTHORS AND COPYRIGHT +This manpage is part of autopkgtest, a tool for testing Debian binary +packages. autopkgtest is Copyright (C) 2006-2007 Canonical Ltd and +others. + +See \fB/usr/share/doc/autopkgtest/CREDITS\fR for the list of +contributors and full copying conditions. -- 2.30.2