From: Ian Jackson Date: Mon, 30 Dec 2019 12:06:47 +0000 (+0000) Subject: subdirmk: Merge new version and fix everything X-Git-Tag: v0.6.0~249 X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?a=commitdiff_plain;h=16bce837ffe3527d4e8ed306f4436a86891d6e60;hp=24eb6b672f1095d5b362b6615f22c17f1136274a;p=secnet.git subdirmk: Merge new version and fix everything git subtree pull, but many incompatible changes - so also fixed up the secnet code to match. Signed-off-by: Ian Jackson --- diff --git a/.gitignore b/.gitignore index b166c42..31d9244 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,9 @@ autom4te.cache TAGS .makefiles.stamp -Subdir.mk +Dir.mk /main.mk +/Final.mk debian/files debian/secnet.debhelper.log diff --git a/Subdir.sd.mk b/Dir.sd.mk similarity index 98% rename from Subdir.sd.mk rename to Dir.sd.mk index e0096df..4111476 100644 --- a/Subdir.sd.mk +++ b/Dir.sd.mk @@ -44,6 +44,8 @@ CPPFLAGS:=@CPPFLAGS@ -DDATAROOTDIR='"$(datarootdir)"' $(EXTRA_CPPFLAGS) LDFLAGS:=@LDFLAGS@ $(EXTRA_LDFLAGS) LDLIBS:=@LIBS@ $(EXTRA_LDLIBS) +&:local+global OBJECTS TARGETS + TARGETS:=secnet OBJECTS:=secnet.o util.o conffile.yy.o conffile.tab.o conffile.o modules.o \ @@ -171,7 +173,7 @@ installdirs: install: installdirs set -e; ok=true; for f in $(STALE_PYTHON_FILES); do \ if test -e $$f; then \ - echo >\&2 "ERROR: $$f still exists "\ + echo >&\&2 "ERROR: $$f still exists "\ "- try \`make install-force'"; \ ok=false; \ fi; \ @@ -208,9 +210,10 @@ distclean:: realclean include subdirmk/regen.mk +&:warn !single-char-var # Release checklist: # -# 0. Use this checklist from Subdir.sd.mk +# 0. Use this checklist from Dir.sd.mk # # 1. Check that the tree has what you want # diff --git a/Perdir.sd.mk b/Suffix.sd.mk similarity index 61% rename from Perdir.sd.mk rename to Suffix.sd.mk index c5ca200..5daff5e 100644 --- a/Perdir.sd.mk +++ b/Suffix.sd.mk @@ -1,5 +1,5 @@ -# &TARGETS_check -# &TARGETS_fullcheck +&TARGETS_check += +&TARGETS_fullcheck += &:include subdirmk/cdeps.sd.mk &:include subdirmk/clean.sd.mk diff --git a/configure b/configure index 51252f1..b069360 100755 --- a/configure +++ b/configure @@ -2344,7 +2344,7 @@ ac_config_headers="$ac_config_headers config.h" - ac_config_files="$ac_config_files main.mk:main.mk.tmp Subdir.mk:Subdir.mk.tmp" + ac_config_files="$ac_config_files main.mk:main.mk.tmp Dir.mk:Dir.mk.tmp Final.mk:Final.mk.tmp" @@ -2360,15 +2360,15 @@ ac_config_headers="$ac_config_headers config.h" subdirmk_subdirs="$subdirmk_subdirs 'test-example/'" - ac_config_files="$ac_config_files test-example/Subdir.mk:test-example/Subdir.mk.tmp" + ac_config_files="$ac_config_files test-example/Dir.mk:test-example/Dir.mk.tmp" subdirmk_subdirs="$subdirmk_subdirs 'mtest/'" - ac_config_files="$ac_config_files mtest/Subdir.mk:mtest/Subdir.mk.tmp" + ac_config_files="$ac_config_files mtest/Dir.mk:mtest/Dir.mk.tmp" subdirmk_subdirs="$subdirmk_subdirs 'stest/'" - ac_config_files="$ac_config_files stest/Subdir.mk:stest/Subdir.mk.tmp" + ac_config_files="$ac_config_files stest/Dir.mk:stest/Dir.mk.tmp" @@ -5437,12 +5437,13 @@ do case $ac_config_target in "config.h") CONFIG_HEADERS="$CONFIG_HEADERS config.h" ;; "main.mk") CONFIG_FILES="$CONFIG_FILES main.mk:main.mk.tmp" ;; - "Subdir.mk") CONFIG_FILES="$CONFIG_FILES Subdir.mk:Subdir.mk.tmp" ;; + "Dir.mk") CONFIG_FILES="$CONFIG_FILES Dir.mk:Dir.mk.tmp" ;; + "Final.mk") CONFIG_FILES="$CONFIG_FILES Final.mk:Final.mk.tmp" ;; "subdirmk/regen.mk") CONFIG_FILES="$CONFIG_FILES subdirmk/regen.mk:subdirmk/regen.mk.in" ;; "subdirmk/usual.mk") CONFIG_FILES="$CONFIG_FILES subdirmk/usual.mk:subdirmk/usual.mk.in" ;; - "test-example/Subdir.mk") CONFIG_FILES="$CONFIG_FILES test-example/Subdir.mk:test-example/Subdir.mk.tmp" ;; - "mtest/Subdir.mk") CONFIG_FILES="$CONFIG_FILES mtest/Subdir.mk:mtest/Subdir.mk.tmp" ;; - "stest/Subdir.mk") CONFIG_FILES="$CONFIG_FILES stest/Subdir.mk:stest/Subdir.mk.tmp" ;; + "test-example/Dir.mk") CONFIG_FILES="$CONFIG_FILES test-example/Dir.mk:test-example/Dir.mk.tmp" ;; + "mtest/Dir.mk") CONFIG_FILES="$CONFIG_FILES mtest/Dir.mk:mtest/Dir.mk.tmp" ;; + "stest/Dir.mk") CONFIG_FILES="$CONFIG_FILES stest/Dir.mk:stest/Dir.mk.tmp" ;; "common.make") CONFIG_FILES="$CONFIG_FILES common.make:common.make.in" ;; "default") CONFIG_COMMANDS="$CONFIG_COMMANDS default" ;; diff --git a/mtest/Subdir.sd.mk b/mtest/Dir.sd.mk similarity index 100% rename from mtest/Subdir.sd.mk rename to mtest/Dir.sd.mk diff --git a/stest/Subdir.sd.mk b/stest/Dir.sd.mk similarity index 100% rename from stest/Subdir.sd.mk rename to stest/Dir.sd.mk diff --git a/subdirmk/.gitignore b/subdirmk/.gitignore index a5b02c2..0b5bd46 100644 --- a/subdirmk/.gitignore +++ b/subdirmk/.gitignore @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. #----- subdirmk-generated ----- /regen.mk diff --git a/subdirmk/README b/subdirmk/README index fcf8c00..3d18dc2 100644 --- a/subdirmk/README +++ b/subdirmk/README @@ -20,13 +20,13 @@ style is not very ergonomic. The main difficulties are: `subdirmk' is an attempt to solve these problems (and it also slightly alleviates some of the boilerplate needed to support out-of-tree -builds well). +builds well, and helps a bit with metaprogramming and rule writing). Basic approach -------------- The developer is expected to write a makefile fragment, in each -relevant subdirectory, called `Subdir.sd.mk'. +relevant subdirectory, called `Dir.sd.mk'. These fragments may contain ordinary make language. Unqualified filenames are relative to the build toplevel, and all commands all run @@ -36,11 +36,11 @@ However, the sigil & is treated specially. By and large, it refers to `the build directory corresponding to this .sd.mk file', etc. There are a variety of convenient constructions. -The result is that to a large extent, the Subdir.sd.mk has an easy way +The result is that to a large extent, the Dir.sd.mk has an easy way to namespace its "local" make variables, and an easy way to refer to its "local" filenames (and filenames in general). -The Subdir.sd.mk's are filtered, fed through autoconf in the usual way +The Dir.sd.mk's are filtered, fed through autoconf in the usual way (for @..@-substitutions) and included by one autogenerated toplevel makefile. @@ -52,9 +52,200 @@ Each subdirectory is also provided with an autogenerated `Makefile' which exists purely to capture ordinary make invocations and arrange for something suitable to happen. -Where there are dependencies between subdirectories, each Subdir.sd.mk +Where there are dependencies between subdirectories, each Dir.sd.mk can simply refer to files in other subdirectories directly. +Substitution syntax +------------------- + +In general & expands to the subdirectory name when used for a +filename, and to the subdirectory name with / replaced with _ for +variable names. (If your variables start with capital letters and +your filenames with lowercase. Otherwise, use &/ or &_.) + +Note that & is processed *even in makefile comments*. The substitutor +does not understand make syntax, or shell syntax, at all. However, +the substitution rules are chosen to work well with constructs which +are common in makefiles. + +In the notation below, we suppose that the substitution is being in +done in a subdirectory sub/dir of the source tree. In the RH column +we describe the expansion at the top level, which is often a special +case (in general in variable names we call that TOP rather than the +empty string). + +&CAPS => sub_dir_CAPS or TOP_CAPS +&lc => sub/dir/lc or lc + Here CAPS is any ASCII letter A-Z and lc is a-z. + The assumption is that filenames are usually lowercase and + variables usually uppercase. Otherwise, use another syntax: + +&/ => sub/dir/ or nothing +&_ => sub_dir_ or TOP_ +&. => sub/dir or . + (This implies that `&./' works roughly like `&/', although + it can produce a needless `./') + +&= => sub_dir or TOP + +&^lc => ${top_srcdir}/sub/dir/lc +&^/ => ${top_srcdir}/sub/dir/ +&^. => ${top_srcdir}/sub/dir + +&~lc => ${top_srcdir}/lc +&~/ => ${top_srcdir}/ +&~. => ${top_srcdir} + +In general: + ^ pathname of this subdirectory in source tree + ~ pathname of top level of source tree + / terminates the path escape } needed if next is + _ terminates the var escape } not letter or space) + . terminates path escape giving dir name (excluding /) + = terminates var escape giving only prefix part (rarely needed) + lwsp starts multi-word processing (see below) + +So pathname syntax is a subset of: + '&' [ '^' | '~' ] [ lc | '/' | '.' ] + +&& => && for convenience in shell runes + +&\& => & general escaping mechanism +&\$ => $ provided for $-doubling regimes +&\NEWLINE eats the newline and vanishes + +&$VARIABLE => ${sub_dir_VARIABLE} or ${TOP_VARIABLE} + VARIABLE is ASCII starting with a letter and matching \w+ + +& thing thing... & +&^ thing thing... & +&~ thing thing... & + Convenience syntax for prefixing multiple filenames. + Introduced by & followed by lwsp where lc could go. + Each lwsp-separated non-ws word is prefixed by &/ etc. + etc. respectively. No other & escapes are recognised. + This processing continues until & preceded by lwsp, + or until EOL (the end of the line), or \ then EOL. + +&: .... + recognised at start of line only (possibly after lwsp) + +&:include filename filename should usually be [&]foo.sd.mk +&:-include filename tolerate nonexistent file + RHS is &-expanded but filenames are relative to the top + srcdir. This implies that unqualified names are like &~/ + whereas &/ is like &^/. &^ and &~ do not work here because + they expand to constructions involving literally + `$(top_srcdir)', but the RHS is not make-expanded. + +&! disables & until EOL (and then disappears) + +&# delete everything to end of line + (useful if the RHS contains unrecognised & constructions) + +&TARGETS_things + Handled specially. If mentioned at the start of a line + (possibly following whitespace), declares that this + subdir ought to have a target `things'. The rule will be + &/things:: $(&TARGETS_things) + + You may extend it by adding more :: rules for the target, + but the preferred style is to do things like this: + &TARGETS_check += & test-passed.stamp + + It is important to mention &TARGETS_things at least once in + the context of each applicable directory, because doing so + arranges that the *parent* will also have a `things' target + which recursively implies this directory's `things'. + + Must be spelled exactly &TARGETS_things. &_TARGETS_things, + for example, is not magic. To make the target exist + without providing any prerequisites for it, write a line + containing just `&TARGETS_things +='. + + `all' is extra special: every directory has an `all' + target, which corresponds to &TARGETS. + +&:warn [!]WARNTAG ... + Suppress (with !) or re-enable (without !) warnings tagged + WARNTAG (see section `Warnings', below). The suppression list + is reset at the start of processing in each subdirectory. + Warnings that appear at the end of processing are controlled + by the final warning state after processing all the toplevel + input files (including Final.sd.mk). + +&:local+global [!][&]VARIABLE ... + Suppresses any warnings relating to forthcoming mentions + to VARIABLE or &VARIABLE, as applicable. Scope ends at + the end of the current directory's Suffix.sd.mk. + Prefixing with ! removes [&]VARIABLE from the suppresion list. + +&:changequote NEWQUOTE + changes the escape sequence from & to literally NEWQUOTE + NEWQUOTE may be any series of of non-whitespace characters, + and is terminated by EOL or lwsp. The whole line is + discarded. + + After this, write NEWQUOTE instead of &, everywhere. + The effect is unscoped and lasts until the next setting, + or until the end of the current directory's Suffix.sd.mk. + It takes effect on &:include'd files too, so maybe set + it back before using &:include. + + Notably + NEWQUOTENEWQUOTE => NEWQUOTENEWQUOTE + NEWQUOTE\NEWQUOTE => NEWQUOTE + NEWQUOTE\$ => $ + NEWQUOTE:changequote & set escape back to & + + +Dollar doubling and macro assistance +------------------------------------ + +&$+ Starts dollar-doubling +&$- Stops dollar-doubling + Both are idempotent and local to the file or context. + +This is useful both for make macrology involving $(eval ...), and +possibly for helping write complicated recipes involving shell +variables, inline Perl code, etc. + +Sometimes we will show $'s being doubled inside another construct. +This means the content of the construct is $-doubled: $-doubling is +locally enabled, and restored afterwards. + +&:macro NAME => define NAME +STUFF $ THINGS .. STUFF $$ THINGS +&:endm .. endef + NAME is processed for & + +&${..$..} => ${eval ${call ..$$..}} + (matches { } pairs to find the end) + content is $-doubled (unless it contains &$- to turn that off) + +Together &:macro and &${...} provide a more reasonable macro facility +than raw make. They solve the problem that make expansions cannot +directly generate multiple rules, variable, etc.; instead, `$(eval )' +must be used, but that re-expands the argument, meaning that all the +literal text must be $-doubled. This applies to the macro text and to +the arguments. Also `$(eval $(call ...))' is an unfortunate syntax. +Hence &:macro and &${...}. + +While dollar-doubling: +- - - - - - - - - - - + +$ => $$ including $'s produced by other + &-expansions not mentioned here + +&\$ => $ +&$( => ${ (expands to { } so it is useable for shell too) +&$NN => ${NN} where N are digits + +A few contexts do not support $-doubling, such as directive arguments +or places where this might imply $-quadrupling. (There is no way to +get $-quadrupling.) + + Invocation, "recursive" per-directory targets --------------------------------------------- @@ -69,7 +260,7 @@ Each subdirectory has its own `all' target. For example a subdirectory `src' has a target `src/all'. The rules for these are automatically generated from the settings of the per-directory &TARGETS variables. &TARGETS is magic in this way. (In -src/Subdir.sd.mk, &TARGETS of course refers to a make variable called +src/Dir.sd.mk, &TARGETS of course refers to a make variable called src_TARGETS.) The `all' target in a parent directory is taken to imply the `all' @@ -83,32 +274,39 @@ targets may be in run in parallel: there is only one `make' invocation at a time. There is no sequencing between subdirectories, only been individual targets (as specified according to their dependencies). -You can define other per-directory recursive targets too: simply -mention (usually, by setting) the variable &TARGETS_zonk, or whatever. -This will create a src/zonk target (for appropriate value of src/). -Unlike `all', these other targets only exist in areas of the project -where at least something mentions them. So for example, if -&TARGETS_zonk is mentioned in src but not lib, `make zonk' in -lib will fail. If you want to make a target exist everywhere, -mention its name in Perdir.sd.mk (see below). +You can define other per-directory recursive targets too: set the +variable &TARGETS_zonk, or whatever (being sure to write &TARGETS_zonk +at the start of a line). This will create a src/zonk target (for +appropriate value of src/). Unlike `all', these other targets only +exist in areas of the project where at least something mentions them. +So for example, if &TARGETS_zonk is set in src but not lib, `make +zonk' in lib will fail. If you want to make a target exist +everywhere, += it with nothing in Prefix.sd.mk or Suffix.sd.mk (see +below). -Perdir.sd.mk, inclusion ------------------------ +Prefix.sd.mk, Suffix.sd.mk, Final.sd.mk, inclusion +-------------------------------------------------- -The file Perdir.sd.mk in the toplevel of the source is automatically -processed after each individual directory's Subdir.sd.mk, and the -&-substituted contents therefore appear once for each subdirectory. +The files Prefix.sd.mk and Suffix.sd.mk in the toplevel of the source +are automatically processed before and after each individual +directory's Dir.sd.mk, and the &-substituted contents therefore +appear once for each subdirectory. This lets you do per-directory boilerplate. Some useful boilerplate is already provided in subdirmk, for you to reference like this: &:include subdirmk/cdeps.sd.mk &:include subdirmk/clean.sd.mk -For example you could put that in Perdir.sd.mk. +For example you could put that in Suffix.sd.mk. -The top-level Subdir.sd.mk is the first makefile included after the +The top-level Dir.sd.mk is the first makefile included after the autogenerated `main.mk' which merely has some basic settings and includes. So if you want to get in early and set global variables, -put them near the top of Subdir.sd.mk. +put them near the top of Dir.sd.mk. + +The file Final.sd.mk in the toplevel directory is processed and +the result included after all the other files. Its subdirmk +filtering context inherits warning suppressions from the toplevel's +Dir.sd.mk etc., but not anything else. subdirmk's filter script itself sets (only) these variables: top_srcdir @@ -118,18 +316,122 @@ subdirmk's filter script itself sets (only) these variables: You are likely to want to define $(PWD), and shorter names for top_srdir and abs_top_srcdir (we suggest $(src) and $(abs_src)). +Warnings +-------- + +subdirmk's `generate' program, which does the acual &-substitution, +can produce some warnings about your .sd.mk files. These can be +suppressed with the &:warn directive. The warning tags are: + + local+global + The same VARNAME was used both with and without an & prefix. + This can be confusing. Also, if you avoid this then you will + get a warning iff you accidentally leave off a needed &. + The generation of this warning depends on scanning your + makefile for things that look like variable references, which + subdirmk does not do completely perfectly. Exciting make + syntax may evade this warning, or require suppressions. + (You can suppress this warning for a particular VARNAME with + the &:local+global directive.) + + single-char-var + A variable expansion like $FBAR. make's expansion rules + interpret this as $(F)BAR. It's normally better to write + it this way, at least if the variable expansion is followed + by more letters. Note that &$FOO works differently to + raw make: it expands to ${sub_dir_FOO}. + + broken-var-ref + An attempt at variable expansion looking like $&... + You probably expected this to mean $(TOP_F)BAR but it + expands to $TOP_FBAR which make thinks means $(T)OP_FBAR. + + unknown-warning + &:warn was used to try to enable a warning that this version + of subdirmk does not understand. (Note that an attempt to + *dis*able an unknown warning is only reported if some other + warning was issued which might have been disabled.) + + +Guides, hints, and further explanations +======================================= + +Incorporating this into your project +------------------------------------ + +Use `git-subtree' to merge the subdirmk/ directory. You may find it +useful to symlink the DEVELOPER-CERTIFICATE file (git can store +symlinks as symlinks - just `git add' the link). And you probably +want to mention the situation in your top-level COPYING and HACKING. + +Symlink autogen.sh into your project toplevel. + +In your configure.ac, say + + m4_include([subdirmk/subdirmk.ac]) + SUBDIRMK_SUBDIRS([...list of subdirectories in relative syntax...]) + +Write a Dir.sd.mk in each directory. See the substitution syntax +reference, above, and the example/ directory here. The toplevel +Dir.sd.mk should probably contain: + + include subdirmk/usual.mk + include subdirmk/regen.mk + +Write a Suffix.sd.mk in the toplevel, if you want. It should probably +have: + + &:include subdirmk/cdeps.sd.mk + &:include subdirmk/clean.sd.mk + + +Hints +----- + +You can convert your project incrementally. Start with the top-level +Makefile.in and rename it to Dir.sd.mk, and add the appropriate +stuff to configure.ac, and fix everything up. Leave the existing +$(MAKE) -C for your existing subdirectories alone. Then you can +convert individual subdirectories, or classes of subdirectories, at +your leisure. (You must be /sure/ that each recursive (non-subdirmk) +subdirectory will be entered only once at a time, but your existing +recursive make descent system should already do that or you already +have concurrency bugs.) + +Aside from this, be very wary of any invocation of $(MAKE) anywhere. +This is a frequent source of concurrency bugs in recursive make build +systems. When combined with nonrecursive make it's all in the same +directory and there is nothing stopping the different invocations +ending up trying to make the same targets at the same time. That +causes hideous racy lossage. There are ways to get this to work +reliably but it is advanced stuff. + +If you make syntax errors, or certain kinds of other errors, in your +makefiles, you may find that just `make' is broken now and cannot get +far enough to regenerate a working set of makefiles. If this happens +just rerun ./config.status by hand. + +If you go back and forth between different versions of your code you +can sometimes find that `make' complains that one of your Dir.sd.mk +files is missing: typically, if iot was used and therefore a +dependency in some other version of your code. If you run `make +clean' (or `make realclean') these dependencies are suppressed, which +will clear up the problem. + + Global definitions ------------------ If want to set global variables, such as CC, that should only be done -once. You can put them in your top-level Subdir.sd.mk, or a separate +once. You can put them in your top-level Dir.sd.mk, or a separate file you `include' and declare using SUBDIRMK_MAKEFILES. If you need different settings of variables like CC for different subdirectories, you should probably do that with target-specific variable settings. See the info node `(make) Target-specific'. -Subdirectory templates `.sd.mk' vs plain autoconf templates `.mk.in' + +Directory templates `.sd.mk' vs plain autoconf templates `.mk.in' -------------------------------------------------------------------- There are two kinds of template files. @@ -141,18 +443,19 @@ There are two kinds of template files. Instantiated Usu. once per subdir Once only - Need to be mentioned No, but Subdir.sd.mk All not in subdirmk/ + Need to be mentioned No, but Dir.sd.mk All not in subdirmk/ in configure.ac? via SUBDIRMK_SUBDIRS via SUBDIRMK_MAKEFILES How to include `&:include foo.sd.mk' `include foo.mk' in all relevant .sd.mk in only one - (but not needed for Subdir.sd.mk - Subdir and Perdir) + (but not needed for Dir.sd.mk + Prefix, Suffix, Final) If you `include subdirmk/regen.mk', dependency management and automatic regeneration for all of this template substitution, and for config.status etc. is done for you. + Tables of file reference syntaxes --------------------------------- @@ -162,7 +465,7 @@ locations: (i) In the build tree, or in the source tree ? - (ii) In (or relative to) the subdirectory to which this Subdir.sd.mk + (ii) In (or relative to) the subdirectory to which this Dir.sd.mk relates, or relative to the project's top level ? (iii) Absolute or relative pathname ? Usually relative pathnames @@ -183,7 +486,7 @@ In more detail, with all the various options laid out: for build source build source This lc &file &^file $(PWD)/&file $(abs_src)/&file - directory any &/file &^/file $(PWD)/&file $(abs_src)/&/file + directory any &/file &^/file $(PWD)/&/file $(abs_src)/&/file several & f g h &^ f g h $(addprefix...) Top lc file &~file @@ -194,115 +497,6 @@ In more detail, with all the various options laid out: (This assumes you have appropriate make variables src, PWD and abs_src.) -Substitution syntax -------------------- - -In general & expands to the subdirectory name when used for a -filename, and to the subdirectory name with / replaced with _ for -variable names. - -Note that & is processed *even in makefile comments*. The substitutor -does not understand make syntax, or shell syntax, at all. However, -the substitution rules are chosen to work well with constructs which -are common in makefiles. - -In the notation below, we suppose that the substitution is being in -done in a subdirectory sub/dir of the source tree. In the RH column -we describe the expansion at the top level, which is often a special -case (in general in variable names we call that TOP rather than the -empty string). - -&CAPS => sub_dir_CAPS or TOP_CAPS -&lc => sub/dir/lc or lc - Here CAPS is any ASCII letter A-Z and lc is a-z. - The assumption is that filenames are usually lowercase and - variables usually uppercase. Otherwise, use another syntax: - -&_ => sub_dir_ or TOP_ -&=_ => sub_dir or TOP - -&/ => sub/dir/ or nothing -&=/ => sub/dir or . - -&^lc => $(top_srcdir)/sub/dir/lc -&^/ => $(top_srcdir)/sub/dir/ - -&~lc => $(top_srcdir)/lc -&~/ => $(top_srcdir)/ - -In general: - = return subdir without delimiter (not allowed with `^' `~') - ^ pathname of this subdirectory in source tree - ~ pathname of top level of source tree - / terminates the escape (needed if next is not lwsp or space) - lwsp starts multi-word processing (see below) - -So pathname syntax is a subset of: - '&' [ '^' | '~' ] [ lc | '/' ] - -&& => && for convenience in shell runes -\& => & general escaping mechanism - -& thing thing... & -&^ thing thing... & -&~ thing thing... & - Convenience syntax for prefixing multiple filenames. - Introduced by & followed by lwsp where lc could go. - Each lwsp-separated non-ws word is prefixed by &/ etc. - etc. respectively. No other & escapes are recognised. - This processing continues until & preceded by lwsp, - or until EOL (the end of the line), or \ then EOL. - -&: .... - recognised at start of line only (possibly after lwsp) - args are processed for & - -&:include filename filename should usually be foo.sd.mk -&:-include filename tolerate nonexistent file - filenames are relative to $(top_srcdir) - -&! disables & until EOL (and then disappears) - -&# delete everything to end of line - (useful if the RHS contains unrecognised & constructions) - -&!STUFF - changes the escape sequence from & to literally STUFF - STUFF may be any series of of non-whitespace characters, - and is terminated by EOL or lwsp. &!STUFF and the lwsp - are discarded. - - After this, write STUFF instead of &, everywhere. - The effect is global and lasts until the next setting. - It takes effect on &:include'd files too, so maybe set - it back before using &:include. - - Notably - STUFFSTUFF => STUFFSTUFF - \STUFF => STUFF - STUFF!& set escape back to & - -&TARGETS_things - Handled specially. If mentioned, declares that this - subdir ought to have a target `things'. The rule will be - &/things:: $(&TARGETS_things) - - You may extend it by adding more :: rules for the target, - but the preferred style is to do things like this: - &TARGETS_check += & test-passed.stamp - - It is important to mention &TARGETS_things at least once in - the context of each applicable directory, because doing so - arranges that the *parent* will also have a `things' target - which recursively implies this directory's `things'. - - Must be spelled exactly &TARGETS_things. &_TARGETS_things, - for example, is not magic. But mentioning &TARGETS_things in - a #-comment *does* work because the & filter does not care - about comments. - - `all' is extra special: every directory has an `all' - target, which corresponds to &TARGETS. Subdirectory and variable naming -------------------------------- @@ -319,68 +513,9 @@ project) are best avoided. If you name your variables in ALL CAPS and your subdirectories in lower case with `-' rather than `_', there will be no confusion. -Incorporating this into your project ------------------------------------- - -Use `git-subtree' to merge the subdirmk/ directory. You may find it -useful to symlink the DEVELOPER-CERTIFICATE file (git can store -symlinks as symlinks - just `git add' the link). And you probably -want to mention the situation in your top-level COPYING. - -Symlink autogen.sh into your project toplevel. - -In your configure.ac, say - - m4_include([subdirmk/subdirmk.ac]) - SUBDIRMK_SUBDIRS([...list of subdirectories in relative syntax...]) - -Write a Subdir.sd.mk in each directory. The toplevel one should -probably contain: - - include subdirmk/usual.mk - include subdirmk/regen.mk - -Write a Perdir.sd.mk in the toplevel, if you want. It should probably -have: - - &:include subdirmk/cdeps.sd.mk - &:include subdirmk/clean.sd.mk - -Hints ------ - -You can convert your project incrementally. Start with the top-level -Makefile.in and rename it to Subdir.sd.mk, and add the appropriate -stuff to configure.ac, and fix everything up. Leave the existing -$(MAKE) -C for your existing subdirectories alone. Then you can -convert individual subdirectories, or classes of subdirectories, at -your leisure. (You must be /sure/ that each subdirectory will be -entered only once at a time, but your existing recursive make descent -system should already do that or you already have concurrency bugs.) - -Aside from this, be very wary of any invocation of $(MAKE) anywhere. -This is a frequent source of concurrency bugs in recursive make build -systems. When combined with nonrecursive make it's all in the same -directory and there is nothing stopping the different invocations -ending up trying to make the same targets at the same time. That -causes hideous racy lossage. There are ways to get this to work -reliably but it is advanced stuff. - -If you make syntax errors, or certain kinds of other errors, in your -makefiles, you may find that just `make' is broken now and cannot get -far enough to regenerate a working set of makefiles. If this happens -just rerun ./config.status by hand. - -If you go back and forth between different versions of your code you -can sometimes find that `make' complains that one of your Subdir.sd.mk -files is missing: typically, if iot was used and therefore a -dependency in some other version of your code. If you run `make -clean' (or `make realclean') these dependencies are suppressed, which -will clear up the problem. - Legal information ------------------ +================= subdirmk is Copyright 2019 Mark Wooding diff --git a/subdirmk/autogen.sh b/subdirmk/autogen.sh index e28e80e..528cfbd 100755 --- a/subdirmk/autogen.sh +++ b/subdirmk/autogen.sh @@ -2,6 +2,7 @@ # subdirmk, autogen.sh (conventional autoconf invocation script) # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. set -e cd ${0%/*} autoconf diff --git a/subdirmk/cdeps.sd.mk b/subdirmk/cdeps.sd.mk index 5a2bfb2..37ea60f 100644 --- a/subdirmk/cdeps.sd.mk +++ b/subdirmk/cdeps.sd.mk @@ -2,10 +2,11 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &# Usage: &# &:include subdirmk/cdeps.sd.mk -&# (probably in Perdir.sd.mk) +&# (probably in Suffix.sd.mk) &# &# Arranges for automatic #include dependency tracking for &# C compilation. The compiler is asked to write the dependencies to diff --git a/subdirmk/clean.sd.mk b/subdirmk/clean.sd.mk index 045561c..a38fb22 100644 --- a/subdirmk/clean.sd.mk +++ b/subdirmk/clean.sd.mk @@ -2,10 +2,11 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &# Usage: &# &:include subdirmk/clean.sd.mk -&# (probably in Perdir.sd.mk) +&# (probably in Suffix.sd.mk) &# &# Provides a per-directory `clean' target, which deletes all the files &# in &CLEAN. &OBJECTS, &DEPFILES and &TARGETS are automatically deleted. @@ -19,7 +20,7 @@ &CLEAN += $(&OBJECTS) &CLEAN += $(&TARGETS) -# &TARGETS_clean +&TARGETS_clean += &/clean:: $(RM) $(&CLEAN) diff --git a/subdirmk/example/.gitignore b/subdirmk/example/.gitignore index a97b413..13c6cfa 100644 --- a/subdirmk/example/.gitignore +++ b/subdirmk/example/.gitignore @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. #----- Build artifacts ----- *.a @@ -21,10 +22,12 @@ .makefiles.stamp /main.mk Makefile -Subdir.mk +Dir.mk +Final.mk *.tmp #----- For our tests ----- /build +/for-test-final.sd.mk /src/for-test.sd.mk /lib/for-test.mk.in diff --git a/subdirmk/example/Subdir.sd.mk b/subdirmk/example/Dir.sd.mk similarity index 73% rename from subdirmk/example/Subdir.sd.mk rename to subdirmk/example/Dir.sd.mk index 5c2040a..045ea90 100644 --- a/subdirmk/example/Subdir.sd.mk +++ b/subdirmk/example/Dir.sd.mk @@ -1,7 +1,8 @@ -# subdirmk example - top-level Subdir.sd.mk +# subdirmk example - top-level Dir.sd.mk # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. INCLUDES += -I&^/lib/ diff --git a/subdirmk/example/Final.sd.mk b/subdirmk/example/Final.sd.mk new file mode 100644 index 0000000..6dd8953 --- /dev/null +++ b/subdirmk/example/Final.sd.mk @@ -0,0 +1,8 @@ +# subdirmk - example Final.sd.mk +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# This is a hook for subdirmk's test suite. +&:-include for-test-final.sd.mk diff --git a/subdirmk/example/Perdir.sd.mk b/subdirmk/example/Suffix.sd.mk similarity index 90% rename from subdirmk/example/Perdir.sd.mk rename to subdirmk/example/Suffix.sd.mk index 65b8630..7eb9880 100644 --- a/subdirmk/example/Perdir.sd.mk +++ b/subdirmk/example/Suffix.sd.mk @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &:include subdirmk/cdeps.sd.mk &:include subdirmk/clean.sd.mk diff --git a/subdirmk/example/configure.ac b/subdirmk/example/configure.ac index b7c07e7..8646e11 100644 --- a/subdirmk/example/configure.ac +++ b/subdirmk/example/configure.ac @@ -3,6 +3,7 @@ dnl subdirmk example - configuration script dnl Copyright 2019 Mark Wooding dnl Copyright 2019 Ian Jackson dnl SPDX-License-Identifier: LGPL-2.0-or-later +dnl There is NO WARRANTY. AC_INIT([mktoy], [0.9.0], [mdw@distorted.org.uk]) AC_CONFIG_SRCDIR([src/toy.c]) @@ -16,6 +17,7 @@ m4_include([subdirmk/subdirmk.ac]) SUBDIRMK_SUBDIRS([lib]) SUBDIRMK_SUBDIRS([lib/t src]) +# This is a hook for subdirmk's test suite. if test -f $srcdir/lib/for-test.mk.in; then SUBDIRMK_MAKEFILES([lib/for-test.mk]) fi diff --git a/subdirmk/example/lib/Subdir.sd.mk b/subdirmk/example/lib/Dir.sd.mk similarity index 72% rename from subdirmk/example/lib/Subdir.sd.mk rename to subdirmk/example/lib/Dir.sd.mk index c2939c5..2639fcb 100644 --- a/subdirmk/example/lib/Subdir.sd.mk +++ b/subdirmk/example/lib/Dir.sd.mk @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &TARGETS += & libtoy.a @@ -10,4 +11,5 @@ &libtoy.a: $(&OBJECTS) $(AR) rc $@ $^ --include &^/lib/for-test.mk +# This is a hook for subdirmk's test suite. +-include &for-test.mk diff --git a/subdirmk/example/lib/t/Subdir.sd.mk b/subdirmk/example/lib/t/Dir.sd.mk similarity index 93% rename from subdirmk/example/lib/t/Subdir.sd.mk rename to subdirmk/example/lib/t/Dir.sd.mk index ae11cd0..9ec08c7 100644 --- a/subdirmk/example/lib/t/Subdir.sd.mk +++ b/subdirmk/example/lib/t/Dir.sd.mk @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &TARGETS_check += & toytest.stamp diff --git a/subdirmk/example/lib/t/toytest.c b/subdirmk/example/lib/t/toytest.c index aa03e1e..1fc5f79 100644 --- a/subdirmk/example/lib/t/toytest.c +++ b/subdirmk/example/lib/t/toytest.c @@ -2,6 +2,7 @@ * subdirmk - example code * Copyright 2019 Mark Wooding * SPDX-License-Identifier: LGPL-2.0-or-later + * There is NO WARRANTY. */ #include diff --git a/subdirmk/example/lib/toylib.c b/subdirmk/example/lib/toylib.c index 6e73f76..afa423d 100644 --- a/subdirmk/example/lib/toylib.c +++ b/subdirmk/example/lib/toylib.c @@ -2,6 +2,7 @@ * subdirmk - example code * Copyright 2019 Mark Wooding * SPDX-License-Identifier: LGPL-2.0-or-later + * There is NO WARRANTY. */ #include "toylib.h" diff --git a/subdirmk/example/lib/toylib.h b/subdirmk/example/lib/toylib.h index 04848d5..2d2dc93 100644 --- a/subdirmk/example/lib/toylib.h +++ b/subdirmk/example/lib/toylib.h @@ -2,6 +2,7 @@ * subdirmk - example code * Copyright 2019 Mark Wooding * SPDX-License-Identifier: LGPL-2.0-or-later + * There is NO WARRANTY. */ #ifndef LIBTOY_H diff --git a/subdirmk/example/src/Subdir.sd.mk b/subdirmk/example/src/Dir.sd.mk similarity index 80% rename from subdirmk/example/src/Subdir.sd.mk rename to subdirmk/example/src/Dir.sd.mk index 402116e..c67a89f 100644 --- a/subdirmk/example/src/Subdir.sd.mk +++ b/subdirmk/example/src/Dir.sd.mk @@ -2,6 +2,7 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. &TARGETS += & toy @@ -11,4 +12,5 @@ &toy: $(&OBJECTS) $(&LIBS) $(LINK) $^ +# This is a hook for subdirmk's test suite. &:-include src/for-test.sd.mk diff --git a/subdirmk/example/src/toy.c b/subdirmk/example/src/toy.c index a70f861..2f5dcb6 100644 --- a/subdirmk/example/src/toy.c +++ b/subdirmk/example/src/toy.c @@ -2,6 +2,7 @@ * subdirmk - example code * Copyright 2019 Mark Wooding * SPDX-License-Identifier: LGPL-2.0-or-later + * There is NO WARRANTY. */ #include diff --git a/subdirmk/generate b/subdirmk/generate index 9275b25..bcbbf79 100755 --- a/subdirmk/generate +++ b/subdirmk/generate @@ -3,14 +3,15 @@ # subdirmk - &-filter (makefile generation program) # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. # # $(srcdir)/subdirmk/generate [--srcdir=SRCDIR] [--] SUBDIR... # -# generates in each subdirectory from in each subdirectory -# Subdir.mk.tmp Subdir.sd.mk -# Makefile and included files -# and in toplevel and in toplevel -# main.mk.tmp Perdir.sd.mk +# generates in each subdirectory +# Dir.mk.tmp +# Makefile +# and in toplevel +# main.mk.tmp use strict; use POSIX; @@ -19,21 +20,40 @@ print "$0 @ARGV\n" or die $!; our $srcdir='.'; +# error handling methods: +# +# Error in input file, while $err_file and $. set, eg in most of +# process_input_mk: +# err "message"; +# +# Other input or usage errors: +# die "subdirmk: $file:$lno: problem\n"; +# die "subdirmk: some problem not locatable in that way\n"; +# +# Usage error: +# die "subdirmk $0: explanation of problem\n"; +# +# System call error (not ENOENT) accessing input/output files: +# die "description of problem eg maybe erbing noun: $!\n"; +# +# Bug detedcted in `generate': +# die "internal error (some information)?"; # or similar + while (@ARGV && $ARGV[0] =~ m/^-/) { $_ = shift @ARGV; last if $_ eq '--'; if (s/^--srcdir=//) { $srcdir=$'; } else { - die "$0: unknown option \`$_'\n"; + die "subdirmk $0: unknown option \`$_'\n"; } } our @subdirs = @ARGV; s{/+$}{} foreach @subdirs; -our $root = [ '.', [ ] ]; -# each node is [ 'relative subdir name', \@children ] +our $root = [ '.', [ ], 1 ]; +# each node is [ 'relative subdir name', \@children, $mentioned ] sub build_tree () { foreach my $subdir (@subdirs) { @@ -47,6 +67,7 @@ sub build_tree () { } $node = $c; } + $node->[2] = 1; } } @@ -59,6 +80,7 @@ our $writing_output; our $buffering_output; our %output_files; our %input_files; +our @output_makefiles; sub close_any_output_file() { return unless defined $writing_output; @@ -67,22 +89,41 @@ sub close_any_output_file() { $writing_output = undef; } -sub o { +sub oraw { + die 'internal error' unless defined $writing_output; + print O @_ or die "error writing $writing_output.tmp: $!\n"; +} + +sub oud { # undoubled if (defined $buffering_output) { $buffering_output .= $_ foreach @_; return; } - die unless defined $writing_output; - print O @_ or die "error writing $writing_output.tmp: $!\n"; + oraw @_; +} + +our $ddbl; + +sub od { # maybe $-doubled + if (!$ddbl) { + oud @_; + return; + } + foreach (@_) { + my $e = $_; + $e =~ s{\$}{\$\$}g; + oud $e; + } } sub start_output_file ($) { close_any_output_file(); ($writing_output) = @_; - die "$writing_output ?" if $output_files{$writing_output}++; + die "internal error ($writing_output?)" + if $output_files{$writing_output}++; my $tmp = "$writing_output.tmp"; open O, ">", $tmp or die "create $tmp: $!\n"; - o "# autogenerated - do not edit\n"; + oraw "# autogenerated - do not edit\n"; } sub install_output_files () { @@ -100,7 +141,7 @@ sub write_makefile ($$) { my $suppress_templates= '$(if $(filter-out clean real-clean, $(subdirmk_targets)),,'. ' MAKEFILE_TEMPLATES=)'; - o < 1 } qw( + local+global + single-char-var + unknown-warning + broken-var-ref +); +our %warn_ena = @warn_ena_dfl; + +our $warned; +our %warn_unk; + +sub err ($) { + my ($m) = @_; + die defined $err_file + ? "subdirmk: ${err_file}:$.: $m\n" + : "subdirmk: $m\n"; +} + +sub wrncore ($$) { + my ($wk,$m) = @_; + return 0 unless $warn_ena{$wk} // warn "internal error $wk ?"; + $warned++; + print STDERR "subdirmk: warning ($wk): $m\n"; + return 1; +} + +sub wrn ($$) { + my ($wk,$m) = @_; + our %warn_dedupe; + return 0 if $warn_dedupe{$err_file,$.,$wk,$m}++; + wrncore($wk, "${err_file}:$.: $m"); +} + +sub ddbl_only ($) { + my ($e) = @_; + return if $ddbl; + err "escape &$e is valid only during \$-doubling"; +} + +sub process_input_mk ($$$$); +sub process_input_mk ($$$$) { + my ($targets, $f, $esclitr, $enoent_ok) = @_; my $caps_re = qr{[A-Z]}; my $lc_re = qr{[a-z]}; @@ -130,55 +230,168 @@ sub process_input_mk ($$$$$$$$) { my $input = new IO::File $f, '<'; if (!$input) { - die "open $f: $!\n" unless $!==ENOENT && $enoent_ok; + err "open $f: $!" unless $!==ENOENT && $enoent_ok; return; } $input_files{$f}++; + local $err_file=$f; + + my %srcdirmap = ( + '^' => "\${top_srcdir}${dir_suffix}", + '~' => "\${top_srcdir}", + ); my %pfxmap = ( '' => $dir_prefix, - '^' => "\$(top_srcdir)${dir_suffix}/", - '~' => "\$(top_srcdir)/", ); + $pfxmap{$_} = $srcdirmap{$_}.'/' foreach keys %srcdirmap; + + local $ddbl; + my @nest = (['']); + my $evalcall_brackets; + + my $push_nest = sub { + my ($nk, $nndbl, $what) = @_; + unshift @nest, [ $nk, $ddbl, $what, $. ]; + $ddbl = $nndbl; + }; + my $pop_nest = sub { + my ($nk) = @_; + err "unexpectedly closed $nk in middle of $nest[0][0] ($nest[0][2])" + unless $nest[0][0] eq $nk; + $ddbl = (shift @nest)[1]; + }; + + # Our detection of variable settings does not have to be completely + # accurate, since it is only going to be used for advice to the user. + my $note_varref = sub { + my ($vn,$amp) = @_; + my $exp = !!$varref_exp{$vn}{$amp}; + $varref{$vn}{$exp}{$amp}{"$f:$."} = 1; + }; while (<$input>) { - if (s#^\s*$esc\:##) { + if (m#^\s*($esc)?(\w+)\s*(?:=|\+=|\?=|:=)# || + m#^\s*(?:$esc\:macro|define)\s+($esc)?(\S+)\s#) { + $note_varref->($2,!!$1); + } + if (s#^\s*$esc\:changequote\s+(\S+)\s+$##) { + $$esclitr = $1; + $set_esc->(); + next; + } elsif (s#^\s*$esc\:endm\s+$##) { + $pop_nest->('macro'); + od "endef\n"; + next; + } elsif (s#^\s*$esc\:warn\s+(\S.*)$##) { + foreach my $wk (split /\s+/, $1) { + my $yes = $wk !~ s{^!}{}; + if (defined $warn_ena{$wk}) { + $warn_ena{$wk} = $yes; + next; + } elsif ($yes) { + wrn 'unknown-warning', + "unknown warning $wk requested"; + } else { + $warn_unk{$wk} //= "$f:$."; + } + } + next; + } elsif (s#^\s*$esc\:local\+global\s+(\S.*)$##) { + foreach my $vn (split /\s+/, $1) { + my $pos = !($vn =~ s{^!}{}); + my $amp = $vn =~ s{^$esc}{}; + $varref_exp{$vn}{!!$amp} = $pos; + } + next; + } elsif (s#^\s*$esc\:(?=(-?)include|macro)##) { $buffering_output=''; + } elsif (m#^\s*$esc\:([a-z][-+0-9a-z_]*)#) { + err "unknown directive &:$1 or bad argumnt syntax"; + } elsif (s{^\s*${esc}TARGETS(?:_([0-9a-zA-Z_]+))?(?=\W)}{}) { + my $t = $1 // 'all'; + my $vn = target_varname($var_prefix, $t); + $note_varref->($vn,1); + od $vn; + $targets->{$t} //= [ ]; } for (;;) { - unless (s{^(.*?)(\\)?(?=$esc)}{}) { o $_; last; } - o $1; - if ($2) { s#^$esc##; o $$esclitr; next; } - s{^$esc}{} or die "$_ ?"; - if (s{^$esc}{}) { o "$$esclitr$$esclitr" } - elsif (s{^TARGETS(?:_([0-9a-zA-Z_]+))?(?=\W)}{}) { - my $t = $1 // 'all'; - o target_varname($var_prefix, $t); - $targets->{$t} //= [ ]; + err 'cannot $-double &-processed RHS of directive' + if $ddbl && defined $buffering_output; + unless ($nest[0][0] eq 'eval' + ? s{^(.*?)($esc|\$|[{}])}{} + : s{^(.*?)($esc|\$)}{}) { od $_; last; } + od $1; + if ($2 eq '{') { + od $2; + $evalcall_brackets++; + next; + } elsif ($2 eq '}') { + od $2; + next if --$evalcall_brackets; + $pop_nest->('eval'); + od '}'; + next; + } elsif ($2 eq '$') { + od $2; + if (s{^\$}{}) { od $&; } + elsif (m{^[a-zA-Z]\w}) { + wrn 'single-char-var', + 'possibly confusing unbracketed single-char $-expansion'; + } + elsif (m{^$esc}) { + wrn 'broken-var-ref', + 'broken $&... expansion; you probably meant &$'; + } + elsif (m{^\(($esc)?([^()\$]+)\)} || + m{^\{($esc)?([^{}\$]+)\}}) { + $note_varref->($2,!!$1); + } + next; + } + if (s{^\\$esc}{}) { od "$$esclitr" } + elsif (s{^\\\$}{}) { oud '$' } + elsif (s{^\\\s+$}{}) { } + elsif (s{^$esc}{}) { od "$$esclitr$$esclitr" } + elsif (m{^(?=$caps_re)}) { od $var_prefix } + elsif (s{^\$([A-Za-z]\w+)}{}) { + $note_varref->($1,1); + od "\${${var_prefix}$1}"; + } + elsif (s{^([~^]?)(?=$lc_re)}{}) { od $pfxmap{$1} } + elsif (s{^_}{}) { od $var_prefix } + elsif (s{^=}{}) { od $var_prefix_name } + elsif (s{^([~^]?)/}{}) { od $pfxmap{$1} } + elsif (s{^\.}{}) { od $dir_name } + elsif (s{^([~^])\.}{}) { od $srcdirmap{$1} } + elsif (s{^\$\-}{}) { $ddbl=undef; } + elsif (s{^\$\+}{}) { $ddbl=1; } + elsif (s{^\$\(}{}) { + ddbl_only($&); oud "\${"; + $note_varref->($2,!!$1) if m{^($esc)?([^()\$]+\))}; } - elsif (m{^(?=$caps_re)}) { o $var_prefix } - elsif (s{^([~^]?)(?=$lc_re)}{}) { o $pfxmap{$1} } - elsif (s{^_}{}) { o $var_prefix } - elsif (s{^=_}{}) { o $var_prefix } - elsif (s{^([~^]?)/}{}) { o $pfxmap{$1} } - elsif (s{^=/}{}) { o $dir_name } - elsif (s{^([~^]?)(?=[ \t])}{}) { - my $prefix = $pfxmap{$1} // die; + elsif (s{^\$(\d+)}{}) { ddbl_only($&); oud "\${$1}"; } + elsif (s{^\$\{}{}) { + err 'macro invocation cannot be re-$-doubled' if $ddbl; + od '${eval ${call '; + $evalcall_brackets = 1; + $push_nest->('eval',1, '&${...}'); + $note_varref->($2,!!$1) if m{^\s*($esc)?([^,{}\$]+)}; + } elsif (s{^([~^]?)(?=[ \t])}{}) { + my $prefix = $pfxmap{$1} // die "internal error ($1?)"; my $after=''; if (m{([ \t])$esc}) { ($_,$after) = ($`, $1.$'); } s{(?<=[ \t])(?=\S)(?!\\\s*$)}{$prefix}g; - o $_; + od $_; $_ = $after; } elsif (s{^\#}{}) { $_ = ''; } elsif (s{^![ \t]+}{}) { - o $_; + od $_; $_ = ''; - } elsif (s{^!(\S+)(?:[ \t]+|$)}{}) { - $$esclitr = $1; - $set_esc->(); } else { - die "bad escape $$esclitr$_ "; + m{^.{0,5}}; + err "bad &-escape \`$$esclitr$&'"; } } if (defined $buffering_output) { @@ -186,22 +399,24 @@ sub process_input_mk ($$$$$$$$) { $buffering_output=undef; if (m#^(-?)include\s+(\S+)\s+$#) { my $subf = "$srcdir/$2"; - process_input_mk($dir_prefix, $dir_suffix, $dir_name, - $var_prefix, $targets, - $subf, $esclitr, $1); - o "\n"; + process_input_mk($targets, $subf, $esclitr, $1); + od "\n"; + } elsif (m#^macro\s+(\S+)\s+$#) { + od "define $1\n"; + $push_nest->('macro', 1, '&:macro'); } else { - die "unknown directive $_ "; + err "bad directive argument syntax"; } } } + die "subdirmk: $f:$nest[0][3]: unclosed $nest[0][0] ($nest[0][2])\n" + if $nest[0][0]; $input->error and die "read $f: $!\n"; close $input or die "close $f: $!\n"; } -sub filter_subdir_mk ($$$$$) { - my ($dir_prefix, $dir_suffix, $dir_name, - $var_prefix, $targets) = @_; +sub filter_subdir_mk ($) { + my ($targets) = @_; #use Data::Dumper; #print STDERR "filter @_\n"; @@ -210,81 +425,141 @@ sub filter_subdir_mk ($$$$$) { my $pi = sub { my ($f, $enoentok) = @_; - process_input_mk($dir_prefix, $dir_suffix, $dir_name, - $var_prefix, $targets, - "${srcdir}/$f", \$esclit, $enoentok); + process_input_mk($targets, "${srcdir}/$f", \$esclit, $enoentok); }; - $pi->("${dir_prefix}Subdir.sd.mk", 0); - $pi->("Perdir.sd.mk", 1); + $pi->("Prefix.sd.mk", 1); + $pi->("${dir_prefix}Dir.sd.mk", 0); + $pi->("Suffix.sd.mk", 1); } sub process_subtree ($$); sub process_subtree ($$) { - # => list of descendants (in form SUBDIR/) + # => list of targets (in form SUBDIR/) # recursive, children first my ($node, $path) = @_; #use Data::Dumper; #print STDERR Dumper(\@_); - my $dir_prefix = join '', map { "$_/" } @$path; - my $dir_suffix = join '', map { "/$_" } @$path; - my $dir_name = join '/', @$path ? @$path : '.'; - my $var_prefix = join '', map { "${_}_" } @$path ? @$path : qw(TOP); + local %varref_exp; + my $dir_prefix = dir_prefix($path); + # ^ this is the only var which we need before we come back from + # the recursion. + + push @output_makefiles, "${dir_prefix}Dir.mk"; write_makefile($dir_prefix, scalar @$path); my %targets = (all => []); foreach my $child (@{ $node->[1] }) { my @childpath = (@$path, $child->[0]); my $child_subdir = join '/', @childpath; - mkdir $child_subdir or $!==EEXIST or die "mkdir $child_subdir: $!"; + mkdir $child_subdir or $!==EEXIST or die "mkdir $child_subdir: $!\n"; + local %warn_ena = @warn_ena_dfl; push @{ $targets{$_} }, $child_subdir foreach process_subtree($child, \@childpath); } - start_output_file("${dir_prefix}Subdir.mk.tmp"); - filter_subdir_mk($dir_prefix, $dir_suffix, $dir_name, - $var_prefix, \%targets); + set_dir_vars($path); + start_output_file("${dir_prefix}Dir.mk.tmp"); + + if ($node->[2]) { + filter_subdir_mk(\%targets); + } else { + my $sdmk = "${dir_prefix}Dir.sd.mk"; + if (stat $sdmk) { + die + "subdirmk: $sdmk unexpectedly exists (${dir_prefix} not mentioned on subdirmk/generate command line, maybe directory is missing from SUBDIRMK_SUBDIRS)"; + } elsif ($!==ENOENT) { + } else { + die "stat $sdmk: $!\n"; + } + } - o "\n"; + oraw "\n"; my @targets = sort keys %targets; foreach my $target (@targets) { my $target_varname = target_varname($var_prefix, $target); - print O "${dir_prefix}${target}:: \$($target_varname)"; + oraw "${dir_prefix}${target}:: \$($target_varname)"; foreach my $child_subdir (@{ $targets{$target} }) { - print O " $child_subdir/$target"; + oraw " $child_subdir/$target"; } - print O "\n"; + oraw "\n"; } if (@targets) { - print O ".PHONY:"; - print O " ${dir_prefix}${_}" foreach @targets; - print O "\n"; + oraw ".PHONY:"; + oraw " ${dir_prefix}${_}" foreach @targets; + oraw "\n"; } return @targets; } +sub process_final ($) { + my ($otargets) = @_; + set_dir_vars([]); + push @output_makefiles, "Final.mk"; + start_output_file("Final.mk.tmp"); + my %ntargets; + my $esclit='&'; + process_input_mk(\%ntargets, "${srcdir}/Final.sd.mk", \$esclit, 1); + delete $ntargets{$_} foreach @$otargets; + my @ntargets = sort keys %ntargets; + die "subdirmk: Final.sd.mk may not introduce new top-level targets". + " (@ntargets)\n" if @ntargets; +} + sub process_tree() { - process_subtree($root, [ ]); + my @targets = process_subtree($root, [ ]); + process_final(\@targets); start_output_file("main.mk.tmp"); foreach my $v (qw(top_srcdir abs_top_srcdir)) { - o "$v=\@$v@\n"; + oraw "$v=\@$v@\n"; } - o "SUBDIRMK_MAKEFILES :=\n"; - o "MAKEFILE_TEMPLATES :=\n"; - o "SUBDIRMK_MAKEFILES += Subdir.mk\n"; - foreach my $subdir (@subdirs) { - o "SUBDIRMK_MAKEFILES += $subdir/Subdir.mk\n"; + oraw "SUBDIRMK_MAKEFILES :=\n"; + oraw "MAKEFILE_TEMPLATES :=\n"; + foreach my $mf (@output_makefiles) { + oraw "SUBDIRMK_MAKEFILES += $mf\n"; } foreach my $input (sort keys %input_files) { - o "MAKEFILE_TEMPLATES += $input\n"; + oraw "MAKEFILE_TEMPLATES += $input\n"; + } + oraw "include \$(SUBDIRMK_MAKEFILES)\n"; +} + +sub flmap ($) { local ($_) = @_; s{:(\d+)$}{ sprintf ":%10d", $1 }e; $_; } + +sub print_varref_warnings () { + foreach my $vn (sort keys %varref) { + my $vv = $varref{$vn}; + next unless $vv->{''}{''} && $vv->{''}{1}; + wrncore 'local+global', "saw both $vn and &$vn" or return; + foreach my $exp ('', 1) { + foreach my $amp ('', 1) { + printf STDERR + ($exp + ? " expectedly saw %s%s at %s\n" + : " saw %s%s at %s\n"), + ($amp ? '&' : ''), $vn, $_ + foreach + sort { flmap($a) cmp flmap($b) } + keys %{ $vv->{$exp}{$amp} }; + } + } + } +} + +sub print_warning_warnings () { + return unless $warned; + foreach my $wk (sort keys %warn_unk) { + wrncore 'unknown-warning', + "$warn_unk{$wk}: attempt to suppress unknown warning(s) \`$wk'"; } - o "include \$(SUBDIRMK_MAKEFILES)"; } build_tree(); process_tree(); +print_varref_warnings(); +print_warning_warnings(); install_output_files(); diff --git a/subdirmk/regen.mk.in b/subdirmk/regen.mk.in index 5f24b08..d11e05c 100644 --- a/subdirmk/regen.mk.in +++ b/subdirmk/regen.mk.in @@ -2,10 +2,11 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. # Usage: # include subdirmk/regen.mk -# (probably in toplevel Subdir.sd.mk) +# (probably in toplevel Dir.sd.mk) # # Arranges that config.status is automatically rerun to update # makefiles from templates, whenever a template *.sd.mk or *.mk.in is @@ -13,9 +14,15 @@ # # If you add includes to configure.ac, add them to CONFIGURE_ACS. # +# Makefiles updated by config.status and passed to SUBDIRMK_MAKEFILES +# in configure.ac are automatically handled too. If you have other +# files updated by config.status (eg, the output of autoheader) you +# need to put them in CONFIG_STATUS_OUTPUTS (before your inclusion +# of regen.mk). +# # Also provides a `realclean::' target at the toplevel which deletes -# the autoconf output. This may be made into a recursive target -# by mentioning &TARGETS_realclean in appropriate .sd.mk. +# the autoconf output. (This is suitable for being part of a recursive +# target creaed by setting &TARGETS_realclean in appropriate .sd.mk.) CONFIGURE ?= configure CONFIGURE_AC ?= $(CONFIGURE).ac @@ -24,19 +31,24 @@ CONFIG_STATUS ?= config.status CONFIGURE_ACS += $(CONFIGURE_AC) CONFIGURE_ACS += subdirmk/subdirmk.ac +# To turn on debugging here, export SUBDIRMK_REGEN_NDEBUG='' +SUBDIRMK_REGEN_NDEBUG ?= @ + $(top_srcdir)/$(CONFIGURE): $(addprefix $(top_srcdir)/,$(CONFIGURE_ACS)) cd $(top_srcdir) && autoconf $(CONFIG_STATUS): $(top_srcdir)/$(CONFIGURE) ./$(CONFIG_STATUS) --recheck -# Normally, generate will add all the inputs to MAKEFILE_TEMPLATES. +# generate will add all its own inputs and outputs to these variables +SUBDIRMK_MAKEFILES += @_SUBDIRMK_MAKEFILES@ MAKEFILE_TEMPLATES += $(addprefix $(top_srcdir)/, $(addsuffix .in, \ @_SUBDIRMK_MAKEFILES@ \ )) main.mk $(SUBDIRMK_MAKEFILES) $(CONFIG_STATUS_OUTPUTS): .makefiles.stamp - @: + $(SUBDIRMK_REGEN_NDEBUG): REGEN STAMP CAUSES TARGET=$@ + .makefiles.stamp: \ $(top_srcdir)/subdirmk/generate \ $(CONFIG_STATUS) \ @@ -44,13 +56,16 @@ main.mk $(SUBDIRMK_MAKEFILES) $(CONFIG_STATUS_OUTPUTS): .makefiles.stamp # This filtering arranges that we can often run config.status to # generate only particular output files. We look for *inputs* that # have changed. If the only inputs that have changed are ones that we -# know affect only one output (Subdir.sd.mk and *.mk.in), we pass -# config.status the corresponding output file names. Otherwise we -# pass nothing and config.status does them all. We need to mention -# Subdir.sd.mk twice because if $(top_srcdir) is `.', make elides the -# directory part from $?. +# know affect only one output (Dir.sd.mk, Final.sd.mk and *.mk.in), +# we pass config.status the corresponding output file names. +# Otherwise we pass nothing and config.status does them all. We need +# to mention Dir.sd.mk twice because if $(top_srcdir) is `.', make +# elides the directory part from $?. Similarly but not identically +# Final.sd.mk. + $(SUBDIRMK_REGEN_NDEBUG): REGEN STAMP WANTS DEPS=$? ./$(CONFIG_STATUS) $(if \ - $(filter-out Subdir.sd.mk %/Subdir.sd.mk \ + $(filter-out Dir.sd.mk %/Dir.sd.mk \ + Final.sd.mk $(top_srcdir)/Final.sd.mk \ %.mk.in \ , $?),, \ $(patsubst $(top_srcdir)/%,%, $(sort \ diff --git a/subdirmk/subdirmk.ac b/subdirmk/subdirmk.ac index bacd1cf..93c77d9 100644 --- a/subdirmk/subdirmk.ac +++ b/subdirmk/subdirmk.ac @@ -3,6 +3,7 @@ dnl subdirmk - autoconf macros dnl Copyright 2019 Mark Wooding dnl Copyright 2019 Ian Jackson dnl SPDX-License-Identifier: LGPL-2.0-or-later +dnl There is NO WARRANTY. _SUBDIRMK_MAKEFILES="" AC_SUBST([_SUBDIRMK_MAKEFILES]) @@ -14,7 +15,8 @@ m4_map_args_w([$1],[_SUBDIRMK_SUBDIR(],[/)])])dnl AC_DEFUN_ONCE([_SUBDIRMK_INIT],[ AC_CONFIG_FILES([ main.mk:main.mk.tmp - Subdir.mk:Subdir.mk.tmp + Dir.mk:Dir.mk.tmp + Final.mk:Final.mk.tmp ],[],[ '$srcdir'/subdirmk/generate --srcdir='$srcdir' $subdirmk_subdirs ]) @@ -23,7 +25,7 @@ AC_DEFUN_ONCE([_SUBDIRMK_INIT],[ AC_DEFUN([_SUBDIRMK_SUBDIR],[ subdirmk_subdirs="$subdirmk_subdirs '$1'" - AC_CONFIG_FILES([$1Subdir.mk:$1Subdir.mk.tmp]) + AC_CONFIG_FILES([$1Dir.mk:$1Dir.mk.tmp]) ]) AC_DEFUN([SUBDIRMK_MAKEFILES], diff --git a/subdirmk/tests/.gitignore b/subdirmk/tests/.gitignore new file mode 100644 index 0000000..4d0028c --- /dev/null +++ b/subdirmk/tests/.gitignore @@ -0,0 +1,7 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +*/log diff --git a/subdirmk/tests/build-common b/subdirmk/tests/build-common new file mode 100644 index 0000000..2f0cb60 --- /dev/null +++ b/subdirmk/tests/build-common @@ -0,0 +1,23 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +SUBDIRMK_REGEN_NDEBUG='' +export SUBDIRMK_REGEN_NDEBUG + +make_copy () { + rm -rf tests/$1/example + mkdir tests/$1/example + + git ls-files -z example \ + | xargs -0 \ + sh -xec 'rsync -R -l "$@" tests/'$1'/' x + + rm tests/$1/example/subdirmk + + git ls-files -z :. :!example \ + | xargs -0 \ + sh -xec 'rsync -R -l "$@" tests/'$1'/example/subdirmk' x +} diff --git a/subdirmk/tests/check b/subdirmk/tests/check index f0914dd..2bb0d68 100755 --- a/subdirmk/tests/check +++ b/subdirmk/tests/check @@ -1,27 +1,15 @@ -#!/bin/sh -set -ex +#!/bin/bash +# subdirmk - toplevel invocation script for the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. -cd example -git clean -xdff -./autogen.sh && ./configure -make -j4 all check -make -j4 clean -make -j4 all check +set -e -git clean -xdff -mkdir build -cd build ->>../src/for-test.sd.mk ->>../lib/for-test.mk.in -.././autogen.sh && ../configure -make -j4 all check +j=$(nproc 2>/dev/null || echo 1) +j=$(( $j * 5 / 4 + 1 )) -echo '# for-check 1' >>../src/for-test.sd.mk -make -j4 -grep '^# for-check 1' src/Subdir.mk || false - -echo '# for-check 2' >>../lib/for-test.mk.in -make -j4 -grep '^# for-check 2' lib/for-test.mk || false - -echo ok. +x () { echo "$@"; "$@"; } +x ${MAKE-make} -f tests/tests.mk -j$j +echo 'ok.' diff --git a/subdirmk/tests/example/.gitignore b/subdirmk/tests/example/.gitignore new file mode 100644 index 0000000..cbd3871 --- /dev/null +++ b/subdirmk/tests/example/.gitignore @@ -0,0 +1,7 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +example diff --git a/subdirmk/tests/example/check b/subdirmk/tests/example/check new file mode 100755 index 0000000..d951eea --- /dev/null +++ b/subdirmk/tests/example/check @@ -0,0 +1,60 @@ +#!/bin/sh +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +set -ex + +. tests/build-common + +make_copy example + +cd tests/example/example + +: ----- out of tree build ----- + +mkdir build +cd build +>>../src/for-test.sd.mk +>>../lib/for-test.mk.in +>>../for-test-final.sd.mk +.././autogen.sh && ../configure +make -j4 all check + +: ----- testing rebuild on input change ----- + +reset_times () { + cd .. + + find ! -path './build/*' -type f -print0 \ + | xargs -0 \ + touch -hmd 'now -2000 seconds' -- + + cd build + + find -type f -print0 \ + | xargs -0 \ + touch -hmd 'now -1000 seconds' -- +} + +: ----- for-check-1 ----- +reset_times +echo 'for-check-1:' >>../src/for-test.sd.mk +make -j4 for-check-1 +grep '^for-check-1:' src/Dir.mk || false + +: ----- for-check-2 ----- +reset_times +echo 'for-check-2:' >>../lib/for-test.mk.in +make -j4 for-check-2 +grep '^for-check-2:' lib/for-test.mk || false + +: ----- for-check-3 ----- +reset_times +echo 'for-check-3:' >>../for-test-final.sd.mk +make -j4 for-check-3 +grep '^for-check-3:' Final.mk + +echo ok. diff --git a/subdirmk/tests/filter/.gitignore b/subdirmk/tests/filter/.gitignore new file mode 100644 index 0000000..7b3ce72 --- /dev/null +++ b/subdirmk/tests/filter/.gitignore @@ -0,0 +1,10 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +Makefile +*.tmp +doctests.sd.mk +doctests.mk.part diff --git a/subdirmk/tests/filter/Dir.mk.expected b/subdirmk/tests/filter/Dir.mk.expected new file mode 100644 index 0000000..c195980 --- /dev/null +++ b/subdirmk/tests/filter/Dir.mk.expected @@ -0,0 +1,30 @@ +# autogenerated - do not edit +# subdirmk - test cases for generate script +# Copyright various contributors - see top level README. +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# Prefix in . + +WARN += 3 +TOP_WARN += 3 +# $WARN +# $(WARN) +# $(TOP_WARN) +# ${TOP_WARN} + +# ${TOP_NOWARN1} $(NOWARN1) +# ${TOP_NOWARN2} $(NOWARN2) + +${eval ${call some-macro, 42, $$x, { $(foreach something) } }} + +$TOP_FBAR + +# doctests: + +# Suffix in . + +all:: $(TOP_TARGETS) sub/all +sometarget1:: $(TOP_TARGETS_sometarget1) sub/sometarget1 +sometarget2:: $(TOP_TARGETS_sometarget2) sub/sometarget2 +.PHONY: all sometarget1 sometarget2 diff --git a/subdirmk/tests/filter/Dir.sd.mk b/subdirmk/tests/filter/Dir.sd.mk new file mode 100644 index 0000000..a9da21f --- /dev/null +++ b/subdirmk/tests/filter/Dir.sd.mk @@ -0,0 +1,22 @@ +&# subdirmk - test cases for generate script +&# Copyright various contributors - see top level README. +&# SPDX-License-Identifier: LGPL-2.0-or-later +&# There is NO WARRANTY. + +WARN += 3 +&WARN += 3 +# $WARN +# $(WARN) +# $(&WARN) +# &$WARN + +&:local+global NOWARN1 &NOWARN2 +# &$NOWARN1 $(NOWARN1) +# &$NOWARN2 $(NOWARN2) + +&${ some-macro, 42, $x, { &$- $(foreach something) } } + +$&FBAR + +# doctests: +&:include &doctests.sd.mk diff --git a/subdirmk/tests/filter/Final.mk.expected b/subdirmk/tests/filter/Final.mk.expected new file mode 100644 index 0000000..0eafd9a --- /dev/null +++ b/subdirmk/tests/filter/Final.mk.expected @@ -0,0 +1,7 @@ +# autogenerated - do not edit +# subdirmk - test cases for generate script +# Copyright various contributors - see top level README. +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# Final diff --git a/subdirmk/tests/filter/Final.sd.mk b/subdirmk/tests/filter/Final.sd.mk new file mode 100644 index 0000000..d5f7e12 --- /dev/null +++ b/subdirmk/tests/filter/Final.sd.mk @@ -0,0 +1,6 @@ +# subdirmk - test cases for generate script +# Copyright various contributors - see top level README. +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# Final &/ diff --git a/subdirmk/tests/filter/Prefix.sd.mk b/subdirmk/tests/filter/Prefix.sd.mk new file mode 100644 index 0000000..e3784af --- /dev/null +++ b/subdirmk/tests/filter/Prefix.sd.mk @@ -0,0 +1,7 @@ +# subdirmk - test cases for generate script +# Copyright various contributors - see top level README. +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# Prefix in &. +&:warn many-requests-for-unknown-warning diff --git a/subdirmk/tests/filter/Suffix.sd.mk b/subdirmk/tests/filter/Suffix.sd.mk new file mode 100644 index 0000000..a398c1b --- /dev/null +++ b/subdirmk/tests/filter/Suffix.sd.mk @@ -0,0 +1,6 @@ +&# subdirmk - test cases for generate script +&# Copyright various contributors - see top level README. +&# SPDX-License-Identifier: LGPL-2.0-or-later +&# There is NO WARRANTY. + +# Suffix in &. diff --git a/subdirmk/tests/filter/check b/subdirmk/tests/filter/check new file mode 100755 index 0000000..7c4e6ac --- /dev/null +++ b/subdirmk/tests/filter/check @@ -0,0 +1,38 @@ +#!/bin/bash +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +set -e +set -o pipefail + +cd tests/filter + +expand <../../README | ./extract-doctests . >/dev/null + +set +e +../../generate sub/dir 2>stderr.tmp +rc=$? +set -e +if [ $rc != 0 ]; then cat stderr.tmp; exit 1; fi + +ok=true + +files=$(find -name \*.expected) +for f in $files; do + i=$f + o=$f.tmp + sed <$i >$o ' + /^# doctests:/ { + r '"${f%/*}/doctests.mk.part"' + a + } + ' + diff -u $f.tmp ${f%.expected}.tmp || ok=false +done + +$ok + +echo ok. diff --git a/subdirmk/tests/filter/extract-doctests b/subdirmk/tests/filter/extract-doctests new file mode 100755 index 0000000..130c237 --- /dev/null +++ b/subdirmk/tests/filter/extract-doctests @@ -0,0 +1,176 @@ +#!/usr/bin/perl -w +# subdirmk - script for extracting doctests from README +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. +# +# usage: +# expand ) { + # adhoc: rely on structure of indented examples in &:changequote part + $in_changequote = (m{^\&\:changequote}...m{^\S}) && m{^\s}; + if (m{^-----|^- - - - -}) { + # parse: rely on underlines for (sub)section headings + $csection = $lastl; + next; + } + $lastl = $_; + my $e = { L => $. }; + # parse: rely on looking for => (and .. on subsequent lines) + next unless m{\=\>} or ($cent and m{ \.\. }); + my $mapop = '=>'; + # adhoc: special case NEWQUOTE here so we recognise things in changequote + if (s{^(\s*)(\&\S+|NEWQUOTE\S+|\$)\s+(\=\>|\.\.)\s+(\S+)\s+}{} || + s{^()(\&\:\w+(?: \S+)*)\s{2,}(\=\>)\s{2,}($withspcs)$}{} || + $cent && s{^()($withspcs)\s{2,}(\.\.)\s{2,}($withspcs)$}{}) { + # adhoc: expected indented iff in changequote part + confess if length($1) xor $in_changequote; + $mapop = $3; + confess if !$cent && $mapop ne '=>'; + $e->{In} = $2; + $e->{Out} = $4; + if (# adhoc: `or ...' introduces the `at toplevel' expansion + s{^or (\S+)$}{}) { + $e->{OutTop} = $1 eq 'nothing' ? '' : $1; + } elsif (# parse: expect other wordish things to be comments + m{^(?!or\b)\(?\w{2,} }) { + } elsif (# adhoc: slightly special case for $(eval $(call + m{^\$\{.*}) { + $e->{Out} .= ' '.$&; + } elsif (m/^$/) { + } else { + confess "unk rhs $_ ?"; + } + $e->{CQ} = $in_changequote; + # adhoc: rely on this specific section title + $e->{DD} = $csection =~ m{^while dollar[- ]doubling}i; + } else { + confess "$_ ?"; + } + if ($mapop eq '=>') { + if ($e->{In} =~ m/\bNN\b/) { + # adhoc: special case NN in examples + confess if defined $cent->{OutTop}; + foreach my $nn (0..11, 999) { + my $f = { %$e }; + foreach my $k (qw(In Out)) { + $f->{$k} = $e->{$k}; + ($f->{$k} =~ s/\bNN\b/$nn/g) == 1 or confess; + } + push @exp, $f; + } + $cent=undef; + } else { + push @exp, $e; + $cent=$e; + } + } elsif ($mapop eq '..') { + confess if defined $cent->{OutTop}; + foreach my $k (qw(In Out)) { + $cent->{$k} .= "\n".$e->{$k}; + } + } +} + +print Dumper(\@exp); + +sub oi { print I @_ or die $!; } +sub oo { print O @_ or die $!; } +sub oh { oi @_; oo @_; } + +sub write_permode ($$$$$;$$) { + my ($dir_prefix, $start, $end, $senl, $what, + $filter, $omap) = @_; + $filter //= sub { 1 }; + $omap //= sub { $_[0] }; + oi $start; + oh "${senl}# ----- $what starts -----\n"; + foreach my $e (@exp) { + next unless $filter->($e); + my $desc = $e->{In}; + $desc =~ s/\&/AMP /g; + $desc =~ s/\$/DOLLAR /g; + $desc =~ s/NEWQUOTE/NEW_QUOTE /g; + my ($f,$pdesc) = $desc =~ m/^(.*)\n/ + ? ("\n# %s:\n%s\n\n", $1) + : ("%-30s: %s .\n", $desc); + my $o; + $o = $e->{OutTop} if $dir_prefix eq ''; + $o //= $e->{Out}; + $o =~ s{/sub/dir}{} if $dir_prefix eq '' && !defined $e->{OutTop}; + $o = $omap->($o, $e); + oi sprintf $f, $pdesc, $e->{In}; + oo sprintf $f, $pdesc, $o; + } + oi $end; + oh "${senl}# ----- $what ends -----\n"; +} + +sub writeout ($) { + my ($dir_prefix) = @_; + open I, '>', "$outdir/${dir_prefix}doctests.sd.mk" or die $!; + open O, '>', "$outdir/${dir_prefix}doctests.mk.part" or die $!; + oh "# doctests start $dir_prefix\n"; + write_permode($dir_prefix, + '','','', 'normal', + sub { !$_[0]{DD} && !$_[0]{CQ} } ); + write_permode($dir_prefix, + '&$+', '&$-', "\n", + 'dollar doubling', + sub { + my ($e) = @_; + # adhoc: skip &:macro in already-doubling part + return 0 if $e->{In} =~ m{^\&\:macro}; + # adhoc: skip &${ ie eval in already-doubling part + return 0 if $e->{In} =~ m{^\&\$\{}; + return 0 if $e->{CQ}; + return $e->{DD} || !grep { + # If there are two entries with the same In, + # use only the one from the `while dollar + # doubling' section. So entries there override + # entries in the rest o the file. + $_ ne $e && $_->{In} eq $e->{In} + } @exp; + }, + sub { + $_=$_[0]; + s/\$/\$\$/g unless $_[1]{DD}; + $_; + } ); + write_permode($dir_prefix, + "&:changequote NEWQUOTE\n", + "NEWQUOTE:changequote &\n", + "", + 'changequote', + sub { $_[0]{CQ} } ); + oh "# doctests end\n"; + close I or die $!; +} + +writeout(''); +writeout('sub/dir/'); diff --git a/subdirmk/tests/filter/main.mk.expected b/subdirmk/tests/filter/main.mk.expected new file mode 100644 index 0000000..9b4931c --- /dev/null +++ b/subdirmk/tests/filter/main.mk.expected @@ -0,0 +1,17 @@ +# autogenerated - do not edit +top_srcdir=@top_srcdir@ +abs_top_srcdir=@abs_top_srcdir@ +SUBDIRMK_MAKEFILES := +MAKEFILE_TEMPLATES := +SUBDIRMK_MAKEFILES += Dir.mk +SUBDIRMK_MAKEFILES += sub/Dir.mk +SUBDIRMK_MAKEFILES += sub/dir/Dir.mk +SUBDIRMK_MAKEFILES += Final.mk +MAKEFILE_TEMPLATES += ./Dir.sd.mk +MAKEFILE_TEMPLATES += ./Final.sd.mk +MAKEFILE_TEMPLATES += ./Prefix.sd.mk +MAKEFILE_TEMPLATES += ./Suffix.sd.mk +MAKEFILE_TEMPLATES += ./doctests.sd.mk +MAKEFILE_TEMPLATES += ./sub/dir/Dir.sd.mk +MAKEFILE_TEMPLATES += ./sub/dir/doctests.sd.mk +include $(SUBDIRMK_MAKEFILES) diff --git a/subdirmk/tests/filter/stderr.expected b/subdirmk/tests/filter/stderr.expected new file mode 100644 index 0000000..3389ef9 --- /dev/null +++ b/subdirmk/tests/filter/stderr.expected @@ -0,0 +1,19 @@ +subdirmk: warning (unknown-warning): ./Prefix.sd.mk:7: unknown warning many-requests-for-unknown-warning requested +subdirmk: warning (single-char-var): ./Dir.sd.mk:8: possibly confusing unbracketed single-char $-expansion +subdirmk: warning (broken-var-ref): ./Dir.sd.mk:19: broken $&... expansion; you probably meant &$ +subdirmk: warning (local+global): saw both NOWARN1 and &NOWARN1 + saw NOWARN1 at ./sub/dir/Dir.sd.mk:24 + saw &NOWARN1 at ./Dir.sd.mk:14 + expectedly saw NOWARN1 at ./Dir.sd.mk:14 +subdirmk: warning (local+global): saw both WARN and &WARN + saw WARN at ./Dir.sd.mk:6 + saw WARN at ./Dir.sd.mk:9 + saw WARN at ./sub/dir/Dir.sd.mk:19 + saw WARN at ./sub/dir/Dir.sd.mk:22 + saw &WARN at ./Dir.sd.mk:7 + saw &WARN at ./Dir.sd.mk:10 + saw &WARN at ./Dir.sd.mk:11 + saw &WARN at ./sub/dir/Dir.sd.mk:18 + saw &WARN at ./sub/dir/Dir.sd.mk:27 + expectedly saw &WARN at ./sub/dir/Dir.sd.mk:21 +subdirmk: warning (unknown-warning): ./sub/dir/Dir.sd.mk:6: attempt to suppress unknown warning(s) `some-unknown-warning' diff --git a/subdirmk/tests/filter/sub/Dir.mk.expected b/subdirmk/tests/filter/sub/Dir.mk.expected new file mode 100644 index 0000000..0cd6e3f --- /dev/null +++ b/subdirmk/tests/filter/sub/Dir.mk.expected @@ -0,0 +1,6 @@ +# autogenerated - do not edit + +sub/all:: $(sub_TARGETS) sub/dir/all +sub/sometarget1:: $(sub_TARGETS_sometarget1) sub/dir/sometarget1 +sub/sometarget2:: $(sub_TARGETS_sometarget2) sub/dir/sometarget2 +.PHONY: sub/all sub/sometarget1 sub/sometarget2 diff --git a/subdirmk/tests/filter/sub/dir/Dir.mk.expected b/subdirmk/tests/filter/sub/dir/Dir.mk.expected new file mode 100644 index 0000000..cde814d --- /dev/null +++ b/subdirmk/tests/filter/sub/dir/Dir.mk.expected @@ -0,0 +1,34 @@ +# autogenerated - do not edit +# subdirmk - test cases for generate script +# Copyright various contributors - see top level README. +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +# Prefix in sub/dir + + +sub/dir/ + +# sub_dir_TARGETS_notarget += 42 +sub_dir_TARGETS_sometarget1 +sub_dir_TARGETS_sometarget2 + +line joining + +sub_dir_WARN += 4 +WARN += 4 +sub_dir_WARN += 5 # this warning suppressed, precisely +WARN += 5 + +$(NOWARN1) + +sub_dir_WARN += 6 + +# doctests: + +# Suffix in sub/dir + +sub/dir/all:: $(sub_dir_TARGETS) +sub/dir/sometarget1:: $(sub_dir_TARGETS_sometarget1) +sub/dir/sometarget2:: $(sub_dir_TARGETS_sometarget2) +.PHONY: sub/dir/all sub/dir/sometarget1 sub/dir/sometarget2 diff --git a/subdirmk/tests/filter/sub/dir/Dir.sd.mk b/subdirmk/tests/filter/sub/dir/Dir.sd.mk new file mode 100644 index 0000000..f358e77 --- /dev/null +++ b/subdirmk/tests/filter/sub/dir/Dir.sd.mk @@ -0,0 +1,30 @@ +&# subdirmk - subdirectory test cases +&# Copyright various contributors - see top level README. +&# SPDX-License-Identifier: LGPL-2.0-or-later +&# There is NO WARRANTY. + +&:warn !some-unknown-warning + +&:changequote & +&/ + +# &TARGETS_notarget += 42 +&TARGETS_sometarget1 +&TARGETS_sometarget2 + +line &\ +joining + +&WARN += 4 +WARN += 4 +&:local+global &WARN +&WARN += 5 # this warning suppressed, precisely +WARN += 5 + +$(NOWARN1) + +&:local+global !&WARN +&WARN += 6 + +# doctests: +&:include &doctests.sd.mk diff --git a/subdirmk/tests/filter/update-expected b/subdirmk/tests/filter/update-expected new file mode 100755 index 0000000..b9c79e7 --- /dev/null +++ b/subdirmk/tests/filter/update-expected @@ -0,0 +1,23 @@ +#!/bin/sh +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. +# +# Usual approach to updating the expected outputs is +# tests/filter/check +# tests/filter/update-expected +# selectively git-add the things that are right, after inspecting them + +set -e +files=$(find tests/filter -name \*.expected.tmp) +for f in $files; do + perl -pe ' + (s/\n//, $stripnl=0) if $stripnl; + next unless /^# doctests start/../^# doctests end/; + $_=""; + $stripnl=1; + ' \ + <${f%.expected.tmp}.tmp >${f%.tmp} +done diff --git a/subdirmk/tests/intree/.gitignore b/subdirmk/tests/intree/.gitignore new file mode 100644 index 0000000..cbd3871 --- /dev/null +++ b/subdirmk/tests/intree/.gitignore @@ -0,0 +1,7 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +example diff --git a/subdirmk/tests/intree/check b/subdirmk/tests/intree/check new file mode 100755 index 0000000..68cfc00 --- /dev/null +++ b/subdirmk/tests/intree/check @@ -0,0 +1,21 @@ +#!/bin/sh +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +set -ex + +. tests/build-common + +make_copy intree + +cd tests/intree/example + +./autogen.sh && ./configure +make -j4 all check +make -j4 clean +make -j4 all check + +echo ok. diff --git a/subdirmk/tests/tests.mk b/subdirmk/tests/tests.mk new file mode 100644 index 0000000..34bb353 --- /dev/null +++ b/subdirmk/tests/tests.mk @@ -0,0 +1,15 @@ +# subdirmk - part of the test suite +# Copyright 2019 Mark Wooding +# Copyright 2019 Ian Jackson +# SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. + +TESTS=$(wildcard tests/*/check) + +all: $(addsuffix .done, $(TESTS)) + +.PHONY: tests/%/check.done all + +tests/%/check.done: + tests/$*/check >tests/$*/log 2>&1 + @echo $* ok. diff --git a/subdirmk/usual.mk.in b/subdirmk/usual.mk.in index 66c71eb..6cb0930 100644 --- a/subdirmk/usual.mk.in +++ b/subdirmk/usual.mk.in @@ -2,10 +2,11 @@ # Copyright 2019 Mark Wooding # Copyright 2019 Ian Jackson # SPDX-License-Identifier: LGPL-2.0-or-later +# There is NO WARRANTY. # Usage: # include subdirmk/usual.mk -# (probably in toplevel Subdir.sd.mk) +# (probably in toplevel Dir.sd.mk) # # Provides various conventional `make' variables, and a # rule for compiling C programs. diff --git a/test-common.sd.mk b/test-common.sd.mk index bf9c6cf..023e41c 100644 --- a/test-common.sd.mk +++ b/test-common.sd.mk @@ -7,7 +7,7 @@ include common.make &DEPS += $(src)/test-common.tcl &DEPS += common.make &DEPS += $(src)/test-common.sd.mk -&DEPS += &/Subdir.mk +&DEPS += &/Dir.mk &check-real: $(foreach t,$(&TESTNAMES),&d-$t/ok) @@ -18,8 +18,8 @@ CHECK_SILENT ?= @ $(CHECK_SILENT) export SECNET_TEST_BUILDDIR=$(topbuilddir); \ export PYTHONBYTECODEBASE=/dev/null; \ cd $(src) && \ - &/t-$* >$(topbuilddir)/&/d-$*/log 2>\&1 \ - || { cat $(topbuilddir)/&/d-$*/log >\&2; false; } + &/t-$* >$(topbuilddir)/&/d-$*/log 2>&\&1 \ + || { cat $(topbuilddir)/&/d-$*/log >&\&2; false; } $(CHECK_SILENT) printf "&/$* " $(CHECK_SILENT) touch $@ diff --git a/test-example/Subdir.sd.mk b/test-example/Dir.sd.mk similarity index 76% rename from test-example/Subdir.sd.mk rename to test-example/Dir.sd.mk index 345b11a..30813b2 100644 --- a/test-example/Subdir.sd.mk +++ b/test-example/Dir.sd.mk @@ -5,7 +5,7 @@ include common.make &/%.key: &^/%.key.b64 base64 -d <$< >$@.new && mv -f $@.new $@ -&sites.conf: $(src)/make-secnet-sites &^/sites &/Subdir.mk +&sites.conf: $(src)/make-secnet-sites &^/sites &/Dir.mk $(src)/make-secnet-sites &^/sites $@ &CLEAN += *.new