syntax.dtx: Allow decorative material following nonterminal name in `grammar'.

[mdwtools] / doafter.dtx

% \begin{meta-comment}
%
% $Id: doafter.dtx,v 1.1 2002/02/03 20:49:03 mdw Exp $
%
% Insert tokens to be read after a group has been processed
%
% (c) 1996 Peter Schmitt and Mark Wooding
%
% \end{meta-comment}
%
% \begin{meta-comment} <general public licence>
%%
%% doafter package -- insert a token really after a group
%% Copyright (c) 1996 Peter Schmitt and Mark Wooding
%<*package>
%%
%% This program is free software; you can redistribute it and/or modify
%% it under the terms of the GNU General Public License as published by
%% the Free Software Foundation; either version 2 of the License, or
%% (at your option) any later version.
%%
%% This program is distributed in the hope that it will be useful,
%% but WITHOUT ANY WARRANTY; without even the implied warranty of
%% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
%% GNU General Public License for more details.
%%
%% You should have received a copy of the GNU General Public License
%% along with this program; if not, write to the Free Software
%% Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
%</package>
%%
% \end{meta-comment}
%
% \begin{meta-comment} <Package preamble>
%<+latex2e>\NeedsTeXFormat{LaTeX2e}
%<+latex2e>\ProvidesPackage{doafter}
%<+latex2e>                [1996/05/08 1.2 Aftergroup hacking (PS/MDW)]
% \end{meta-comment}
%
% \CheckSum{259}
%\iffalse
%<*package>
%\fi
%% \CharacterTable
%%  {Upper-case    \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z
%%   Lower-case    \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z
%%   Digits        \0\1\2\3\4\5\6\7\8\9
%%   Exclamation   \!     Double quote  \"     Hash (number) \#
%%   Dollar        \$     Percent       \%     Ampersand     \&
%%   Acute accent  \'     Left paren    \(     Right paren   \)
%%   Asterisk      \*     Plus          \+     Comma         \,
%%   Minus         \-     Point         \.     Solidus       \/
%%   Colon         \:     Semicolon     \;     Less than     \<
%%   Equals        \=     Greater than  \>     Question mark \?
%%   Commercial at \@     Left bracket  \[     Backslash     \\
%%   Right bracket \]     Circumflex    \^     Underscore    \_
%%   Grave accent  \`     Left brace    \{     Vertical bar  \|
%%   Right brace   \}     Tilde         \~}
%%
%\iffalse
%</package>
%\fi
%
% \begin{meta-comment} <driver>
%
%<*driver>
\input{mdwtools}
\describespackage{doafter}
\author{Peter Schmitt\thanks{%
  Peter came up with the basic implementation after I posed the problem
  in the \texttt{comp.text.tex} newsgroup.  I fixed some really piddly little
  things, to improve it a bit, wrote the documentation, and turned the code
  into a nice \package{doc}ced package.  Then Peter gave me an updated
  version, and I upgraded this from memory.  Then he gave me some more tweaks
  which I haven't incorporated.}
  \and Mark Wooding}
\def\author#1{}
\mdwdoc
%</driver>
%
% \end{meta-comment}
%
% \section{Description}
%
% \subsection{What it's all about}
%
% \DescribeMacro{\doafter}
% It's common for the \TeX\ primitive |\aftergroup| to be used to `tidy up'
% after a group.  For example, \LaTeX's colour handling uses this to insert
% appropriate |\special|s when the scope of a colour change ends.  This
% causes several problems, though; for example, extra grouping must be added
% within boxes to ensure that the |\special|s don't `leak' out of their box
% and appear in odd places in the document.  \LaTeX\ usually solves this
% problem by reading the box contents as an argument, although this isn't
% particularly desirable.  The |\doafter| macro provided here will solve the
% problem in a different way, by allowing a macro to regain control after
% all the |\aftergroup| things have been processed.
%
% The macro works like this:
% \begin{grammar}
% <doafter-cmd> ::= \[[
%   "\\doafter" <token> <group>
% \]]
% \end{grammar}
% The \<token> can be any token you like, except an explicit braces, since
% it's read as an undelimited macro argument.  The \<group> is a normal
% \TeX\ group, surrounded by either implicit or explicit braces, or by
% |\begingroup| and |\endgroup| tokens.  Once the final closing token of the
% \<group> is read, and any tokens saved up by |\aftergroup| have been
% processed, the \<token> is inserted and processed.  Under normal
% circumstances, this will be a macro.
%
% There are some subtle problems with the current implementation, which you
% may need to be aware of:
%
% \begin{itemize}
%
% \item Since we're inserting things after all the |\aftergroup| tokens,
%       those tokens might read something they're not expecting if they try
%       to look ahead at the text after the group (e.g., with |\futurelet|).
%       This is obviously totally unavoidable.
%
% \item Implicit braces (like |\bgroup| and |\egroup|) inserted using
%       |\aftergroup| may be turned into \emph{explicit} $|{|_1$ and $|}|_2$
%       characters within a |\doafter| group.  This can cause probems under
%       very specialised circumstances.  The names |\bgroup| and |\egroup|
%       are treated specially, and they will work normally (remaining as
%       implicit braces).  This should minimise problems caused by this
%       slight difference.  (This only applies to the last |\aftergroup|
%       token in a group.)
%
% \item To handle the |\aftergroup| tokens properly, |\doafter| has to insert
%       some |\aftergroup| tokens of its own.  It will then process the
%       other tokens some more, and set them up to be read again.  This does
%       mean that after the group ends, some assignments and other `stomach
%       operations' will be performed, which may cause problems in
%       alignments and similar places.
%
% \end{itemize}
%
%
% \subsection{Package options}
%
% There are a fair few \textsf{docstrip} options provided by this packge:
%
% \begin{description}
% \item [driver] extracts the documentation driver.  This isn't usually
%       necessary.
% \item [package] extracts the code as a standalone package, formatted for
%       either \LaTeXe\ or Plain~\TeX.
% \item [latex2e] inserts extra identification code for a \LaTeXe\ package.
% \item [plain] inserts some extra code for a Plain \TeX\ package.
% \item [macro] just extracts the raw code, for inclusion in another package.
% \item [test] extracts some code for testing the current implementation.
% \end{description}
%
%
% \implementation
%
% \section{Implementation}
%
% \subsection{The main macro}
%
% We start outputting code here.  If this is a Plain~\TeX\ package, we must
% make \lit{@} into a letter.
%
%    \begin{macrocode}
%<*macro|package>
%<+plain>\catcode`\@=11
%    \end{macrocode}
%
% \begin{macro}{\doafter}
%
% The idea is to say \syntax{"\\doafter" <token> <group>} and expect the
% \synt{token} to be processed after the group has finished its stuff,
% even if it contains |\aftergroup| things.  My eternal gratitude goes to
% Peter Schmitt, who came up with most of the solution implemented here;
% I've just tidied up some very minor niggles and things later.
%
% Let's start with some preamble.  I'll save the (hopefully) primitive
% |\aftergroup| in a different token.
%
%    \begin{macrocode}
\let\@@aftergroup\aftergroup
%    \end{macrocode}
%
% Now to define the `user' interface.  It takes a normal undelimited
% argument, although this must be a single token; otherwise eveything will
% go wrong.  It assumes that the token following is some kind of group
% opening thing (an explicit or implicit character with catcode~1, or
% a |\begingroup| token).  To make this work, I'll save the token,
% together with an |\@@aftergroup| (to save an |\expandafter| later) in
% a temporary macro which no-one will mind me using, and then look ahead at
% the beginning-group token.
%
%    \begin{macrocode}
\def\doafter#1{%
  \def\@tempa{\@@aftergroup#1}%
  \afterassignment\doafter@i\let\@let@token%
}
%    \end{macrocode}
%
% I now have the token in |\@let@token|, so I'll put that in.  I'll then
% make |\aftergroup| do my thing rather than the normal thing, and queue
% the tokens |\@prepare@after| and the |\doafter| argument for later use.
%
%    \begin{macrocode}
\def\doafter@i{%
  \@let@token%
  \let\aftergroup\@my@aftergroup%
  \@@aftergroup\@prepare@after\@tempa%
}
%    \end{macrocode}
%
% \end{macro}
%
% \begin{macro}{\@my@aftergroup}
%
% Now the cleverness begins.  We keep two macros (Peter's original used
% count registers) which keep counts of the numbers of |\aftergroup|s,
% both locally and globally.  Let's call the local counter~$n$ and the
% global one $N$.  Every time we get a call to our |\aftergroup| hack,
% we set~$n := n+1$ and~$N := n$, and leave the token given to us for later
% processing.  When we actually process an |\aftergroup| token properly,
% set~$N := N-1$ to indicate that it's been handled; when they're all done,
% we'll have $N=n$, which is exactly what we'd have if there weren't any
% to begin with.
%
%    \begin{macrocode}
\def\ag@cnt@local{0 }
\let\ag@cnt@global\ag@cnt@local
%    \end{macrocode}
%
% Now we come to the definition of my version of |\aftergroup|.  I'll just
% add the token |\@after@token| before every |\aftergroup| token I find.
% This means there's two calls to |\aftergroup| for every one the user makes,
% but these things aren't all that common, so it's OK really.  I'll also
% bump the local counter, and synchronise them.
%
%    \begin{macrocode}
\def\@my@aftergroup{%
  \begingroup%
    \count@\ag@cnt@local%
    \advance\count@\@ne%
    \xdef\ag@cnt@global{\the\count@\space}%
  \endgroup%
  \let\ag@cnt@local\ag@cnt@global%
  \@@aftergroup\@after@token\@@aftergroup%
}
%    \end{macrocode}
%
% \end{macro}
%
% Now what does |\@after@token| we inserted above actually do?  Well, this
% is more exciting.  There are actually two different variants of the
% macro, which are used at different times.
%
% \begin{macro}{\@after@token}
%
% The default |\@after@token| starts a group, which will `catch'
% |\aftergroup| tokens which I throw at it.  I put the two counters into
% some scratch count registers.  (There's a slight problem here: Plain \TeX\
% only gives us one.  For the sake of evilness I'll use |\clubpenalty| as the
% other one.  Eeeek.)  I then redefine |\@after@token| to the second
% variant, and execute it.  The |\@start@after@group| macro starts the
% group, because this code is shared with |\@prepare@after| below.
%
%    \begin{macrocode}
\def\@after@token{%
  \@start@after@group%
  \@after@token%
}
\def\@start@after@group{%
  \begingroup%
  \count@\ag@cnt@global%
  \clubpenalty\ag@cnt@local%
  \let\@after@token\@after@token@i%
}
%    \end{macrocode}
%
% \end{macro}
%
% \begin{macro}{\@after@token@i}
%
% I have $|\count@| = N$ and $|\@tempcnta| = n$.  I'll decrement~$N$,
% and if I have $N = n$, I know that this is the last token to do, so I
% must insert an |\@after@all| after the token.  This will close the group,
% and maybe insert the original |\doafter| token if appropriate.
%
%    \begin{macrocode}
\def\@after@token@i{%
  \advance\count@\m@ne%
  \ifnum\count@=\clubpenalty%
    \global\let\ag@cnt@global\ag@cnt@local%
    \expandafter\@after@aftertoken\expandafter\@after@all%
  \else%
    \expandafter\@@aftergroup%
  \fi%
}
%    \end{macrocode}
%
% Finally, establish a default meaning for |\@after@all|.
%
%    \begin{macrocode}
\let\@after@all\endgroup
%    \end{macrocode}
%
% \end{macro}
%
% \begin{macro}{\@prepare@after}
%
% If this group is handled by |\doafter|, then the first |\aftergroup| token
% isn't |\@after@token|; it's |\@prepare@after|.
%
% There are some extra cases to deal with:
% \begin{itemize}
% \item If $N=n$ then there were no |\aftergroup| tokens, so we have an easy
%       job.  I'll just let the token do its stuff directly.
% \item Otherwise, $N>n$, and there are |\aftergroup| tokens.  I'll open
%       the group, and let |\@after@token| do all the handling.
% \end{itemize}
%
%    \begin{macrocode}
\def\@prepare@after{%
  \ifx\ag@cnt@local\ag@cnt@global\else%
    \expandafter\@prepare@after@i%
  \fi%
}
\def\@prepare@after@i#1{%
  \@start@after@group%
  \def\@after@all{\@@aftergroup#1\endgroup}%
}
%    \end{macrocode}
%
% \end{macro}
%
% \begin{macro}{\@after@aftertoken}
%
% This is where all the difficulty lies.  The next token in the stream is
% an |\aftergroup| one, which could be more or less anything.  We have an
% argument, which is some code to do \emph{after} the token has been
% |\aftergroup|ed.
%
% If the token is anything other than a brace (i.e., an  explicit character
% of category~1 or~2) then I have no problem; I can scoop up the token with
% an undelimited macro argument.  But the only way I can decide if this token
% is a brace (nondestructively) is with |\futurelet|, which makes the token
% implicit, so I can't decide whether it's really dangerous.
%
% There is a possible way of doing this\footnote{Due to Peter Schmitt,
% again.} which relates to nobbling the offending token with |\string| and
% sifting through the results.  The problem here involves scooping up all the
% tokens of a |\string|ed control sequence, which may turn out to be
% `|\csname\endcsname|' or something equally horrid.
%
% The solution I've used is much simpler: I'll change |\bgroup| and |\egroup|
% to stop them from being implicit braces before comparing.
%
%    \begin{macrocode}
\def\@after@aftertoken#1{%
  \let\bgroup\relax\let\egroup\relax%
  \toks@{#1}%
  \futurelet\@let@token\@after@aftertoken@i%
}
\def\@after@aftertoken@i{%
  \ifcat\noexpand\@let@token{%
    \@@aftergroup{%
  \else\ifcat\noexpand\@let@token}%
    \@@aftergroup}%
  \else%
    \def\@tempa##1{\@@aftergroup##1\the\toks@}%
    \expandafter\expandafter\expandafter\@tempa%
  \fi\fi%
}
%    \end{macrocode}
%
% \end{macro}
%
%
% Phew!
%
%    \begin{macrocode}
%<+plain>\catcode`\@=12
%</macro|package>
%    \end{macrocode}
%
% \subsection{Test code}
%
% The following code gives |\doafter| a bit of a testing.  It's based on
% the test suite I gave to comp.text.tex, although it's been improved a
% little since then.
%
% The first thing to do is define a control sequence with an \lit{@} sign
% in its name, so we can test catcode changes.  This also hides an
% |\aftergroup| within a macro, making life more difficult for prospective
% implementations.
%
%    \begin{macrocode}
%<*test>
\catcode`\@=11
\def\at@name{\aftergroup\saynine}
\def\saynine{\say{ix}}
\catcode`\@=12
%    \end{macrocode}
%
% Now define a command to write a string to the terminal.  The name will
% probably be familiar to REXX hackers.
%
%    \begin{macrocode}
\def\say{\immediate\write16}
%    \end{macrocode}
%
% Test one: This is really easy; it just tests that the thing works at all.
% If your implementation fails this, it's time for a major rethink.
%
%    \begin{macrocode}
\say{Test one... (1--2)}
\def\saytwo{\say{ii}}
\doafter\saytwo{\say{i}}
%    \end{macrocode}
%
% Test two: Does |\aftergroup| work?
%
%    \begin{macrocode}
\say{Test two... (1--4)}
\def\saythree{\say{iii}}
\def\sayfour{\say{iv}}
\doafter\sayfour{\say{i}\aftergroup\saythree\say{ii}}
%    \end{macrocode}
%
% Test three: Test braces and |\iffalse| working as they should.  Several
% proposed solutions based on |\write|ing the group to a file get upset by
% this test, although I forgot to include it in the torture test.  It also
% tests whether literal braces can be |\aftergroup|ed properly.  (Added a new
% test here, making sure that |\bgroup| is left as an implicit token.)
%
%    \begin{macrocode}
\say{Test three... (1--4, `\string\bgroup', 5)}
\def\sayfive{\say{v}}
\doafter\sayfive{%
  \say{i}%
  \aftergroup\say%
  \aftergroup{%
  \aftergroup\romannumeral\aftergroup3%
  \aftergroup}%
  \iffalse}\fi%
  \aftergroup\def%
  \aftergroup\sayfouretc%
  \aftergroup{%
  \aftergroup\say%
  \aftergroup{%
  \aftergroup i%
  \aftergroup v%
  \aftergroup}%
  \aftergroup\say%
  \aftergroup{%
  \aftergroup\string%
  \aftergroup\bgroup%
  \aftergroup}%
  \aftergroup}%
  \aftergroup\sayfouretc%
  \say{ii}%
}
%    \end{macrocode}
%
% Test four: Make sure the implementation isn't leaking things.  This just
% makes sure that |\aftergroup| is its normal reasonable self.
%
%    \begin{macrocode}
\say{Test four... (1--3)}
{\say{i}\aftergroup\saythree\say{ii}}
%    \end{macrocode}
%
% Test five: Nesting, aftergroup, catcodes, grouping.  This is the `torture'
% test I gave to comp.text.tex, slightly corrected (oops) and amended.  It
% ensures that nested groups and |\doafter|s work properly (the latter is
% actually more likely than might be imagined).
%
%    \begin{macrocode}
\say{Test five... (1--14)}
\def\sayten{\say{x}}
\def\saythirteen{\say{xiii}}
\def\sayfourteen{\say{xiv}}
\doafter\sayfourteen\begingroup%
  \say{i}%
  {\say{ii}\aftergroup\sayfour\say{iii}}%
  \def\saynum{\say{viii}}%
  \doafter\sayten{%
    \say{v}%
    \def\saynum{\say{vii}}%
    \catcode`\@=11%
    \aftergroup\saynum%
    \say{vi}%
    \at@name%
    \saynum%
  }%
  \say{xi}%
  \aftergroup\saythirteen%
  \say{xii}%
\endgroup
\end
%</test>
%    \end{macrocode}
%
% That's it.  All present and correct.
%
% \Finale
%
\endinput