src/lexer-{proto,impl}.lisp: Enhance `skip-until' to match token values.

diff --git a/src/lexer-impl.lisp b/src/lexer-impl.lisp
index 6a9b2cec0d590f7ccbc631043afc2b07b189d509..16861793b9a269cb027a2b91d9ee6aae38ee9940 100644 (file)
--- a/src/lexer-impl.lisp
+++ b/src/lexer-impl.lisp
@@ -50,7 +50,16 @@ (defun %skip-until (scanner token-types
                    &key (keep-end (not (null (cdr token-types)))))
   "This is the implementation of the `skip-until' parser."
   (do ((consumedp nil t))
-      ((member (token-type scanner) token-types)
+      ((let ((type (token-type scanner))
+            (value (token-value scanner)))
+        (some (lambda (spec)
+                (multiple-value-bind (want-type want-value)
+                    (cond ((listp spec) (values (car spec) (cadr spec)))
+                          (t (values spec t)))
+                  (and (eq want-type type)
+                       (or (eq want-value t)
+                           (equal want-value value)))))
+              token-types))
        (unless keep-end (scanner-step scanner))
        (values nil t (or keep-end consumedp)))
     (when (scanner-at-eof-p scanner)

diff --git a/src/lexer-proto.lisp b/src/lexer-proto.lisp
index 22d61c01b26bb0e36bea923ce39eda64972d99ed..60235ff7f0934f75c00746ca05e3b8413355cd3b 100644 (file)
--- a/src/lexer-proto.lisp
+++ b/src/lexer-proto.lisp
@@ -112,6 +112,11 @@ (defparse skip-until (:context (context token-scanner-context)
                      &rest token-types)
   "Discard tokens until we find one listed in TOKEN-TYPES.
 
+   Each of the TOKEN-TYPES is an expression which evaluates to either a
+   two-item list (TYPE VALUE), or a singleton TYPE; the latter is equivalent
+   to a list (TYPE t).  Such a pair matches a token with the corresponding
+   TYPE and VALUE, except that a VALUE of `t' matches any token value.
+
    If KEEP-END is true then retain the found token for later; otherwise
    discard it.  KEEP-END defaults to true if multiple TOKEN-TYPES are given;
    otherwise false.  If end-of-file is encountered then the indicator list is