From 1c2db39a35b4efe99594b2a53737d4f0971d01d8 Mon Sep 17 00:00:00 2001 Message-Id: <1c2db39a35b4efe99594b2a53737d4f0971d01d8.1719159403.git.mdw@distorted.org.uk> From: Mark Wooding Date: Sun, 26 Mar 2017 15:16:18 +0100 Subject: [PATCH 1/1] src/lexer-{proto,impl}.lisp: Enhance `skip-until' to match token values. Organization: Straylight/Edgeware From: Mark Wooding --- src/lexer-impl.lisp | 11 ++++++++++- src/lexer-proto.lisp | 5 +++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/lexer-impl.lisp b/src/lexer-impl.lisp index 6a9b2ce..1686179 100644 --- a/src/lexer-impl.lisp +++ b/src/lexer-impl.lisp @@ -50,7 +50,16 @@ (defun %skip-until (scanner token-types &key (keep-end (not (null (cdr token-types))))) "This is the implementation of the `skip-until' parser." (do ((consumedp nil t)) - ((member (token-type scanner) token-types) + ((let ((type (token-type scanner)) + (value (token-value scanner))) + (some (lambda (spec) + (multiple-value-bind (want-type want-value) + (cond ((listp spec) (values (car spec) (cadr spec))) + (t (values spec t))) + (and (eq want-type type) + (or (eq want-value t) + (equal want-value value))))) + token-types)) (unless keep-end (scanner-step scanner)) (values nil t (or keep-end consumedp))) (when (scanner-at-eof-p scanner) diff --git a/src/lexer-proto.lisp b/src/lexer-proto.lisp index 22d61c0..60235ff 100644 --- a/src/lexer-proto.lisp +++ b/src/lexer-proto.lisp @@ -112,6 +112,11 @@ (defparse skip-until (:context (context token-scanner-context) &rest token-types) "Discard tokens until we find one listed in TOKEN-TYPES. + Each of the TOKEN-TYPES is an expression which evaluates to either a + two-item list (TYPE VALUE), or a singleton TYPE; the latter is equivalent + to a list (TYPE t). Such a pair matches a token with the corresponding + TYPE and VALUE, except that a VALUE of `t' matches any token value. + If KEEP-END is true then retain the found token for later; otherwise discard it. KEEP-END defaults to true if multiple TOKEN-TYPES are given; otherwise false. If end-of-file is encountered then the indicator list is -- [mdw]