src/lexer-impl.lisp: Don't always skip a token.

It's usually, or at least often, the wrong thing to do.  Consider, for
example, how to recover from trying to parse

	( list[x] )

when presented with

	( )

Instead, add a `:force-progress' argument to the `error' parser.  The
default is /not/ to consume a token; I think this is right.

diff --git a/doc/parsing.tex b/doc/parsing.tex
index d0671a2af67326fdf586cccd1cdd5fe295a5400f..439d6d9bf4eb5f78caee6a1be7154f15c1167018 100644 (file)
--- a/doc/parsing.tex
+++ b/doc/parsing.tex
@@ -792,7 +792,8 @@ file-location protocols.
 \end{describe}
 
 \begin{describe}{parseform}
 \end{describe}
 
 \begin{describe}{parseform}

-    {error (@[[ :ignore-unconsumed @<flag> @]])                 \\ \ind
+    {error (@[[ :ignore-unconsumed @<flag> @!
+                :force-process @<flag> @]])                     \\ \ind

       @<sub-parser> @<recover-parser>}
 \end{describe}
 
       @<sub-parser> @<recover-parser>}
 \end{describe}
 

diff --git a/src/lexer-impl.lisp b/src/lexer-impl.lisp
index 821849bad202d9764cfdbd02a1a9db25d1e798b5..f00994a4913afbf5e8dd854efe95cff97a5cf1ba 100644 (file)
--- a/src/lexer-impl.lisp
+++ b/src/lexer-impl.lisp
@@ -59,7 +59,8 @@ (defun skip-until (scanner token-types &key keep-end)
       (return (values token-types nil consumedp)))
     (scanner-step scanner)))
 
       (return (values token-types nil consumedp)))
     (scanner-step scanner)))
 

-(defun parse-error-recover (scanner parser recover &key ignore-unconsumed)
+(defun parse-error-recover (scanner parser recover
+                           &key ignore-unconsumed force-progress)

   "This is the implementation of the `error' parser."
   (multiple-value-bind (result win consumedp) (funcall parser)
     (cond ((or win
   "This is the implementation of the `error' parser."
   (multiple-value-bind (result win consumedp) (funcall parser)
     (cond ((or win


@@ -84,8 +85,8 @@ (defun parse-error-recover (scanner parser recover &key ignore-unconsumed)
           ;; current token.  Finally, if we are at EOF then our best bet is
           ;; simply to propagate the current failure back to the caller, but
           ;; we handled that case above.
           ;; current token.  Finally, if we are at EOF then our best bet is
           ;; simply to propagate the current failure back to the caller, but
           ;; we handled that case above.

-          (syntax-error scanner result :continuep t)
-          (unless consumedp (scanner-step scanner))
+          (syntax-error scanner result)
+          (when (and force-progress (not consumedp)) (scanner-step scanner))

           (funcall recover)))))
 
 ;;;--------------------------------------------------------------------------
           (funcall recover)))))
 
 ;;;--------------------------------------------------------------------------

diff --git a/src/lexer-proto.lisp b/src/lexer-proto.lisp
index c6d6f2889d774049d4a43434369e7445e3435923..349e2a0ee5c325f181337ed4c461e55cd1310b96 100644 (file)
--- a/src/lexer-proto.lisp
+++ b/src/lexer-proto.lisp
@@ -116,7 +116,7 @@ (defparse skip-until (:context (context token-scanner-context)
 
 (export 'error)
 (defparse error (:context (context token-scanner-context)
 
 (export 'error)
 (defparse error (:context (context token-scanner-context)

-                (&key ignore-unconsumed)
+                (&key ignore-unconsumed force-progress)

                 sub &optional (recover t))
   "Try to parse SUB; if it fails then report an error, and parse RECOVER.
 
                 sub &optional (recover t))
   "Try to parse SUB; if it fails then report an error, and parse RECOVER.
 


@@ -135,7 +135,8 @@ (defparse error (:context (context token-scanner-context)
   `(parse-error-recover ,(parser-scanner context)
                        (parser () ,sub)
                        (parser () ,recover)
   `(parse-error-recover ,(parser-scanner context)
                        (parser () ,sub)
                        (parser () ,recover)

-                       :ignore-unconsumed ,ignore-unconsumed))
+                       :ignore-unconsumed ,ignore-unconsumed
+                       :force-progress ,force-progress))

 
 ;;;--------------------------------------------------------------------------
 ;;; Lexical analysis utilities.
 
 ;;;--------------------------------------------------------------------------
 ;;; Lexical analysis utilities.