3 ;;; Scanner protocol definitions.
5 ;;; (c) 2009 Straylight/Edgeware
8 ;;;----- Licensing notice ---------------------------------------------------
10 ;;; This file is part of the Sensible Object Design, an object system for C.
12 ;;; SOD is free software; you can redistribute it and/or modify
13 ;;; it under the terms of the GNU General Public License as published by
14 ;;; the Free Software Foundation; either version 2 of the License, or
15 ;;; (at your option) any later version.
17 ;;; SOD is distributed in the hope that it will be useful,
18 ;;; but WITHOUT ANY WARRANTY; without even the implied warranty of
19 ;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 ;;; GNU General Public License for more details.
22 ;;; You should have received a copy of the GNU General Public License
23 ;;; along with SOD; if not, write to the Free Software Foundation,
24 ;;; Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
26 (cl:in-package #:sod-parser)
28 ;;;--------------------------------------------------------------------------
29 ;;; Scanner context protocol.
31 (export 'parser-scanner)
32 (defgeneric parser-scanner (context)
34 "Return the symbol naming the CONTEXT's run-time scanner."))
36 (export 'scanner-context)
37 (defclass scanner-context ()
38 ((scanner :initarg :scanner :type symbol :reader parser-scanner))
40 "Base class for scanner contexts.
42 A scanner is simply an object maintaining the run-time state of a parsing
43 operation, in the same way as a parser context maintains the compile-time
44 state. So the scanner context is a compile-time context which expands to
45 calls to use the run-time scanner. See?
47 This class provides common compile-time behaviour for `parser-at-eof-p'
48 and friends by invoking corresponding methods on the scanner object at
51 ;;;--------------------------------------------------------------------------
52 ;;; Basic scanner protocol.
54 (export 'scanner-at-eof-p)
55 (defgeneric scanner-at-eof-p (scanner)
57 "Answer whether the SCANNER is at end-of-file.
59 It is an error to query the current item when at end-of-file."))
61 (export 'scanner-step)
62 (defgeneric scanner-step (scanner)
64 "Advance the SCANNER to the next item.
66 The precise nature of the items isn't known at this level, so a protocol
67 for accessing them is left for later."))
69 ;;;--------------------------------------------------------------------------
70 ;;; Scanner place-capture protocol.
72 (export 'scanner-capture-place)
73 (defgeneric scanner-capture-place (scanner)
75 "Capture the SCANNER's current place and return it.")
77 (error "Scanner ~S doesn't support rewinding" scanner)))
79 (export 'scanner-restore-place)
80 (defgeneric scanner-restore-place (scanner place)
82 "`Rewind' the SCANNER to the captured PLACE.
84 The place was previously captured by `scanner-capture-place'."))
86 (export 'scanner-release-place)
87 (defgeneric scanner-release-place (scanner place)
89 "Release a PLACE captured from the SCANNER.
91 The place was previously captured by `scanner-capture-place'.")
92 (:method (scanner place) nil))
94 (export 'with-scanner-place)
95 (defmacro with-scanner-place ((place scanner) &body body)
96 "Evaluate BODY with PLACE bound to the captured current place.
98 Automatically releases the place when the BODY finishes. Note that
99 if you wanted to circumvent the cleanup then you should have used
100 `with-parser-place', which does all of this in the meta-level."
102 (multiple-value-bind (docs decls body) (parse-body body :docp nil)
103 (declare (ignore docs))
104 `(let ((,place (scanner-capture-place ,scanner)))
106 (unwind-protect (progn ,@body)
107 (when ,place (scanner-release-place ,scanner ,place)))))))
109 ;;;--------------------------------------------------------------------------
110 ;;; Character scanner protocol.
112 (export 'character-scanner)
113 (defclass character-scanner ()
115 (:documentation "Base class for character scanners."))
117 (export 'character-scanner-context)
118 (defclass character-scanner-context
119 (scanner-context character-parser-context)
122 "A context for a richer character-oriented scanner."))
124 (export 'scanner-current-char)
125 (defgeneric scanner-current-char (scanner)
127 "Returns the SCANNER's current character.
129 You advance to the next one using `scanner-step'."))
131 (export 'scanner-unread)
132 (defgeneric scanner-unread (scanner char)
134 "Rewind SCANNER by one character, specifically CHAR.
136 CHAR must be the character most recently stepped over by `scanner-step' --
137 it is an error to unread before the first call to `scanner-step'. It is
138 also an error to unread after encountering end-of-file."))
140 (export 'scanner-interval)
141 (defgeneric scanner-interval (scanner place-a &optional place-b)
143 "Return the characters from PLACE-A up to (but not including) PLACE-B.
145 The characters are returned as a string. If PLACE-B is omitted, return
146 the characters up to (but not including) the current position. It is an
147 error if PLACE-B precedes PLACE-A or they are from different scanners."))
149 (export '(scanner-filename scanner-line scanner-column))
150 (defgeneric scanner-filename (scanner)
151 (:documentation "Return the filename backing the SCANNER.")
152 (:method (scanner) nil))
153 (defgeneric scanner-line (scanner)
154 (:documentation "Return the SCANNER's current line number.")
155 (:method (scanner) nil))
156 (defgeneric scanner-column (scanner)
157 (:documentation "Return the SCANNER's current column number.")
158 (:method (scanner) nil))
160 (defun scanner-file-location (scanner)
161 "Capture the current location of the SCANNER.
163 This uses the generic functions `scanner-filename', `scanner-line' and
164 `scanner-column' to compute its result. There are default methods on
165 these functions which make up dummy results.
167 There is a method for `file-location' defined on `character-scanner' which
168 simply calls this function; but since some scanners are structure-objects
169 rather than standard-objects they can't include `character-scanner' as a
171 (make-file-location (scanner-filename scanner)
172 (scanner-line scanner)
173 (scanner-column scanner)))
175 ;;;--------------------------------------------------------------------------
176 ;;; Token scanner protocol.
178 ;; The token scanner base class and parser context.
180 (export '(token-scanner token-type token-value))
181 (defclass token-scanner ()
182 ((%type :reader token-type)
183 (value :reader token-value)
184 (captures :initform 0 :type fixnum)
185 (tail :initform nil :type (or token-scanner-place null))
186 (filename :initarg :filename :type string :reader scanner-filename)
187 (line :initarg :line :initform 1 :type fixnum :accessor scanner-line)
188 (column :initarg :column :initform 0
189 :type fixnum :accessor scanner-column))
191 "A rewindable scanner for tokenizing.
193 The scanner should be used via the parser protocol; see also the token
194 scanner protocol, which explains the model.
196 Subclasses must provide the detailed scanning behaviour -- most notably
197 the `scanner-token' generic function -- and also implement a method on
198 `file-location' to return the location. The `scanner-token' method should
199 also update the `line' and `column' slots to track the position in the
200 underlying source, if appropriate. This class will handle the remaining
201 details, such as dealing correctly with rewinding."))
203 (export 'token-scanner-context)
204 (defclass token-scanner-context (scanner-context token-parser-context)
207 "A parser context for a richer token-based scanners."))
211 (export '(token-scanner-place token-scanner-place-p))
212 (defstruct (token-scanner-place
213 (:constructor make-token-scanner-place
214 (&key scanner next type value line column
216 "A link in the chain of lookahead tokens; capturable as a place.
218 If the scanner's place is captured, it starts to maintain a list of
219 lookahead tokens. The list contains internal links -- it works out
220 slightly easier that way. This is basically a simpler version of the
221 charbuf scanner (q.v.); most significantly, the chain links here do double
222 duty as place markers.
224 The details of this structure are not a defined part of the token scanner
227 (scanner nil :type token-scanner :read-only t)
228 (next nil :type (or token-scanner-place null))
229 (%type nil :read-only t)
230 (value nil :read-only t)
231 (line 1 :type (or fixnum null) :read-only t)
232 (column 0 :type (or fixnum null) :read-only t))
233 (define-access-wrapper token-scanner-place-type token-scanner-place-%type
238 (export 'scanner-token)
239 (defgeneric scanner-token (scanner)
241 "Internal protocol: read the next token from the SCANNER.
243 This function is called by `scanner-step' to actually read the next token
244 if necessary. It should return two values: the token's `type' and its
247 ;;;--------------------------------------------------------------------------
248 ;;; Character scanner streams.
250 ;;; This seems like an abstraction inversion, but it's important if we're to
251 ;;; `read' from a character scanner.
253 (export 'character-scanner-stream)
254 (defclass character-scanner-stream (fundamental-character-input-stream)
255 ((scanner :initarg :scanner))
257 "A stream which reads from a character scanner.
259 The SCANNER must implement the character scanner protcol, including
260 `scanner-current-char', `scanner-step', and `scanner-unread'; it is not
261 necessary that the scanner implement the place-capture protocol.
263 The stream can be made more efficient by implementing
264 `stream-read-sequence' and `stream-read-line' in a scanner-specific
267 (export 'make-scanner-stream)
268 (defgeneric make-scanner-stream (scanner)
270 "Return a stream which reads from the SCANNER.
272 The default method simply constructs a `character-scanner-stream'
273 instance. Subclasses of `character-scanner' can override this method in
274 order to return instances of more efficient stream subclasses.")
275 (:method ((scanner character-scanner))
276 (make-instance 'character-scanner-stream :scanner scanner)))
278 ;;;----- That's all, folks --------------------------------------------------