1 /* iobuf.h - I/O buffer
2 * Copyright (C) 1998, 1999, 2000, 2001, 2003,
3 * 2010 Free Software Foundation, Inc.
5 * This file is part of GnuPG.
7 * This file is free software; you can redistribute it and/or modify
8 * it under the terms of either
10 * - the GNU Lesser General Public License as published by the Free
11 * Software Foundation; either version 3 of the License, or (at
12 * your option) any later version.
16 * - the GNU General Public License as published by the Free
17 * Software Foundation; either version 2 of the License, or (at
18 * your option) any later version.
20 * or both in parallel, as here.
22 * This file is distributed in the hope that it will be useful,
23 * but WITHOUT ANY WARRANTY; without even the implied warranty of
24 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
25 * GNU General Public License for more details.
27 * You should have received a copy of the GNU General Public License
28 * along with this program; if not, see <https://www.gnu.org/licenses/>.
31 #ifndef GNUPG_COMMON_IOBUF_H
32 #define GNUPG_COMMON_IOBUF_H
34 /* An iobuf is basically a filter in a pipeline.
36 Consider the following command, which consists of three filters
37 that are chained together:
39 $ cat file | base64 --decode | gunzip
41 The first filter reads the file from the file system and sends that
42 data to the second filter. The second filter decodes
43 base64-encoded data and sends the data to the third and last
44 filter. The last filter decompresses the data and the result is
45 displayed on the terminal. The iobuf system works in the same way
46 where each iobuf is a filter and the individual iobufs can be
49 There are number of predefined filters. iobuf_open(), for
50 instance, creates a filter that reads from a specified file. And,
51 iobuf_temp_with_content() creates a filter that returns some
52 specified contents. There are also filters for writing content.
53 iobuf_openrw opens a file for writing. iobuf_temp creates a filter
54 that writes data to a fixed-sized buffer.
56 To chain filters together, you use the iobuf_push_filter()
57 function. The filters are chained together using the chain field
60 A pipeline can only be used for reading (IOBUF_INPUT) or for
61 writing (IOBUF_OUTPUT / IOBUF_OUTPUT_TEMP). When reading, data
62 flows from the last filter towards the first. That is, the user
63 calls iobuf_read(), the module reads from the first filter, which
64 gets its input from the second filter, etc. When writing, data
65 flows from the first filter towards the last. In this case, when
66 the user calls iobuf_write(), the data is written to the first
67 filter, which writes the transformed data to the second filter,
70 An iobuf_t contains some state about the filter. For instance, it
71 indicates if the filter has already returned EOF (filter_eof) and
72 the next filter in the pipeline, if any (chain). It also contains
73 a function pointer, filter. This is a generic function. It is
74 called when input is needed or output is available. In this case
75 it is passed a pointer to some filter-specific persistent state
76 (filter_ov), the actual operation, the next filter in the chain, if
77 any, and a buffer that either contains the contents to write, if
78 the pipeline is setup to write data, or is the place to store data,
79 if the pipeline is setup to read data.
82 Unlike a Unix pipeline, an IOBUF pipeline can return EOF multiple
83 times. This is similar to the following:
85 { cat file1; cat file2; } | grep foo
87 However, instead of grep seeing a single stream, grep would see
88 each byte stream followed by an EOF marker. (When a filter returns
89 EOF, the EOF is returned to the user exactly once and then the
90 filter is removed from the pipeline.) */
93 #include <gpg-error.h>
95 #include "../common/types.h"
96 #include "../common/sysutils.h"
98 #define DBG_IOBUF iobuf_debug_mode
100 /* Filter control modes. */
105 IOBUFCTRL_UNDERFLOW = 3,
108 IOBUFCTRL_CANCEL = 6,
113 /* Command codes for iobuf_ioctl. */
116 IOBUF_IOCTL_KEEP_OPEN = 1, /* Uses intval. */
117 IOBUF_IOCTL_INVALIDATE_CACHE = 2, /* Uses ptrval. */
118 IOBUF_IOCTL_NO_CACHE = 3, /* Uses intval. */
119 IOBUF_IOCTL_FSYNC = 4 /* Uses ptrval. */
124 /* Pipeline is in input mode. The data flows from the end to the
125 beginning. That is, when reading from the pipeline, the first
126 filter gets its input from the second filter, etc. */
128 /* Pipeline is in input mode. The last filter in the pipeline is
129 a temporary buffer from which the data is "read". */
131 /* Pipeline is in output mode. The data flows from the beginning
132 to the end. That is, when writing to the pipeline, the user
133 writes to the first filter, which transforms the data and sends
134 it to the second filter, etc. */
136 /* Pipeline is in output mode. The last filter in the pipeline is
137 a temporary buffer that grows as necessary. */
142 typedef struct iobuf_struct *iobuf_t;
143 typedef struct iobuf_struct *IOBUF; /* Compatibility with gpg 1.4. */
145 /* fixme: we should hide most of this stuff */
148 /* The type of filter. Either IOBUF_INPUT, IOBUF_OUTPUT or
149 IOBUF_OUTPUT_TEMP. */
152 /* nlimit can be changed using iobuf_set_limit. If non-zero, it is
153 the number of additional bytes that can be read from the filter
154 before EOF is forcefully returned. */
156 /* nbytes if the number of bytes that have been read (using
157 iobuf_get / iobuf_readbyte / iobuf_read) since the last call to
161 /* The number of bytes read prior to the last call to
162 iobuf_set_limit. Thus, the total bytes read (i.e., the position
163 of stream) is ntotal + nbytes. */
166 /* Whether we need to read from the filter one byte at a time or
167 whether we can do bulk reads. We need to read one byte at a time
168 if a limit (set via iobuf_set_limit) is active. */
171 /* A buffer for unread/unwritten data.
173 For an output pipeline (IOBUF_OUTPUT), this is the data that has
174 not yet been written to the filter. Consider a simple pipeline
175 consisting of a single stage, which writes to a file. When you
176 write to the pipeline (iobuf_writebyte or iobuf_write), the data
177 is first stored in this buffer. Only when the buffer is full or
178 you call iobuf_flush() is FILTER actually called and the data
181 For an input pipeline (IOBUF_INPUT), this is the data that has
182 been read from this filter, but not yet been read from the
183 preceding filter (or the user, if this filter is the head of the
184 pipeline). Again, consider a simple pipeline consisting of a
185 single stage. This stage reads from a file. If you read a
186 single byte (iobuf_get) and the buffer is empty, then FILTER is
187 called to fill the buffer. In this case, a single byte is not
188 requested, but the whole buffer is filled (if possible). */
191 /* Size of the buffer. */
193 /* Number of bytes at the beginning of the buffer that have
194 already been consumed. (In other words: the index of the first
195 byte that hasn't been consumed.) This is only non-zero for
198 /* The number of bytes in the buffer including any bytes that have
201 /* The buffer itself. */
205 /* When FILTER is called to read some data, it may read some data
206 and then return EOF. We can't return the EOF immediately.
207 Instead, we note that we observed the EOF and when the buffer is
208 finally empty, we return the EOF. */
210 /* Like filter_eof, when FILTER is called to read some data, it may
211 read some data and then return an error. We can't return the
212 error (in the form of an EOF) immediately. Instead, we note that
213 we observed the error and when the buffer is finally empty, we
217 /* The callback function to read data from the filter, etc. See
218 iobuf_filter_push for details. */
219 int (*filter) (void *opaque, int control,
220 iobuf_t chain, byte * buf, size_t * len);
221 /* An opaque pointer that can be used for local filter state. This
222 is passed as the first parameter to FILTER. */
224 /* Whether the iobuf code should free(filter_ov) when destroying the
228 /* When using iobuf_open, iobuf_create, iobuf_openrw to open a file,
229 the file's name is saved here. This is used to delete the file
230 when an output pipeline (IOBUF_OUPUT) is canceled
234 /* The next filter in the pipeline. */
237 /* This field is for debugging. Each time a filter is allocated
238 (via iobuf_alloc()), a monotonically increasing counter is
239 incremented and this field is set to the new value. This field
240 should only be accessed via the iobuf_io macro. */
243 /* The number of filters in the pipeline following (not including)
244 this one. When you call iobuf_push_filter or iobuf_push_filter2,
245 this value is used to check the length of the pipeline if the
246 pipeline already contains 65 stages then these functions fail.
247 This amount of nesting typically indicates corrupted data or an
248 active denial of service attack. */
252 #ifndef EXTERN_UNLESS_MAIN_MODULE
253 #if defined (__riscos__) && !defined (INCLUDED_BY_MAIN_MODULE)
254 #define EXTERN_UNLESS_MAIN_MODULE extern
256 #define EXTERN_UNLESS_MAIN_MODULE
259 EXTERN_UNLESS_MAIN_MODULE int iobuf_debug_mode;
262 /* Returns whether the specified filename corresponds to a pipe. In
263 particular, this function checks if FNAME is "-" and, if special
264 filenames are enabled (see check_special_filename), whether
265 FNAME is a special filename. */
266 int iobuf_is_pipe_filename (const char *fname);
268 /* Allocate a new filter. This filter doesn't have a function
269 assigned to it. Thus you need to manually set IOBUF->FILTER and
270 IOBUF->FILTER_OV, if required. This function is intended to help
271 create a new primary source or primary sink, i.e., the last filter
274 USE is IOBUF_INPUT, IOBUF_INPUT_TEMP, IOBUF_OUTPUT or
277 BUFSIZE is the desired internal buffer size (that is, the size of
278 the typical read / write request). */
279 iobuf_t iobuf_alloc (int use, size_t bufsize);
281 /* Create an output filter that simply buffers data written to it.
282 This is useful for collecting data for later processing. The
283 buffer can be written to in the usual way (iobuf_write, etc.). The
284 data can later be extracted using iobuf_write_temp() or
285 iobuf_temp_to_buffer(). */
286 iobuf_t iobuf_temp (void);
288 /* Create an input filter that contains some data for reading. */
289 iobuf_t iobuf_temp_with_content (const char *buffer, size_t length);
291 /* Create an input file filter that reads from a file. If FNAME is
292 '-', reads from stdin. If special filenames are enabled
293 (iobuf_enable_special_filenames), then interprets special
295 iobuf_t iobuf_open (const char *fname);
297 /* Create an output file filter that writes to a file. If FNAME is
298 NULL or '-', writes to stdout. If special filenames are enabled
299 (iobuf_enable_special_filenames), then interprets special
300 filenames. If FNAME is not NULL, '-' or a special filename, the
301 file is opened for writing. If the file exists, it is truncated.
302 If MODE700 is TRUE, the file is created with mode 600. Otherwise,
304 iobuf_t iobuf_create (const char *fname, int mode700);
306 /* Create an output file filter that writes to a specified file.
307 Neither '-' nor special file names are recognized. */
308 iobuf_t iobuf_openrw (const char *fname);
310 /* Create a file filter using an existing file descriptor. If MODE
311 contains the letter 'w', creates an output filter. Otherwise,
312 creates an input filter. Note: MODE must reflect the file
313 descriptors actual mode! When the filter is destroyed, the file
314 descriptor is closed. */
315 iobuf_t iobuf_fdopen (int fd, const char *mode);
317 /* Like iobuf_fdopen, but doesn't close the file descriptor when the
318 filter is destroyed. */
319 iobuf_t iobuf_fdopen_nc (int fd, const char *mode);
321 /* Create a filter using an existing estream. If MODE contains the
322 letter 'w', creates an output filter. Otherwise, creates an input
323 filter. If KEEP_OPEN is TRUE, then the stream is not closed when
324 the filter is destroyed. Otherwise, the stream is closed when the
325 filter is destroyed. */
326 iobuf_t iobuf_esopen (estream_t estream, const char *mode, int keep_open);
328 /* Create a filter using an existing socket. On Windows creates a
329 special socket filter. On non-Windows systems simply, this simply
330 calls iobuf_fdopen. */
331 iobuf_t iobuf_sockopen (int fd, const char *mode);
333 /* Set various options / perform different actions on a PIPELINE. See
334 the IOBUF_IOCTL_* macros above. */
335 int iobuf_ioctl (iobuf_t a, iobuf_ioctl_t cmd, int intval, void *ptrval);
337 /* Close a pipeline. The filters in the pipeline are first flushed
338 using iobuf_flush, if they are output filters, and then
339 IOBUFCTRL_FREE is called on each filter.
341 If any filter returns a non-zero value in response to the
342 IOBUFCTRL_FREE, that first such non-zero value is returned. Note:
343 processing is not aborted in this case. If all filters are freed
344 successfully, 0 is returned. */
345 int iobuf_close (iobuf_t iobuf);
347 /* Calls IOBUFCTRL_CANCEL on each filter in the pipeline. Then calls
348 io_close() on the pipeline. Finally, if the pipeline is an output
349 pipeline, deletes the file. Returns the result of calling
350 iobuf_close on the pipeline. */
351 int iobuf_cancel (iobuf_t iobuf);
353 /* Add a new filter to the front of a pipeline. A is the head of the
354 pipeline. F is the filter implementation. OV is an opaque pointer
355 that is passed to F and is normally used to hold any internal
356 state, such as a file pointer.
358 Note: you may only maintain a reference to an iobuf_t as a
359 reference to the head of the pipeline. That is, don't think about
360 setting a pointer in OV to point to the filter's iobuf_t. This is
361 because when we add a new filter to a pipeline, we memcpy the state
362 in A into new buffer. This has the advantage that there is no need
363 to update any references to the pipeline when a filter is added or
364 removed, but it also means that a filter's state moves around in
367 The behavior of the filter function is determined by the value of
368 the control parameter:
370 IOBUFCTRL_INIT: Called this value just before the filter is
371 linked into the pipeline. This can be used to initialize
372 internal data structures.
374 IOBUFCTRL_FREE: Called with this value just before the filter is
375 removed from the pipeline. Normally used to release internal
376 data structures, close a file handle, etc.
378 IOBUFCTRL_UNDERFLOW: Called with this value to fill the passed
379 buffer with more data. *LEN is the size of the buffer. Before
380 returning, it should be set to the number of bytes which were
381 written into the buffer. The function must return 0 to
382 indicate success, -1 on EOF and a GPG_ERR_xxxxx code for any
385 Note: this function may both return data and indicate an error
386 or EOF. In this case, it simply writes the data to BUF, sets
387 *LEN and returns the appropriate return code. The implication
388 is that if an error occurs and no data has yet been written, it
389 is essential that *LEN be set to 0!
391 IOBUFCTRL_FLUSH: Called with this value to write out any
392 collected data. *LEN is the number of bytes in BUF that need
393 to be written out. Returns 0 on success and a GPG_ERR_* code
394 otherwise. *LEN must be set to the number of bytes that were
397 IOBUFCTRL_CANCEL: Called with this value when iobuf_cancel() is
398 called on the pipeline.
400 IOBUFCTRL_DESC: Called with this value to get a human-readable
401 description of the filter. *LEN is the size of the buffer.
402 The description is filled into BUF, NUL-terminated. Always
405 int iobuf_push_filter (iobuf_t a, int (*f) (void *opaque, int control,
406 iobuf_t chain, byte * buf,
407 size_t * len), void *ov);
408 /* This variant of iobuf_push_filter allows the called to indicate
409 that OV should be freed when this filter is freed. That is, if
410 REL_OV is TRUE, then when the filter is popped or freed OV will be
411 freed after the filter function is called with control set to
413 int iobuf_push_filter2 (iobuf_t a,
414 int (*f) (void *opaque, int control, iobuf_t chain,
415 byte * buf, size_t * len), void *ov,
418 /* Pop the top filter. The top filter must have the filter function F
419 and the cookie OV. The cookie check is ignored if OV is NULL. */
420 int iobuf_pop_filter (iobuf_t a,
421 int (*f) (void *opaque, int control,
422 iobuf_t chain, byte * buf, size_t * len),
425 /* Used for debugging. Prints out the chain using log_debug if
426 IOBUF_DEBUG_MODE is not 0. */
427 int iobuf_print_chain (iobuf_t a);
429 /* Indicate that some error occurred on the specified filter. */
430 #define iobuf_set_error(a) do { (a)->error = 1; } while(0)
432 /* Return any pending error on filter A. */
433 #define iobuf_error(a) ((a)->error)
435 /* Limit the amount of additional data that may be read from the
436 filter. That is, if you've already read 100 bytes from A and you
437 set the limit to 50, then you can read up to an additional 50 bytes
438 (i.e., a total of 150 bytes) before EOF is forcefully returned.
439 Setting NLIMIT to 0 removes any active limit.
441 Note: using iobuf_seek removes any currently enforced limit! */
442 void iobuf_set_limit (iobuf_t a, off_t nlimit);
444 /* Returns the number of bytes that have been read from the pipeline.
445 Note: the result is undefined for IOBUF_OUTPUT and IOBUF_OUTPUT_TEMP
447 off_t iobuf_tell (iobuf_t a);
449 /* There are two cases:
451 - If A is an INPUT or OUTPUT pipeline, then the last filter in the
452 pipeline is found. If that is not a file filter, -1 is returned.
453 Otherwise, an fseek(..., SEEK_SET) is performed on the file
456 - If A is a TEMP pipeline and the *first* (and thus only filter) is
457 a TEMP filter, then the "file position" is effectively unchanged.
458 That is, data is appended to the buffer and the seek does not
459 cause the size of the buffer to grow.
461 If no error occurred, then any limit previous set by
462 iobuf_set_limit() is cleared. Further, any error on the filter
463 (the file filter or the temp filter) is cleared.
465 Returns 0 on success and -1 if an error occurs. */
466 int iobuf_seek (iobuf_t a, off_t newpos);
468 /* Read a single byte. If a filter has no more data, returns -1 to
469 indicate the EOF. Generally, you don't want to use this function,
470 but instead prefer the iobuf_get macro, which is faster if there is
471 data in the internal buffer. */
472 int iobuf_readbyte (iobuf_t a);
474 /* Get a byte from the iobuf; must check for eof prior to this
475 function. This function returns values in the range 0 .. 255 or -1
476 to indicate EOF. iobuf_get_noeof() does not return -1 to indicate
477 EOF, but masks the returned value to be in the range 0 .. 255. */
478 #define iobuf_get(a) \
479 ( ((a)->nofast || (a)->d.start >= (a)->d.len )? \
480 iobuf_readbyte((a)) : ( (a)->nbytes++, (a)->d.buf[(a)->d.start++] ) )
481 #define iobuf_get_noeof(a) (iobuf_get((a))&0xff)
483 /* Fill BUF with up to BUFLEN bytes. If a filter has no more data,
484 returns -1 to indicate the EOF. Otherwise returns the number of
486 int iobuf_read (iobuf_t a, void *buf, unsigned buflen);
488 /* Read a line of input (including the '\n') from the pipeline.
490 The semantics are the same as for fgets(), but if the buffer is too
491 short a larger one will be allocated up to *MAX_LENGTH and the end
492 of the line except the trailing '\n' discarded. (Thus,
493 *ADDR_OF_BUFFER must be allocated using malloc().) If the buffer
494 is enlarged, then *LENGTH_OF_BUFFER will be updated to reflect the
495 new size. If the line is truncated, then *MAX_LENGTH will be set
496 to 0. If *ADDR_OF_BUFFER is NULL, a buffer is allocated using
499 A line is considered a byte stream ending in a '\n'. Returns the
500 number of characters written to the buffer (i.e., excluding any
501 discarded characters due to truncation). Thus, use this instead of
502 strlen(buffer) to determine the length of the string as this is
503 unreliable if the input contains NUL characters.
505 EOF is indicated by a line of length zero.
507 The last LF may be missing due to an EOF. */
508 unsigned iobuf_read_line (iobuf_t a, byte ** addr_of_buffer,
509 unsigned *length_of_buffer, unsigned *max_length);
511 /* Read up to BUFLEN bytes from pipeline A. Note: this function can't
512 return more than the pipeline's internal buffer size. The return
513 value is the number of bytes actually written to BUF. If the
514 filter returns EOF, then this function returns -1.
516 This function does not clear any pending EOF. That is, if the
517 pipeline consists of two filters and the first one returns EOF
518 during the peek, then the subsequent iobuf_read* will still return
519 EOF before returning the data from the second filter. */
520 int iobuf_peek (iobuf_t a, byte * buf, unsigned buflen);
522 /* Write a byte to the pipeline. Returns 0 on success and an error
524 int iobuf_writebyte (iobuf_t a, unsigned c);
526 /* Alias for iobuf_writebyte. */
527 #define iobuf_put(a,c) iobuf_writebyte(a,c)
529 /* Write a sequence of bytes to the pipeline. Returns 0 on success
530 and an error code otherwise. */
531 int iobuf_write (iobuf_t a, const void *buf, unsigned buflen);
533 /* Write a string (not including the NUL terminator) to the pipeline.
534 Returns 0 on success and an error code otherwise. */
535 int iobuf_writestr (iobuf_t a, const char *buf);
537 /* Flushes the pipeline removing all filters but the sink (the last
538 filter) in the process. */
539 void iobuf_flush_temp (iobuf_t temp);
541 /* Flushes the pipeline SOURCE removing all filters but the sink (the
542 last filter) in the process (i.e., it calls
543 iobuf_flush_temp(source)) and then writes the data to the pipeline
544 DEST. Note: this doesn't free (iobuf_close()) SOURCE. Both SOURCE
545 and DEST must be output pipelines. */
546 int iobuf_write_temp (iobuf_t dest, iobuf_t source);
548 /* Flushes each filter in the pipeline (i.e., sends any buffered data
549 to the filter by calling IOBUFCTRL_FLUSH). Then, copies up to the
550 first BUFLEN bytes from the last filter's internal buffer (which
551 will only be non-empty if it is a temp filter) to the buffer
552 BUFFER. Returns the number of bytes actually copied. */
553 size_t iobuf_temp_to_buffer (iobuf_t a, byte * buffer, size_t buflen);
555 /* Copies the data from the input iobuf SOURCE to the output iobuf
556 DEST until either an error is encountered or EOF is reached.
557 Returns the number of bytes successfully written. If an error
558 occurred, then any buffered bytes are not returned to SOURCE and are
559 effectively lost. To check if an error occurred, use
561 size_t iobuf_copy (iobuf_t dest, iobuf_t source);
563 /* Return the size of any underlying file. This only works with
564 file_filter based pipelines.
566 On Win32, it is sometimes not possible to determine the size of
567 files larger than 4GB. In this case, *OVERFLOW (if not NULL) is
568 set to 1. Otherwise, *OVERFLOW is set to 0. */
569 off_t iobuf_get_filelength (iobuf_t a, int *overflow);
570 #define IOBUF_FILELENGTH_LIMIT 0xffffffff
572 /* Return the file descriptor designating the underlying file. This
573 only works with file_filter based pipelines. */
574 int iobuf_get_fd (iobuf_t a);
576 /* Return the real filename, if available. This only supports
577 pipelines that end in file filters. Returns NULL if not
579 const char *iobuf_get_real_fname (iobuf_t a);
581 /* Return the filename or a description thereof. For instance, for
582 iobuf_open("-"), this will return "[stdin]". This only supports
583 pipelines that end in file filters. Returns NULL if not
585 const char *iobuf_get_fname (iobuf_t a);
587 /* Like iobuf_getfname, but instead of returning NULL if no
588 description is available, return "[?]". */
589 const char *iobuf_get_fname_nonnull (iobuf_t a);
591 /* Pushes a filter on the pipeline that interprets the datastream as
592 an OpenPGP data block whose length is encoded using partial body
593 length headers (see Section 4.2.2.4 of RFC 4880). Concretely, it
594 just returns / writes the data and finishes the packet with an
596 void iobuf_set_partial_body_length_mode (iobuf_t a, size_t len);
598 /* If PARTIAL is set, then read from the pipeline until the first EOF
601 If PARTIAL is 0, then read up to N bytes or until the first EOF is
604 Recall: a filter can return EOF. In this case, it and all
605 preceding filters are popped from the pipeline and the next read is
606 from the following filter (which may or may not return EOF). */
607 void iobuf_skip_rest (iobuf_t a, unsigned long n, int partial);
609 #define iobuf_where(a) "[don't know]"
611 /* Each time a filter is allocated (via iobuf_alloc()), a
612 monotonically increasing counter is incremented and this field is
613 set to the new value. This macro returns that number. */
614 #define iobuf_id(a) ((a)->no)
616 #define iobuf_get_temp_buffer(a) ( (a)->d.buf )
617 #define iobuf_get_temp_length(a) ( (a)->d.len )
619 /* Whether the filter uses an in-memory buffer. */
620 #define iobuf_is_temp(a) ( (a)->use == IOBUF_OUTPUT_TEMP )
622 #endif /*GNUPG_COMMON_IOBUF_H*/