+From ijackson@chiark.greenend.org.uk Wed Aug 12 19:36:07 BST 1998
+Article: 742 of ucam.comp.unix
+Path: ewrotcd!not-for-mail
+From: ijackson@chiark.greenend.org.uk (Ian Jackson)
+Newsgroups: ucam.comp.unix,comp.sys.sun.admin
+Subject: Buffering programs (was Re: Speeding up network dumps)
+Date: 21 Jul 1998 12:23:23 +0100 (BST)
+Organization: Linux Unlimited
+Lines: 233
+Message-ID: <xWb*k6rBn@news.chiark.greenend.org.uk>
+References: <6oiapl$6k5$1@pegasus.csx.cam.ac.uk> <6ol7kf$cln$2@pegasus.csx.cam.ac.uk>
+NNTP-Posting-Host: chiark.greenend.org.uk
+X-Server-Date: 21 Jul 1998 11:23:25 GMT
+Originator: ijackson@news.chiark.greenend.org.uk ([127.0.0.1])
+Xref: news.chiark.greenend.org.uk ucam.comp.unix:742 comp.sys.sun.admin:14526
+
+I said I'd post here the buffering programs that I use, so here they
+are.
+
+They're perfectly functional and (I believe) correct, but they do lack
+sophistication. The size of the buffer is a #define, and the
+diagnostics lack verbosity. I haven't compiled them on anything but
+Linux/i386 libc5, but they ought to work on other reasonable Unices.
+
+writebuffer is intended for writing to tapes; when its buffer is empty
+it will write no more data until it gets 3/4 of its buffer filled, or
+EOF.
+
+readbuffer is intended for reading from tapes; when its buffer becomes
+full it stops reading until it's down to 1/4.
+
+Remember that using shell pipes loses the exit status of all but the
+last command, and so can cause failures to go unnoticed.
+
+
+/*
+ * writebuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#define BUFFER 16*1024*1024
+#define WAITFILL ((BUFFER*3)/4)
+
+static inline int min(int a, int b) { return a<=b ? a : b; }
+
+static void nonblock(int fd) {
+ int r;
+ r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
+ r |= O_NDELAY;
+ if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
+}
+
+int main(void) {
+ static unsigned char buf[BUFFER];
+
+ unsigned char *wp, *rp;
+ int used,r,writing,seeneof;
+ fd_set readfds;
+ fd_set writefds;
+
+ used=0; wp=rp=buf; writing=0; seeneof=0;
+ nonblock(0); nonblock(1);
+ while (!seeneof || used) {
+ FD_ZERO(&readfds); if (!seeneof && used+1<BUFFER) FD_SET(0,&readfds);
+ FD_ZERO(&writefds); if (writing) FD_SET(1,&writefds);
+/*fprintf(stderr,"used %6d writing %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
+ used,writing,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ r= select(2,&readfds,&writefds,0,0);
+/*fprintf(stderr,"\t readable %d writeable %d\n",
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ if (r == -1) {
+ if (errno == EINTR) continue;
+ perror("select"); exit(1);
+ }
+ if (FD_ISSET(1,&writefds) && !FD_ISSET(0,&readfds) && !used) {
+ writing= 0;
+ FD_CLR(1,&writefds);
+/*fprintf(stderr,"\t buffers empty - stopping\n");*/
+ }
+ if (FD_ISSET(0,&readfds)) {
+ r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
+ if (!r) {
+/*fprintf(stderr,"\t eof detected\n");*/
+ seeneof=1; writing=1;
+ } else if (r<0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
+fprintf(stderr,"\t read transient error\n");
+ } else {
+/*fprintf(stderr,"\t read %d\n",r);*/
+ used+= r;
+ rp+= r;
+ if (rp == buf+BUFFER) rp=buf;
+/*fprintf(stderr,"\t starting writes\n");*/
+ }
+ if (used > WAITFILL) writing=1;
+ }
+ if (FD_ISSET(1,&writefds) && used) {
+ r= write(1,wp,min(used,buf+BUFFER-wp));
+ if (r<=0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
+/*fprintf(stderr,"\t write transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t wrote %d\n",r);*/
+ used-= r;
+ wp+= r;
+ if (wp == buf+BUFFER) wp=buf;
+ }
+ }
+ }
+ exit(0);
+}
+
+
+/*
+ * readbuffer.c
+ *
+ * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
+ *
+ * This is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2,
+ * or (at your option) any later version.
+ *
+ * This is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public
+ * License along with this file; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ *
+ */
+
+#include <sys/time.h>
+#include <sys/types.h>
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <errno.h>
+#include <unistd.h>
+
+#define BUFFER 16*1024*1024
+#define WAITEMPTY ((BUFFER*1)/4)
+
+static inline int min(int a, int b) { return a<=b ? a : b; }
+
+static void nonblock(int fd) {
+ int r;
+ r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
+ r |= O_NDELAY;
+ if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
+}
+
+int main(void) {
+ static unsigned char buf[BUFFER];
+
+ unsigned char *wp, *rp;
+ int used,r,reading,seeneof;
+ fd_set readfds;
+ fd_set writefds;
+
+ used=0; wp=rp=buf; reading=1; seeneof=0;
+ nonblock(0); nonblock(1);
+ while (!seeneof || used) {
+ FD_ZERO(&readfds);
+ if (reading) {
+ if (used<BUFFER-1) {
+ FD_SET(0,&readfds);
+ } else {
+/*fprintf(stderr,"\t buffers full - stopping\n");*/
+ reading=0;
+ }
+ }
+ FD_ZERO(&writefds); if (used) FD_SET(1,&writefds);
+/*fprintf(stderr,"used %6d reading %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
+ used,reading,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ r= select(2,&readfds,&writefds,0,0);
+/*fprintf(stderr,"\t readable %d writeable %d\n",
+ FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
+ if (r == -1) {
+ if (errno == EINTR) continue;
+ perror("select"); exit(1);
+ }
+ if (FD_ISSET(0,&readfds)) {
+ r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
+ if (!r) {
+/*fprintf(stderr,"\t eof detected\n");*/
+ seeneof=1; reading=0;
+ } else if (r<0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
+/*fprintf(stderr,"\t read transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t read %d\n",r);*/
+ used+= r;
+ rp+= r;
+ if (rp == buf+BUFFER) rp=buf;
+ }
+ }
+ if (FD_ISSET(1,&writefds)) {
+ assert(used);
+ r= write(1,wp,min(used,buf+BUFFER-wp));
+ if (r<=0) {
+ if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
+/*fprintf(stderr,"\t write transient error\n");*/
+ } else {
+/*fprintf(stderr,"\t wrote %d\n",r);*/
+ used-= r;
+ wp+= r;
+ if (wp == buf+BUFFER) wp=buf;
+ }
+ if (used < WAITEMPTY && !seeneof) {
+/*fprintf(stderr,"\t starting writes\n");*/
+ reading=1;
+ }
+ }
+ }
+ exit(0);
+}
+
+--
+Ian Jackson personal email: <ijackson@chiark.greenend.org.uk>
+These opinions are my own. http://www.chiark.greenend.org.uk/~ijackson/
+PGP2 public key id 0x23f5addb, fingerprint 5906F687 BD03ACAD 0D8E602E FCF37657
+
+