From ijackson@chiark.greenend.org.uk Wed Aug 12 19:36:07 BST 1998
Article: 742 of ucam.comp.unix
Path: ewrotcd!not-for-mail
From: ijackson@chiark.greenend.org.uk (Ian Jackson)
Newsgroups: ucam.comp.unix,comp.sys.sun.admin
Subject: Buffering programs (was Re: Speeding up network dumps)
Date: 21 Jul 1998 12:23:23 +0100 (BST)
Organization: Linux Unlimited
Lines: 233
Message-ID: <xWb*k6rBn@news.chiark.greenend.org.uk>
References: <6oiapl$6k5$1@pegasus.csx.cam.ac.uk> <6ol7kf$cln$2@pegasus.csx.cam.ac.uk>
NNTP-Posting-Host: chiark.greenend.org.uk
X-Server-Date: 21 Jul 1998 11:23:25 GMT
Originator: ijackson@news.chiark.greenend.org.uk ([127.0.0.1])
Xref: news.chiark.greenend.org.uk ucam.comp.unix:742 comp.sys.sun.admin:14526

I said I'd post here the buffering programs that I use, so here they
are.

They're perfectly functional and (I believe) correct, but they do lack
sophistication.  The size of the buffer is a #define, and the
diagnostics lack verbosity.  I haven't compiled them on anything but
Linux/i386 libc5, but they ought to work on other reasonable Unices.

writebuffer is intended for writing to tapes; when its buffer is empty
it will write no more data until it gets 3/4 of its buffer filled, or
EOF.

readbuffer is intended for reading from tapes; when its buffer becomes
full it stops reading until it's down to 1/4.

Remember that using shell pipes loses the exit status of all but the
last command, and so can cause failures to go unnoticed.


/*
 * writebuffer.c
 *
 * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
 *
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2,
 * or (at your option) any later version.
 *
 * This is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this file; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#include <sys/time.h>
#include <sys/types.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>

#define BUFFER 16*1024*1024
#define WAITFILL ((BUFFER*3)/4)

static inline int min(int a, int b) { return a<=b ? a : b; }

static void nonblock(int fd) {
  int r;
  r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
  r |= O_NDELAY;
  if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
}

int main(void) {
  static unsigned char buf[BUFFER];
  
  unsigned char *wp, *rp;
  int used,r,writing,seeneof;
  fd_set readfds;
  fd_set writefds;

  used=0; wp=rp=buf; writing=0; seeneof=0;
  nonblock(0); nonblock(1);
  while (!seeneof || used) {
    FD_ZERO(&readfds); if (!seeneof && used+1<BUFFER) FD_SET(0,&readfds);
    FD_ZERO(&writefds); if (writing) FD_SET(1,&writefds);
/*fprintf(stderr,"used %6d writing %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
        used,writing,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
        FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
    r= select(2,&readfds,&writefds,0,0);
/*fprintf(stderr,"\t readable %d writeable %d\n",
        FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
    if (r == -1) {
      if (errno == EINTR) continue;
      perror("select"); exit(1);
    }
    if (FD_ISSET(1,&writefds) && !FD_ISSET(0,&readfds) && !used) {
      writing= 0;
      FD_CLR(1,&writefds);
/*fprintf(stderr,"\t buffers empty - stopping\n");*/
    }
    if (FD_ISSET(0,&readfds)) {
      r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
      if (!r) {
/*fprintf(stderr,"\t eof detected\n");*/
        seeneof=1; writing=1;
      } else if (r<0) {
        if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
fprintf(stderr,"\t read transient error\n");
      } else {
/*fprintf(stderr,"\t read %d\n",r);*/
        used+= r;
        rp+= r;
        if (rp == buf+BUFFER) rp=buf;
/*fprintf(stderr,"\t starting writes\n");*/
      }
      if (used > WAITFILL) writing=1;
    }
    if (FD_ISSET(1,&writefds) && used) {
      r= write(1,wp,min(used,buf+BUFFER-wp));
      if (r<=0) {
        if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
/*fprintf(stderr,"\t write transient error\n");*/
      } else {
/*fprintf(stderr,"\t wrote %d\n",r);*/
        used-= r;
        wp+= r;
        if (wp == buf+BUFFER) wp=buf;
      }
    }
  }
  exit(0);
}


/*
 * readbuffer.c
 *
 * Copyright (C) 1997,1998 Ian Jackson <ian@chiark.greenend.org.uk>
 *
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2,
 * or (at your option) any later version.
 *
 * This is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this file; if not, write to the Free Software
 * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
 *
 */

#include <sys/time.h>
#include <sys/types.h>
#include <assert.h>
#include <fcntl.h>
#include <stdio.h>
#include <errno.h>
#include <unistd.h>

#define BUFFER 16*1024*1024
#define WAITEMPTY ((BUFFER*1)/4)

static inline int min(int a, int b) { return a<=b ? a : b; }

static void nonblock(int fd) {
  int r;
  r= fcntl(fd,F_GETFL,0); if (r == -1) { perror("fcntl getfl"); exit(1); }
  r |= O_NDELAY;
  if (fcntl(fd,F_SETFL,r) == -1) { perror("fcntl setfl"); exit(1); }
}

int main(void) {
  static unsigned char buf[BUFFER];
  
  unsigned char *wp, *rp;
  int used,r,reading,seeneof;
  fd_set readfds;
  fd_set writefds;

  used=0; wp=rp=buf; reading=1; seeneof=0;
  nonblock(0); nonblock(1);
  while (!seeneof || used) {
    FD_ZERO(&readfds);
    if (reading) {
      if (used<BUFFER-1) {
	FD_SET(0,&readfds);
      } else {
/*fprintf(stderr,"\t buffers full - stopping\n");*/
	reading=0;
      }
    }
    FD_ZERO(&writefds); if (used) FD_SET(1,&writefds);
/*fprintf(stderr,"used %6d reading %d seeneof %d wp %8lx rp %8lx read %d write %d\n",
        used,reading,seeneof,(unsigned long)(wp-buf),(unsigned long)(rp-buf),
        FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
    r= select(2,&readfds,&writefds,0,0);
/*fprintf(stderr,"\t readable %d writeable %d\n",
        FD_ISSET(0,&readfds),FD_ISSET(1,&writefds));*/
    if (r == -1) {
      if (errno == EINTR) continue;
      perror("select"); exit(1);
    }
    if (FD_ISSET(0,&readfds)) {
      r= read(0,rp,min(BUFFER-1-used,buf+BUFFER-rp));
      if (!r) {
/*fprintf(stderr,"\t eof detected\n");*/
        seeneof=1; reading=0;
      } else if (r<0) {
        if (!(errno == EAGAIN || errno == EINTR)) { perror("read"); exit(1); }
/*fprintf(stderr,"\t read transient error\n");*/
      } else {
/*fprintf(stderr,"\t read %d\n",r);*/
        used+= r;
        rp+= r;
        if (rp == buf+BUFFER) rp=buf;
      }
    }
    if (FD_ISSET(1,&writefds)) {
      assert(used);
      r= write(1,wp,min(used,buf+BUFFER-wp));
      if (r<=0) {
        if (!(errno == EAGAIN || errno == EINTR)) { perror("write"); exit(1); }
/*fprintf(stderr,"\t write transient error\n");*/
      } else {
/*fprintf(stderr,"\t wrote %d\n",r);*/
        used-= r;
        wp+= r;
        if (wp == buf+BUFFER) wp=buf;
      }
      if (used < WAITEMPTY && !seeneof) {
/*fprintf(stderr,"\t starting writes\n");*/
	reading=1;
      }
    }
  }
  exit(0);
}

-- 
Ian Jackson                  personal email: <ijackson@chiark.greenend.org.uk>
These opinions are my own.        http://www.chiark.greenend.org.uk/~ijackson/
PGP2 public key id 0x23f5addb, fingerprint 5906F687 BD03ACAD 0D8E602E FCF37657