X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ian/git?p=chiark-utils.git;a=blobdiff_plain;f=cprogs%2Frcopy-repeatedly.c;h=fc2741d31b038bbcd2e7626bd20fa1e648a3256c;hp=86cd3170388c2d9a66d32b7925616411a8a38874;hb=236653b7bc419c519da8d479abc0c78d0a748c4c;hpb=f4ebcf48e13ce42543ce941c27946fe296eed319 diff --git a/cprogs/rcopy-repeatedly.c b/cprogs/rcopy-repeatedly.c index 86cd317..fc2741d 100644 --- a/cprogs/rcopy-repeatedly.c +++ b/cprogs/rcopy-repeatedly.c @@ -1,21 +1,69 @@ /* * rcopy-repeatedly + * + * You say rcopy-repeatedly local-file user@host:remote-file + * and it polls for changes to local-file and copies them to + * remote-file. rcopy-repeatedly must be installed at the far end. + * You can copy in either direction but not between two remote + * locations. + * + * Limitations: + * * Cannot cope with files which are modified between us opening + * and statting them for the first time; if the file shrinks + * we may bomb out. Workaround: use rename-in-place. + * * When transferring large files, bandwidth limiter will + * be `lumpy' as the whole file is transferred and then we + * sleep. + * * Cannot copy between two local files. Workaround: a symlink + * (but presumably there was some reason you weren't doing that) + * * No ability to synchronise more than just exactly one file + * * Polls. It would be nice to use inotify or something. + * + * Inherent limitations: + * * Can only copy plain files. + * + * See the --help for options. */ - + +/* + * rcopy-repeatedly is + * Copyright (C) 2008 Ian Jackson + * and the option parser we use is + * Copyright (C) 1994,1995 Ian Jackson + * + * This is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 3, + * or (at your option) any later version. + * + * This is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this file; if not, consult the Free Software + * Foundation's website at www.fsf.org, or the GNU Project website at + * www.gnu.org. + */ + /* * protocol is: * server sends banner * - "#rcopy-repeatedly#\n" * - length of declaration, as 4 hex digits, zero prefixed, - * and a space [5 bytes]. In this protocol version this + * and a newline [5 bytes]. In this protocol version this * will be "0002" but client _must_ parse it. * server sends declaration * - one of "u " or "d" [1 byte] * - optionally, some more ascii text, reserved for future use - * must be ignored by client (but not sent by server) + * must be ignored by declaree (but not sent by declarer) * - a newline [1 byte] * client sends - * - 0x01 go + * - 0x02 START + * n 2 bytes big endian declaration length + * ... client's declaration (ascii text, including newline) + 8 see above * then for each update * sender sends one of * - 0x03 destination file should be deleted @@ -24,7 +72,7 @@ * - 0x04 complete new destination file follows, 64-bit length * l 8 bytes big endian length * ... l bytes data - * receiver must then reply with 0x01 GO + * receiver must then reply with 0x01 ACK */ #define _GNU_SOURCE @@ -46,7 +94,8 @@ #include "myopt.h" -#define REPLMSG_GO 0x01 +#define REPLMSG_ACK 0x01 +#define REPLMSG_START 0x02 #define REPLMSG_RM 0x03 #define REPLMSG_FILE64 0x04 @@ -54,8 +103,7 @@ static const char banner[]= "#rcopy-repeatedly#\n"; static FILE *commsi, *commso; -static double max_bw_prop_mean= 0.2; -static double max_bw_prop_burst= 0.8; +static double max_bw_prop= 0.2; static int txblocksz= INT_MAX, verbose=1; static int min_interval_usec= 100000; /* 100ms */ @@ -68,7 +116,7 @@ static int server_upcopy=-1; /* -1 means not yet known; 0 means download */ /* `up' means towards the client, * since we regard the subprocess as `down' */ -static double stream_allow_secsperbyte= 1/1e6; /* for initial transfer */ +static int udchar; static char mainbuf[65536]; /* must be at least 2^16 */ @@ -143,6 +191,13 @@ static void sendbyte(int c) { die_badsend(); } +static void mfreadcommsi(void *buf, int l, const char *what) { + int r= fread(buf,1,l,commsi); if (r!=l) die_badrecv(what); +} +static void mfwritecommso(const void *buf, int l) { + int r= fwrite(buf,1,l,commso); if (r!=l) die_badsend(); +} + static void mpipe(int p[2]) { if (pipe(p)) diee("could not create pipe"); } static void mdup2(int fd, int fd2) { if (dup2(fd,fd2)!=fd2) diee("could not dup2(%d,%d)",fd,fd2); @@ -194,19 +249,16 @@ static void bandlimit_sendend(uint64_t bytes, int *interval_usec_update) { double elapsed= mgettime_elapsed(ts_sendstart, &ts_buf); double secsperbyte_observed= elapsed / bytes; - stream_allow_secsperbyte= - secsperbyte_observed * max_bw_prop_mean / max_bw_prop_burst; - - double min_update= elapsed / max_bw_prop_mean; + double min_update= elapsed / max_bw_prop; if (min_update > 1e3) min_update= 1e3; int min_update_usec= min_update * 1e6; - if (*interval_usec_update > min_update_usec) + if (*interval_usec_update < min_update_usec) *interval_usec_update= min_update_usec; - verbosespinprintf("%12lluby %10.3fs %13.2fkby/s", + verbosespinprintf("%12lluby %10.3fs %13.2fkby/s %8dms", (unsigned long long)bytes, elapsed, - 1e-3/secsperbyte_observed); + 1e-3/secsperbyte_observed, *interval_usec_update/1000); } static void copyfile(FILE *sf, copyfile_die_fn *sdie, const char *sxi, @@ -222,24 +274,16 @@ static void copyfile(FILE *sf, copyfile_die_fn *sdie, const char *sxi, now= l < sizeof(mainbuf) ? l : sizeof(mainbuf); if (now > txblocksz) now= txblocksz; + r= fread(mainbuf,1,now,sf); if (r!=now) sdie(sf,sxi); + r= fwrite(mainbuf,1,now,df); if (r!=now) ddie(df,dxi); + l -= now; + done += now; + if (verbose) { fprintf(stderr," %3d%% \r", (int)(done*100.0/lstart)); flushstderr(); } - - if (amsender) { - double elapsed_want= now * stream_allow_secsperbyte; - double elapsed= mgettime_elapsed(ts_last, &ts_last); - double needwait= elapsed_want - elapsed; - if (needwait > 1) needwait= 1; - if (needwait > 0) usleep(ceil(needwait * 1e6)); - } - - r= fread(mainbuf,1,now,sf); if (r!=now) sdie(sf,sxi); - r= fwrite(mainbuf,1,now,df); if (r!=now) ddie(df,dxi); - l -= now; - done += now; } } @@ -256,6 +300,24 @@ static void copydie_commso(FILE *f, const char *what) { die_badsend(); } +static int generate_declaration(void) { + /* returns length; declaration is left in mainbuf */ + char *p= mainbuf; + *p++= udchar; + *p++= '\n'; + return p - mainbuf; +} + +static void read_declaration(int decllen) { + assert(decllen <= sizeof(mainbuf)); + if (decllen<2) die_protocol("declaration too short"); + mfreadcommsi(mainbuf,decllen,"declaration"); + if (mainbuf[decllen-1] != '\n') + die_protocol("declaration missing final newline"); + if (mainbuf[0] != udchar) + die_protocol("declaration incorrect direction indicator"); +} + static void receiver(const char *filename) { FILE *newfile; char *tmpfilename; @@ -296,7 +358,7 @@ static void receiver(const char *filename) { if (!newfile) diee("could not create temporary receiving file `%s'", tmpfilename); uint8_t lbuf[8]; - r= fread(lbuf,1,8,commsi); if (r!=8) die_badrecv("FILE64 l"); + mfreadcommsi(lbuf,8,"FILE64 l"); uint64_t l= (lbuf[0] << 28 << 28) | @@ -318,7 +380,7 @@ static void receiver(const char *filename) { diee("could not install new version of destination file `%s'", filename); - sendbyte(REPLMSG_GO); + sendbyte(REPLMSG_ACK); break; default: @@ -362,7 +424,8 @@ static void sender(const char *filename) { if (!f) { if (errno!=ENOENT) diee("could not access source file `%s'",filename); if (told != told_remove) { - verbosespinprintf(" ENOENT "); + verbosespinprintf + (" ENOENT "); sendbyte(REPLMSG_RM); told= told_remove; } @@ -391,8 +454,8 @@ static void sender(const char *filename) { }; bandlimit_sendstart(); - - r= fwrite(hbuf,1,9,commso); if (r!=9) die_badsend(); + + mfwritecommso(hbuf,9); copyfile(f, copydie_inputfile,filename, commso, copydie_commso,0, @@ -401,7 +464,7 @@ static void sender(const char *filename) { send_flush(); c= fgetc(commsi); if (c==EOF) die_badrecv("ack"); - if (c!=REPLMSG_GO) die_protocol("got %#02x instead of GO",c); + if (c!=REPLMSG_ACK) die_protocol("got %#02x instead of ACK",c); bandlimit_sendend(stab.st_size, &interval_usec); @@ -410,15 +473,6 @@ static void sender(const char *filename) { } } -void usagemessage(void) { - puts("usage: rcopy-repeatedly [] \n" - " may be or [@]:\n" - " exactly one of each of the two forms must be provided\n" - " a file is taken as remote if it has a : before the first /\n" - "options\n" - " --help\n"); -} - typedef struct { const char *userhost, *path; } FileSpecification; @@ -460,27 +514,64 @@ static void of_server_int(const struct cmdinfo *ci, const char *val) { *(int*)ci->parg= of__server_int(ci,val); } +void usagemessage(void) { + printf( + "usage: rcopy-repeatedly [] \n" + " may be or [@]:\n" + " exactly one of each of the two forms must be provided\n" + " a file is taken as remote if it has a : before the first /\n" + "general options:\n" + " --help\n" + " --quiet | -q\n" + "options for bandwidth (and cpu time) control:\n" + " --max-bandwidth-percent (default %d)\n" + " --tx-block-size (default/max %d)\n" + " --min-interval-usec (default %d)\n" + "options for finding programs:\n" + " --rcopy-repeatedly (default: rcopy-repeatedly)\n" + " --rsh-program (default: $RCOPY_REPEATEDLY_RSH or $RSYNC_RSH or ssh)\n" + "options passed to server side via ssh:\n" + " --receiver --sender, bandwidth control options\n", + (int)(max_bw_prop*100), (int)sizeof(mainbuf), min_interval_usec); +} + static const struct cmdinfo cmdinfos[]= { { "help", .call= of_help }, - { "max-bandwidth-percent-mean", 0,1,.call=of_bw,.parg=&max_bw_prop_mean }, - { "max-bandwidth-percent-burst",0,1,.call=of_bw,.parg=&max_bw_prop_burst }, - { "tx-block-size", 0,1,.call=of_server_int, .parg=&txblocksz }, - { "min-interval-usec", 0,1,.call=of_server_int, .parg=&min_interval_usec }, - { "rcopy-repeatedly", 0,1, .sassignto=&rcopy_repeatedly_program }, - { "ssh-program", 0,1, .sassignto=&rsh_program }, - { "receiver", .iassignto=&server_upcopy, .arg=0 }, - { "sender", .iassignto=&server_upcopy, .arg=1 }, + { "max-bandwidth-percent", 0,1,.call=of_bw,.parg=&max_bw_prop }, + { "tx-block-size",0, 1,.call=of_server_int, .parg=&txblocksz }, + { "min-interval-usec",0, 1,.call=of_server_int, .parg=&min_interval_usec }, + { "rcopy-repeatedly",0, 1, .sassignto=&rcopy_repeatedly_program }, + { "rsh-program",0, 1, .sassignto=&rsh_program }, + { "quiet",'q', .iassignto= &verbose, .arg=0 }, + { "receiver", .iassignto= &server_upcopy, .arg=0 }, + { "sender", .iassignto= &server_upcopy, .arg=1 }, { 0 } }; static void server(const char *filename) { - int c; + int c, l; + char buf[2]; + + udchar= server_upcopy?'u':'d'; + commsi= stdin; commso= stdout; - fprintf(commso, "%s0002 %c\n", banner, server_upcopy?'u':'d'); + l= generate_declaration(); + fprintf(commso, "%s%04x\n", banner, l); + mfwritecommso(mainbuf, l); send_flush(); - c= fgetc(commsi); if (c==EOF) die_badrecv("initial go"); - if (c!=REPLMSG_GO) die_protocol("initial go message was %#02x instead",c); + + c= fgetc(commsi); + if (c==EOF) { + if (feof(commsi)) exit(14); + assert(ferror(commsi)); die_badrecv("initial START message"); + } + if (c!=REPLMSG_START) die_protocol("initial START was %#02x instead",c); + + mfreadcommsi(buf,2,"START l"); + l= (buf[0] << 8) | buf[1]; + + read_declaration(l); if (server_upcopy) sender(filename); @@ -491,12 +582,21 @@ static void server(const char *filename) { static void client(void) { int uppipe[2], downpipe[2], r; pid_t child; + FileSpecification *remotespec; + const char *remotemode; mpipe(uppipe); mpipe(downpipe); - FileSpecification *remotespec= srcspec.userhost ? &srcspec : &dstspec; - const char *remotemode= srcspec.userhost ? "--sender" : "--receiver"; + if (srcspec.userhost) { + udchar= 'u'; + remotespec= &srcspec; + remotemode= "--sender"; + } else { + udchar= 'd'; + remotespec= &dstspec; + remotemode= "--receiver"; + } sargs= xrealloc(sargs, sizeof(*sargs) * (7 + nsargs)); memmove(sargs+5, sargs, sizeof(*sargs) * nsargs); @@ -532,7 +632,7 @@ static void client(void) { if (r!=sizeof(banbuf)-1 || memcmp(banbuf,banner,sizeof(banner)-1) || - banbuf[sizeof(banner)-1 + 4] != ' ') { + banbuf[sizeof(banner)-1 + 4] != '\n') { const char **sap; int count=0; for (count=0, sap=sargs; *sap; sap++) count+= strlen(*sap)+1; @@ -555,19 +655,16 @@ static void client(void) { banbuf[sizeof(banbuf)-1]= 0; char *ep; long decllen= strtoul(banbuf + sizeof(banner)-1, &ep, 16); - if (ep!=banbuf + sizeof(banner)-1 + 4 || *ep!=' ') - die_protocol("declaration length syntax error (`%s')",ep); - assert(decllen <= sizeof(mainbuf)); - if (decllen<2) die_protocol("declaration too short"); + if (ep != banbuf + sizeof(banner)-1 + 4) + die_protocol("declaration length syntax error"); - r= fread(mainbuf,1,decllen,commsi); - if (r!=decllen) die_badrecv("declaration"); - if (mainbuf[decllen-1] != '\n') - die_protocol("declaration missing final newline"); - if (mainbuf[0] != (remotespec==&srcspec ? 'u' : 'd')) - die_protocol("declaration incorrect direction indicator"); + read_declaration(decllen); - sendbyte(REPLMSG_GO); + int l= generate_declaration(); + sendbyte(REPLMSG_START); + sendbyte((l >> 8) & 0x0ff); + sendbyte( l & 0x0ff); + mfwritecommso(mainbuf,l); if (remotespec==&srcspec) receiver(dstspec.path); @@ -606,10 +703,6 @@ int main(int argc, const char *const *argv) { if (!rsh_program) rsh_program= getenv("RSYNC_RSH"); if (!rsh_program) rsh_program= "ssh"; - if (max_bw_prop_burst / max_bw_prop_mean < 1.1) - badusage("max bandwidth prop burst must be at least 1.1x" - " max bandwidth prop mean"); - if (txblocksz<1) badusage("transmit block size must be at least 1"); if (min_interval_usec<0) badusage("minimum update interval may not be -ve");