4 * You say rcopy-repeatedly local-file user@host:remote-file
5 * and it polls for changes to local-file and copies them to
6 * remote-file. rcopy-repeatedly must be installed at the far end.
7 * You can copy in either direction but not between two remote
11 * * Cannot cope with files which are modified between us opening
12 * and statting them for the first time; if the file shrinks
13 * we may bomb out. Workaround: use rename-in-place.
14 * * When transferring large files, bandwidth limiter will
15 * be `lumpy' as the whole file is transferred and then we
17 * * Cannot copy between two local files. Workaround: a symlink
18 * (but presumably there was some reason you weren't doing that)
19 * * No ability to synchronise more than just exactly one file
20 * * Polls. It would be nice to use inotify or something.
22 * Inherent limitations:
23 * * Can only copy plain files.
25 * See the --help for options.
30 * Copyright (C) 2008 Ian Jackson <ian@davenant.greenend.org.uk>
31 * and the option parser we use is
32 * Copyright (C) 1994,1995 Ian Jackson <ian@davenant.greenend.org.uk>
34 * This is free software; you can redistribute it and/or modify
35 * it under the terms of the GNU General Public License as
36 * published by the Free Software Foundation; either version 3,
37 * or (at your option) any later version.
39 * This is distributed in the hope that it will be useful, but
40 * WITHOUT ANY WARRANTY; without even the implied warranty of
41 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
42 * GNU General Public License for more details.
44 * You should have received a copy of the GNU General Public
45 * License along with this file; if not, consult the Free Software
46 * Foundation's website at www.fsf.org, or the GNU Project website at
53 * - "#rcopy-repeatedly#\n"
54 * - length of declaration, as 4 hex digits, zero prefixed,
55 * and a newline [5 bytes]. In this protocol version this
56 * will be "0002" but client _must_ parse it.
57 * server sends declaration
58 * - one of "u " or "d" [1 byte]
59 * - optionally, some more ascii text, reserved for future use
60 * must be ignored by declaree (but not sent by declarer)
61 * - a newline [1 byte]
64 * n 2 bytes big endian declaration length
65 * ... client's declaration (ascii text, including newline)
67 * then for each update
69 * - 0x03 destination file should be deleted
70 * but note that contents must be retained by receiver
71 * as it may be used for rle updates
72 * - 0x04 complete new destination file follows, 64-bit length
73 * l 8 bytes big endian length
75 * receiver must then reply with 0x01 ACK
91 #include <sys/types.h>
97 #define REPLMSG_ACK 0x01
98 #define REPLMSG_START 0x02
99 #define REPLMSG_RM 0x03
100 #define REPLMSG_FILE64 0x04
102 static const char banner[]= "#rcopy-repeatedly#\n";
104 static FILE *commsi, *commso;
106 static double max_bw_prop= 0.2;
107 static int txblocksz= INT_MAX, verbose=1;
108 static int min_interval_usec= 100000; /* 100ms */
111 static const char **sargs;
113 static const char *rsh_program= 0;
114 static const char *rcopy_repeatedly_program= "rcopy-repeatedly";
115 static int server_upcopy=-1; /* -1 means not yet known; 0 means download */
116 /* `up' means towards the client,
117 * since we regard the subprocess as `down' */
121 static char mainbuf[65536]; /* must be at least 2^16 */
123 #define NORETURN __attribute__((noreturn))
125 static void vdie(int ec, const char *pfx, int eno,
126 const char *fmt, va_list al) NORETURN;
127 static void vdie(int ec, const char *pfx, int eno,
128 const char *fmt, va_list al) {
129 fputs("rcopy-repeatedly: ",stderr);
130 if (server_upcopy>=0) fputs("server: ",stderr);
131 if (pfx) fprintf(stderr,"%s: ",pfx);
132 vfprintf(stderr,fmt,al);
133 if (eno!=-1) fprintf(stderr,": %s",strerror(eno));
137 static void die(int ec, const char *pfx, int eno,
138 const char *fmt, ...) NORETURN;
139 static void die(int ec, const char *pfx, int eno,
140 const char *fmt, ...) {
143 vdie(ec,pfx,eno,fmt,al);
146 static void diem(void) NORETURN;
147 static void diem(void) { die(16,0,errno,"malloc failed"); }
148 static void *xmalloc(size_t sz) {
154 static void *xrealloc(void *p, size_t sz) {
161 static void diee(const char *fmt, ...) NORETURN;
162 static void diee(const char *fmt, ...) {
165 vdie(12,0,errno,fmt,al);
167 static void die_protocol(const char *fmt, ...) NORETURN;
168 static void die_protocol(const char *fmt, ...) {
171 vdie(10,"protocol error",-1,fmt,al);
174 static void die_badrecv(const char *what) NORETURN;
175 static void die_badrecv(const char *what) {
176 if (ferror(commsi)) diee("communication failed while receiving %s", what);
177 if (feof(commsi)) die_protocol("receiver got unexpected EOF in %s", what);
180 static void die_badsend(void) NORETURN;
181 static void die_badsend(void) {
182 diee("transmission failed");
185 static void send_flush(void) {
186 if (ferror(commso) || fflush(commso))
189 static void sendbyte(int c) {
190 if (putc(c,commso)==EOF)
194 static void mfreadcommsi(void *buf, int l, const char *what) {
195 int r= fread(buf,1,l,commsi); if (r!=l) die_badrecv(what);
197 static void mfwritecommso(const void *buf, int l) {
198 int r= fwrite(buf,1,l,commso); if (r!=l) die_badsend();
201 static void mpipe(int p[2]) { if (pipe(p)) diee("could not create pipe"); }
202 static void mdup2(int fd, int fd2) {
203 if (dup2(fd,fd2)!=fd2) diee("could not dup2(%d,%d)",fd,fd2);
206 typedef void copyfile_die_fn(FILE *f, const char *xi);
208 struct timespec ts_sendstart;
210 static void mgettime(struct timespec *ts) {
211 int r= clock_gettime(CLOCK_MONOTONIC, ts);
212 if (r) diee("clock_gettime failed");
215 static void bandlimit_sendstart(void) {
216 mgettime(&ts_sendstart);
219 static double mgettime_elapsed(struct timespec ts_base,
220 struct timespec *ts_ret) {
222 return (ts_ret->tv_sec - ts_base.tv_sec) +
223 (ts_ret->tv_nsec - ts_base.tv_nsec)*1e-9;
226 static void flushstderr(void) {
227 if (ferror(stderr) || fflush(stderr))
228 diee("could not write progress to stderr");
231 static void verbosespinprintf(const char *fmt, ...) {
232 static const char spinnerchars[]= "/-\\";
233 static int spinnerchar;
240 fprintf(stderr," %c ",spinnerchars[spinnerchar]);
241 spinnerchar++; spinnerchar %= sizeof(spinnerchars)-1;
242 vfprintf(stderr,fmt,al);
247 static void bandlimit_sendend(uint64_t bytes, int *interval_usec_update) {
248 struct timespec ts_buf;
249 double elapsed= mgettime_elapsed(ts_sendstart, &ts_buf);
250 double secsperbyte_observed= elapsed / bytes;
252 double min_update= elapsed / max_bw_prop;
253 if (min_update > 1e3) min_update= 1e3;
254 int min_update_usec= min_update * 1e6;
256 if (*interval_usec_update < min_update_usec)
257 *interval_usec_update= min_update_usec;
259 verbosespinprintf("%12lluby %10.3fs %13.2fkby/s %8dms",
260 (unsigned long long)bytes, elapsed,
261 1e-3/secsperbyte_observed, *interval_usec_update/1000);
264 static void copyfile(FILE *sf, copyfile_die_fn *sdie, const char *sxi,
265 FILE *df, copyfile_die_fn *ddie, const char *dxi,
266 uint64_t lstart, int amsender) {
267 struct timespec ts_last;
269 uint64_t l=lstart, done=0;
271 ts_last= ts_sendstart;
274 now= l < sizeof(mainbuf) ? l : sizeof(mainbuf);
275 if (now > txblocksz) now= txblocksz;
277 r= fread(mainbuf,1,now,sf); if (r!=now) sdie(sf,sxi);
278 r= fwrite(mainbuf,1,now,df); if (r!=now) ddie(df,dxi);
283 fprintf(stderr," %3d%% \r",
284 (int)(done*100.0/lstart));
290 static void copydie_inputfile(FILE *f, const char *filename) {
291 diee("read failed on source file `%s'", filename);
293 static void copydie_tmpwrite(FILE *f, const char *tmpfilename) {
294 diee("write failed to temporary receiving file `%s'", tmpfilename);
296 static void copydie_commsi(FILE *f, const char *what) {
299 static void copydie_commso(FILE *f, const char *what) {
303 static int generate_declaration(void) {
304 /* returns length; declaration is left in mainbuf */
311 static void read_declaration(int decllen) {
312 assert(decllen <= sizeof(mainbuf));
313 if (decllen<2) die_protocol("declaration too short");
314 mfreadcommsi(mainbuf,decllen,"declaration");
315 if (mainbuf[decllen-1] != '\n')
316 die_protocol("declaration missing final newline");
317 if (mainbuf[0] != udchar)
318 die_protocol("declaration incorrect direction indicator");
321 static void receiver(const char *filename) {
326 char *lastslash= strrchr(filename,'/');
328 r= asprintf(&tmpfilename, ".rcopy-repeatedly.#%s#", filename);
330 r= asprintf(&tmpfilename, "%.*s/.rcopy-repeatedly.#%s#",
331 (int)(lastslash-filename), filename, lastslash+1);
334 r= unlink(tmpfilename);
335 if (r && errno!=ENOENT)
336 diee("could not remove temporary receiving file `%s'", tmpfilename);
345 if (ferror(commsi)) die_badrecv("transfer message code");
346 assert(feof(commsi));
351 if (r && errno!=ENOENT)
352 diee("source file removed but could not remove destination file `%s'",
357 newfile= fopen(tmpfilename, "wb");
358 if (!newfile) diee("could not create temporary receiving file `%s'",
361 mfreadcommsi(lbuf,8,"FILE64 l");
364 (lbuf[0] << 28 << 28) |
365 (lbuf[1] << 24 << 24) |
366 (lbuf[2] << 16 << 24) |
367 (lbuf[3] << 8 << 24) |
373 copyfile(commsi, copydie_commsi,"FILE64 file data",
374 newfile, copydie_tmpwrite,tmpfilename,
377 if (fclose(newfile)) diee("could not flush and close temporary"
378 " receiving file `%s'", tmpfilename);
379 if (rename(tmpfilename, filename))
380 diee("could not install new version of destination file `%s'",
383 sendbyte(REPLMSG_ACK);
387 die_protocol("unknown transfer message code 0x%02x",c);
393 static void sender(const char *filename) {
395 int interval_usec, r, c;
396 struct stat stabtest, stab;
397 enum { told_nothing, told_file, told_remove } told;
406 usleep(interval_usec);
408 interval_usec= min_interval_usec;
410 r= stat(filename, &stabtest);
414 if (told == told_file &&
415 stabtest.st_mode == stab.st_mode &&
416 stabtest.st_dev == stab.st_dev &&
417 stabtest.st_ino == stab.st_ino &&
418 stabtest.st_mtime == stab.st_mtime &&
419 stabtest.st_size == stab.st_size)
421 f= fopen(filename, "rb");
425 if (errno!=ENOENT) diee("could not access source file `%s'",filename);
426 if (told != told_remove) {
429 sendbyte(REPLMSG_RM);
435 if (fold) fclose(fold);
438 r= fstat(fileno(f),&stab);
439 if (r) diee("could not fstat source file `%s'",filename);
441 if (!S_ISREG(stab.st_mode))
442 die(8,0,-1,"source file `%s' is not a plain file",filename);
446 stab.st_size >> 28 >> 28,
447 stab.st_size >> 24 >> 24,
448 stab.st_size >> 16 >> 24,
449 stab.st_size >> 8 >> 24,
456 bandlimit_sendstart();
458 mfwritecommso(hbuf,9);
460 copyfile(f, copydie_inputfile,filename,
461 commso, copydie_commso,0,
466 c= fgetc(commsi); if (c==EOF) die_badrecv("ack");
467 if (c!=REPLMSG_ACK) die_protocol("got %#02x instead of ACK",c);
469 bandlimit_sendend(stab.st_size, &interval_usec);
477 const char *userhost, *path;
480 static FileSpecification srcspec, dstspec;
482 static void of__server(const struct cmdinfo *ci, const char *val) {
483 int ncount= nsargs + 1 + !!val;
484 sargs= xrealloc(sargs, sizeof(*sargs) * ncount);
485 sargs[nsargs++]= ci->olong;
487 sargs[nsargs++]= val;
490 static int of__server_int(const struct cmdinfo *ci, const char *val) {
494 errno= 0; v= strtol(val,&ep,10);
495 if (!*val || *ep || errno || v<INT_MIN || v>INT_MAX)
496 badusage("bad integer argument `%s' for --%s",val,ci->olong);
500 static void of_help(const struct cmdinfo *ci, const char *val) {
502 if (ferror(stdout)) diee("could not write usage message to stdout");
506 static void of_bw(const struct cmdinfo *ci, const char *val) {
507 int pct= of__server_int(ci,val);
508 if (pct<1 || pct>100)
509 badusage("bandwidth percentage must be between 1 and 100 inclusive");
510 *(double*)ci->parg= pct * 0.01;
513 static void of_server_int(const struct cmdinfo *ci, const char *val) {
514 *(int*)ci->parg= of__server_int(ci,val);
517 void usagemessage(void) {
519 "usage: rcopy-repeatedly [<options>] <file> <file>\n"
520 " <file> may be <local-file> or [<user>@]<host>:<file>\n"
521 " exactly one of each of the two forms must be provided\n"
522 " a file is taken as remote if it has a : before the first /\n"
526 "options for bandwidth (and cpu time) control:\n"
527 " --max-bandwidth-percent (default %d)\n"
528 " --tx-block-size (default/max %d)\n"
529 " --min-interval-usec (default %d)\n"
530 "options for finding programs:\n"
531 " --rcopy-repeatedly (default: rcopy-repeatedly)\n"
532 " --rsh-program (default: $RCOPY_REPEATEDLY_RSH or $RSYNC_RSH or ssh)\n"
533 "options passed to server side via ssh:\n"
534 " --receiver --sender, bandwidth control options\n",
535 (int)(max_bw_prop*100), (int)sizeof(mainbuf), min_interval_usec);
538 static const struct cmdinfo cmdinfos[]= {
539 { "help", .call= of_help },
540 { "max-bandwidth-percent", 0,1,.call=of_bw,.parg=&max_bw_prop },
541 { "tx-block-size",0, 1,.call=of_server_int, .parg=&txblocksz },
542 { "min-interval-usec",0, 1,.call=of_server_int, .parg=&min_interval_usec },
543 { "rcopy-repeatedly",0, 1, .sassignto=&rcopy_repeatedly_program },
544 { "rsh-program",0, 1, .sassignto=&rsh_program },
545 { "quiet",'q', .iassignto= &verbose, .arg=0 },
546 { "receiver", .iassignto= &server_upcopy, .arg=0 },
547 { "sender", .iassignto= &server_upcopy, .arg=1 },
551 static void server(const char *filename) {
555 udchar= server_upcopy?'u':'d';
559 l= generate_declaration();
560 fprintf(commso, "%s%04x\n", banner, l);
561 mfwritecommso(mainbuf, l);
566 if (feof(commsi)) exit(14);
567 assert(ferror(commsi)); die_badrecv("initial START message");
569 if (c!=REPLMSG_START) die_protocol("initial START was %#02x instead",c);
571 mfreadcommsi(buf,2,"START l");
572 l= (buf[0] << 8) | buf[1];
582 static void client(void) {
583 int uppipe[2], downpipe[2], r;
585 FileSpecification *remotespec;
586 const char *remotemode;
591 if (srcspec.userhost) {
593 remotespec= &srcspec;
594 remotemode= "--sender";
597 remotespec= &dstspec;
598 remotemode= "--receiver";
601 sargs= xrealloc(sargs, sizeof(*sargs) * (7 + nsargs));
602 memmove(sargs+5, sargs, sizeof(*sargs) * nsargs);
603 sargs[0]= rsh_program;
604 sargs[1]= remotespec->userhost;
605 sargs[2]= rcopy_repeatedly_program;
606 sargs[3]= remotemode;
608 sargs[5+nsargs]= remotespec->path;
612 if (child==-1) diee("fork failed");
614 mdup2(downpipe[0],0);
616 close(uppipe[0]); close(downpipe[0]);
617 close(uppipe[1]); close(downpipe[1]);
619 execvp(rsh_program, (char**)sargs);
620 diee("failed to execute rsh program `%s'",rsh_program);
623 commso= fdopen(downpipe[1],"wb");
624 commsi= fdopen(uppipe[0],"rb");
625 if (!commso || !commsi) diee("fdopen failed");
629 char banbuf[sizeof(banner)-1 + 5 + 1];
630 r= fread(banbuf,1,sizeof(banbuf)-1,commsi);
631 if (ferror(commsi)) die_badrecv("read banner");
633 if (r!=sizeof(banbuf)-1 ||
634 memcmp(banbuf,banner,sizeof(banner)-1) ||
635 banbuf[sizeof(banner)-1 + 4] != '\n') {
638 for (count=0, sap=sargs; *sap; sap++) count+= strlen(*sap)+1;
639 char *cmdline= xmalloc(count+1);
641 for (sap=sargs; *sap; sap++) {
643 strcat(cmdline,*sap);
646 die(8,0,-1,"did not receive banner as expected -"
647 " shell dirty? ssh broken?\n"
650 " and expect the first line to be\n"
655 banbuf[sizeof(banbuf)-1]= 0;
657 long decllen= strtoul(banbuf + sizeof(banner)-1, &ep, 16);
658 if (ep != banbuf + sizeof(banner)-1 + 4)
659 die_protocol("declaration length syntax error");
661 read_declaration(decllen);
663 int l= generate_declaration();
664 sendbyte(REPLMSG_START);
665 sendbyte((l >> 8) & 0x0ff);
666 sendbyte( l & 0x0ff);
667 mfwritecommso(mainbuf,l);
669 if (remotespec==&srcspec)
670 receiver(dstspec.path);
672 sender(srcspec.path);
675 static void parse_file_specification(FileSpecification *fs, const char *arg,
679 if (!arg) badusage("too few arguments - missing %s\n",what);
681 for (colon=arg; ; colon++) {
682 if (!*colon || *colon=='/') {
688 char *uh= xmalloc(colon-arg + 1);
689 memcpy(uh,arg, colon-arg); uh[colon-arg]= 0;
697 int main(int argc, const char *const *argv) {
698 setvbuf(stderr,0,_IOLBF,BUFSIZ);
700 myopt(&argv, cmdinfos);
702 if (!rsh_program) rsh_program= getenv("RCOPY_REPEATEDLY_RSH");
703 if (!rsh_program) rsh_program= getenv("RSYNC_RSH");
704 if (!rsh_program) rsh_program= "ssh";
706 if (txblocksz<1) badusage("transmit block size must be at least 1");
707 if (min_interval_usec<0) badusage("minimum update interval may not be -ve");
709 if (server_upcopy>=0) {
710 if (!argv[0] || argv[1])
711 badusage("server mode must have just the filename as non-option arg");
714 parse_file_specification(&srcspec, argv[0], "source");
715 parse_file_specification(&dstspec, argv[1], "destination");
716 if (argv[2]) badusage("too many non-option arguments");
717 if (!!srcspec.userhost == !!dstspec.userhost)
718 badusage("need exactly one remote file argument");