4 * You say rcopy-repeatedly local-file user@host:remote-file
5 * and it polls for changes to local-file and copies them to
6 * remote-file. rcopy-repeatedly must be installed at the far end.
7 * You can copy in either direction but not between two remote
11 * * Cannot cope with files which are modified between us opening
12 * and statting them for the first time; if the file shrinks
13 * we may bomb out. Workaround: use rename-in-place.
14 * * When transferring large files, bandwidth limiter will
15 * be `lumpy' as the whole file is transferred and then we
17 * * Cannot copy between two local files. Workaround: a symlink
18 * (but presumably there was some reason you weren't doing that)
19 * * No ability to synchronise more than just exactly one file
20 * * Polls. It would be nice to use inotify or something.
22 * Inherent limitations:
23 * * Can only copy plain files.
25 * See the --help for options.
29 * Copyright (C) 2008 Ian Jackson <ian@davenant.greenend.org.uk>
30 * and the option parser we use is
31 * Copyright (C) 1994,1995 Ian Jackson <ian@davenant.greenend.org.uk>
33 * This is free software; you can redistribute it and/or modify
34 * it under the terms of the GNU General Public License as
35 * published by the Free Software Foundation; either version 3,
36 * or (at your option) any later version.
38 * This is distributed in the hope that it will be useful, but
39 * WITHOUT ANY WARRANTY; without even the implied warranty of
40 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
41 * GNU General Public License for more details.
43 * You should have received a copy of the GNU General Public
44 * License along with this file; if not, consult the Free Software
45 * Foundation's website at www.fsf.org, or the GNU Project website at
51 * - "#rcopy-repeatedly#\n"
52 * - length of declaration, as 4 hex digits, zero prefixed,
53 * and a newline [5 bytes]. In this protocol version this
54 * will be "0002" but client _must_ parse it.
55 * server sends declaration
56 * - one of "u " or "d" [1 byte]
57 * - optionally, some more ascii text, reserved for future use
58 * must be ignored by declaree (but not sent by declarer)
59 * - a newline [1 byte]
62 * n 2 bytes big endian declaration length
63 * ... client's declaration (ascii text, including newline)
65 * then for each update
67 * - 0x03 destination file should be deleted
68 * but note that contents must be retained by receiver
69 * as it may be used for rle updates
70 * - 0x04 complete new destination file follows, 64-bit length
71 * l 8 bytes big endian length
73 * receiver must then reply with 0x01 ACK
89 #include <sys/types.h>
95 #define REPLMSG_ACK 0x01
96 #define REPLMSG_START 0x02
97 #define REPLMSG_RM 0x03
98 #define REPLMSG_FILE64 0x04
100 static const char banner[]= "#rcopy-repeatedly#\n";
102 static FILE *commsi, *commso;
104 static double max_bw_prop= 0.2;
105 static int txblocksz= INT_MAX, verbose=1;
106 static int min_interval_usec= 100000; /* 100ms */
109 static const char **sargs;
111 static const char *rsh_program= 0;
112 static const char *rcopy_repeatedly_program= "rcopy-repeatedly";
113 static int server_upcopy=-1; /* -1 means not yet known; 0 means download */
114 /* `up' means towards the client,
115 * since we regard the subprocess as `down' */
119 static char mainbuf[65536]; /* must be at least 2^16 */
121 #define NORETURN __attribute__((noreturn))
123 static void vdie(int ec, const char *pfx, int eno,
124 const char *fmt, va_list al) NORETURN;
125 static void vdie(int ec, const char *pfx, int eno,
126 const char *fmt, va_list al) {
127 fputs("rcopy-repeatedly: ",stderr);
128 if (server_upcopy>=0) fputs("server: ",stderr);
129 if (pfx) fprintf(stderr,"%s: ",pfx);
130 vfprintf(stderr,fmt,al);
131 if (eno!=-1) fprintf(stderr,": %s",strerror(eno));
135 static void die(int ec, const char *pfx, int eno,
136 const char *fmt, ...) NORETURN;
137 static void die(int ec, const char *pfx, int eno,
138 const char *fmt, ...) {
141 vdie(ec,pfx,eno,fmt,al);
144 static void diem(void) NORETURN;
145 static void diem(void) { die(16,0,errno,"malloc failed"); }
146 static void *xmalloc(size_t sz) {
152 static void *xrealloc(void *p, size_t sz) {
159 static void diee(const char *fmt, ...) NORETURN;
160 static void diee(const char *fmt, ...) {
163 vdie(12,0,errno,fmt,al);
165 static void die_protocol(const char *fmt, ...) NORETURN;
166 static void die_protocol(const char *fmt, ...) {
169 vdie(10,"protocol error",-1,fmt,al);
172 static void die_badrecv(const char *what) NORETURN;
173 static void die_badrecv(const char *what) {
174 if (ferror(commsi)) diee("communication failed while receiving %s", what);
175 if (feof(commsi)) die_protocol("receiver got unexpected EOF in %s", what);
178 static void die_badsend(void) NORETURN;
179 static void die_badsend(void) {
180 diee("transmission failed");
183 static void send_flush(void) {
184 if (ferror(commso) || fflush(commso))
187 static void sendbyte(int c) {
188 if (putc(c,commso)==EOF)
192 static void mfreadcommsi(void *buf, int l, const char *what) {
193 int r= fread(buf,1,l,commsi); if (r!=l) die_badrecv(what);
195 static void mfwritecommso(const void *buf, int l) {
196 int r= fwrite(buf,1,l,commso); if (r!=l) die_badsend();
199 static void mpipe(int p[2]) { if (pipe(p)) diee("could not create pipe"); }
200 static void mdup2(int fd, int fd2) {
201 if (dup2(fd,fd2)!=fd2) diee("could not dup2(%d,%d)",fd,fd2);
204 typedef void copyfile_die_fn(FILE *f, const char *xi);
206 struct timespec ts_sendstart;
208 static void mgettime(struct timespec *ts) {
209 int r= clock_gettime(CLOCK_MONOTONIC, ts);
210 if (r) diee("clock_gettime failed");
213 static void bandlimit_sendstart(void) {
214 mgettime(&ts_sendstart);
217 static double mgettime_elapsed(struct timespec ts_base,
218 struct timespec *ts_ret) {
220 return (ts_ret->tv_sec - ts_base.tv_sec) +
221 (ts_ret->tv_nsec - ts_base.tv_nsec)*1e-9;
224 static void flushstderr(void) {
225 if (ferror(stderr) || fflush(stderr))
226 diee("could not write progress to stderr");
229 static void verbosespinprintf(const char *fmt, ...) {
230 static const char spinnerchars[]= "/-\\";
231 static int spinnerchar;
238 fprintf(stderr," %c ",spinnerchars[spinnerchar]);
239 spinnerchar++; spinnerchar %= sizeof(spinnerchars)-1;
240 vfprintf(stderr,fmt,al);
245 static void bandlimit_sendend(uint64_t bytes, int *interval_usec_update) {
246 struct timespec ts_buf;
247 double elapsed= mgettime_elapsed(ts_sendstart, &ts_buf);
248 double secsperbyte_observed= elapsed / bytes;
250 double min_update= elapsed / max_bw_prop;
251 if (min_update > 1e3) min_update= 1e3;
252 int min_update_usec= min_update * 1e6;
254 if (*interval_usec_update < min_update_usec)
255 *interval_usec_update= min_update_usec;
257 verbosespinprintf("%12lluby %10.3fs %13.2fkby/s %8dms",
258 (unsigned long long)bytes, elapsed,
259 1e-3/secsperbyte_observed, *interval_usec_update/1000);
262 static void copyfile(FILE *sf, copyfile_die_fn *sdie, const char *sxi,
263 FILE *df, copyfile_die_fn *ddie, const char *dxi,
264 uint64_t lstart, int amsender) {
266 uint64_t l=lstart, done=0;
269 now= l < sizeof(mainbuf) ? l : sizeof(mainbuf);
270 if (now > txblocksz) now= txblocksz;
272 r= fread(mainbuf,1,now,sf); if (r!=now) sdie(sf,sxi);
273 r= fwrite(mainbuf,1,now,df); if (r!=now) ddie(df,dxi);
278 fprintf(stderr," %3d%% \r",
279 (int)(done*100.0/lstart));
285 static void copydie_inputfile(FILE *f, const char *filename) {
286 diee("read failed on source file `%s'", filename);
288 static void copydie_tmpwrite(FILE *f, const char *tmpfilename) {
289 diee("write failed to temporary receiving file `%s'", tmpfilename);
291 static void copydie_commsi(FILE *f, const char *what) {
294 static void copydie_commso(FILE *f, const char *what) {
298 static int generate_declaration(void) {
299 /* returns length; declaration is left in mainbuf */
306 static void read_declaration(int decllen) {
307 assert(decllen <= sizeof(mainbuf));
308 if (decllen<2) die_protocol("declaration too short");
309 mfreadcommsi(mainbuf,decllen,"declaration");
310 if (mainbuf[decllen-1] != '\n')
311 die_protocol("declaration missing final newline");
312 if (mainbuf[0] != udchar)
313 die_protocol("declaration incorrect direction indicator");
316 static void receiver(const char *filename) {
321 char *lastslash= strrchr(filename,'/');
323 r= asprintf(&tmpfilename, ".rcopy-repeatedly.#%s#", filename);
325 r= asprintf(&tmpfilename, "%.*s/.rcopy-repeatedly.#%s#",
326 (int)(lastslash-filename), filename, lastslash+1);
329 r= unlink(tmpfilename);
330 if (r && errno!=ENOENT)
331 diee("could not remove temporary receiving file `%s'", tmpfilename);
340 if (ferror(commsi)) die_badrecv("transfer message code");
341 assert(feof(commsi));
346 if (r && errno!=ENOENT)
347 diee("source file removed but could not remove destination file `%s'",
352 newfile= fopen(tmpfilename, "wb");
353 if (!newfile) diee("could not create temporary receiving file `%s'",
356 mfreadcommsi(lbuf,8,"FILE64 l");
359 (lbuf[0] << 28 << 28) |
360 (lbuf[1] << 24 << 24) |
361 (lbuf[2] << 16 << 24) |
362 (lbuf[3] << 8 << 24) |
368 copyfile(commsi, copydie_commsi,"FILE64 file data",
369 newfile, copydie_tmpwrite,tmpfilename,
372 if (fclose(newfile)) diee("could not flush and close temporary"
373 " receiving file `%s'", tmpfilename);
374 if (rename(tmpfilename, filename))
375 diee("could not install new version of destination file `%s'",
378 sendbyte(REPLMSG_ACK);
382 die_protocol("unknown transfer message code 0x%02x",c);
388 static void sender(const char *filename) {
390 int interval_usec, r, c;
391 struct stat stabtest, stab;
392 enum { told_nothing, told_file, told_remove } told;
401 usleep(interval_usec);
403 interval_usec= min_interval_usec;
405 r= stat(filename, &stabtest);
409 if (told == told_file &&
410 stabtest.st_mode == stab.st_mode &&
411 stabtest.st_dev == stab.st_dev &&
412 stabtest.st_ino == stab.st_ino &&
413 stabtest.st_mtime == stab.st_mtime &&
414 stabtest.st_size == stab.st_size)
416 f= fopen(filename, "rb");
420 if (errno!=ENOENT) diee("could not access source file `%s'",filename);
421 if (told != told_remove) {
424 sendbyte(REPLMSG_RM);
430 if (fold) fclose(fold);
433 r= fstat(fileno(f),&stab);
434 if (r) diee("could not fstat source file `%s'",filename);
436 if (!S_ISREG(stab.st_mode))
437 die(8,0,-1,"source file `%s' is not a plain file",filename);
441 stab.st_size >> 28 >> 28,
442 stab.st_size >> 24 >> 24,
443 stab.st_size >> 16 >> 24,
444 stab.st_size >> 8 >> 24,
451 bandlimit_sendstart();
453 mfwritecommso(hbuf,9);
455 copyfile(f, copydie_inputfile,filename,
456 commso, copydie_commso,0,
461 c= fgetc(commsi); if (c==EOF) die_badrecv("ack");
462 if (c!=REPLMSG_ACK) die_protocol("got %#02x instead of ACK",c);
464 bandlimit_sendend(stab.st_size, &interval_usec);
472 const char *userhost, *path;
475 static FileSpecification srcspec, dstspec;
477 static void of__server(const struct cmdinfo *ci, const char *val) {
478 int ncount= nsargs + 1 + !!val;
479 sargs= xrealloc(sargs, sizeof(*sargs) * ncount);
480 sargs[nsargs++]= ci->olong;
482 sargs[nsargs++]= val;
485 static int of__server_int(const struct cmdinfo *ci, const char *val) {
489 errno= 0; v= strtol(val,&ep,10);
490 if (!*val || *ep || errno || v<INT_MIN || v>INT_MAX)
491 badusage("bad integer argument `%s' for --%s",val,ci->olong);
495 static void of_help(const struct cmdinfo *ci, const char *val) {
497 if (ferror(stdout)) diee("could not write usage message to stdout");
501 static void of_bw(const struct cmdinfo *ci, const char *val) {
502 int pct= of__server_int(ci,val);
503 if (pct<1 || pct>100)
504 badusage("bandwidth percentage must be between 1 and 100 inclusive");
505 *(double*)ci->parg= pct * 0.01;
508 static void of_server_int(const struct cmdinfo *ci, const char *val) {
509 *(int*)ci->parg= of__server_int(ci,val);
512 void usagemessage(void) {
514 "usage: rcopy-repeatedly [<options>] <file> <file>\n"
515 " <file> may be <local-file> or [<user>@]<host>:<file>\n"
516 " exactly one of each of the two forms must be provided\n"
517 " a file is taken as remote if it has a : before the first /\n"
521 "options for bandwidth (and cpu time) control:\n"
522 " --max-bandwidth-percent (default %d)\n"
523 " --tx-block-size (default/max %d)\n"
524 " --min-interval-usec (default %d)\n"
525 "options for finding programs:\n"
526 " --rcopy-repeatedly (default: rcopy-repeatedly)\n"
527 " --rsh-program (default: $RCOPY_REPEATEDLY_RSH or $RSYNC_RSH or ssh)\n"
528 "options passed to server side via ssh:\n"
529 " --receiver --sender, bandwidth control options\n",
530 (int)(max_bw_prop*100), (int)sizeof(mainbuf), min_interval_usec);
533 static const struct cmdinfo cmdinfos[]= {
534 { "help", .call= of_help },
535 { "max-bandwidth-percent", 0,1,.call=of_bw,.parg=&max_bw_prop },
536 { "tx-block-size",0, 1,.call=of_server_int, .parg=&txblocksz },
537 { "min-interval-usec",0, 1,.call=of_server_int, .parg=&min_interval_usec },
538 { "rcopy-repeatedly",0, 1, .sassignto=&rcopy_repeatedly_program },
539 { "rsh-program",0, 1, .sassignto=&rsh_program },
540 { "quiet",'q', .iassignto= &verbose, .arg=0 },
541 { "receiver", .iassignto= &server_upcopy, .arg=0 },
542 { "sender", .iassignto= &server_upcopy, .arg=1 },
546 static void server(const char *filename) {
550 udchar= server_upcopy?'u':'d';
554 l= generate_declaration();
555 fprintf(commso, "%s%04x\n", banner, l);
556 mfwritecommso(mainbuf, l);
561 if (feof(commsi)) exit(14);
562 assert(ferror(commsi)); die_badrecv("initial START message");
564 if (c!=REPLMSG_START) die_protocol("initial START was %#02x instead",c);
566 mfreadcommsi(buf,2,"START l");
567 l= (buf[0] << 8) | buf[1];
577 static void client(void) {
578 int uppipe[2], downpipe[2], r;
580 FileSpecification *remotespec;
581 const char *remotemode;
586 if (srcspec.userhost) {
588 remotespec= &srcspec;
589 remotemode= "--sender";
592 remotespec= &dstspec;
593 remotemode= "--receiver";
596 sargs= xrealloc(sargs, sizeof(*sargs) * (7 + nsargs));
597 memmove(sargs+5, sargs, sizeof(*sargs) * nsargs);
598 sargs[0]= rsh_program;
599 sargs[1]= remotespec->userhost;
600 sargs[2]= rcopy_repeatedly_program;
601 sargs[3]= remotemode;
603 sargs[5+nsargs]= remotespec->path;
607 if (child==-1) diee("fork failed");
609 mdup2(downpipe[0],0);
611 close(uppipe[0]); close(downpipe[0]);
612 close(uppipe[1]); close(downpipe[1]);
614 execvp(rsh_program, (char**)sargs);
615 diee("failed to execute rsh program `%s'",rsh_program);
618 commso= fdopen(downpipe[1],"wb");
619 commsi= fdopen(uppipe[0],"rb");
620 if (!commso || !commsi) diee("fdopen failed");
624 char banbuf[sizeof(banner)-1 + 5 + 1];
625 r= fread(banbuf,1,sizeof(banbuf)-1,commsi);
626 if (ferror(commsi)) die_badrecv("read banner");
628 if (r!=sizeof(banbuf)-1 ||
629 memcmp(banbuf,banner,sizeof(banner)-1) ||
630 banbuf[sizeof(banner)-1 + 4] != '\n') {
633 for (count=0, sap=sargs; *sap; sap++) count+= strlen(*sap)+1;
634 char *cmdline= xmalloc(count+1);
636 for (sap=sargs; *sap; sap++) {
638 strcat(cmdline,*sap);
641 die(8,0,-1,"did not receive banner as expected -"
642 " shell dirty? ssh broken?\n"
645 " and expect the first line to be\n"
650 banbuf[sizeof(banbuf)-1]= 0;
652 long decllen= strtoul(banbuf + sizeof(banner)-1, &ep, 16);
653 if (ep != banbuf + sizeof(banner)-1 + 4)
654 die_protocol("declaration length syntax error");
656 read_declaration(decllen);
658 int l= generate_declaration();
659 sendbyte(REPLMSG_START);
660 sendbyte((l >> 8) & 0x0ff);
661 sendbyte( l & 0x0ff);
662 mfwritecommso(mainbuf,l);
664 if (remotespec==&srcspec)
665 receiver(dstspec.path);
667 sender(srcspec.path);
670 static void parse_file_specification(FileSpecification *fs, const char *arg,
674 if (!arg) badusage("too few arguments - missing %s\n",what);
676 for (colon=arg; ; colon++) {
677 if (!*colon || *colon=='/') {
683 char *uh= xmalloc(colon-arg + 1);
684 memcpy(uh,arg, colon-arg); uh[colon-arg]= 0;
692 int main(int argc, const char *const *argv) {
693 setvbuf(stderr,0,_IOLBF,BUFSIZ);
695 myopt(&argv, cmdinfos);
697 if (!rsh_program) rsh_program= getenv("RCOPY_REPEATEDLY_RSH");
698 if (!rsh_program) rsh_program= getenv("RSYNC_RSH");
699 if (!rsh_program) rsh_program= "ssh";
701 if (txblocksz<1) badusage("transmit block size must be at least 1");
702 if (min_interval_usec<0) badusage("minimum update interval may not be -ve");
704 if (server_upcopy>=0) {
705 if (!argv[0] || argv[1])
706 badusage("server mode must have just the filename as non-option arg");
709 parse_file_specification(&srcspec, argv[0], "source");
710 parse_file_specification(&dstspec, argv[1], "destination");
711 if (argv[2]) badusage("too many non-option arguments");
712 if (!!srcspec.userhost == !!dstspec.userhost)
713 badusage("need exactly one remote file argument");