3 # Date: 26 Jun 1999 17:59:00 +0200
4 # From: kaih=7Jbfpa7mw-B@khms.westfalen.de (Kai Henningsen)
5 # Newsgroups: news.software.nntp
6 # Message-ID: <7Jbfpa7mw-B@khms.westfalen.de>
7 # Subject: Re: Version of pullnews that support authentication?
10 # I'm appending a script I wrote (called backupfeed.pl for some reason). Hmm
11 # ... oh, I hereby put that into the public domain. Use as you see fit. If
12 # it breaks, you get to keep all the parts.
14 # Needs the newer Net::NNTP versions for the MODE READER fix.
16 # This thing is both faster and uses far less memory than suck. And it
17 # inserts a predictable Path: entry (in case the host you pull from
20 # It's in production use as a backup to regular feeds, so it specifically
21 # fetches only old articles unless you say -p 1 (default is -p 0.6666...).
28 use vars qw($Group $Host $Pos $Rc %Rc $Starttime
29 $opt_S $opt_T $opt_d $opt_p $opt_s $opt_t);
31 my ( @groups, $localhost, $remotehost, $accepted, $rejected, $lockf,
32 $history, $acc, $rej, $his, @parms, $from, $to, $art, %err );
36 $opt_S = 10; # sleep between groups
37 $opt_T = 10000; # max running time
38 $opt_d = 0; # debugging
39 $opt_p = 2/3; # how many articles to fetch
40 $opt_s = 0; # sleep between articles
41 $opt_t = 0; # timeout for NNTP connections
42 getopts("dt:p:s:S:T:");
44 die <<USAGE if @ARGV < 2;
45 Usage: $0 hostname /groups/wanted [ userid password ]
49 -p nn how many articles (0.0 .. 1.0)
50 -s s sleep between articles
51 -S s sleep between groups
55 my ($GroupsWanted, $userid, $password);
56 ($Host, $GroupsWanted, $userid, $password) = @ARGV;
58 chdir("/var/local/lib/backupfeed") or die "chdir: $!";
59 $lockf = "/var/lock/lock-backupfeed-$Host";
60 system("/usr/lib/news/bin/shlock -p $$ -f $lockf")==0 or exit 0;
62 open LOG, ">> /var/log/news/backupfeed.$Host" or die "normal log: $!";
65 open ERR, ">> /var/log/news/backupfeed.$Host.errors" or die "error log: $!";
68 print LOG scalar(localtime), " $0 starting for $Host\n";
69 print ERR scalar(localtime), " $0 starting for $Host\n";
71 open GUP, $GroupsWanted or die "Groups Wanted: $GroupsWanted: $!";
77 $localhost = Net::NNTP->new("localhost", "Debug", $opt_d, "Timeout", $opt_t, "Reader", 0) or die "localhost: $!";
79 $remotehost = Net::NNTP->new($Host, "Debug", $opt_d, "Timeout", $opt_t) or die "remotehost: $!";
81 &lifecheck($remotehost, $Host);
82 $remotehost->authinfo($userid, $password) if ($userid);
83 &lifecheck($remotehost, $Host);
85 tie %Rc, "DB_File", "$Host.bfrc" or die "$Host.bfrc: $!";
91 my $restart = $Rc{'=restart='};
92 $restart='' unless ($restart);
94 my @before = grep $_ lt $restart, @groups;
95 my @after = grep $_ ge $restart, @groups;
96 @groups = ( @after, @before );
98 ($acc, $rej, $his) = (0, 0, 0);
99 foreach $Group (@groups) {
101 (@parms = $remotehost->group($Group)) or next;
102 &lifecheck($remotehost, $Host);
103 next if ($#parms < 3);
104 $Rc{'=restart='} = $Group;
105 print LOG scalar(localtime), " \t<$Group>\n";
107 if (!defined $Rc{$Group});
112 $to = $from + ($to - $from) * $opt_p;
113 if ($to < $Rc{$Group}) {
114 print LOG scalar(localtime), " \t watermark high, reset\n";
115 $Rc{$Group} = $from-1;
117 $Rc{$Group} = $from-1
118 if ($from > $Rc{$Group});
119 # print LOG scalar(localtime), " \t\t",$Rc{$Group}+1,"-$to\n";
120 $remotehost->nntpstat($Rc{$Group}+1);
121 # print LOG scalar(localtime), " \t\t",$remotehost->message,"\n";
122 &lifecheck($remotehost, $Host);
123 $art = $remotehost->nntpstat;
124 &lifecheck($remotehost, $Host);
125 $remotehost->message =~ /^(\d+)/;
132 while ($art = $remotehost->next) {
133 &lifecheck($remotehost, $Host);
134 $remotehost->message =~ /^(\d+)/;
140 &lifecheck($remotehost, $Host);
141 print LOG scalar(localtime), " \taccepted=$accepted rejected=$rejected history=$history\n";
149 sleep $opt_S if $opt_S;
162 system("echo $Host $Group $Pos > $Host.status");
163 if ($localhost->ihave($_[0])) {
164 &lifecheck($localhost, 'localhost');
165 my $article = $remotehost->article;
167 #open ART1, "> art1";
168 #print ART1 @$article;
171 while ($i <= @$article && !($$article[$i] =~ /^Path:/i)) {
174 $$article[$i] =~ s/^(Path:\s*)/$1NNTP-from-$Host!/i;
175 #open ART2, "> art2";
176 #print ART2 @$article;
179 $localhost->datasend($article);
180 if ($localhost->dataend) {
185 $err{" local " . $localhost->code . " " . $localhost->message} ++;
191 $err{" remote " . $remotehost->code . " " . $remotehost->message} ++;
193 sleep $opt_s if $opt_s;
196 if ($localhost->status == 4) {
197 if ($localhost->code == 435) {
198 $err{" local " . $localhost->code . " " . $localhost->message} ++;
201 $err{" local " . $localhost->code . " " . $localhost->message} ++;
202 print LOG scalar(localtime), " local ", $localhost->code, " ", $localhost->message, "\n";
206 &lifecheck($localhost, 'localhost');
214 unless (defined $_[0]->code and $_[0]->code > 0) {
215 print LOG scalar(localtime), " Connection to $_[1] dropped\n";
216 print ERR scalar(localtime), " Connection to $_[1] dropped\n";
219 #print "time=",time," starttime=$Starttime\n";
220 kill 'TERM', $$ if time-$Starttime > $opt_T;
225 print LOG scalar(localtime), " Caught sig: ", Data::Dumper::Dumper(@_), "\n";
226 print ERR scalar(localtime), " Caught sig: ", Data::Dumper::Dumper(@_), "\n";
240 print LOG scalar(localtime), " $0 $Host accepted=$acc rejected=$rej history=$his\n";
241 foreach my $e (sort keys %err) {
242 print ERR $err{$e}, $e, "\n";
244 print ERR scalar(localtime), " $0 $Host accepted=$acc rejected=$rej history=$his\n";