4 # with-lock-ex -f data/news/lock sh -xc \
5 # "rm data/news/*.rrd; ./newstailer -Odata/news/ -D \
6 # /var/log/news/news.notice.{6,5,4,3,2,1}.gz /var/log/news/news.notice{.0,} ''"
8 use strict qw(refs vars);
23 open DEBUG, ">/dev/null" or die $!;
25 while (@ARGV && $ARGV[0] =~ m/^\-./) {
29 if (s/^-h//) { $dohosts{$_}=1; last; }
30 elsif (s/^-O//) { $outpfx=$_; last; }
31 elsif (s/^-s//) { $sep=$_; last; }
32 elsif (s/^-i([1-9]\d{0,5})$//) { $interval= $1; }
33 elsif (s/^-D/-/) { $debug++; }
34 else { die "bad option $_"; }
39 my ($sysname, $nodename) = POSIX::uname();
40 die unless defined $nodename;
41 $dohosts{$nodename}= 1;
45 my $totail= pop @ARGV;
47 if ($debug) { open DEBUG, ">& STDOUT" or die $!; }
55 DstArguments => "7200:0:U",
57 Archives => [ [ 3600*4, 60 ], # 4hr, 1min resolution
58 [ 3600*25, 180 ], # 25h, 3min resolution
59 [ 86400*8, 600 ], # 8d, 10min resolution
60 [ 86400*7*14, 3600*2 ], # 14wks, 2hr resolution
61 [ 86400*366, 3600*6 ], # 1yr, 6hr resolution
62 [ 86400*366*3, 3600*24 ] ], # 3yr, 1d resolution
65 our @fields_in= qw(seconds accepted refused rejected duplicate
66 accepted_size duplicate_size);
68 Fields => \@fields_in,
72 our @fields_out= qw(missing offered deferred
73 accepted unwanted rejected body_missing);
75 Fields => \@fields_out,
80 our ($time,$host,$peer,$conn,$stats);
83 my ($inout, $path) = @_;
85 my $details= $details{$inout};
87 my @sargs= ($path, '--start','now-1y', '--step',$details->{Step});
89 push @largs, "DS:$_:ABSOLUTE:$details->{DstArguments}"
90 foreach @{ $details{$inout}{Fields} };
91 foreach (@{ $details->{Archives} }) {
92 my ($whole,$reso) = @$_;
93 my $steps= $reso / $details->{Step};
94 my $rows= $whole / $reso;
95 push @largs, "RRA:AVERAGE:$details->{Xff}:$steps:$rows";
97 print DEBUG join(" \\\n ", "creating @sargs", @largs),"\n";
98 RRDs::create(@sargs,@largs);
100 die "$err [@sargs @largs]" if defined $err;
103 sub get_rrd_info ($$) {
104 my ($rrdupdate, $path) = @_;
105 my $h= RRDs::info($path);
106 die "$path $! ".(RRDs::error) unless $h;
107 die "$path ?" unless $h->{'last_update'};
108 $rrdupdate->{DoneUpto}= $h->{'last_update'};
111 sub find_or_create_rrd ($) {
114 Path => "${outpfx}${host}${sep}${peer}_${inout}.rrd",
116 if (stat $rrd->{Path}) {
117 get_rrd_info($rrd, $rrd->{Path});
119 $!==&ENOENT or die "$rrd->{Path} $!";
120 create_rrd($inout, $rrd->{Path});
127 our @rrd_blockedupdates;
128 our $rrd_blockedupdate_time;
130 sub update_rrd ($$) {
131 my ($inout,$vals) = @_;
133 my $rrd= $rrds{$host,$peer,$inout};
135 $rrd= $rrds{$host,$peer,$inout}= find_or_create_rrd($inout);
136 $rrd->{Template}= join ':', @{ $details{$inout}{Fields} };
138 return if $time <= $rrd->{DoneUpto};
140 my $blocked= $rrd->{BlockedUpdate};
141 if (defined $blocked) {
142 for (my $ix=0; $ix<@$vals; $ix++) {
143 my $old= $blocked->[$ix];
144 my $new= $vals->[$ix];
145 $blocked->[$ix]= ($old eq 'U' || $new eq 'U') ? 'U' : $old + $new;
149 $rrd->{BlockedUpdate}= $vals;
150 $rrd_blockedupdate_time= $time;
151 push @rrd_blockedupdates, $rrd;
154 sub actually_update_rrds () {
155 return unless defined $rrd_blockedupdate_time;
156 return if $time == $rrd_blockedupdate_time;
158 while (my $rrd= shift @rrd_blockedupdates) {
159 my $vals= $rrd->{BlockedUpdate};
161 delete $rrd->{BlockedUpdate};
163 my @args= ($rrd->{Path}, '--template',$rrd->{Template},
164 join(':',$rrd_blockedupdate_time,@$vals));
165 print DEBUG "update @args\n" if $debug>=2;
167 my $err= RRDs::error;
168 die "$err [@args]" if defined $err;
171 $rrd_blockedupdate_time= undef;
176 sub inbound_connected () {
177 print DEBUG "inbound $time connected $host $peer $conn\n" if $debug>=2;
178 $in_conns{$host,$peer,$conn} = [ (0) x @fields_in ];
180 sub inbound_closed () {
181 print DEBUG "inbound $time closed $host $peer $conn\n" if $debug>=2;
182 delete $in_conns{$host,$peer,$conn};
184 sub inbound_stats () {
186 s/(?<=[a-z]) (?=[a-z])/_/g;
187 my $hpc= $in_conns{$host,$peer,$conn};
189 print DEBUG "inbound $time UNKNOWN $host $peer $conn $stats\n";
190 $in_conns{$host,$peer,$conn}= $hpc= [ (undef) x @fields_in ];
192 print DEBUG "inbound $time stats $host $peer $conn $stats\n"
196 while (s/^([a-z_]+) (\d+)\s//) { $s{$1}= $2; }
198 foreach my $f (@fields_in) {
200 if (!defined $this) {
205 my $last= $hpc->[@v];
207 push @v, defined($last) ? $this - $last : 'U';
209 update_rrd('in',\@v);
212 sub outbound_stats () {
213 print DEBUG "outbound $time stats $host $peer $stats\n" if $debug>=2;
215 s/missing(?=\=\d+ \()/body_missing/;
218 while (s/ ([a-z]\w+)\=(\d+) / /) { $s{$1}= $2; }
220 foreach my $f (@fields_out) {
222 push @v, defined($this) ? $this : 'U';
224 update_rrd('out',\@v);
229 my $parser= new Parse::Syslog $object, repeat=>0, arrayref=>1;
230 my $host_re= '[-.0-9a-z]+';
231 my $conn_re= '[1-9]\d{0,5}';
232 my ($process,$pid,$msg,$cc,$sl);
233 while ($sl= $parser->next) {
234 ($time,$host,$process,$pid,$msg) = @$sl;
235 actually_update_rrds();
236 next unless exists $dohosts{$host};
237 print DEBUG "logfile ",
238 join("|", map { defined($_) ? $_ : "<undef>" } @$sl), "\n"
240 if ($process eq 'innd' && !defined $pid) {
241 if (($peer,$conn) = $msg =~
242 m/^($host_re) connected ($conn_re)(?: streaming allowed)?$/) {
244 } elsif (($peer,$conn,$cc,$stats) = $msg =~
245 m/^($host_re):($conn_re) (closed|checkpoint) (seconds .*)$/) {
247 inbound_closed() if $cc eq 'closed';
249 } elsif ($process eq 'innduct') {
250 if (($peer,$stats) = $msg =~
251 m/^($host_re)\| notice: (?:completed|processed) \S+ (read=.*)$/) {
258 #seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+) duplicate (\d+) accepted size (\d+) duplicate size (\d+)
261 foreach my $staticpath (@ARGV) {
262 if ($staticpath =~ m/\.gz$/) {
263 my $fh= new IO::Handle;
264 open $fh, '-|', 'gunzip', '-c', '--', $staticpath or die $!;
266 !$fh->error or die "$staticpath $!";
267 $!=0;$?=0; close $fh or die "$staticpath $! $?";
269 my $fh= new IO::File $staticpath, '<' or die $!;
271 !$fh->error or die "$staticpath $!";
272 close $fh or die "$staticpath $!";
276 exit 0 if $totail eq '';
278 my $tailer= new File::Tail name=>$totail,
279 interval=>$interval, adjustafter=>2, ignore_nonexistant=>1, tail=>-1