wip rrd updater

diff --git a/.gitignore b/.gitignore
index cac239706689481e71fddb383bb5b23dc05eb4d0..baef3eb11d1a155e7539848022f174fbab5b148f 100644 (file)
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 *~
 testlogs
+chiark_*

diff --git a/newstailer b/newstailer
index 3ff6b72140d056b24e11f9b258cc75cd754fca6b..2b482e70fe6fda00010f65ba08e528c347ce4923 100755 (executable)
--- a/newstailer
+++ b/newstailer
@@ -7,20 +7,162 @@ use IO::Handle;
 use IO::File;
 use File::Tail;
 use Parse::Syslog;
+use RRDs;
 
-die unless @ARGV;
-die if $ARGV[0] =~ m/^\-/;
+our %dohosts;
+our $outpfx= './';
+our $sep= '_';
+our $debug= 0;
+
+open DEBUG, ">/dev/null" or die $!;
+
+while (@ARGV && $ARGV[0] =~ m/^\-./) {
+    $_= shift @ARGV;
+    last if $_ eq '--';
+    while (m/^\-./) {
+       if (s/^-h//) { $dohosts{$_}=1; last; }
+       elsif (s/^-O//) { $outpfx=$_; last; }
+       elsif (s/^-s//) { $sep=$_; last; }
+       elsif (s/^-D/-/) { $debug++; }
+       else { die "bad option $_"; }
+    }
+}
+
+if (!keys %dohosts) {
+    my ($sysname, $nodename) = POSIX::uname();
+    die unless defined $nodename;
+    $dohosts{$nodename}= 1;
+}
 
+die unless @ARGV;
 my $totail= pop @ARGV;
 
+if ($debug) { open DEBUG, ">& STDERR" or die $!; }
+
+our @fields_in= qw(seconds accepted refused rejected duplicate
+                  accepted_size duplicate_size);
+our $tmpl_in= join ':',@fields_in;
+our %details= (
+    'in' => {
+       Step => 60,
+       DstArguments => "7200:0:U",
+       Xff => 0.5,
+       Archives => [ [ 3600*4,           60 ],   # 4hr, 1min resolution
+                     [ 3600*25,         180 ],   # 25h, 3min resolution
+                     [ 86400*14*5,      3600 ],  # 14wks, 1hr resolution
+                     [ 86400*370*2, 3600*24 ] ], # 2yr+, 1day resolution
+    }
+);
+
+our ($time,$host,$peer,$conn,$stats);
+
+sub path_rrd ($) {
+    my ($inout) = @_;
+    return "${outpfx}${host}${sep}${peer}_${inout}.rrd";
+}
+
+sub perhaps_create_rrd ($$) {
+    my ($inout, $fields) = @_;
+    my $path= path_rrd($inout);
+    return if stat $path;
+    $!==&ENOENT or die "$path $!";
+    my $details= $details{$inout};
+
+    my @sargs= ($path, '--start','now-1y', '--step',$details->{Step});
+    my @largs;
+    push @largs, "DS:$_:ABSOLUTE:$details->{DstArguments}" foreach @$fields;
+    foreach (@{ $details->{Archives} }) {
+       my ($whole,$reso) = @$_;
+       my $steps= $reso / $details->{Step};
+       my $rows= $whole / $reso;
+       push @largs, "RRA:AVERAGE:$details->{Xff}:$steps:$rows";
+    } 
+    print DEBUG join("            \\\n  ", "creating @sargs", @largs),"\n";
+    RRDs::create(@sargs,@largs);
+    my $err= RRDs::error;
+    die "$err [@sargs @largs]" if defined $err;
+}
+
+sub update_rrd ($$$) {
+    my ($inout,$tmpl,$vals) = @_;
+    my $path= path_rrd($inout);
+    my @args= ($path, '--template',$tmpl, join(':',$time,@$vals));
+    print DEBUG "update @args\n";
+    RRDs::update(@args);
+    my $err= RRDs::error;
+    die "$err [@args]" if defined $err;
+}
+
+our %in_conns;
+
+sub inbound_connected () {
+    #print "$host $peer $conn START\n";
+    perhaps_create_rrd('in',\@fields_in);
+    $in_conns{$host,$peer,$conn} = [ (0) x @fields_in ];
+}
+sub inbound_closed () {
+    #print "$host $peer $conn STOP\n";
+    delete $in_conns{$host,$peer,$conn};
+}
+sub inbound_stats () {
+    $_= $stats.' ';
+    my %s;
+    s/(?<=[a-z]) (?=[a-z])/_/g;
+    my $hpc= $in_conns{$host,$peer,$conn};
+    if (!$hpc) {
+       perhaps_create_rrd('in',\@fields_in);
+       $in_conns{$host,$peer,$conn}= $hpc= [ (undef) x @fields_in ];
+    }
+    while (s/^([a-z_]+) (\d+)\s//) { $s{$1}= $2; }
+    my @v;
+    foreach my $f (@fields_in) {
+       my $this= $s{$f};
+       if (!defined $this) {
+           delete $hpc->[@v];
+           push @v, 'U';
+           next;
+       }
+       my $last= $hpc->[@v];
+       $hpc->[@v]= $this;
+       push @v, defined($last) ? $this - $last : 'U';
+    }
+    update_rrd('in',$tmpl_in,\@v);
+}
+
+sub outbound_stats () {
+    print "$host $peer OUT $stats\n";
+}
+
 sub run ($) {
     my ($object) = @_;
     my $parser= new Parse::Syslog $object, repeat=>0, arrayref=>1;
-    while (my $sl= $parser->next) {
-       print join("|", @$sl), "\n";
+    my $host_re= '[-.0-9a-z]+';
+    my $conn_re= '[1-9]\d{0,5}';
+    my ($process,$pid,$msg,$cc,$sl);
+    while ($sl= $parser->next) {
+       ($time,$host,$process,$pid,$msg) = @$sl;
+       next unless exists $dohosts{$host};
+       #print join("|", map { defined($_) ? $_ : "<undef>" } @$sl), "\n";
+       if ($process eq 'innd' && !defined $pid) {
+           if (($peer,$conn) = $msg =~ m/^($host_re) connected ($conn_re)$/) {
+               inbound_connected()
+           } elsif (($peer,$conn,$cc,$stats) = $msg =~
+            m/^($host_re):($conn_re) (closed|checkpoint) (seconds .*)$/) {
+               inbound_stats();
+               inbound_closed() if $cc eq 'closed';
+           }
+       } elsif ($process eq 'innduct') {
+           if (($peer,$stats) = $msg =~
+               m/^($host_re)\| (?:completed|processed) \S+ (read=.*)$/) {
+               outbound_stats();
+           }
+       }
     }
 }
 
+#seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+) duplicate (\d+) accepted size (\d+) duplicate size (\d+) 
+
+       
 foreach my $staticpath (@ARGV) {
     if ($staticpath =~ m/\.gz$/) {
        my $fh= new IO::Handle;
@@ -29,13 +171,15 @@ foreach my $staticpath (@ARGV) {
        !$fh->error or die "$staticpath $!";
        $!=0;$?=0; close $fh or die "$staticpath $! $?";
     } else {
-       my $fh= new IO::File '<', $staticpath or die $!;
+       my $fh= new IO::File $staticpath, '<' or die $!;
        run($staticpath);
        !$fh->error or die "$staticpath $!";
        close $fh or die "$staticpath $!";
     }
 }
 
+exit 0 if $totail eq '';
+
 my $tailer= new File::Tail name=>$totail,
     interval=>60, adjustafter=>2, ignore_nonexistant=>1, tail=>-1
     or die "$totail $!";