From: Ian Jackson <ian@liberator.relativity.greenend.org.uk>
Date: Sun, 27 Jun 2010 16:19:35 +0000 (+0100)
Subject: wip rrd updater
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~ijackson/git?a=commitdiff_plain;h=e0ad2a121e7c3f9ccfce1c6e374dd2a9232cbb43;p=rrd-graphs.git

wip rrd updater
---

diff --git a/.gitignore b/.gitignore
index cac2397..baef3eb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
 *~
 testlogs
+chiark_*
diff --git a/newstailer b/newstailer
index 3ff6b72..2b482e7 100755
--- a/newstailer
+++ b/newstailer
@@ -7,20 +7,162 @@ use IO::Handle;
 use IO::File;
 use File::Tail;
 use Parse::Syslog;
+use RRDs;
 
-die unless @ARGV;
-die if $ARGV[0] =~ m/^\-/;
+our %dohosts;
+our $outpfx= './';
+our $sep= '_';
+our $debug= 0;
+
+open DEBUG, ">/dev/null" or die $!;
+
+while (@ARGV && $ARGV[0] =~ m/^\-./) {
+    $_= shift @ARGV;
+    last if $_ eq '--';
+    while (m/^\-./) {
+	if (s/^-h//) { $dohosts{$_}=1; last; }
+	elsif (s/^-O//) { $outpfx=$_; last; }
+	elsif (s/^-s//) { $sep=$_; last; }
+	elsif (s/^-D/-/) { $debug++; }
+	else { die "bad option $_"; }
+    }
+}
+
+if (!keys %dohosts) {
+    my ($sysname, $nodename) = POSIX::uname();
+    die unless defined $nodename;
+    $dohosts{$nodename}= 1;
+}
 
+die unless @ARGV;
 my $totail= pop @ARGV;
 
+if ($debug) { open DEBUG, ">& STDERR" or die $!; }
+
+our @fields_in= qw(seconds accepted refused rejected duplicate
+		   accepted_size duplicate_size);
+our $tmpl_in= join ':',@fields_in;
+our %details= (
+    'in' => {
+	Step => 60,
+	DstArguments => "7200:0:U",
+	Xff => 0.5,
+	Archives => [ [ 3600*4,           60 ],   # 4hr, 1min resolution
+		      [ 3600*25,         180 ],   # 25h, 3min resolution
+		      [ 86400*14*5,      3600 ],  # 14wks, 1hr resolution
+		      [ 86400*370*2, 3600*24 ] ], # 2yr+, 1day resolution
+    }
+);
+
+our ($time,$host,$peer,$conn,$stats);
+
+sub path_rrd ($) {
+    my ($inout) = @_;
+    return "${outpfx}${host}${sep}${peer}_${inout}.rrd";
+}
+
+sub perhaps_create_rrd ($$) {
+    my ($inout, $fields) = @_;
+    my $path= path_rrd($inout);
+    return if stat $path;
+    $!==&ENOENT or die "$path $!";
+    my $details= $details{$inout};
+
+    my @sargs= ($path, '--start','now-1y', '--step',$details->{Step});
+    my @largs;
+    push @largs, "DS:$_:ABSOLUTE:$details->{DstArguments}" foreach @$fields;
+    foreach (@{ $details->{Archives} }) {
+	my ($whole,$reso) = @$_;
+	my $steps= $reso / $details->{Step};
+	my $rows= $whole / $reso;
+	push @largs, "RRA:AVERAGE:$details->{Xff}:$steps:$rows";
+    } 
+    print DEBUG join("            \\\n  ", "creating @sargs", @largs),"\n";
+    RRDs::create(@sargs,@largs);
+    my $err= RRDs::error;
+    die "$err [@sargs @largs]" if defined $err;
+}
+
+sub update_rrd ($$$) {
+    my ($inout,$tmpl,$vals) = @_;
+    my $path= path_rrd($inout);
+    my @args= ($path, '--template',$tmpl, join(':',$time,@$vals));
+    print DEBUG "update @args\n";
+    RRDs::update(@args);
+    my $err= RRDs::error;
+    die "$err [@args]" if defined $err;
+}
+
+our %in_conns;
+
+sub inbound_connected () {
+    #print "$host $peer $conn START\n";
+    perhaps_create_rrd('in',\@fields_in);
+    $in_conns{$host,$peer,$conn} = [ (0) x @fields_in ];
+}
+sub inbound_closed () {
+    #print "$host $peer $conn STOP\n";
+    delete $in_conns{$host,$peer,$conn};
+}
+sub inbound_stats () {
+    $_= $stats.' ';
+    my %s;
+    s/(?<=[a-z]) (?=[a-z])/_/g;
+    my $hpc= $in_conns{$host,$peer,$conn};
+    if (!$hpc) {
+	perhaps_create_rrd('in',\@fields_in);
+	$in_conns{$host,$peer,$conn}= $hpc= [ (undef) x @fields_in ];
+    }
+    while (s/^([a-z_]+) (\d+)\s//) { $s{$1}= $2; }
+    my @v;
+    foreach my $f (@fields_in) {
+	my $this= $s{$f};
+	if (!defined $this) {
+	    delete $hpc->[@v];
+	    push @v, 'U';
+	    next;
+	}
+	my $last= $hpc->[@v];
+	$hpc->[@v]= $this;
+	push @v, defined($last) ? $this - $last : 'U';
+    }
+    update_rrd('in',$tmpl_in,\@v);
+}
+
+sub outbound_stats () {
+    print "$host $peer OUT $stats\n";
+}
+
 sub run ($) {
     my ($object) = @_;
     my $parser= new Parse::Syslog $object, repeat=>0, arrayref=>1;
-    while (my $sl= $parser->next) {
-	print join("|", @$sl), "\n";
+    my $host_re= '[-.0-9a-z]+';
+    my $conn_re= '[1-9]\d{0,5}';
+    my ($process,$pid,$msg,$cc,$sl);
+    while ($sl= $parser->next) {
+	($time,$host,$process,$pid,$msg) = @$sl;
+	next unless exists $dohosts{$host};
+	#print join("|", map { defined($_) ? $_ : "<undef>" } @$sl), "\n";
+	if ($process eq 'innd' && !defined $pid) {
+	    if (($peer,$conn) = $msg =~ m/^($host_re) connected ($conn_re)$/) {
+		inbound_connected()
+	    } elsif (($peer,$conn,$cc,$stats) = $msg =~
+            m/^($host_re):($conn_re) (closed|checkpoint) (seconds .*)$/) {
+		inbound_stats();
+		inbound_closed() if $cc eq 'closed';
+	    }
+	} elsif ($process eq 'innduct') {
+	    if (($peer,$stats) = $msg =~
+		m/^($host_re)\| (?:completed|processed) \S+ (read=.*)$/) {
+		outbound_stats();
+	    }
+	}
     }
 }
 
+#seconds (\d+) accepted (\d+) refused (\d+) rejected (\d+) duplicate (\d+) accepted size (\d+) duplicate size (\d+) 
+
+	
 foreach my $staticpath (@ARGV) {
     if ($staticpath =~ m/\.gz$/) {
 	my $fh= new IO::Handle;
@@ -29,13 +171,15 @@ foreach my $staticpath (@ARGV) {
 	!$fh->error or die "$staticpath $!";
 	$!=0;$?=0; close $fh or die "$staticpath $! $?";
     } else {
-	my $fh= new IO::File '<', $staticpath or die $!;
+	my $fh= new IO::File $staticpath, '<' or die $!;
 	run($staticpath);
 	!$fh->error or die "$staticpath $!";
 	close $fh or die "$staticpath $!";
     }
 }
 
+exit 0 if $totail eq '';
+
 my $tailer= new File::Tail name=>$totail,
     interval=>60, adjustafter=>2, ignore_nonexistant=>1, tail=>-1
     or die "$totail $!";