From: Ian Jackson <ijackson@chiark.greenend.org.uk>
Date: Mon, 2 Nov 2009 15:51:34 +0000 (+0000)
Subject: Move referential integrity check into CommodsDatabase
X-Git-Tag: 6.0~20
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.main.git;a=commitdiff_plain;h=84acb589eaec42201d8488eb909e5a6e262ff149

Move referential integrity check into CommodsDatabase
---

diff --git a/yarrg/CommodsDatabase.pm b/yarrg/CommodsDatabase.pm
index c510080..6ed0b74 100644
--- a/yarrg/CommodsDatabase.pm
+++ b/yarrg/CommodsDatabase.pm
@@ -45,7 +45,8 @@ BEGIN {
     @ISA         = qw(Exporter);
     @EXPORT      = qw(&db_setocean &db_writer &db_connect $dbh
 		      &db_filename &db_doall &db_onconflict
-		      &dbr_filename &dbr_connect &db_connect_core);
+		      &dbr_filename &dbr_connect &db_connect_core
+		      &db_chkcommit &db_check_referential_integrity);
     %EXPORT_TAGS = ( );
 
     @EXPORT_OK   = qw();
@@ -119,4 +120,111 @@ sub db_doall ($) {
     }
 }
 
+#---------- referential integrity constraints ----------
+
+# SQLite doesn't support foreign key constraints so we do it by steam:
+
+sub nooutput ($) {
+    my ($stmts) = @_;
+    my $ekindcount= 0;
+    my $letxt= '';
+    foreach my $stmt (split /\;/, $stmts) {
+	next unless $stmt =~ /\S/;
+
+	my $etxt= '';
+	$stmt =~ s/^([ \t]*\#.*)$/ $etxt .= $1."\n"; ''; /mge;
+	$etxt= $letxt unless length $etxt;
+	$letxt= $etxt;
+	
+	$stmt =~ s/^\s+//; $stmt =~ s/\s+$//;
+	my $sth= $dbh->prepare($stmt);
+	$sth->execute();
+	my $row;
+	my $ecount= 0;
+	my @cols= @{ $sth->{NAME_lc} };
+	my $w= 11;
+	while ($row= $sth->fetchrow_hashref) {
+	    if (!$ecount++) {
+		print STDERR "REFERENTIAL INTEGRITY ERROR\n";
+		print STDERR "\n$etxt\n $stmt\n\n";
+		printf STDERR "|%-${w}s", $_ foreach @cols; print STDERR "|\n";
+		print STDERR "+",('-'x$w)  foreach @cols; print STDERR "+\n";
+	    }
+	    if ($ecount>5) { print STDERR "...\n"; last; }
+	    printf STDERR "|%-$w.${w}s",
+	        (defined $row->{$_} ? $row->{$_} : 'NULL')
+		    foreach @cols;
+	    print STDERR "\n";
+	}
+	next unless $ecount;
+	
+	$ekindcount++;
+	print STDERR "\n\n";
+    }
+    die "REFERENTIAL INTEGRITY ERRORS $ekindcount\n"
+	if $ekindcount;
+}
+
+sub db_check_referential_integrity () {
+    foreach my $bs (qw(buy sell)) {
+	nooutput(<<END);
+
+ # Every buy/sell must refer to an entry in commods, islands, and stalls:
+ SELECT * FROM $bs NATURAL LEFT JOIN commods WHERE commodname IS NULL;
+ SELECT * FROM $bs NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+ SELECT * FROM $bs LEFT JOIN STALLS USING (stallid) WHERE stallname IS NULL;
+
+ # Every buy/sell must be part of an upload:
+ SELECT * FROM $bs NATURAL LEFT JOIN uploads WHERE timestamp IS NULL;
+
+ # The islandid in stalls must be the same as the islandid in buy/sell:
+ SELECT * FROM $bs JOIN stalls USING (stallid)
+	WHERE $bs.islandid != stalls.islandid;
+
+END
+    }
+    foreach my $end (qw(aiid biid)) {
+	foreach my $tab (qw(dists routes)) {
+	    nooutput(<<END);
+
+ # Every row in dists and routes must refer to two existing rows in islands:
+ SELECT * FROM $tab d LEFT JOIN islands ON (d.$end=islandid)
+	WHERE islandname IS NULL;
+
+END
+	}
+    }
+    nooutput(<<END);
+
+ # Every pair of islands must have an entry in dists:
+ SELECT * FROM islands ia JOIN islands ib LEFT JOIN dists
+	ON (ia.islandid=aiid and ib.islandid=biid)
+	WHERE dist IS NULL;
+
+ # Every stall and upload must refer to an island:
+ SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+ SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+
+ # Every commod which refers to a commodclass refers to an existing one:
+ SELECT * FROM commods WHERE commodclass NOT IN
+	(SELECT commodclass FROM commodclasses);
+
+ # There are no empty commodclasses:
+ SELECT * FROM commodclasses NATURAL LEFT JOIN commods
+	WHERE commodname IS NULL;
+
+ # Ordvals which are not zero are unique:
+ SELECT ordval,count(*) FROM COMMODS
+	WHERE ordval IS NOT NULL AND ordval != 0
+	GROUP BY ordval
+	HAVING count(*) > 1;
+
+END
+}
+
+sub db_chkcommit () {
+    db_check_referential_integrity();
+    $dbh->commit();
+}
+
 1;
diff --git a/yarrg/db-idempotent-populate b/yarrg/db-idempotent-populate
index 2cce5c7..2e0b0ae 100755
--- a/yarrg/db-idempotent-populate
+++ b/yarrg/db-idempotent-populate
@@ -55,113 +55,6 @@ db_connect();
 
 $dbh->trace(1) if $trace;
 
-#---------- referential integrity constraints ----------
-
-# SQLite doesn't support foreign key constraints so we do it by steam:
-
-sub nooutput ($) {
-    my ($stmts) = @_;
-    my $ekindcount= 0;
-    my $letxt= '';
-    foreach my $stmt (split /\;/, $stmts) {
-	next unless $stmt =~ /\S/;
-
-	my $etxt= '';
-	$stmt =~ s/^([ \t]*\#.*)$/ $etxt .= $1."\n"; ''; /mge;
-	$etxt= $letxt unless length $etxt;
-	$letxt= $etxt;
-	
-	$stmt =~ s/^\s+//; $stmt =~ s/\s+$//;
-	my $sth= $dbh->prepare($stmt);
-	$sth->execute();
-	my $row;
-	my $ecount= 0;
-	my @cols= @{ $sth->{NAME_lc} };
-	my $w= 11;
-	while ($row= $sth->fetchrow_hashref) {
-	    if (!$ecount++) {
-		print STDERR "REFERENTIAL INTEGRITY ERROR\n";
-		print STDERR "\n$etxt\n $stmt\n\n";
-		printf STDERR "|%-${w}s", $_ foreach @cols; print STDERR "|\n";
-		print STDERR "+",('-'x$w)  foreach @cols; print STDERR "+\n";
-	    }
-	    if ($ecount>5) { print STDERR "...\n"; last; }
-	    printf STDERR "|%-$w.${w}s",
-	        (defined $row->{$_} ? $row->{$_} : 'NULL')
-		    foreach @cols;
-	    print STDERR "\n";
-	}
-	next unless $ecount;
-	
-	$ekindcount++;
-	print STDERR "\n\n";
-    }
-    die "REFERENTIAL INTEGRITY ERRORS $ekindcount\n"
-	if $ekindcount;
-}
-
-sub check_referential_integrity () {
-    foreach my $bs (qw(buy sell)) {
-	nooutput(<<END);
-
- # Every buy/sell must refer to an entry in commods, islands, and stalls:
- SELECT * FROM $bs NATURAL LEFT JOIN commods WHERE commodname IS NULL;
- SELECT * FROM $bs NATURAL LEFT JOIN islands WHERE islandname IS NULL;
- SELECT * FROM $bs LEFT JOIN STALLS USING (stallid) WHERE stallname IS NULL;
-
- # Every buy/sell must be part of an upload:
- SELECT * FROM $bs NATURAL LEFT JOIN uploads WHERE timestamp IS NULL;
-
- # The islandid in stalls must be the same as the islandid in buy/sell:
- SELECT * FROM $bs JOIN stalls USING (stallid)
-	WHERE $bs.islandid != stalls.islandid;
-
-END
-    }
-    foreach my $end (qw(aiid biid)) {
-	foreach my $tab (qw(dists routes)) {
-	    nooutput(<<END);
-
- # Every row in dists and routes must refer to two existing rows in islands:
- SELECT * FROM $tab d LEFT JOIN islands ON (d.$end=islandid)
-	WHERE islandname IS NULL;
-
-END
-	}
-    }
-    nooutput(<<END);
-
- # Every pair of islands must have an entry in dists:
- SELECT * FROM islands ia JOIN islands ib LEFT JOIN dists
-	ON (ia.islandid=aiid and ib.islandid=biid)
-	WHERE dist IS NULL;
-
- # Every stall and upload must refer to an island:
- SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
- SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
-
- # Every commod which refers to a commodclass refers to an existing one:
- SELECT * FROM commods WHERE commodclass NOT IN
-	(SELECT commodclass FROM commodclasses);
-
- # There are no empty commodclasses:
- SELECT * FROM commodclasses NATURAL LEFT JOIN commods
-	WHERE commodname IS NULL;
-
- # Ordvals which are not zero are unique:
- SELECT ordval,count(*) FROM COMMODS
-	WHERE ordval IS NOT NULL AND ordval != 0
-	GROUP BY ordval
-	HAVING count(*) > 1;
-
-END
-}
-
-sub chkcommit () {
-    check_referential_integrity();
-    $dbh->commit();
-}
-
 #---------- schema ----------
 
 foreach my $bs (qw(buy sell)) {
@@ -282,7 +175,7 @@ db_doall(<<END)
 END
     ;
 
-chkcommit();
+db_chkcommit();
 
 #---------- commodity list ----------
 
@@ -359,7 +252,7 @@ END
     foreach my $cl (sort keys %incl) {
 	$addclass->execute($cl, $incl{$cl});	
     }
-    chkcommit();
+    db_chkcommit();
 }
 
 #---------- vessel types ----------
@@ -377,5 +270,5 @@ END
 	my @qa= ($name, $shotdamage, map { $v->{$_} } qw(Mass Volume));
 	$idempotent->execute(@qa);
     }
-    chkcommit();
+    db_chkcommit();
 }
diff --git a/yarrg/yppedia-chart-parser b/yarrg/yppedia-chart-parser
index 41ef985..e6e22b5 100755
--- a/yarrg/yppedia-chart-parser
+++ b/yarrg/yppedia-chart-parser
@@ -772,6 +772,7 @@ for (;;) {
 	    print STDERR "*** --stdin-chart, aborting!\n";
 	    exit 1;
 	}
+	progress("checking database");        db_check_referential_integrity();
 	progress("committing database");       $dbh->commit();
 	progress("committing _ocean-*.txt");   localtopo_commit();
 	exit 0;