From: Ian Jackson <ijackson@chiark.greenend.org.uk>
Date: Thu, 5 Nov 2009 16:27:42 +0000 (+0000)
Subject: Do only a quick referential integrity check on market data updates
X-Git-Tag: 6.0~12
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.main.git;a=commitdiff_plain;h=663db396d271b241a8ec76bd44fbda9ec432e10f;ds=sidebyside

Do only a quick referential integrity check on market data updates
---

diff --git a/yarrg/CommodsDatabase.pm b/yarrg/CommodsDatabase.pm
index e14c0eb..3cb543d 100644
--- a/yarrg/CommodsDatabase.pm
+++ b/yarrg/CommodsDatabase.pm
@@ -178,7 +178,12 @@ sub nooutput ($) {
 	if $ekindcount;
 }
 
-sub db_check_referential_integrity () {
+sub db_check_referential_integrity ($) {
+    my ($full) = @_;
+    # non-full is done only for market data updates; it avoids
+    # detecting errors which are essentially missing metadata and
+    # old schemas, etc.
+
     foreach my $bs (qw(buy sell)) {
 	nooutput(<<END);
 
@@ -196,28 +201,33 @@ sub db_check_referential_integrity () {
 
 END
     }
-    foreach my $end (qw(aiid biid)) {
-	foreach my $tab (qw(dists routes)) {
-	    nooutput(<<END);
+
+    nooutput(<<END);
+
+ # Every stall and upload must refer to an island:
+ SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+ SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+
+END
+    if ($full) {
+	foreach my $end (qw(aiid biid)) {
+	    foreach my $tab (qw(dists routes)) {
+		nooutput(<<END);
 
  # Every row in dists and routes must refer to two existing rows in islands:
  SELECT * FROM $tab d LEFT JOIN islands ON d.$end=islandid
 	WHERE islandname IS NULL;
 
 END
+	    }
 	}
-    }
-    nooutput(<<END);
+	nooutput(<<END);
 
  # Every pair of islands must have an entry in dists:
  SELECT * FROM islands ia JOIN islands ib LEFT JOIN dists
 	ON ia.islandid=aiid and ib.islandid=biid
 	WHERE dist IS NULL;
 
- # Every stall and upload must refer to an island:
- SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
- SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
-
  # Every commod must refers to a commodclass and vice versa:
  SELECT * FROM commods NATURAL LEFT JOIN commodclasses
 	WHERE commodclass IS NULL;
@@ -248,10 +258,12 @@ END
 	WHERE posinclass < 0 OR posinclass > maxposinclass;
 
 END
+    }
 }
 
-sub db_chkcommit () {
-    db_check_referential_integrity();
+sub db_chkcommit ($) {
+    my ($full) = @_;
+    db_check_referential_integrity($full);
     $dbh->commit();
 }
 
diff --git a/yarrg/commod-email-processor b/yarrg/commod-email-processor
index 8acf4c1..a75864a 100755
--- a/yarrg/commod-email-processor
+++ b/yarrg/commod-email-processor
@@ -195,7 +195,7 @@ sub main () {
     pipethrough_run_finish($pt, 'gunzip <$deduped_tsv.gz');
 
 #    print "\n";
-    db_chkcommit();
+    db_chkcommit(0);
 
     # select * from ((buy natural join commods) natural join stalls) natural join islands;
     # select * from ((sell natural join commods) natural join stalls) natural join islands;
diff --git a/yarrg/db-idempotent-populate b/yarrg/db-idempotent-populate
index 691f77b..bb0448c 100755
--- a/yarrg/db-idempotent-populate
+++ b/yarrg/db-idempotent-populate
@@ -477,7 +477,7 @@ END
     ;
 }
 
-db_check_referential_integrity();
+db_check_referential_integrity(1);
 
 #---------- compact IDs ----------
 
@@ -539,7 +539,7 @@ END
 
 #---------- put it all into effect ----------
 
-db_chkcommit();
+db_chkcommit(1);
 
 {
     local $dbh->{AutoCommit} = 1;