Do only a quick referential integrity check on market data updates

diff --git a/yarrg/CommodsDatabase.pm b/yarrg/CommodsDatabase.pm
index e14c0eb401e10a0f299b9b2c0564ce69d492266a..3cb543dc43b3866971cff2cb28e39de7f6061631 100644 (file)
--- a/yarrg/CommodsDatabase.pm
+++ b/yarrg/CommodsDatabase.pm
@@ -178,7 +178,12 @@ sub nooutput ($) {
        if $ekindcount;
 }
 
        if $ekindcount;
 }
 

-sub db_check_referential_integrity () {
+sub db_check_referential_integrity ($) {
+    my ($full) = @_;
+    # non-full is done only for market data updates; it avoids
+    # detecting errors which are essentially missing metadata and
+    # old schemas, etc.
+

     foreach my $bs (qw(buy sell)) {
        nooutput(<<END);
 
     foreach my $bs (qw(buy sell)) {
        nooutput(<<END);
 


@@ -196,28 +201,33 @@ sub db_check_referential_integrity () {
 
 END
     }
 
 END
     }

-    foreach my $end (qw(aiid biid)) {
-       foreach my $tab (qw(dists routes)) {
-           nooutput(<<END);
+
+    nooutput(<<END);
+
+ # Every stall and upload must refer to an island:
+ SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+ SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
+
+END
+    if ($full) {
+       foreach my $end (qw(aiid biid)) {
+           foreach my $tab (qw(dists routes)) {
+               nooutput(<<END);

 
  # Every row in dists and routes must refer to two existing rows in islands:
  SELECT * FROM $tab d LEFT JOIN islands ON d.$end=islandid
        WHERE islandname IS NULL;
 
 END
 
  # Every row in dists and routes must refer to two existing rows in islands:
  SELECT * FROM $tab d LEFT JOIN islands ON d.$end=islandid
        WHERE islandname IS NULL;
 
 END

+           }

        }
        }

-    }
-    nooutput(<<END);
+       nooutput(<<END);

 
  # Every pair of islands must have an entry in dists:
  SELECT * FROM islands ia JOIN islands ib LEFT JOIN dists
        ON ia.islandid=aiid and ib.islandid=biid
        WHERE dist IS NULL;
 
 
  # Every pair of islands must have an entry in dists:
  SELECT * FROM islands ia JOIN islands ib LEFT JOIN dists
        ON ia.islandid=aiid and ib.islandid=biid
        WHERE dist IS NULL;
 

- # Every stall and upload must refer to an island:
- SELECT * FROM stalls NATURAL LEFT JOIN islands WHERE islandname IS NULL;
- SELECT * FROM uploads NATURAL LEFT JOIN islands WHERE islandname IS NULL;
-

  # Every commod must refers to a commodclass and vice versa:
  SELECT * FROM commods NATURAL LEFT JOIN commodclasses
        WHERE commodclass IS NULL;
  # Every commod must refers to a commodclass and vice versa:
  SELECT * FROM commods NATURAL LEFT JOIN commodclasses
        WHERE commodclass IS NULL;


@@ -248,10 +258,12 @@ END
        WHERE posinclass < 0 OR posinclass > maxposinclass;
 
 END
        WHERE posinclass < 0 OR posinclass > maxposinclass;
 
 END

+    }

 }
 
 }
 

-sub db_chkcommit () {
-    db_check_referential_integrity();
+sub db_chkcommit ($) {
+    my ($full) = @_;
+    db_check_referential_integrity($full);

     $dbh->commit();
 }
 
     $dbh->commit();
 }
 

diff --git a/yarrg/commod-email-processor b/yarrg/commod-email-processor
index 8acf4c105d118c38d99f49425eea89806c0a0f76..a75864a57e2ff4f0d3d48514697f78d900a14d24 100755 (executable)
--- a/yarrg/commod-email-processor
+++ b/yarrg/commod-email-processor
@@ -195,7 +195,7 @@ sub main () {
     pipethrough_run_finish($pt, 'gunzip <$deduped_tsv.gz');
 
 #    print "\n";
     pipethrough_run_finish($pt, 'gunzip <$deduped_tsv.gz');
 
 #    print "\n";

-    db_chkcommit();
+    db_chkcommit(0);

 
     # select * from ((buy natural join commods) natural join stalls) natural join islands;
     # select * from ((sell natural join commods) natural join stalls) natural join islands;
 
     # select * from ((buy natural join commods) natural join stalls) natural join islands;
     # select * from ((sell natural join commods) natural join stalls) natural join islands;

diff --git a/yarrg/db-idempotent-populate b/yarrg/db-idempotent-populate
index 691f77ba8af294d0f8e47e3463137df30d2205f5..bb0448ce478389c1c030220514c1cdabe648f588 100755 (executable)
--- a/yarrg/db-idempotent-populate
+++ b/yarrg/db-idempotent-populate
@@ -477,7 +477,7 @@ END
     ;
 }
 
     ;
 }
 

-db_check_referential_integrity();
+db_check_referential_integrity(1);

 
 #---------- compact IDs ----------
 
 
 #---------- compact IDs ----------
 


@@ -539,7 +539,7 @@ END
 
 #---------- put it all into effect ----------
 
 
 #---------- put it all into effect ----------
 

-db_chkcommit();
+db_chkcommit(1);

 
 {
     local $dbh->{AutoCommit} = 1;
 
 {
     local $dbh->{AutoCommit} = 1;