From: Ian Jackson <ian@liberator.relativity.greenend.org.uk>
Date: Sat, 17 Oct 2009 16:19:24 +0000 (+0100)
Subject: routesearch: concurrency limit
X-Git-Tag: 5.0^2~25
X-Git-Url: http://www.chiark.greenend.org.uk/ucgi/~yarrgweb/git?p=ypp-sc-tools.db-test.git;a=commitdiff_plain;h=d892748541adab0a53bbd641a1440f4c65c9cb63;ds=sidebyside

routesearch: concurrency limit
---

diff --git a/yarrg/CommodsWeb.pm b/yarrg/CommodsWeb.pm
index 00c6f24..0f3f43d 100644
--- a/yarrg/CommodsWeb.pm
+++ b/yarrg/CommodsWeb.pm
@@ -50,7 +50,7 @@ BEGIN {
     @ISA         = qw(Exporter);
     @EXPORT      = qw(&dbw_connect &dbw_filename &ocean_list &sourcebasedir
 		      &to_json_shim &to_json_protecttags
-		      &set_ctype_utf8
+		      &set_ctype_utf8 &webdatadir
 		      &expected_error &dbw_lookup_string
 		      &prettyprint_age &meta_prettyprint_age);
     %EXPORT_TAGS = ( );
@@ -73,20 +73,25 @@ sub sourcebasedir () {
     return dotperllibdir().'/..';
 }
 
-sub datadir () {
-    my $edir= $ENV{'YARRG_DATA_DIR'};
+sub some_datadir ($) {
+    my ($what) = @_;
+    my $edir= $ENV{"YARRG_${what}_DIR"};
     return $edir if defined $edir;
     my $dir= dotperllibdir();
-    if (stat "$dir/DATA") {
-	return "$dir/DATA";
+    my $dirwhat= "$dir/$what";
+    if (stat $dirwhat) {
+	return $dirwhat;
     } elsif ($!==&ENOENT) {
 	return "$dir";
     } else {
-	die "stat $dir/DATA $!";
+	die "stat $dirwhat $!";
     }
     return '.';
 }
 
+sub webdatadir () { return some_datadir('WEBDATA'); }
+sub datadir () { return some_datadir('DATA'); }
+
 my @ocean_list;
 
 sub ocean_list () {
diff --git a/yarrg/TODO b/yarrg/TODO
index a23fc65..ed81ded 100644
--- a/yarrg/TODO
+++ b/yarrg/TODO
@@ -1,7 +1,3 @@
-routesearch:
-
-	concurrency limit option
-
 query_routesearch:
 
 	Doesn't spot routesearch dying
@@ -11,7 +7,6 @@ query_routesearch:
 
 	links to per-route pages
 
-	concurrency limit
 	nice routesearch
 
 	sort arrows on table
diff --git a/yarrg/rsmain.c b/yarrg/rsmain.c
index 437a13d..774f507 100644
--- a/yarrg/rsmain.c
+++ b/yarrg/rsmain.c
@@ -31,6 +31,8 @@ int main(int argc, const char **argv) {
   int i, ap;
   int granui;
   const char *database=0;
+  const char *concur_base=0, *concur_rhs=0;
+  int concur_lim=-1;
 
 #ifndef debug_flags
   debug_flags= ~( dbg_sql2 );
@@ -41,6 +43,10 @@ int main(int argc, const char **argv) {
     if (arg[0] != '-') break;
     if (!strcmp(arg,"-d")) {
       database= *++argv;
+    } else if (!strcmp(arg,"-C")) {
+      concur_base= *++argv;
+      concur_rhs= *++argv;
+      concur_lim= atoi(*++argv);
     } else if (!strcmp(arg,"-g")) {
       granus= atoi(*++argv);
       assert(granus>=1 && granus<=GRANUS);
@@ -92,6 +98,28 @@ int main(int argc, const char **argv) {
 
   min_trade_maxprofit= atoi(*argv++);
 
+  if (concur_base) {
+    for (i=0; i<concur_lim; i++) {
+      char *concfn= masprintf("%s%02d%s", concur_base, i, concur_rhs);
+      int concfd, r;
+      sysassert( (concfd= open(concfn, O_RDWR|O_CREAT|O_TRUNC, 0600)) >= 0);
+      struct flock fl;
+      memset(&fl,0,sizeof(fl));
+      fl.l_type= F_WRLCK;
+      r= fcntl(concfd, F_SETLK, &fl);
+      free(concfn);
+      if (!r) goto concur_ok;
+      sysassert( errno == EWOULDBLOCK );
+      close(concfd);
+    }
+    fprintf(output,"@@@ concurrency limit exceeded (%d)\n", concur_lim);
+    exit(0);
+
+  concur_ok:
+    /* deliberately leak concfd */
+    fprintf(output,"concurrency slot %d\n", i);
+  }
+
   setup_sql(database);
   setup_value();
   setup_search();
diff --git a/yarrg/web/query_routesearch b/yarrg/web/query_routesearch
index b9b7348..8937f28 100644
--- a/yarrg/web/query_routesearch
+++ b/yarrg/web/query_routesearch
@@ -54,6 +54,7 @@ my @islandids;
 
 my $maxmaxdist=35;
 my $maxcpu=90;
+my $concur_lim=5;
 
 my $qa= \%ARGS;
 my $routeparams= { EmsgRef => \$emsg, SayRequiredCapacity => 1 };
@@ -125,7 +126,8 @@ my $isleinfo = sub {
 
 #---------- compute the results ----------
 
-my @rsargs= qw(-DN);
+my @rsargs= ($concur_lim, '-DN');
+my $concur_fail;
 
 foreach my $k (qw(MaxMass MaxVolume MaxCapital)) {
 	my $v= $routeparams->{$k};
@@ -144,8 +146,13 @@ if ($qa->{'debug'}) {
 <%perl>
 }
 
-unshift @rsargs, sourcebasedir().'/yarrg/routesearch',
-	'-d', dbw_filename($qa->{'Ocean'});
+unshift @rsargs,
+	sourcebasedir().'/yarrg/routesearch',
+	'-d', dbw_filename($qa->{'Ocean'}),
+	'-C', webdatadir().'/_concur.', '.lock';
+
+# touch _concur.0{0,1,2,3,4}.lock
+# really chgrp www-data _concur.0?.lock
 
 my %results; # $results{$ap}{"5 6 9 10"} = { stuff }
 
@@ -166,7 +173,12 @@ while (<$fh>) {
 <% $_ |h %>
 <%perl>
 	}
-	next unless m/^ \@ *\d+ ([ap])\# *\d+ \|.*\| *(\d+)lg *\| *\d+ +(\d+) +(\d+) *\| ([0-9 ]+)$/;
+	next unless m/^\s*\@/;
+	if (m/^\@\@\@ concurrency limit exceeded/) {
+		$concur_fail= 1;
+		last;
+	}
+	die unless m/^ \@ *\d+ ([ap])\# *\d+ \|.*\| *(\d+)lg *\| *\d+ +(\d+) +(\d+) *\| ([0-9 ]+)$/;
 	my ($ap,$isles) = (uc $1,$5);
 	next if $results{$ap} && %{$results{$ap}} >= $maxcountea;
 	my $item= { A => $3, P => $4, Leagues => $2 };
@@ -192,12 +204,36 @@ if ($qa->{'debug'}) {
 	print "</pre>\n";
 }
 
+if ($concur_fail) {
+</%perl>
+<h2>Server too busy</h2>
+
+Sorry, but there are already <% $concur_lim |h %> route searches
+running.  We limit the number which can run at once to avoid
+overloading the server system and to make sure that the rest of the
+YARRG website still runs quickly.
+<p>
+
+If you submitted several searches and gave up on them (eg by hitting
+`back' or `stop' in your browser), be aware that that doesn't
+generally stop the search process at the server end.  So it's best to
+avoid asking for large searches that you're not sure about.
+
+<p>
+Otherwise, please try later.  Searches are limited to <% $maxcpu |h %>
+seconds of CPU time so more processing resources should be available soon.
+
+<%perl>
+	return;
+}
+
 </%perl>
 % foreach my $ap (qw(A P)) {
 <h2>ap=<% $ap %></h2>
 <table rules=groups>
 <colgroup span=2>
 <colgroup span=1>
+<colgroup span=1>
 <colgroup span=3>
 <tbody>
 <tr>