From 5933e39a714521fd5144b6fb8d033065bfbda1be Mon Sep 17 00:00:00 2001 From: Ian Jackson Date: Sun, 9 Aug 2009 17:16:09 +0100 Subject: [PATCH] Add developer documentation for upload format. --- yarrg/Commods.pm | 3 + yarrg/README.devel | 156 +++++++++++++++++++++++++++++++++++ yarrg/commod-email-processor | 19 ----- yarrg/commod-update-receiver | 14 ---- 4 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 yarrg/README.devel diff --git a/yarrg/Commods.pm b/yarrg/Commods.pm index 5003037..b90e6d7 100644 --- a/yarrg/Commods.pm +++ b/yarrg/Commods.pm @@ -220,6 +220,9 @@ sub get_our_version ($$) { chomp($version); $aref->{"${prefix}version"}= $version; return $aref; + # clientname "ypp-sc-tools" + # clientversion 2.1-g2e06a26 [from git-describe --tags HEAD] + # clientfixes "lastpage" [space separated list] } sub pipethrough_prep () { diff --git a/yarrg/README.devel b/yarrg/README.devel new file mode 100644 index 0000000..ec1a1f1 --- /dev/null +++ b/yarrg/README.devel @@ -0,0 +1,156 @@ +Format for submissions to the YARRG server +------------------------------------------ + +The yarrg server takes submissions as HTTP requests to one of: + http://upload.yarrg.chiark.net/commod-update-receiver main server + http://upload.yarrg.chiark.net/test/commod-update-receiver testing copy + +The HTTP request should be a CGI form submission as if from the +following form: +
+ + + + + + + + +The fields are as follows: + + Information about your upload client. These are checked by the + upload server to make sure the client is approved and is not out of + date. + + clientname + The name of the upload client; different clients maintained + by different people have different names. Please talk to + the yarrg server maintainer to have your client added to + the approved list. + clientversion + Version number of the specific version of the upload client. + Should be ideally generated from your version control system + if you have one. + clientfixes + Space-separated list of features your client supports and + fixes for bugs which that specific version has had applied. + Should be empty at first :-). + + Information about the upload: + + ocean + The ocean name. Must be in the correct mixture of lower + and upper case letters, eg `Midnight'. + + island + The full island name. Must be the full name in the + correct mixture of lower and upper case, eg `Alpha Island'. + + timestamp + Oldest age of any data in this upload. Your client should + collect a timestamp from the yarrg server (see below) before + starting the screenscraping, and then include the value here. + The syntax is a string of digits. Please do not use your + local computer clock to generate it. + + The upload data itself: + + data + Should be a file upload with filename `deduped.tsv.gz'. + The Content-Type should be application/octet-stream. + + The actual data should be a gzipped tab-separated-values file + with Unix line endings. Each line should be: + commodity stallname buyprice buyqty sellprice sellqty + with tabs in between each item. This is just as the data + appears in the Puzzle Pirates trade commodities interfance. + If there is no buy offer, or no sell offer, supply empty + strings. Where the YPP client says `>1000', pass exactly + that string. + + Commodity names must be the full name as displayed in the YPP + screen; they are checked against the master yarrg database. + Each commodity/stall pair should appear only once in the data. + +When you submit this successfully you should get + 400 Bad commodity update + with a text/plain document giving the error +or + 200 OK + with a text/plain document starting with `OK'. (If you get `200 + OK' but the document doesn't start `OK' something has gone wrong.) +In either case you should show the resulting text to your user. + + +In order to fetch a timestamp, submit this form: + + + + + + +The return value will be a text/plain document of the form + OK . +Strip the OK and the full stop and then use that as the value +for `timestamp'. + + +Format for emails from the yarrg exploder +----------------------------------------- + +Uploads checked and accepted by the yarrg server are distributed by +email to operator(s) of databases doing interesting commodity stuff. +Currently there's only one such database, the yarrg pirate trader +website. (Work in progress.) + +But if you want to get copies, please just email us saying what you +plan to do. + +These emails are MIME multipart/mixed documents, with two parts you +should look for: + +One important part looks like this: + + Content-Type: text/plain; charset="utf-8"; name="metadata" + Content-Disposition: inline; filename="metadata" + +This part is a text file giving metadata about the upload in a simple +textual format. Each line has a keyword at the beginning, a single +tab character, and then a value. The keywords are: + + Passed through from the client (after checking): + clientname clientversion clientfixes + ocean island timestamp + + Information about the server which processed the request: + servername serverversion serverfixes + instance + + Aggregated information about the client and server versions: + clientspec serverspec + each is the three values name, version, fixes, separated by tabs. + +`instance' specifies which installation of the upload server +generated the email. There are two official yarrg servers, with +instance values of `yarrg.chiark.net' and `yarrg.chiark.net/test'. +You must check this value. + +For convenience of your email filtering, the instance value is +reproduced in the Subject line of the whole email. + +The other important part looks like this: + + Content-Type: application/octet-stream; name="deduped.tsv.gz" + Content-Disposition: attachment; filename="deduped.tsv.gz" + +Your email processor should look at the names attached to the parts, +not at the order. Future updates may send out emails with additional +parts, or additional metadata keywords, so your software should ignore +any such additional information. + + +Contacting the yarrg server operator +------------------------------------ + +Talk to Aristarchus on Midnight, or ask any officer of the crew +Special Circumstances, or email ijackson@chiark.greenend.org.uk. diff --git a/yarrg/commod-email-processor b/yarrg/commod-email-processor index de68fba..0c16cd1 100755 --- a/yarrg/commod-email-processor +++ b/yarrg/commod-email-processor @@ -25,25 +25,6 @@ # are used without permission. This program is not endorsed or # sponsored by Three Rings. -# Emails are: -# multipart/mixed, containing -# text/plain; name="metadata"; charset="utf-8" -# Content-Disposition: inline; filename="metadata" -# ocean\t canonical mixed case -# island\t canonical mixed case -# timestamp\t time_t (non-leap secs since start of 1970 UTC) -# clientname\t may contain spaces -# clientversion\t may contain spaces -# clientfixes\t space-delimited list -# clientspec\t \t\t -# servername\t may contain spaces -# serverversion\t may contain spaces -# serverfixes\t space-delimited list -# serverspec\t \t\t -# application/octet-stream; name="deduped.tsv.gz" -# Content-Disposition: attachment; filename="deduped.tsv.gz" -# - use strict (qw(vars)); use POSIX; diff --git a/yarrg/commod-update-receiver b/yarrg/commod-update-receiver index 9c8dd30..889913d 100755 --- a/yarrg/commod-update-receiver +++ b/yarrg/commod-update-receiver @@ -25,20 +25,6 @@ # are used without permission. This program is not endorsed or # sponsored by Three Rings. -# All calls contain: -# clientname "ypp-sc-tools" -# clientversion 2.1-g2e06a26 [from git-describe --tags HEAD] -# clientfixes "lastpage" [space separated list] -# -# Timestamp requests contain: -# requesttimestamp -# -# Uploads contain: -# ocean canonical mixed case -# island canonical mixed case -# data filename=deduped.tsv.gz output of yarrg --tsv - - use strict (qw(vars)); use POSIX; use MIME::Entity; -- 2.30.2