add robots.txt to stop crawlers delving into lookup/ and doing searches etc.

diff --git a/yarrg/web/robots.txt b/yarrg/web/robots.txt
new file mode 100755 (executable)
index 0000000..38a5ff7
--- /dev/null
+++ b/yarrg/web/robots.txt
@@ -0,0 +1,11 @@
+<%flags>
+inherit => undef
+</%flags>
+<%perl>
+$r->content_type('text/plain');
+</%perl>
+User-Agent: *
+Disallow: /lookup
+Disallow: /test/code/lookup
+Disallow: /test/data/lookup
+Disallow: /test/both/lookup