From 024f4a767b43a66e6ff004532f039b874e401b80 Mon Sep 17 00:00:00 2001
From: Ian Jackson <ijackson@chiark.greenend.org.uk>
Date: Tue, 7 Sep 2010 20:03:09 +0100
Subject: [PATCH] add robots.txt to stop crawlers delving into lookup/ and
 doing searches etc.

---
 yarrg/web/robots.txt | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100755 yarrg/web/robots.txt

diff --git a/yarrg/web/robots.txt b/yarrg/web/robots.txt
new file mode 100755
index 0000000..38a5ff7
--- /dev/null
+++ b/yarrg/web/robots.txt
@@ -0,0 +1,11 @@
+<%flags>
+inherit => undef
+</%flags>
+<%perl>
+$r->content_type('text/plain');
+</%perl>
+User-Agent: *
+Disallow: /lookup
+Disallow: /test/code/lookup
+Disallow: /test/data/lookup
+Disallow: /test/both/lookup
-- 
2.30.2