From 1abc6da19bec21222c48fb6fd57d27bfcb0b9bd4 Mon Sep 17 00:00:00 2001
From: Jake Howard <git@theorangeone.net>
Date: Fri, 19 Aug 2022 14:35:38 +0100
Subject: [PATCH] Add robots.txt

---
 website/common/templates/robots.txt |  5 +++++
 website/common/tests/test_views.py  | 10 +++++++++-
 website/common/views.py             | 11 +++++++++++
 website/urls.py                     |  5 +++--
 4 files changed, 28 insertions(+), 3 deletions(-)
 create mode 100644 website/common/templates/robots.txt

diff --git a/website/common/templates/robots.txt b/website/common/templates/robots.txt
new file mode 100644
index 0000000..0e16ab0
--- /dev/null
+++ b/website/common/templates/robots.txt
@@ -0,0 +1,5 @@
+User-agent: *
+Allow: /
+Disallow: {% url "wagtailadmin_home" %}
+
+Sitemap: {{ sitemap }}
diff --git a/website/common/tests/test_views.py b/website/common/tests/test_views.py
index ede6b6c..fb6889e 100644
--- a/website/common/tests/test_views.py
+++ b/website/common/tests/test_views.py
@@ -1,4 +1,4 @@
-from django.test import TestCase
+from django.test import SimpleTestCase, TestCase
 from django.urls import reverse
 
 
@@ -12,3 +12,11 @@ class Error404PageTestCase(TestCase):
     def test_queries(self) -> None:
         with self.assertNumQueries(10):
             self.client.get(self.url)
+
+
+class RobotsViewTestCase(SimpleTestCase):
+    url = reverse("robotstxt")
+
+    def test_accessible(self) -> None:
+        response = self.client.get(self.url)
+        self.assertEqual(response.status_code, 200)
diff --git a/website/common/views.py b/website/common/views.py
index 9541896..2a33383 100644
--- a/website/common/views.py
+++ b/website/common/views.py
@@ -1,6 +1,7 @@
 from typing import Any
 
 from django.http.response import HttpResponse
+from django.urls import reverse
 from django.views.defaults import ERROR_404_TEMPLATE_NAME
 from django.views.generic import TemplateView
 
@@ -21,3 +22,13 @@ class Error404View(TemplateView):
 
 
 page_not_found = Error404View.as_view()
+
+
+class RobotsView(TemplateView):
+    template_name = "robots.txt"
+    content_type = "text/plain"
+
+    def get_context_data(self, **kwargs: dict) -> dict:
+        context = super().get_context_data(**kwargs)
+        context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
+        return context
diff --git a/website/urls.py b/website/urls.py
index bedf18b..657ef04 100644
--- a/website/urls.py
+++ b/website/urls.py
@@ -6,7 +6,7 @@ from wagtail.contrib.sitemaps.views import sitemap
 from wagtail.documents import urls as wagtaildocs_urls
 from wagtail.images.views.serve import ServeView
 
-from website.common.views import page_not_found
+from website.common.views import RobotsView, page_not_found
 
 urlpatterns = [
     path("admin/", include(wagtailadmin_urls)),
@@ -20,7 +20,8 @@ urlpatterns = [
         ServeView.as_view(action="redirect"),
         name="wagtailimages_serve",
     ),
-    path("sitemap.xml", sitemap),
+    path("sitemap.xml", sitemap, name="sitemap"),
+    path("robots.txt", RobotsView.as_view(), name="robotstxt"),
     path("404/", page_not_found, name="404"),
 ]