From 2639d6eb1caf07fec2b9097036ce0cc6b3357054 Mon Sep 17 00:00:00 2001
From: Jake Howard <git@theorangeone.net>
Date: Sun, 26 May 2024 17:52:48 +0100
Subject: [PATCH] Block AI bots

---
 website/common/templates/robots.txt |  9 ++++-----
 website/common/utils.py             | 10 ++++++++++
 website/common/views.py             |  2 ++
 3 files changed, 16 insertions(+), 5 deletions(-)

diff --git a/website/common/templates/robots.txt b/website/common/templates/robots.txt
index a6f1eec..00566f3 100644
--- a/website/common/templates/robots.txt
+++ b/website/common/templates/robots.txt
@@ -1,8 +1,7 @@
 User-agent: *
-{% if SEO_INDEX %}
-Allow: /
-{% else %}
-Disallow: /
-{% endif %}
+{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
+
+# https://github.com/ai-robots-txt/ai.robots.txt
+{{ ai_robots_txt }}
 
 Sitemap: {{ sitemap }}
diff --git a/website/common/utils.py b/website/common/utils.py
index cfe4c6a..4a8db0b 100644
--- a/website/common/utils.py
+++ b/website/common/utils.py
@@ -112,3 +112,13 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
         return queryset.get()
     except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
         return None
+
+
+@django_cache_decorator(time=21600)
+def get_ai_robots_txt() -> str:
+    """
+    https://github.com/ai-robots-txt/ai.robots.txt
+    """
+    return requests_session.get(
+        "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
+    ).content.decode()
diff --git a/website/common/views.py b/website/common/views.py
index f9288c3..8a36572 100644
--- a/website/common/views.py
+++ b/website/common/views.py
@@ -23,6 +23,7 @@ from website.search.models import SearchPage
 
 from .feed_generators import CustomFeed
 from .models import BaseListingPage, BasePage
+from .utils import get_ai_robots_txt
 
 
 class Error404View(TemplateView):
@@ -52,6 +53,7 @@ class RobotsView(TemplateView):
     def get_context_data(self, **kwargs: dict) -> dict:
         context = super().get_context_data(**kwargs)
         context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
+        context["ai_robots_txt"] = get_ai_robots_txt()
         return context