Block AI bots

website/common/templates/robots.txt

@@ -1,8 +1,7 @@
 User-agent: *
-{% if SEO_INDEX %}
+{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
-Allow: /
+
-{% else %}
+# https://github.com/ai-robots-txt/ai.robots.txt
-Disallow: /
+{{ ai_robots_txt }}
-{% endif %}
 
 Sitemap: {{ sitemap }}

website/common/utils.py

@@ -112,3 +112,13 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
         return queryset.get()
     except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
         return None
+
+
+@django_cache_decorator(time=21600)
+def get_ai_robots_txt() -> str:
+    """
+    https://github.com/ai-robots-txt/ai.robots.txt
+    """
+    return requests_session.get(
+        "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
+    ).content.decode()

website/common/views.py

@@ -23,6 +23,7 @@ from website.search.models import SearchPage
 
 from .feed_generators import CustomFeed
 from .models import BaseListingPage, BasePage
+from .utils import get_ai_robots_txt
 
 
 class Error404View(TemplateView):
@@ -52,6 +53,7 @@ class RobotsView(TemplateView):
     def get_context_data(self, **kwargs: dict) -> dict:
         context = super().get_context_data(**kwargs)
         context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
+        context["ai_robots_txt"] = get_ai_robots_txt()
         return context