Block AI bots

This commit is contained in:
Jake Howard 2024-05-26 17:52:48 +01:00
parent 43503921db
commit 2639d6eb1c
Signed by: jake
GPG key ID: 57AFB45680EDD477
3 changed files with 16 additions and 5 deletions

View file

@ -1,8 +1,7 @@
User-agent: *
{% if SEO_INDEX %}
Allow: /
{% else %}
Disallow: /
{% endif %}
{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
# https://github.com/ai-robots-txt/ai.robots.txt
{{ ai_robots_txt }}
Sitemap: {{ sitemap }}

View file

@ -112,3 +112,13 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
return queryset.get()
except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
return None
@django_cache_decorator(time=21600)
def get_ai_robots_txt() -> str:
"""
https://github.com/ai-robots-txt/ai.robots.txt
"""
return requests_session.get(
"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
).content.decode()

View file

@ -23,6 +23,7 @@ from website.search.models import SearchPage
from .feed_generators import CustomFeed
from .models import BaseListingPage, BasePage
from .utils import get_ai_robots_txt
class Error404View(TemplateView):
@ -52,6 +53,7 @@ class RobotsView(TemplateView):
def get_context_data(self, **kwargs: dict) -> dict:
context = super().get_context_data(**kwargs)
context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
context["ai_robots_txt"] = get_ai_robots_txt()
return context