diff --git a/website/common/templates/robots.txt b/website/common/templates/robots.txt index a6f1eec..00566f3 100644 --- a/website/common/templates/robots.txt +++ b/website/common/templates/robots.txt @@ -1,8 +1,7 @@ User-agent: * -{% if SEO_INDEX %} -Allow: / -{% else %} -Disallow: / -{% endif %} +{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %} + +# https://github.com/ai-robots-txt/ai.robots.txt +{{ ai_robots_txt }} Sitemap: {{ sitemap }} diff --git a/website/common/utils.py b/website/common/utils.py index cfe4c6a..4a8db0b 100644 --- a/website/common/utils.py +++ b/website/common/utils.py @@ -112,3 +112,13 @@ def get_or_none(queryset: models.QuerySet) -> models.Model: return queryset.get() except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned): return None + + +@django_cache_decorator(time=21600) +def get_ai_robots_txt() -> str: + """ + https://github.com/ai-robots-txt/ai.robots.txt + """ + return requests_session.get( + "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt" + ).content.decode() diff --git a/website/common/views.py b/website/common/views.py index f9288c3..8a36572 100644 --- a/website/common/views.py +++ b/website/common/views.py @@ -23,6 +23,7 @@ from website.search.models import SearchPage from .feed_generators import CustomFeed from .models import BaseListingPage, BasePage +from .utils import get_ai_robots_txt class Error404View(TemplateView): @@ -52,6 +53,7 @@ class RobotsView(TemplateView): def get_context_data(self, **kwargs: dict) -> dict: context = super().get_context_data(**kwargs) context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap")) + context["ai_robots_txt"] = get_ai_robots_txt() return context