From 54aaca308742d786794fd19aefa1034ce7fa0c9a Mon Sep 17 00:00:00 2001
From: Jake Howard <git@theorangeone.net>
Date: Sun, 26 May 2024 15:45:58 +0100
Subject: [PATCH] Remove explicit URLs from robots.txt

Rely instead on the `robots` metatag on the respective pages
---
 website/common/templates/robots.txt | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/website/common/templates/robots.txt b/website/common/templates/robots.txt
index 00a94bf..a6f1eec 100644
--- a/website/common/templates/robots.txt
+++ b/website/common/templates/robots.txt
@@ -1,11 +1,8 @@
-{% if SEO_INDEX %}
 User-agent: *
+{% if SEO_INDEX %}
 Allow: /
 {% else %}
-User-agent: *
 Disallow: /
 {% endif %}
 
-Disallow: {% url "wagtailadmin_home" %}
-Disallow: {% url "api:index" %}
 Sitemap: {{ sitemap }}