Require approval before opening renovate PRs

Still allow them to be followed

Procfile

@@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080
 watch-js: npm run build:js -- --watch
 watch-css: npm run build:css -- --watch
 watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
-rqworker: ./manage.py rqworker --with-scheduler
+worker: ./manage.py db_worker --interval 5

docker/dev/docker-compose.yml

@@ -5,7 +5,6 @@ services:
       context: ../../
       target: dev
     environment:
-      - QUEUE_STORE_URL=redis://redis/0
       - DEBUG=true
       - SECRET_KEY=super-secret-key
       - DATABASE_URL=postgres://website:website@db/website
@@ -14,15 +13,11 @@ services:
     tmpfs:
       - /tmp
     depends_on:
-      - redis
       - db
     ports:
       - 127.0.0.1:8000:8000
       - 127.0.0.1:8080:8080
 
-  redis:
-    image: redis:6-alpine
-
   db:
     image: postgres:14-alpine
     environment:

etc/s6-rc.d/rq/run

@@ -4,4 +4,4 @@ set -e
 
 cd /app
 
-exec python manage.py rqworker --with-scheduler
+exec python manage.py db_worker -v3 --interval 10

renovate.json

@@ -1,16 +1,11 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base"
+    "config:base",
+    "replacements:all",
+    "workarounds:all"
   ],
   "prConcurrentLimit": 0,
-  "packageRules": [
-    {
-      "matchUpdateTypes": ["patch"],
-      "schedule": ["every weekend"],
-      "enabled": false
-    }
-  ],
   "regexManagers": [
     {
       "fileMatch": ["^Dockerfile$"],
@@ -18,5 +13,6 @@
       "depNameTemplate": "just-containers/s6-overlay",
       "datasourceTemplate": "github-releases"
     }
-  ]
+  ],
+  "dependencyDashboardApproval": true
 }

requirements.txt

@@ -7,7 +7,6 @@ beautifulsoup4
 lxml==5.2.1
 requests
 wagtail-generic-chooser==0.6
-django-rq==2.10.1
 django-redis==5.4.0
 gunicorn==22.0.0
 psycopg==3.1.18
@@ -29,6 +28,9 @@ django-enforce-host==1.1.0
 django-proxy==1.2.2
 wagtail-lite-youtube-embed==0.1.0
 django-minify-html==1.7.1
+metadata-parser==0.12.1
+django-tasks==0.2.0
+lightningcss==0.2.0
 
 # DRF OpenAPI dependencies
 uritemplate

static/src/scss/_commento.scss

@@ -1,35 +0,0 @@
-#commento {
-  .commento-profile-button {
-    @include dark-mode {
-      fill: $dark-mode-text;
-    }
-  }
-
-  .commento-name,
-  #commento-mod-tools-lock-button,
-  .commento-login-text,
-  .commento-anonymous-checkbox-container label {
-    @include dark-mode {
-      color: $dark-mode-text;
-    }
-  }
-
-  #commento-textarea-root,
-  #commento-guest-details-input-root,
-  .commento-textarea-container textarea {
-    @include dark-mode {
-      background-color: $black;
-      color: $dark-mode-text;
-    }
-  }
-
-  .commento-card {
-    @include dark-mode {
-      border-top: 1px solid color.adjust($black, $alpha: -0.4);
-
-      p {
-        color: $dark-mode-text;
-      }
-    }
-  }
-}

static/src/scss/_comments.scss

@@ -0,0 +1,46 @@
+#comments {
+  margin-top: 2rem;
+}
+
+.comentario-text-muted,
+.comentario-root,
+.comentario-card .comentario-name {
+  @include dark-mode {
+    color: $dark-mode-text !important;
+  }
+}
+
+.comentario-add-comment-host,
+.comentario-comment-editor textarea,
+.comentario-toolbar.comentario-disabled {
+  background-color: transparent !important;
+}
+
+.comentario-add-comment-host:not(.comentario-editor-inserted) {
+  border: 1px solid color.adjust($white, $alpha: -0.5) !important;
+}
+
+.comentario-comment-editor textarea {
+  background-color: transparent !important;
+  @include dark-mode {
+    color: $dark-mode-text !important;
+  }
+}
+
+.comentario-footer a {
+  color: $link !important;
+}
+
+.comentario-btn-link {
+  color: $link !important;
+}
+
+.comentario-card {
+  @include dark-mode {
+    border-top: 1px solid color.adjust($white, $alpha: -0.5);
+
+    p {
+      color: $dark-mode-text;
+    }
+  }
+}

static/src/scss/_listing.scss

@@ -18,6 +18,17 @@
 
   .title {
     margin-bottom: 0;
+
+    a {
+      display: flex;
+      flex-direction: row;
+      align-items: center;
+      justify-content: space-between;
+
+      i {
+        font-size: 50%;
+      }
+    }
   }
 
   .content-details {

static/src/scss/_similar_content.scss

@@ -15,8 +15,8 @@ section#similar-content {
   }
 
   .media {
-    @include desktop {
-      transform: scale(85%);
-    }
+    transform: scale(85%);
+    margin-top: 0;
+    padding-top: 0;
   }
 }

static/src/scss/base.scss

@@ -19,7 +19,7 @@
 @import "spotify";
 @import "404";
 @import "password_required";
-@import "commento";
+@import "comments";
 @import "similar_content";
 @import "support_pill";
 

website/blog/factories.py

@@ -1,3 +1,5 @@
+import factory
+
 from website.common.factories import BaseContentFactory, BaseListingFactory
 
 from . import models
@@ -11,3 +13,11 @@ class BlogPostListPageFactory(BaseListingFactory):
 class BlogPostPageFactory(BaseContentFactory):
     class Meta:
         model = models.BlogPostPage
+
+
+class ExternalBlogPostPageFactory(BaseContentFactory):
+    external_url = factory.Faker("url")
+
+    class Meta:
+        model = models.ExternalBlogPostPage
+        exclude = ["subtitle"]

website/blog/migrations/0006_externalblogpostpage.py

@@ -0,0 +1,45 @@
+# Generated by Django 5.0.4 on 2024-05-29 21:10
+
+import django.db.models.deletion
+import django.utils.timezone
+import modelcluster.fields
+import wagtailmetadata.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("blog", "0005_auto_20230602_1236"),
+        ("wagtailcore", "0089_log_entry_data_json_null_to_object"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="ExternalBlogPostPage",
+            fields=[
+                (
+                    "page_ptr",
+                    models.OneToOneField(
+                        auto_created=True,
+                        on_delete=django.db.models.deletion.CASCADE,
+                        parent_link=True,
+                        primary_key=True,
+                        serialize=False,
+                        to="wagtailcore.page",
+                    ),
+                ),
+                ("external_url", models.URLField()),
+                ("date", models.DateField(default=django.utils.timezone.now)),
+                (
+                    "tags",
+                    modelcluster.fields.ParentalManyToManyField(
+                        blank=True, to="blog.blogposttagpage"
+                    ),
+                ),
+            ],
+            options={
+                "abstract": False,
+            },
+            bases=("wagtailcore.page", wagtailmetadata.models.MetadataMixin),
+        ),
+    ]

website/blog/models.py

@@ -1,18 +1,26 @@
 from typing import Any, Optional
+from urllib.parse import urlsplit
 
 from django.contrib.postgres.search import TrigramSimilarity
 from django.db import models
-from django.db.models.functions import Cast
+from django.db.models.functions import Cast, Coalesce
+from django.http import HttpRequest, HttpResponse, HttpResponsePermanentRedirect
 from django.utils import timezone
 from django.utils.functional import cached_property
+from metadata_parser import ParsedResult
 from modelcluster.fields import ParentalManyToManyField
 from wagtail.admin.panels import FieldPanel
-from wagtail.models import PageQuerySet
+from wagtail.models import Page, PageQuerySet, Site
 from wagtail.search import index
 from wagtailautocomplete.edit_handlers import AutocompletePanel
 
-from website.common.models import BaseContentPage, BaseListingPage
-from website.common.utils import TocEntry
+from website.common.models import BaseContentPage, BaseListingPage, BasePage
+from website.common.utils import (
+    TocEntry,
+    extend_query_params,
+    get_page_metadata,
+    get_url_mime_type,
+)
 from website.contrib.singleton_page.utils import SingletonPageCache
 
 
@@ -23,6 +31,8 @@ class BlogPostListPage(BaseListingPage):
         "blog.BlogPostTagListPage",
         "blog.BlogPostCollectionListPage",
         "blog.BlogPostCollectionPage",
+        "blog.BlogPostCollectionPage",
+        "blog.ExternalBlogPostPage",
     ]
 
     @cached_property
@@ -31,9 +41,12 @@ class BlogPostListPage(BaseListingPage):
 
     def get_listing_pages(self) -> models.QuerySet:
         return (
-            BlogPostPage.objects.descendant_of(self)
-            .live()
+            Page.objects.live()
             .public()
+            .annotate(date=Coalesce("blogpostpage__date", "externalblogpostpage__date"))
+            .descendant_of(self)
+            .type(BlogPostPage, ExternalBlogPostPage)
+            .specific()
             .order_by("-date", "title")
         )
 
@@ -80,36 +93,40 @@ class BlogPostPage(BaseContentPage):
         return SingletonPageCache.get_url(BlogPostListPage)
 
     def get_similar_posts(self) -> models.QuerySet:
-        try:
-            listing_pages = BlogPostListPage.objects.get().get_listing_pages()
-        except BlogPostListPage.DoesNotExist:
-            return BlogPostPage.objects.none()
+        listing_pages = BlogPostListPage.objects.get().get_listing_pages()
 
         similar_posts = listing_pages.exclude(id=self.id).alias(
             title_similarity=TrigramSimilarity("title", self.title),
-            # If this page has no subtitle, ignore it as part of similarity
-            subtitle_similarity=TrigramSimilarity("subtitle", self.subtitle)
-            if self.subtitle
-            else models.Value(1),
         )
 
         page_tags = list(self.tags.public().live().values_list("id", flat=True))
+        # If this page has no tags, ignore it as part of similarity
+        divisor = len(page_tags) if page_tags else models.Value(1)
         similar_posts = similar_posts.alias(
-            # If this page has no tags, ignore it as part of similarity
             # NB: Cast to a float, because `COUNT` returns a `bigint`.
-            tag_similarity=Cast(
-                models.Count("tags", filter=models.Q(tags__in=page_tags)),
+            _blog_tag_similarity=Cast(
+                models.Count(
+                    "blogpostpage__tags",
+                    filter=models.Q(blogpostpage__tags__in=page_tags),
+                ),
                 output_field=models.FloatField(),
             )
-            / len(page_tags)
-            if page_tags
-            else models.Value(1)
+            / divisor,
+            _external_tag_similarity=Cast(
+                models.Count(
+                    "externalblogpostpage__tags",
+                    filter=models.Q(externalblogpostpage__tags__in=page_tags),
+                ),
+                output_field=models.FloatField(),
+            )
+            / divisor,
+            tag_similarity=models.F("_blog_tag_similarity")
+            + models.F("_external_tag_similarity"),
         )
 
         similar_posts = similar_posts.annotate(
             similarity=(models.F("tag_similarity") * 2)
             + (models.F("title_similarity") * 10)
-            + (models.F("subtitle_similarity"))
         ).order_by("-similarity")[:3]
 
         return similar_posts
@@ -137,7 +154,12 @@ class BlogPostTagPage(BaseListingPage):
 
     def get_listing_pages(self) -> models.QuerySet:
         blog_list_page = BlogPostListPage.objects.get()
-        return blog_list_page.get_listing_pages().filter(tags=self)
+        listing_pages = blog_list_page.get_listing_pages()
+
+        return listing_pages.filter(
+            models.Q(blogpostpage__tags=self)
+            | models.Q(externalblogpostpage__tags=self)
+        ).distinct()
 
 
 class BlogPostCollectionListPage(BaseListingPage):
@@ -167,3 +189,101 @@ class BlogPostCollectionPage(BaseListingPage):
             .public()
             .order_by("-date", "title")
         )
+
+
+class ExternalBlogPostPage(BaseContentPage):
+    subpage_types: list[Any] = []
+    parent_page_types = [BlogPostListPage]
+    preview_modes: list[Any] = []
+
+    is_external = True
+
+    # Some `BaseContentPage` fields aren't relevant
+    body = None
+    subtitle = None
+    hero_image = None
+    hero_unsplash_photo = None
+
+    external_url = models.URLField()
+
+    tags = ParentalManyToManyField("blog.BlogPostTagPage", blank=True)
+    date = models.DateField(default=timezone.now)
+
+    content_panels = BasePage.content_panels + [FieldPanel("external_url")]
+
+    promote_panels = BaseContentPage.promote_panels + [
+        FieldPanel("date"),
+        AutocompletePanel("tags"),
+    ]
+
+    search_fields = BaseContentPage.search_fields + [
+        index.RelatedFields("tags", [index.SearchField("title", boost=1)]),
+        index.SearchField("external_url"),
+    ]
+
+    @cached_property
+    def tag_list_page_url(self) -> Optional[str]:
+        return SingletonPageCache.get_url(BlogPostTagListPage)
+
+    @cached_property
+    def tags_list(self) -> models.QuerySet:
+        """
+        Use this to get a page's tags.
+        """
+        tags = self.tags.order_by("slug")
+
+        # In drafts, `django-modelcluster` doesn't support these filters
+        if isinstance(tags, PageQuerySet):
+            return tags.public().live()
+
+        return tags
+
+    @cached_property
+    def metadata(self) -> ParsedResult:
+        return get_page_metadata(self.external_url)
+
+    @cached_property
+    def _body_html(self) -> str:
+        try:
+            return self.metadata.get_metadatas("description")[0]
+        except (KeyError, IndexError, TypeError):
+            return ""
+
+    @cached_property
+    def plain_text(self) -> str:
+        # The metadata is already just text
+        return self._body_html
+
+    def hero_url(
+        self, image_size: str, wagtail_image_spec_extra: Optional[str] = None
+    ) -> Optional[str]:
+        try:
+            return self.metadata.get_metadatas("image")[0]
+        except (KeyError, IndexError, TypeError):
+            return None
+
+    @cached_property
+    def hero_image_url(self) -> str:
+        return ""
+
+    @cached_property
+    def hero_image_alt(self) -> str:
+        return ""
+
+    def get_meta_image_mime(self) -> Optional[str]:
+        return get_url_mime_type(self.hero_url(""))
+
+    def get_url(
+        self, request: HttpRequest | None = None, current_site: Site | None = None
+    ) -> str:
+        return self.get_full_url(request)
+
+    def get_full_url(self, request: HttpRequest | None = None) -> str:
+        full_url = urlsplit(super().get_full_url(request))
+        return extend_query_params(self.external_url, {"utm_source": full_url.netloc})
+
+    def serve(self, request: HttpRequest, *args: tuple, **kwargs: dict) -> HttpResponse:
+        """
+        Send the user directly to the external page
+        """
+        return HttpResponsePermanentRedirect(self.get_full_url(request))

website/blog/templates/blog/blog_post_page.html

@@ -2,6 +2,16 @@
 
 {% load wagtail_cache navbar_tags %}
 
+{% block post_toc %}
+  <hr class="dropdown-divider" />
+  <li>
+    <a href="#similar-content">Similar content</a>
+  </li>
+  <li>
+    <a href="#comments">Comments</a>
+  </li>
+{% endblock %}
+
 {% block post_content %}
   {% if not request.is_preview %}
     {% include "common/shareon.html" %}

website/blog/templates/blog/external_blog_post_page.html

@@ -0,0 +1,16 @@
+{% comment %}
+This template is never used, but exists just in case.
+{% endcomment %}
+
+<!DOCTYPE html>
+<html lang="en-GB">
+  <head>
+    <title>Redirecting...</title>
+    <link rel="canonical" href="{{ page.external_url }}" />
+    <meta charset="utf-8" />
+    <meta http-equiv="refresh" content="0; url={{ page.external_url }}" />
+  </head>
+  <body>
+    <p>Redirecting...</p>
+  </body>
+</html>

website/blog/tests.py

@@ -1,9 +1,15 @@
+import pickle
+
 from django.test import TestCase
 from django.urls import reverse
 
 from website.home.models import HomePage
 
-from .factories import BlogPostListPageFactory, BlogPostPageFactory
+from .factories import (
+    BlogPostListPageFactory,
+    BlogPostPageFactory,
+    ExternalBlogPostPageFactory,
+)
 
 
 class BlogPostPageTestCase(TestCase):
@@ -69,14 +75,15 @@ class BlogPostListPageTestCase(TestCase):
 
         BlogPostPageFactory(parent=cls.page)
         BlogPostPageFactory(parent=cls.page)
+        ExternalBlogPostPageFactory(parent=cls.page, external_url="https://example.com")
 
     def test_accessible(self) -> None:
         response = self.client.get(self.page.url)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.context["listing_pages"]), 2)
+        self.assertEqual(len(response.context["listing_pages"]), 3)
 
     def test_queries(self) -> None:
-        with self.assertNumQueries(39):
+        with self.assertNumQueries(43):
             self.client.get(self.page.url)
 
     def test_feed_accessible(self) -> None:
@@ -84,3 +91,31 @@ class BlogPostListPageTestCase(TestCase):
         self.assertRedirects(
             response, reverse("feed"), status_code=301, fetch_redirect_response=True
         )
+
+
+class ExternalBlogPostPageTestCase(TestCase):
+    @classmethod
+    def setUpTestData(cls) -> None:
+        cls.home_page = HomePage.objects.get()
+        cls.blog_post_list_page = BlogPostListPageFactory(parent=cls.home_page)
+        cls.page = ExternalBlogPostPageFactory(
+            parent=cls.blog_post_list_page, external_url="https://example.com"
+        )
+
+    def test_redirects(self) -> None:
+        with self.assertNumQueries(10):
+            response = self.client.get(self.page.url)
+        self.assertRedirects(
+            response,
+            self.page.external_url + "?utm_source=localhost",
+            status_code=301,
+            fetch_redirect_response=False,
+        )
+
+    def test_metadata(self) -> None:
+        metadata = self.page.metadata
+
+        self.assertIsNone(metadata.soup)
+
+        # Confirm it can pickle
+        pickle.dumps(metadata)

website/common/models.py

@@ -1,7 +1,6 @@
 from datetime import timedelta
 from math import ceil
 from typing import Any, Optional
-from urllib.parse import urlencode
 
 from django.core.paginator import EmptyPage, Paginator
 from django.core.paginator import Page as PaginatorPage
@@ -31,6 +30,7 @@ from .serializers import PaginationSerializer
 from .streamfield import add_heading_anchors, get_blocks, get_content_html
 from .utils import (
     TocEntry,
+    extend_query_params,
     extract_text,
     get_site_title,
     get_table_of_contents,
@@ -181,6 +181,7 @@ class BaseContentPage(BasePage, MetadataMixin):
             for size, width in UNSPLASH_SIZES.items()
         }
 
+    @cached_property
     def hero_image_url(self) -> Optional[str]:
         return self.hero_url("regular")
 
@@ -286,10 +287,7 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
 
         url = super().get_meta_url()
 
-        if not query_data:
-            return url
-
-        return url + "?" + urlencode(query_data)
+        return extend_query_params(url, query_data)
 
     @route(r"^feed/$")
     def feed(self, request: HttpRequest) -> HttpResponse:
@@ -299,8 +297,11 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
     def random(self, request: HttpRequest) -> HttpResponse:
         page = self.get_listing_pages().order_by("?").first()
         if page is None:
-            return redirect(self.get_url(request=request), permanent=False)
-        return redirect(page.get_url(request=request), permanent=False)
+            response = redirect(self.get_url(request=request), permanent=False)
+        else:
+            response = redirect(page.get_url(request=request), permanent=False)
+        response.headers["X-Robots-Tag"] = "noindex"
+        return response
 
 
 class ListingPage(BaseListingPage):

website/common/templates/common/comments.html

@@ -1,5 +1,5 @@
 <section class="container" id="comments">
-  <div id="commento"></div>
+  <comentario-comments></comentario-comments>
 </section>
 
-<script async defer src="https://commento.theorangeone.net/js/commento.js"></script>
+<script async defer src="https://comentario.theorangeone.net/comentario.js"></script>

website/common/templates/common/content_page.html

@@ -10,6 +10,13 @@
   {% endif %}
 {% endblock %}
 
+{% block post_toc %}
+  <hr class="dropdown-divider" />
+  <li>
+    <a href="#comments">Comments</a>
+  </li>
+{% endblock %}
+
 {% block post_content %}
   {% if not request.is_preview %}
     {% include "common/shareon.html" %}

website/common/templates/common/listing-item.html

@@ -16,7 +16,10 @@
           {% include "common/breadcrumbs.html" with parents=page.get_parent_pages %}
         {% endif %}
         <h2 class="title is-3">
-          <a href="{% pageurl page %}">{{ page.title }}</a>
+          <a href="{% pageurl page %}">
+            {{ page.title }}
+            {% if page.is_external %}<i class="fa-solid fa-arrow-up-right-from-square" title="This page is from a external source"></i>{% endif %}
+          </a>
         </h2>
         {% include "common/content-details.html" %}
         <p>{{ page.summary }}</p>

website/common/templates/robots.txt

@@ -1,11 +1,7 @@
-{% if SEO_INDEX %}
 User-agent: *
-Allow: /
-{% else %}
-User-agent: *
-Disallow: /
-{% endif %}
+{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
+
+# https://github.com/ai-robots-txt/ai.robots.txt
+{{ ai_robots_txt }}
 
-Disallow: {% url "wagtailadmin_home" %}
-Disallow: {% url "api:index" %}
 Sitemap: {{ sitemap }}

website/common/templates/wagtail_base.html

@@ -56,6 +56,7 @@
                         {% for toc_item in page.table_of_contents %}
                           {% include "common/toc-item.html" %}
                         {% endfor %}
+                        {% block post_toc %}{% endblock %}
                       </ul>
                     </div>
                   </div>

website/common/tests/test_utils.py

@@ -3,6 +3,7 @@ from django.test import SimpleTestCase
 from wagtail.rich_text import features as richtext_feature_registry
 
 from website.common.utils import (
+    extend_query_params,
     extract_text,
     get_table_of_contents,
     heading_id,
@@ -111,3 +112,25 @@ class HeadingIDTestCase(SimpleTestCase):
         self.assertEqual(heading_id("123"), "ref-123")
         self.assertEqual(heading_id("test"), "test")
         self.assertEqual(heading_id("Look, a title!"), "look-a-title")
+
+
+class ExtendQueryParamsTestCase(SimpleTestCase):
+    def test_params(self) -> None:
+        self.assertEqual(
+            extend_query_params("https://example.com", {"foo": "bar"}),
+            "https://example.com?foo=bar",
+        )
+        self.assertEqual(
+            extend_query_params("https://example.com?foo=bar", {"bar": "foo"}),
+            "https://example.com?foo=bar&bar=foo",
+        )
+        self.assertEqual(
+            extend_query_params("https://example.com?foo=baz", {"foo": "baz"}),
+            "https://example.com?foo=baz",
+        )
+
+    def test_removes_param(self) -> None:
+        self.assertEqual(
+            extend_query_params("https://example.com?foo=bar", {"foo": None}),
+            "https://example.com",
+        )

website/common/utils.py

@@ -1,14 +1,17 @@
 from dataclasses import dataclass
 from itertools import pairwise
-from typing import Optional, Type
+from typing import Any, Optional, Type
+from urllib.parse import urlsplit, urlunsplit
 
 import requests
 from bs4 import BeautifulSoup, SoupStrainer
 from django.conf import settings
 from django.db import models
+from django.http import QueryDict
 from django.http.request import HttpRequest
 from django.utils.text import slugify
 from django_cache_decorator import django_cache_decorator
+from metadata_parser import MetadataParser, ParsedResult
 from wagtail.models import Page, Site
 from wagtail.models import get_page_models as get_wagtail_page_models
 
@@ -112,3 +115,36 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
         return queryset.get()
     except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
         return None
+
+
+@django_cache_decorator(time=21600)
+def get_ai_robots_txt() -> str:
+    """
+    https://github.com/ai-robots-txt/ai.robots.txt
+    """
+    return requests_session.get(
+        "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
+    ).content.decode()
+
+
+@django_cache_decorator(time=21600)
+def get_page_metadata(url: str) -> ParsedResult:
+    metadata = MetadataParser(url=url, search_head_only=True).parsed_result
+
+    # HACK: BeautifulSoup doesn't pickle nicely, and so can't be cached
+    metadata.soup = None
+
+    return metadata
+
+
+def extend_query_params(url: str, params: dict[str, Any]) -> str:
+    scheme, netloc, path, query, fragment = urlsplit(url)
+    query_dict = QueryDict(query, mutable=True)
+
+    for k, v in params.items():
+        if v is None:
+            del query_dict[k]
+        else:
+            query_dict[k] = v
+
+    return urlunsplit((scheme, netloc, path, query_dict.urlencode(), fragment))

website/common/views.py

@@ -23,6 +23,7 @@ from website.search.models import SearchPage
 
 from .feed_generators import CustomFeed
 from .models import BaseListingPage, BasePage
+from .utils import extend_query_params, get_ai_robots_txt
 
 
 class Error404View(TemplateView):
@@ -52,6 +53,7 @@ class RobotsView(TemplateView):
     def get_context_data(self, **kwargs: dict) -> dict:
         context = super().get_context_data(**kwargs)
         context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
+        context["ai_robots_txt"] = get_ai_robots_txt()
         return context
 
 
@@ -114,7 +116,9 @@ class AllPagesFeed(Feed):
         return item.title
 
     def item_link(self, item: BasePage) -> str:
-        return item.get_full_url(request=self.request) + "?utm_medium=rss"
+        return extend_query_params(
+            item.get_full_url(request=self.request), {"utm_medium": "rss"}
+        )
 
     def item_pubdate(self, item: BasePage) -> datetime:
         if item_date := getattr(item, "date", None):

website/contrib/code_block/views.py

@@ -1,21 +1,16 @@
+import lightningcss
 from django.http import HttpRequest, HttpResponse
-from django.utils.datastructures import OrderedSet
 from django.views.decorators.cache import cache_control
 from pygments.formatters.html import HtmlFormatter
 
 
 @cache_control(max_age=3600)
 def pygments_styles(request: HttpRequest) -> HttpResponse:
-    default_styles = (
-        HtmlFormatter(style="default")
-        .get_style_defs("html:not(.dark-mode) .highlight")
-        .split("\n")
+    default_styles = HtmlFormatter(style="default").get_style_defs(
+        "html:not(.dark-mode) .highlight"
     )
-    dark_styles = (
-        HtmlFormatter(style="monokai")
-        .get_style_defs("html.dark-mode .highlight")
-        .split("\n")
-    )
-    return HttpResponse(
-        "".join(OrderedSet(default_styles + dark_styles)), content_type="text/css"
+    dark_styles = HtmlFormatter(style="monokai").get_style_defs(
+        "html.dark-mode .highlight"
     )
+    compressed = lightningcss.process_stylesheet(default_styles + dark_styles)
+    return HttpResponse(compressed, content_type="text/css")

website/contrib/unsplash/management/commands/update_unsplash_photos.py

@@ -3,13 +3,15 @@ from datetime import timedelta
 
 from django.core.management.base import BaseCommand
 from django.utils import timezone
+from django_tasks import task
 
 from website.contrib.unsplash.models import UnsplashPhoto
 from website.contrib.unsplash.utils import get_unsplash_photo
-from website.utils.queue import enqueue_or_sync
 
 
-def update_photo(photo: UnsplashPhoto) -> None:
+@task()
+def update_photo(photo_id: int) -> None:
+    photo = UnsplashPhoto.objects.get(id=photo_id)
     photo.data = get_unsplash_photo(photo.unsplash_id)
     photo.data_last_updated = timezone.now()
     photo.save()
@@ -25,6 +27,6 @@ class Command(BaseCommand):
         photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
         self.stdout.write(f"Found {photos.count()} photos to update.")
 
-        for photo in photos:
-            self.stdout.write(f"Updating {photo.unsplash_id}")
-            enqueue_or_sync(update_photo, args=[photo])
+        for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"):
+            self.stdout.write(f"Updating {unsplash_id}")
+            update_photo.enqueue(photo_id)

website/search/views.py

@@ -7,7 +7,7 @@ from wagtail.search.utils import parse_query_string
 from wagtail_favicon.models import FaviconSettings
 from wagtail_favicon.utils import get_rendition_url
 
-from website.common.utils import get_or_none, get_site_title
+from website.common.utils import extend_query_params, get_or_none, get_site_title
 from website.contrib.singleton_page.utils import SingletonPageCache
 
 from .models import SearchPage
@@ -87,4 +87,4 @@ class GoView(RedirectView):
         if slug_match := get_or_none(pages.filter(slug__iexact=query)):
             return slug_match.get_url(request=self.request)
 
-        return f"{search_page_url}?{self.request.GET.urlencode()}"
+        return extend_query_params(search_page_url, self.request.GET)

website/settings.py

@@ -69,7 +69,6 @@ INSTALLED_APPS = [
     "generic_chooser",
     "wagtail_draftail_snippet",
     "wagtailautocomplete",
-    "django_rq",
     "rest_framework",
     "corsheaders",
     "wagtail_favicon",
@@ -80,6 +79,7 @@ INSTALLED_APPS = [
     "django_otp",
     "django_otp.plugins.otp_totp",
     "django_minify_html",
+    "django_tasks.backends.database",
     "health_check",
     "health_check.db",
     "health_check.cache",
@@ -146,16 +146,10 @@ CACHES = {
 # https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
 WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
 
-RQ_QUEUES = {}
-
-USE_REDIS_QUEUE = False
-if queue_store := env.cache(
-    "QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache"
-):
-    CACHES["rq"] = queue_store
-    USE_REDIS_QUEUE = True
-    RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"}
-
+if TEST:
+    TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}}
+else:
+    TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}}
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.0/topics/i18n/
@@ -320,14 +314,15 @@ if DEBUG:
     INSTALLED_APPS.append("django_browser_reload")
     MIDDLEWARE.append("django_browser_reload.middleware.BrowserReloadMiddleware")
 
-    # Add django-debug-toolbar
-    INSTALLED_APPS.append("debug_toolbar")
-    MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
-    DEBUG_TOOLBAR_CONFIG = {
-        "SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
-        "RESULTS_CACHE_SIZE": 5,
-        "SHOW_COLLAPSED": True,
-    }
+    if not TEST:
+        # Add django-debug-toolbar
+        INSTALLED_APPS.append("debug_toolbar")
+        MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
+        DEBUG_TOOLBAR_CONFIG = {
+            "SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
+            "RESULTS_CACHE_SIZE": 5,
+            "SHOW_COLLAPSED": True,
+        }
 
     # Add Wagtail styleguide
     INSTALLED_APPS.append("wagtail.contrib.styleguide")
@@ -391,6 +386,11 @@ LOGGING = {
             "level": "WARNING",
             "propagate": False,
         },
+        "metadata_parser": {
+            "handlers": ["console"],
+            "level": "CRITICAL",
+            "propagate": False,
+        },
     },
 }
 
@@ -442,12 +442,11 @@ if sentry_dsn := env("SENTRY_DSN"):
     import sentry_sdk
     from sentry_sdk.integrations.django import DjangoIntegration
     from sentry_sdk.integrations.redis import RedisIntegration
-    from sentry_sdk.integrations.rq import RqIntegration
     from sentry_sdk.utils import get_default_release
 
     sentry_kwargs = {
         "dsn": sentry_dsn,
-        "integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()],
+        "integrations": [DjangoIntegration(), RedisIntegration()],
         "release": get_default_release(),
     }
 

website/spotify/management/commands/refresh_spotify_playlists.py

@@ -1,10 +1,11 @@
 from django.core.cache import cache
 from django.core.management.base import BaseCommand
+from django_tasks import task
 
 from website.spotify.models import SpotifyPlaylistPage
-from website.utils.queue import enqueue_or_sync
 
 
+@task()
 def refresh_cache(page_id: int) -> None:
     page = SpotifyPlaylistPage.objects.get(id=page_id)
     cache.delete(page.playlist_cache_key)
@@ -15,5 +16,7 @@ def refresh_cache(page_id: int) -> None:
 
 class Command(BaseCommand):
     def handle(self, *args: list, **options: dict) -> None:
-        for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator():
-            enqueue_or_sync(refresh_cache, args=[page.id])
+        for page_id in (
+            SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator()
+        ):
+            refresh_cache.enqueue(page_id)

website/utils/queue.py

@@ -1,21 +0,0 @@
-from typing import Callable
-
-from django.conf import settings
-from django_rq import get_queue
-
-
-def enqueue_or_sync(
-    job_func: Callable, args: list | None = None, kwargs: dict | None = None
-) -> None:
-    """
-    Run a task now, or put in RQ
-    """
-    if args is None:
-        args = []
-    if kwargs is None:
-        kwargs = {}
-
-    if settings.USE_REDIS_QUEUE:
-        get_queue().enqueue(job_func, args=args, kwargs=kwargs)
-    else:
-        job_func(*args, **kwargs)