Update dependency eslint to v8.57.0

Procfile

@@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080
 watch-js: npm run build:js -- --watch
 watch-css: npm run build:css -- --watch
 watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
-worker: ./manage.py db_worker --interval 5
+rqworker: ./manage.py rqworker --with-scheduler

docker/dev/docker-compose.yml

@@ -5,6 +5,7 @@ services:
       context: ../../
       target: dev
     environment:
+      - QUEUE_STORE_URL=redis://redis/0
       - DEBUG=true
       - SECRET_KEY=super-secret-key
       - DATABASE_URL=postgres://website:website@db/website
@@ -13,11 +14,15 @@ services:
     tmpfs:
       - /tmp
     depends_on:
+      - redis
       - db
     ports:
       - 127.0.0.1:8000:8000
       - 127.0.0.1:8080:8080
 
+  redis:
+    image: redis:6-alpine
+
   db:
     image: postgres:14-alpine
     environment:

etc/s6-rc.d/rq/run

@@ -4,4 +4,4 @@ set -e
 
 cd /app
 
-exec python manage.py db_worker -v3 --interval 10
+exec python manage.py rqworker --with-scheduler

package.json

@@ -20,7 +20,7 @@
   },
   "author": "Jake Howard",
   "devDependencies": {
-    "eslint": "8.55.0",
+    "eslint": "8.57.0",
     "eslint-plugin-unicorn": "49.0.0",
     "prettier": "2.7.1",
     "stylelint": "14.16.1",

renovate.json

@@ -1,11 +1,16 @@
 {
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "extends": [
-    "config:base",
-    "replacements:all",
-    "workarounds:all"
+    "config:base"
   ],
   "prConcurrentLimit": 0,
+  "packageRules": [
+    {
+      "matchUpdateTypes": ["patch"],
+      "schedule": ["every weekend"],
+      "enabled": false
+    }
+  ],
   "regexManagers": [
     {
       "fileMatch": ["^Dockerfile$"],
@@ -13,6 +18,5 @@
       "depNameTemplate": "just-containers/s6-overlay",
       "datasourceTemplate": "github-releases"
     }
-  ],
-  "dependencyDashboardApproval": true
+  ]
 }

requirements.txt

@@ -7,6 +7,7 @@ beautifulsoup4
 lxml==5.2.1
 requests
 wagtail-generic-chooser==0.6
+django-rq==2.10.1
 django-redis==5.4.0
 gunicorn==22.0.0
 psycopg==3.1.18
@@ -28,9 +29,6 @@ django-enforce-host==1.1.0
 django-proxy==1.2.2
 wagtail-lite-youtube-embed==0.1.0
 django-minify-html==1.7.1
-metadata-parser==0.12.1
-django-tasks==0.2.0
-lightningcss==0.2.0
 
 # DRF OpenAPI dependencies
 uritemplate

static/src/scss/_commento.scss

@@ -0,0 +1,35 @@
+#commento {
+  .commento-profile-button {
+    @include dark-mode {
+      fill: $dark-mode-text;
+    }
+  }
+
+  .commento-name,
+  #commento-mod-tools-lock-button,
+  .commento-login-text,
+  .commento-anonymous-checkbox-container label {
+    @include dark-mode {
+      color: $dark-mode-text;
+    }
+  }
+
+  #commento-textarea-root,
+  #commento-guest-details-input-root,
+  .commento-textarea-container textarea {
+    @include dark-mode {
+      background-color: $black;
+      color: $dark-mode-text;
+    }
+  }
+
+  .commento-card {
+    @include dark-mode {
+      border-top: 1px solid color.adjust($black, $alpha: -0.4);
+
+      p {
+        color: $dark-mode-text;
+      }
+    }
+  }
+}

static/src/scss/_comments.scss

@@ -1,46 +0,0 @@
-#comments {
-  margin-top: 2rem;
-}
-
-.comentario-text-muted,
-.comentario-root,
-.comentario-card .comentario-name {
-  @include dark-mode {
-    color: $dark-mode-text !important;
-  }
-}
-
-.comentario-add-comment-host,
-.comentario-comment-editor textarea,
-.comentario-toolbar.comentario-disabled {
-  background-color: transparent !important;
-}
-
-.comentario-add-comment-host:not(.comentario-editor-inserted) {
-  border: 1px solid color.adjust($white, $alpha: -0.5) !important;
-}
-
-.comentario-comment-editor textarea {
-  background-color: transparent !important;
-  @include dark-mode {
-    color: $dark-mode-text !important;
-  }
-}
-
-.comentario-footer a {
-  color: $link !important;
-}
-
-.comentario-btn-link {
-  color: $link !important;
-}
-
-.comentario-card {
-  @include dark-mode {
-    border-top: 1px solid color.adjust($white, $alpha: -0.5);
-
-    p {
-      color: $dark-mode-text;
-    }
-  }
-}

static/src/scss/_listing.scss

@@ -18,17 +18,6 @@
 
   .title {
     margin-bottom: 0;
-
-    a {
-      display: flex;
-      flex-direction: row;
-      align-items: center;
-      justify-content: space-between;
-
-      i {
-        font-size: 50%;
-      }
-    }
   }
 
   .content-details {

static/src/scss/_similar_content.scss

@@ -15,8 +15,8 @@ section#similar-content {
   }
 
   .media {
-    transform: scale(85%);
-    margin-top: 0;
-    padding-top: 0;
+    @include desktop {
+      transform: scale(85%);
+    }
   }
 }

static/src/scss/base.scss

@@ -19,7 +19,7 @@
 @import "spotify";
 @import "404";
 @import "password_required";
-@import "comments";
+@import "commento";
 @import "similar_content";
 @import "support_pill";
 

website/blog/factories.py

@@ -1,5 +1,3 @@
-import factory
-
 from website.common.factories import BaseContentFactory, BaseListingFactory
 
 from . import models
@@ -13,11 +11,3 @@ class BlogPostListPageFactory(BaseListingFactory):
 class BlogPostPageFactory(BaseContentFactory):
     class Meta:
         model = models.BlogPostPage
-
-
-class ExternalBlogPostPageFactory(BaseContentFactory):
-    external_url = factory.Faker("url")
-
-    class Meta:
-        model = models.ExternalBlogPostPage
-        exclude = ["subtitle"]

website/blog/migrations/0006_externalblogpostpage.py

@@ -1,45 +0,0 @@
-# Generated by Django 5.0.4 on 2024-05-29 21:10
-
-import django.db.models.deletion
-import django.utils.timezone
-import modelcluster.fields
-import wagtailmetadata.models
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-    dependencies = [
-        ("blog", "0005_auto_20230602_1236"),
-        ("wagtailcore", "0089_log_entry_data_json_null_to_object"),
-    ]
-
-    operations = [
-        migrations.CreateModel(
-            name="ExternalBlogPostPage",
-            fields=[
-                (
-                    "page_ptr",
-                    models.OneToOneField(
-                        auto_created=True,
-                        on_delete=django.db.models.deletion.CASCADE,
-                        parent_link=True,
-                        primary_key=True,
-                        serialize=False,
-                        to="wagtailcore.page",
-                    ),
-                ),
-                ("external_url", models.URLField()),
-                ("date", models.DateField(default=django.utils.timezone.now)),
-                (
-                    "tags",
-                    modelcluster.fields.ParentalManyToManyField(
-                        blank=True, to="blog.blogposttagpage"
-                    ),
-                ),
-            ],
-            options={
-                "abstract": False,
-            },
-            bases=("wagtailcore.page", wagtailmetadata.models.MetadataMixin),
-        ),
-    ]

website/blog/models.py

@@ -1,26 +1,18 @@
 from typing import Any, Optional
-from urllib.parse import urlsplit
 
 from django.contrib.postgres.search import TrigramSimilarity
 from django.db import models
-from django.db.models.functions import Cast, Coalesce
-from django.http import HttpRequest, HttpResponse, HttpResponsePermanentRedirect
+from django.db.models.functions import Cast
 from django.utils import timezone
 from django.utils.functional import cached_property
-from metadata_parser import ParsedResult
 from modelcluster.fields import ParentalManyToManyField
 from wagtail.admin.panels import FieldPanel
-from wagtail.models import Page, PageQuerySet, Site
+from wagtail.models import PageQuerySet
 from wagtail.search import index
 from wagtailautocomplete.edit_handlers import AutocompletePanel
 
-from website.common.models import BaseContentPage, BaseListingPage, BasePage
-from website.common.utils import (
-    TocEntry,
-    extend_query_params,
-    get_page_metadata,
-    get_url_mime_type,
-)
+from website.common.models import BaseContentPage, BaseListingPage
+from website.common.utils import TocEntry
 from website.contrib.singleton_page.utils import SingletonPageCache
 
 
@@ -31,8 +23,6 @@ class BlogPostListPage(BaseListingPage):
         "blog.BlogPostTagListPage",
         "blog.BlogPostCollectionListPage",
         "blog.BlogPostCollectionPage",
-        "blog.BlogPostCollectionPage",
-        "blog.ExternalBlogPostPage",
     ]
 
     @cached_property
@@ -41,12 +31,9 @@ class BlogPostListPage(BaseListingPage):
 
     def get_listing_pages(self) -> models.QuerySet:
         return (
-            Page.objects.live()
+            BlogPostPage.objects.descendant_of(self)
+            .live()
             .public()
-            .annotate(date=Coalesce("blogpostpage__date", "externalblogpostpage__date"))
-            .descendant_of(self)
-            .type(BlogPostPage, ExternalBlogPostPage)
-            .specific()
             .order_by("-date", "title")
         )
 
@@ -93,40 +80,36 @@ class BlogPostPage(BaseContentPage):
         return SingletonPageCache.get_url(BlogPostListPage)
 
     def get_similar_posts(self) -> models.QuerySet:
-        listing_pages = BlogPostListPage.objects.get().get_listing_pages()
+        try:
+            listing_pages = BlogPostListPage.objects.get().get_listing_pages()
+        except BlogPostListPage.DoesNotExist:
+            return BlogPostPage.objects.none()
 
         similar_posts = listing_pages.exclude(id=self.id).alias(
             title_similarity=TrigramSimilarity("title", self.title),
+            # If this page has no subtitle, ignore it as part of similarity
+            subtitle_similarity=TrigramSimilarity("subtitle", self.subtitle)
+            if self.subtitle
+            else models.Value(1),
         )
 
         page_tags = list(self.tags.public().live().values_list("id", flat=True))
-        # If this page has no tags, ignore it as part of similarity
-        divisor = len(page_tags) if page_tags else models.Value(1)
         similar_posts = similar_posts.alias(
+            # If this page has no tags, ignore it as part of similarity
             # NB: Cast to a float, because `COUNT` returns a `bigint`.
-            _blog_tag_similarity=Cast(
-                models.Count(
-                    "blogpostpage__tags",
-                    filter=models.Q(blogpostpage__tags__in=page_tags),
-                ),
+            tag_similarity=Cast(
+                models.Count("tags", filter=models.Q(tags__in=page_tags)),
                 output_field=models.FloatField(),
             )
-            / divisor,
-            _external_tag_similarity=Cast(
-                models.Count(
-                    "externalblogpostpage__tags",
-                    filter=models.Q(externalblogpostpage__tags__in=page_tags),
-                ),
-                output_field=models.FloatField(),
-            )
-            / divisor,
-            tag_similarity=models.F("_blog_tag_similarity")
-            + models.F("_external_tag_similarity"),
+            / len(page_tags)
+            if page_tags
+            else models.Value(1)
         )
 
         similar_posts = similar_posts.annotate(
             similarity=(models.F("tag_similarity") * 2)
             + (models.F("title_similarity") * 10)
+            + (models.F("subtitle_similarity"))
         ).order_by("-similarity")[:3]
 
         return similar_posts
@@ -154,12 +137,7 @@ class BlogPostTagPage(BaseListingPage):
 
     def get_listing_pages(self) -> models.QuerySet:
         blog_list_page = BlogPostListPage.objects.get()
-        listing_pages = blog_list_page.get_listing_pages()
-
-        return listing_pages.filter(
-            models.Q(blogpostpage__tags=self)
-            | models.Q(externalblogpostpage__tags=self)
-        ).distinct()
+        return blog_list_page.get_listing_pages().filter(tags=self)
 
 
 class BlogPostCollectionListPage(BaseListingPage):
@@ -189,101 +167,3 @@ class BlogPostCollectionPage(BaseListingPage):
             .public()
             .order_by("-date", "title")
         )
-
-
-class ExternalBlogPostPage(BaseContentPage):
-    subpage_types: list[Any] = []
-    parent_page_types = [BlogPostListPage]
-    preview_modes: list[Any] = []
-
-    is_external = True
-
-    # Some `BaseContentPage` fields aren't relevant
-    body = None
-    subtitle = None
-    hero_image = None
-    hero_unsplash_photo = None
-
-    external_url = models.URLField()
-
-    tags = ParentalManyToManyField("blog.BlogPostTagPage", blank=True)
-    date = models.DateField(default=timezone.now)
-
-    content_panels = BasePage.content_panels + [FieldPanel("external_url")]
-
-    promote_panels = BaseContentPage.promote_panels + [
-        FieldPanel("date"),
-        AutocompletePanel("tags"),
-    ]
-
-    search_fields = BaseContentPage.search_fields + [
-        index.RelatedFields("tags", [index.SearchField("title", boost=1)]),
-        index.SearchField("external_url"),
-    ]
-
-    @cached_property
-    def tag_list_page_url(self) -> Optional[str]:
-        return SingletonPageCache.get_url(BlogPostTagListPage)
-
-    @cached_property
-    def tags_list(self) -> models.QuerySet:
-        """
-        Use this to get a page's tags.
-        """
-        tags = self.tags.order_by("slug")
-
-        # In drafts, `django-modelcluster` doesn't support these filters
-        if isinstance(tags, PageQuerySet):
-            return tags.public().live()
-
-        return tags
-
-    @cached_property
-    def metadata(self) -> ParsedResult:
-        return get_page_metadata(self.external_url)
-
-    @cached_property
-    def _body_html(self) -> str:
-        try:
-            return self.metadata.get_metadatas("description")[0]
-        except (KeyError, IndexError, TypeError):
-            return ""
-
-    @cached_property
-    def plain_text(self) -> str:
-        # The metadata is already just text
-        return self._body_html
-
-    def hero_url(
-        self, image_size: str, wagtail_image_spec_extra: Optional[str] = None
-    ) -> Optional[str]:
-        try:
-            return self.metadata.get_metadatas("image")[0]
-        except (KeyError, IndexError, TypeError):
-            return None
-
-    @cached_property
-    def hero_image_url(self) -> str:
-        return ""
-
-    @cached_property
-    def hero_image_alt(self) -> str:
-        return ""
-
-    def get_meta_image_mime(self) -> Optional[str]:
-        return get_url_mime_type(self.hero_url(""))
-
-    def get_url(
-        self, request: HttpRequest | None = None, current_site: Site | None = None
-    ) -> str:
-        return self.get_full_url(request)
-
-    def get_full_url(self, request: HttpRequest | None = None) -> str:
-        full_url = urlsplit(super().get_full_url(request))
-        return extend_query_params(self.external_url, {"utm_source": full_url.netloc})
-
-    def serve(self, request: HttpRequest, *args: tuple, **kwargs: dict) -> HttpResponse:
-        """
-        Send the user directly to the external page
-        """
-        return HttpResponsePermanentRedirect(self.get_full_url(request))

website/blog/templates/blog/blog_post_page.html

@@ -2,16 +2,6 @@
 
 {% load wagtail_cache navbar_tags %}
 
-{% block post_toc %}
-  <hr class="dropdown-divider" />
-  <li>
-    <a href="#similar-content">Similar content</a>
-  </li>
-  <li>
-    <a href="#comments">Comments</a>
-  </li>
-{% endblock %}
-
 {% block post_content %}
   {% if not request.is_preview %}
     {% include "common/shareon.html" %}

website/blog/templates/blog/external_blog_post_page.html

@@ -1,16 +0,0 @@
-{% comment %}
-This template is never used, but exists just in case.
-{% endcomment %}
-
-<!DOCTYPE html>
-<html lang="en-GB">
-  <head>
-    <title>Redirecting...</title>
-    <link rel="canonical" href="{{ page.external_url }}" />
-    <meta charset="utf-8" />
-    <meta http-equiv="refresh" content="0; url={{ page.external_url }}" />
-  </head>
-  <body>
-    <p>Redirecting...</p>
-  </body>
-</html>

website/blog/tests.py

@@ -1,15 +1,9 @@
-import pickle
-
 from django.test import TestCase
 from django.urls import reverse
 
 from website.home.models import HomePage
 
-from .factories import (
-    BlogPostListPageFactory,
-    BlogPostPageFactory,
-    ExternalBlogPostPageFactory,
-)
+from .factories import BlogPostListPageFactory, BlogPostPageFactory
 
 
 class BlogPostPageTestCase(TestCase):
@@ -75,15 +69,14 @@ class BlogPostListPageTestCase(TestCase):
 
         BlogPostPageFactory(parent=cls.page)
         BlogPostPageFactory(parent=cls.page)
-        ExternalBlogPostPageFactory(parent=cls.page, external_url="https://example.com")
 
     def test_accessible(self) -> None:
         response = self.client.get(self.page.url)
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(len(response.context["listing_pages"]), 3)
+        self.assertEqual(len(response.context["listing_pages"]), 2)
 
     def test_queries(self) -> None:
-        with self.assertNumQueries(43):
+        with self.assertNumQueries(39):
             self.client.get(self.page.url)
 
     def test_feed_accessible(self) -> None:
@@ -91,31 +84,3 @@ class BlogPostListPageTestCase(TestCase):
         self.assertRedirects(
             response, reverse("feed"), status_code=301, fetch_redirect_response=True
         )
-
-
-class ExternalBlogPostPageTestCase(TestCase):
-    @classmethod
-    def setUpTestData(cls) -> None:
-        cls.home_page = HomePage.objects.get()
-        cls.blog_post_list_page = BlogPostListPageFactory(parent=cls.home_page)
-        cls.page = ExternalBlogPostPageFactory(
-            parent=cls.blog_post_list_page, external_url="https://example.com"
-        )
-
-    def test_redirects(self) -> None:
-        with self.assertNumQueries(10):
-            response = self.client.get(self.page.url)
-        self.assertRedirects(
-            response,
-            self.page.external_url + "?utm_source=localhost",
-            status_code=301,
-            fetch_redirect_response=False,
-        )
-
-    def test_metadata(self) -> None:
-        metadata = self.page.metadata
-
-        self.assertIsNone(metadata.soup)
-
-        # Confirm it can pickle
-        pickle.dumps(metadata)

website/common/models.py

@@ -1,6 +1,7 @@
 from datetime import timedelta
 from math import ceil
 from typing import Any, Optional
+from urllib.parse import urlencode
 
 from django.core.paginator import EmptyPage, Paginator
 from django.core.paginator import Page as PaginatorPage
@@ -30,7 +31,6 @@ from .serializers import PaginationSerializer
 from .streamfield import add_heading_anchors, get_blocks, get_content_html
 from .utils import (
     TocEntry,
-    extend_query_params,
     extract_text,
     get_site_title,
     get_table_of_contents,
@@ -181,7 +181,6 @@ class BaseContentPage(BasePage, MetadataMixin):
             for size, width in UNSPLASH_SIZES.items()
         }
 
-    @cached_property
     def hero_image_url(self) -> Optional[str]:
         return self.hero_url("regular")
 
@@ -287,7 +286,10 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
 
         url = super().get_meta_url()
 
-        return extend_query_params(url, query_data)
+        if not query_data:
+            return url
+
+        return url + "?" + urlencode(query_data)
 
     @route(r"^feed/$")
     def feed(self, request: HttpRequest) -> HttpResponse:
@@ -297,11 +299,8 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
     def random(self, request: HttpRequest) -> HttpResponse:
         page = self.get_listing_pages().order_by("?").first()
         if page is None:
-            response = redirect(self.get_url(request=request), permanent=False)
-        else:
-            response = redirect(page.get_url(request=request), permanent=False)
-        response.headers["X-Robots-Tag"] = "noindex"
-        return response
+            return redirect(self.get_url(request=request), permanent=False)
+        return redirect(page.get_url(request=request), permanent=False)
 
 
 class ListingPage(BaseListingPage):

website/common/templates/common/comments.html

@@ -1,5 +1,5 @@
 <section class="container" id="comments">
-  <comentario-comments></comentario-comments>
+  <div id="commento"></div>
 </section>
 
-<script async defer src="https://comentario.theorangeone.net/comentario.js"></script>
+<script async defer src="https://commento.theorangeone.net/js/commento.js"></script>

website/common/templates/common/content_page.html

@@ -10,13 +10,6 @@
   {% endif %}
 {% endblock %}
 
-{% block post_toc %}
-  <hr class="dropdown-divider" />
-  <li>
-    <a href="#comments">Comments</a>
-  </li>
-{% endblock %}
-
 {% block post_content %}
   {% if not request.is_preview %}
     {% include "common/shareon.html" %}

website/common/templates/common/listing-item.html

@@ -16,10 +16,7 @@
           {% include "common/breadcrumbs.html" with parents=page.get_parent_pages %}
         {% endif %}
         <h2 class="title is-3">
-          <a href="{% pageurl page %}">
-            {{ page.title }}
-            {% if page.is_external %}<i class="fa-solid fa-arrow-up-right-from-square" title="This page is from a external source"></i>{% endif %}
-          </a>
+          <a href="{% pageurl page %}">{{ page.title }}</a>
         </h2>
         {% include "common/content-details.html" %}
         <p>{{ page.summary }}</p>

website/common/templates/robots.txt

@@ -1,7 +1,11 @@
+{% if SEO_INDEX %}
 User-agent: *
-{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
-
-# https://github.com/ai-robots-txt/ai.robots.txt
-{{ ai_robots_txt }}
+Allow: /
+{% else %}
+User-agent: *
+Disallow: /
+{% endif %}
 
+Disallow: {% url "wagtailadmin_home" %}
+Disallow: {% url "api:index" %}
 Sitemap: {{ sitemap }}

website/common/templates/wagtail_base.html

@@ -56,7 +56,6 @@
                         {% for toc_item in page.table_of_contents %}
                           {% include "common/toc-item.html" %}
                         {% endfor %}
-                        {% block post_toc %}{% endblock %}
                       </ul>
                     </div>
                   </div>

website/common/tests/test_utils.py

@@ -3,7 +3,6 @@ from django.test import SimpleTestCase
 from wagtail.rich_text import features as richtext_feature_registry
 
 from website.common.utils import (
-    extend_query_params,
     extract_text,
     get_table_of_contents,
     heading_id,
@@ -112,25 +111,3 @@ class HeadingIDTestCase(SimpleTestCase):
         self.assertEqual(heading_id("123"), "ref-123")
         self.assertEqual(heading_id("test"), "test")
         self.assertEqual(heading_id("Look, a title!"), "look-a-title")
-
-
-class ExtendQueryParamsTestCase(SimpleTestCase):
-    def test_params(self) -> None:
-        self.assertEqual(
-            extend_query_params("https://example.com", {"foo": "bar"}),
-            "https://example.com?foo=bar",
-        )
-        self.assertEqual(
-            extend_query_params("https://example.com?foo=bar", {"bar": "foo"}),
-            "https://example.com?foo=bar&bar=foo",
-        )
-        self.assertEqual(
-            extend_query_params("https://example.com?foo=baz", {"foo": "baz"}),
-            "https://example.com?foo=baz",
-        )
-
-    def test_removes_param(self) -> None:
-        self.assertEqual(
-            extend_query_params("https://example.com?foo=bar", {"foo": None}),
-            "https://example.com",
-        )

website/common/utils.py

@@ -1,17 +1,14 @@
 from dataclasses import dataclass
 from itertools import pairwise
-from typing import Any, Optional, Type
-from urllib.parse import urlsplit, urlunsplit
+from typing import Optional, Type
 
 import requests
 from bs4 import BeautifulSoup, SoupStrainer
 from django.conf import settings
 from django.db import models
-from django.http import QueryDict
 from django.http.request import HttpRequest
 from django.utils.text import slugify
 from django_cache_decorator import django_cache_decorator
-from metadata_parser import MetadataParser, ParsedResult
 from wagtail.models import Page, Site
 from wagtail.models import get_page_models as get_wagtail_page_models
 
@@ -115,36 +112,3 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
         return queryset.get()
     except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
         return None
-
-
-@django_cache_decorator(time=21600)
-def get_ai_robots_txt() -> str:
-    """
-    https://github.com/ai-robots-txt/ai.robots.txt
-    """
-    return requests_session.get(
-        "https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
-    ).content.decode()
-
-
-@django_cache_decorator(time=21600)
-def get_page_metadata(url: str) -> ParsedResult:
-    metadata = MetadataParser(url=url, search_head_only=True).parsed_result
-
-    # HACK: BeautifulSoup doesn't pickle nicely, and so can't be cached
-    metadata.soup = None
-
-    return metadata
-
-
-def extend_query_params(url: str, params: dict[str, Any]) -> str:
-    scheme, netloc, path, query, fragment = urlsplit(url)
-    query_dict = QueryDict(query, mutable=True)
-
-    for k, v in params.items():
-        if v is None:
-            del query_dict[k]
-        else:
-            query_dict[k] = v
-
-    return urlunsplit((scheme, netloc, path, query_dict.urlencode(), fragment))

website/common/views.py

@@ -23,7 +23,6 @@ from website.search.models import SearchPage
 
 from .feed_generators import CustomFeed
 from .models import BaseListingPage, BasePage
-from .utils import extend_query_params, get_ai_robots_txt
 
 
 class Error404View(TemplateView):
@@ -53,7 +52,6 @@ class RobotsView(TemplateView):
     def get_context_data(self, **kwargs: dict) -> dict:
         context = super().get_context_data(**kwargs)
         context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
-        context["ai_robots_txt"] = get_ai_robots_txt()
         return context
 
 
@@ -116,9 +114,7 @@ class AllPagesFeed(Feed):
         return item.title
 
     def item_link(self, item: BasePage) -> str:
-        return extend_query_params(
-            item.get_full_url(request=self.request), {"utm_medium": "rss"}
-        )
+        return item.get_full_url(request=self.request) + "?utm_medium=rss"
 
     def item_pubdate(self, item: BasePage) -> datetime:
         if item_date := getattr(item, "date", None):

website/contrib/code_block/views.py

@@ -1,16 +1,21 @@
-import lightningcss
 from django.http import HttpRequest, HttpResponse
+from django.utils.datastructures import OrderedSet
 from django.views.decorators.cache import cache_control
 from pygments.formatters.html import HtmlFormatter
 
 
 @cache_control(max_age=3600)
 def pygments_styles(request: HttpRequest) -> HttpResponse:
-    default_styles = HtmlFormatter(style="default").get_style_defs(
-        "html:not(.dark-mode) .highlight"
+    default_styles = (
+        HtmlFormatter(style="default")
+        .get_style_defs("html:not(.dark-mode) .highlight")
+        .split("\n")
     )
-    dark_styles = HtmlFormatter(style="monokai").get_style_defs(
-        "html.dark-mode .highlight"
+    dark_styles = (
+        HtmlFormatter(style="monokai")
+        .get_style_defs("html.dark-mode .highlight")
+        .split("\n")
+    )
+    return HttpResponse(
+        "".join(OrderedSet(default_styles + dark_styles)), content_type="text/css"
     )
-    compressed = lightningcss.process_stylesheet(default_styles + dark_styles)
-    return HttpResponse(compressed, content_type="text/css")

website/contrib/unsplash/management/commands/update_unsplash_photos.py

@@ -3,15 +3,13 @@ from datetime import timedelta
 
 from django.core.management.base import BaseCommand
 from django.utils import timezone
-from django_tasks import task
 
 from website.contrib.unsplash.models import UnsplashPhoto
 from website.contrib.unsplash.utils import get_unsplash_photo
+from website.utils.queue import enqueue_or_sync
 
 
-@task()
-def update_photo(photo_id: int) -> None:
-    photo = UnsplashPhoto.objects.get(id=photo_id)
+def update_photo(photo: UnsplashPhoto) -> None:
     photo.data = get_unsplash_photo(photo.unsplash_id)
     photo.data_last_updated = timezone.now()
     photo.save()
@@ -27,6 +25,6 @@ class Command(BaseCommand):
         photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
         self.stdout.write(f"Found {photos.count()} photos to update.")
 
-        for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"):
-            self.stdout.write(f"Updating {unsplash_id}")
-            update_photo.enqueue(photo_id)
+        for photo in photos:
+            self.stdout.write(f"Updating {photo.unsplash_id}")
+            enqueue_or_sync(update_photo, args=[photo])

website/search/views.py

@@ -7,7 +7,7 @@ from wagtail.search.utils import parse_query_string
 from wagtail_favicon.models import FaviconSettings
 from wagtail_favicon.utils import get_rendition_url
 
-from website.common.utils import extend_query_params, get_or_none, get_site_title
+from website.common.utils import get_or_none, get_site_title
 from website.contrib.singleton_page.utils import SingletonPageCache
 
 from .models import SearchPage
@@ -87,4 +87,4 @@ class GoView(RedirectView):
         if slug_match := get_or_none(pages.filter(slug__iexact=query)):
             return slug_match.get_url(request=self.request)
 
-        return extend_query_params(search_page_url, self.request.GET)
+        return f"{search_page_url}?{self.request.GET.urlencode()}"

website/settings.py

@@ -69,6 +69,7 @@ INSTALLED_APPS = [
     "generic_chooser",
     "wagtail_draftail_snippet",
     "wagtailautocomplete",
+    "django_rq",
     "rest_framework",
     "corsheaders",
     "wagtail_favicon",
@@ -79,7 +80,6 @@ INSTALLED_APPS = [
     "django_otp",
     "django_otp.plugins.otp_totp",
     "django_minify_html",
-    "django_tasks.backends.database",
     "health_check",
     "health_check.db",
     "health_check.cache",
@@ -146,10 +146,16 @@ CACHES = {
 # https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
 WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
 
-if TEST:
-    TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}}
-else:
-    TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}}
+RQ_QUEUES = {}
+
+USE_REDIS_QUEUE = False
+if queue_store := env.cache(
+    "QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache"
+):
+    CACHES["rq"] = queue_store
+    USE_REDIS_QUEUE = True
+    RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"}
+
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.0/topics/i18n/
@@ -314,15 +320,14 @@ if DEBUG:
     INSTALLED_APPS.append("django_browser_reload")
     MIDDLEWARE.append("django_browser_reload.middleware.BrowserReloadMiddleware")
 
-    if not TEST:
-        # Add django-debug-toolbar
-        INSTALLED_APPS.append("debug_toolbar")
-        MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
-        DEBUG_TOOLBAR_CONFIG = {
-            "SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
-            "RESULTS_CACHE_SIZE": 5,
-            "SHOW_COLLAPSED": True,
-        }
+    # Add django-debug-toolbar
+    INSTALLED_APPS.append("debug_toolbar")
+    MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
+    DEBUG_TOOLBAR_CONFIG = {
+        "SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
+        "RESULTS_CACHE_SIZE": 5,
+        "SHOW_COLLAPSED": True,
+    }
 
     # Add Wagtail styleguide
     INSTALLED_APPS.append("wagtail.contrib.styleguide")
@@ -386,11 +391,6 @@ LOGGING = {
             "level": "WARNING",
             "propagate": False,
         },
-        "metadata_parser": {
-            "handlers": ["console"],
-            "level": "CRITICAL",
-            "propagate": False,
-        },
     },
 }
 
@@ -442,11 +442,12 @@ if sentry_dsn := env("SENTRY_DSN"):
     import sentry_sdk
     from sentry_sdk.integrations.django import DjangoIntegration
     from sentry_sdk.integrations.redis import RedisIntegration
+    from sentry_sdk.integrations.rq import RqIntegration
     from sentry_sdk.utils import get_default_release
 
     sentry_kwargs = {
         "dsn": sentry_dsn,
-        "integrations": [DjangoIntegration(), RedisIntegration()],
+        "integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()],
         "release": get_default_release(),
     }
 

website/spotify/management/commands/refresh_spotify_playlists.py

@@ -1,11 +1,10 @@
 from django.core.cache import cache
 from django.core.management.base import BaseCommand
-from django_tasks import task
 
 from website.spotify.models import SpotifyPlaylistPage
+from website.utils.queue import enqueue_or_sync
 
 
-@task()
 def refresh_cache(page_id: int) -> None:
     page = SpotifyPlaylistPage.objects.get(id=page_id)
     cache.delete(page.playlist_cache_key)
@@ -16,7 +15,5 @@ def refresh_cache(page_id: int) -> None:
 
 class Command(BaseCommand):
     def handle(self, *args: list, **options: dict) -> None:
-        for page_id in (
-            SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator()
-        ):
-            refresh_cache.enqueue(page_id)
+        for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator():
+            enqueue_or_sync(refresh_cache, args=[page.id])

website/utils/queue.py

@@ -0,0 +1,21 @@
+from typing import Callable
+
+from django.conf import settings
+from django_rq import get_queue
+
+
+def enqueue_or_sync(
+    job_func: Callable, args: list | None = None, kwargs: dict | None = None
+) -> None:
+    """
+    Run a task now, or put in RQ
+    """
+    if args is None:
+        args = []
+    if kwargs is None:
+        kwargs = {}
+
+    if settings.USE_REDIS_QUEUE:
+        get_queue().enqueue(job_func, args=args, kwargs=kwargs)
+    else:
+        job_func(*args, **kwargs)