Replace RQ with django-tasks

Procfile

@@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080
 watch-js: npm run build:js -- --watch
 watch-css: npm run build:css -- --watch
 watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
-rqworker: ./manage.py rqworker --with-scheduler
+worker: ./manage.py db_worker --interval 5

docker/dev/docker-compose.yml

@@ -5,7 +5,6 @@ services:
       context: ../../
       target: dev
     environment:
-      - QUEUE_STORE_URL=redis://redis/0
       - DEBUG=true
       - SECRET_KEY=super-secret-key
       - DATABASE_URL=postgres://website:website@db/website
@@ -20,9 +19,6 @@ services:
       - 127.0.0.1:8000:8000
       - 127.0.0.1:8080:8080
 
-  redis:
-    image: redis:6-alpine
-
   db:
     image: postgres:14-alpine
     environment:

etc/s6-rc.d/rq/run

@@ -4,4 +4,4 @@ set -e
 
 cd /app
 
-exec python manage.py rqworker --with-scheduler
+exec python manage.py db_worker -v3 --interval 10

requirements.txt

@@ -7,7 +7,6 @@ beautifulsoup4
 lxml==5.2.1
 requests
 wagtail-generic-chooser==0.6
-django-rq==2.10.1
 django-redis==5.4.0
 gunicorn==22.0.0
 psycopg==3.1.18
@@ -41,3 +40,6 @@ git+https://github.com/RealOrangeOne/wagtail-favicon@b892165e047b35c46d7244109b9
 
 # Use custom `wagtail-draftail-snippet` with support for Wagtail 5.x
 git+https://github.com/RealOrangeOne/wagtail-draftail-snippet@74ed858bd958a066d5aee295c9848257107b1546
+
+# Background tasks!
+git+https://github.com/RealOrangeOne/django-tasks@92ec776fccb6a9f8cc5305e75e8846633e26b643

website/contrib/unsplash/management/commands/update_unsplash_photos.py

@@ -3,13 +3,15 @@ from datetime import timedelta
 
 from django.core.management.base import BaseCommand
 from django.utils import timezone
+from django_tasks import task
 
 from website.contrib.unsplash.models import UnsplashPhoto
 from website.contrib.unsplash.utils import get_unsplash_photo
-from website.utils.queue import enqueue_or_sync
 
 
-def update_photo(photo: UnsplashPhoto) -> None:
+@task()
+def update_photo(photo_id: int) -> None:
+    photo = UnsplashPhoto.objects.get(id=photo_id)
     photo.data = get_unsplash_photo(photo.unsplash_id)
     photo.data_last_updated = timezone.now()
     photo.save()
@@ -25,6 +27,6 @@ class Command(BaseCommand):
         photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
         self.stdout.write(f"Found {photos.count()} photos to update.")
 
-        for photo in photos:
-            self.stdout.write(f"Updating {photo.unsplash_id}")
-            enqueue_or_sync(update_photo, args=[photo])
+        for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"):
+            self.stdout.write(f"Updating {unsplash_id}")
+            update_photo.enqueue(photo_id)

website/settings.py

@@ -69,7 +69,6 @@ INSTALLED_APPS = [
     "generic_chooser",
     "wagtail_draftail_snippet",
     "wagtailautocomplete",
-    "django_rq",
     "rest_framework",
     "corsheaders",
     "wagtail_favicon",
@@ -80,6 +79,7 @@ INSTALLED_APPS = [
     "django_otp",
     "django_otp.plugins.otp_totp",
     "django_minify_html",
+    "django_tasks.backends.database",
     "health_check",
     "health_check.db",
     "health_check.cache",
@@ -146,16 +146,10 @@ CACHES = {
 # https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
 WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
 
-RQ_QUEUES = {}
-
-USE_REDIS_QUEUE = False
-if queue_store := env.cache(
-    "QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache"
-):
-    CACHES["rq"] = queue_store
-    USE_REDIS_QUEUE = True
-    RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"}
-
+if TEST:
+    TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}}
+else:
+    TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}}
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.0/topics/i18n/
@@ -448,12 +442,11 @@ if sentry_dsn := env("SENTRY_DSN"):
     import sentry_sdk
     from sentry_sdk.integrations.django import DjangoIntegration
     from sentry_sdk.integrations.redis import RedisIntegration
-    from sentry_sdk.integrations.rq import RqIntegration
     from sentry_sdk.utils import get_default_release
 
     sentry_kwargs = {
         "dsn": sentry_dsn,
-        "integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()],
+        "integrations": [DjangoIntegration(), RedisIntegration()],
         "release": get_default_release(),
     }
 

website/spotify/management/commands/refresh_spotify_playlists.py

@@ -1,10 +1,11 @@
 from django.core.cache import cache
 from django.core.management.base import BaseCommand
+from django_tasks import task
 
 from website.spotify.models import SpotifyPlaylistPage
-from website.utils.queue import enqueue_or_sync
 
 
+@task()
 def refresh_cache(page_id: int) -> None:
     page = SpotifyPlaylistPage.objects.get(id=page_id)
     cache.delete(page.playlist_cache_key)
@@ -15,5 +16,7 @@ def refresh_cache(page_id: int) -> None:
 
 class Command(BaseCommand):
     def handle(self, *args: list, **options: dict) -> None:
-        for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator():
-            enqueue_or_sync(refresh_cache, args=[page.id])
+        for page_id in (
+            SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator()
+        ):
+            refresh_cache.enqueue(page_id)

website/utils/queue.py

@@ -1,21 +0,0 @@
-from typing import Callable
-
-from django.conf import settings
-from django_rq import get_queue
-
-
-def enqueue_or_sync(
-    job_func: Callable, args: list | None = None, kwargs: dict | None = None
-) -> None:
-    """
-    Run a task now, or put in RQ
-    """
-    if args is None:
-        args = []
-    if kwargs is None:
-        kwargs = {}
-
-    if settings.USE_REDIS_QUEUE:
-        get_queue().enqueue(job_func, args=args, kwargs=kwargs)
-    else:
-        job_func(*args, **kwargs)