Replace RQ with django-tasks

This commit is contained in:
Jake Howard 2024-06-08 12:52:52 +01:00
parent a7ec3a8f8e
commit 5d31c16a3c
Signed by: jake
GPG key ID: 57AFB45680EDD477
8 changed files with 24 additions and 49 deletions

View file

@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080
watch-js: npm run build:js -- --watch
watch-css: npm run build:css -- --watch
watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
rqworker: ./manage.py rqworker --with-scheduler
worker: ./manage.py db_worker --interval 5

View file

@ -5,7 +5,6 @@ services:
context: ../../
target: dev
environment:
- QUEUE_STORE_URL=redis://redis/0
- DEBUG=true
- SECRET_KEY=super-secret-key
- DATABASE_URL=postgres://website:website@db/website
@ -20,9 +19,6 @@ services:
- 127.0.0.1:8000:8000
- 127.0.0.1:8080:8080
redis:
image: redis:6-alpine
db:
image: postgres:14-alpine
environment:

View file

@ -4,4 +4,4 @@ set -e
cd /app
exec python manage.py rqworker --with-scheduler
exec python manage.py db_worker -v3 --interval 10

View file

@ -7,7 +7,6 @@ beautifulsoup4
lxml==5.2.1
requests
wagtail-generic-chooser==0.6
django-rq==2.10.1
django-redis==5.4.0
gunicorn==22.0.0
psycopg==3.1.18
@ -41,3 +40,6 @@ git+https://github.com/RealOrangeOne/wagtail-favicon@b892165e047b35c46d7244109b9
# Use custom `wagtail-draftail-snippet` with support for Wagtail 5.x
git+https://github.com/RealOrangeOne/wagtail-draftail-snippet@74ed858bd958a066d5aee295c9848257107b1546
# Background tasks!
git+https://github.com/RealOrangeOne/django-tasks@92ec776fccb6a9f8cc5305e75e8846633e26b643

View file

@ -3,13 +3,15 @@ from datetime import timedelta
from django.core.management.base import BaseCommand
from django.utils import timezone
from django_tasks import task
from website.contrib.unsplash.models import UnsplashPhoto
from website.contrib.unsplash.utils import get_unsplash_photo
from website.utils.queue import enqueue_or_sync
def update_photo(photo: UnsplashPhoto) -> None:
@task()
def update_photo(photo_id: int) -> None:
photo = UnsplashPhoto.objects.get(id=photo_id)
photo.data = get_unsplash_photo(photo.unsplash_id)
photo.data_last_updated = timezone.now()
photo.save()
@ -25,6 +27,6 @@ class Command(BaseCommand):
photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
self.stdout.write(f"Found {photos.count()} photos to update.")
for photo in photos:
self.stdout.write(f"Updating {photo.unsplash_id}")
enqueue_or_sync(update_photo, args=[photo])
for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"):
self.stdout.write(f"Updating {unsplash_id}")
update_photo.enqueue(photo_id)

View file

@ -69,7 +69,6 @@ INSTALLED_APPS = [
"generic_chooser",
"wagtail_draftail_snippet",
"wagtailautocomplete",
"django_rq",
"rest_framework",
"corsheaders",
"wagtail_favicon",
@ -80,6 +79,7 @@ INSTALLED_APPS = [
"django_otp",
"django_otp.plugins.otp_totp",
"django_minify_html",
"django_tasks.backends.database",
"health_check",
"health_check.db",
"health_check.cache",
@ -146,16 +146,10 @@ CACHES = {
# https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
RQ_QUEUES = {}
USE_REDIS_QUEUE = False
if queue_store := env.cache(
"QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache"
):
CACHES["rq"] = queue_store
USE_REDIS_QUEUE = True
RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"}
if TEST:
TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}}
else:
TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}}
# Internationalization
# https://docs.djangoproject.com/en/4.0/topics/i18n/
@ -448,12 +442,11 @@ if sentry_dsn := env("SENTRY_DSN"):
import sentry_sdk
from sentry_sdk.integrations.django import DjangoIntegration
from sentry_sdk.integrations.redis import RedisIntegration
from sentry_sdk.integrations.rq import RqIntegration
from sentry_sdk.utils import get_default_release
sentry_kwargs = {
"dsn": sentry_dsn,
"integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()],
"integrations": [DjangoIntegration(), RedisIntegration()],
"release": get_default_release(),
}

View file

@ -1,10 +1,11 @@
from django.core.cache import cache
from django.core.management.base import BaseCommand
from django_tasks import task
from website.spotify.models import SpotifyPlaylistPage
from website.utils.queue import enqueue_or_sync
@task()
def refresh_cache(page_id: int) -> None:
page = SpotifyPlaylistPage.objects.get(id=page_id)
cache.delete(page.playlist_cache_key)
@ -15,5 +16,7 @@ def refresh_cache(page_id: int) -> None:
class Command(BaseCommand):
def handle(self, *args: list, **options: dict) -> None:
for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator():
enqueue_or_sync(refresh_cache, args=[page.id])
for page_id in (
SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator()
):
refresh_cache.enqueue(page_id)

View file

@ -1,21 +0,0 @@
from typing import Callable
from django.conf import settings
from django_rq import get_queue
def enqueue_or_sync(
job_func: Callable, args: list | None = None, kwargs: dict | None = None
) -> None:
"""
Run a task now, or put in RQ
"""
if args is None:
args = []
if kwargs is None:
kwargs = {}
if settings.USE_REDIS_QUEUE:
get_queue().enqueue(job_func, args=args, kwargs=kwargs)
else:
job_func(*args, **kwargs)