diff --git a/Procfile b/Procfile index 0da497d..241b899 100644 --- a/Procfile +++ b/Procfile @@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080 watch-js: npm run build:js -- --watch watch-css: npm run build:css -- --watch watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done -rqworker: ./manage.py rqworker --with-scheduler +worker: ./manage.py db_worker --interval 5 diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index 4b3eb69..d617b51 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -5,7 +5,6 @@ services: context: ../../ target: dev environment: - - QUEUE_STORE_URL=redis://redis/0 - DEBUG=true - SECRET_KEY=super-secret-key - DATABASE_URL=postgres://website:website@db/website @@ -20,9 +19,6 @@ services: - 127.0.0.1:8000:8000 - 127.0.0.1:8080:8080 - redis: - image: redis:6-alpine - db: image: postgres:14-alpine environment: diff --git a/etc/s6-rc.d/rq/run b/etc/s6-rc.d/rq/run index bb99115..3c260bf 100644 --- a/etc/s6-rc.d/rq/run +++ b/etc/s6-rc.d/rq/run @@ -4,4 +4,4 @@ set -e cd /app -exec python manage.py rqworker --with-scheduler +exec python manage.py db_worker -v3 --interval 10 diff --git a/requirements.txt b/requirements.txt index 89c6cb2..43e8615 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,7 +7,6 @@ beautifulsoup4 lxml==5.2.1 requests wagtail-generic-chooser==0.6 -django-rq==2.10.1 django-redis==5.4.0 gunicorn==22.0.0 psycopg==3.1.18 @@ -41,3 +40,6 @@ git+https://github.com/RealOrangeOne/wagtail-favicon@b892165e047b35c46d7244109b9 # Use custom `wagtail-draftail-snippet` with support for Wagtail 5.x git+https://github.com/RealOrangeOne/wagtail-draftail-snippet@74ed858bd958a066d5aee295c9848257107b1546 + +# Background tasks! +git+https://github.com/RealOrangeOne/django-tasks@92ec776fccb6a9f8cc5305e75e8846633e26b643 diff --git a/website/contrib/unsplash/management/commands/update_unsplash_photos.py b/website/contrib/unsplash/management/commands/update_unsplash_photos.py index 7b7d36c..ece060b 100644 --- a/website/contrib/unsplash/management/commands/update_unsplash_photos.py +++ b/website/contrib/unsplash/management/commands/update_unsplash_photos.py @@ -3,13 +3,15 @@ from datetime import timedelta from django.core.management.base import BaseCommand from django.utils import timezone +from django_tasks import task from website.contrib.unsplash.models import UnsplashPhoto from website.contrib.unsplash.utils import get_unsplash_photo -from website.utils.queue import enqueue_or_sync -def update_photo(photo: UnsplashPhoto) -> None: +@task() +def update_photo(photo_id: int) -> None: + photo = UnsplashPhoto.objects.get(id=photo_id) photo.data = get_unsplash_photo(photo.unsplash_id) photo.data_last_updated = timezone.now() photo.save() @@ -25,6 +27,6 @@ class Command(BaseCommand): photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age) self.stdout.write(f"Found {photos.count()} photos to update.") - for photo in photos: - self.stdout.write(f"Updating {photo.unsplash_id}") - enqueue_or_sync(update_photo, args=[photo]) + for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"): + self.stdout.write(f"Updating {unsplash_id}") + update_photo.enqueue(photo_id) diff --git a/website/settings.py b/website/settings.py index 8489994..97535bc 100644 --- a/website/settings.py +++ b/website/settings.py @@ -69,7 +69,6 @@ INSTALLED_APPS = [ "generic_chooser", "wagtail_draftail_snippet", "wagtailautocomplete", - "django_rq", "rest_framework", "corsheaders", "wagtail_favicon", @@ -80,6 +79,7 @@ INSTALLED_APPS = [ "django_otp", "django_otp.plugins.otp_totp", "django_minify_html", + "django_tasks.backends.database", "health_check", "health_check.db", "health_check.cache", @@ -146,16 +146,10 @@ CACHES = { # https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects WAGTAIL_REDIRECTS_FILE_STORAGE = "cache" -RQ_QUEUES = {} - -USE_REDIS_QUEUE = False -if queue_store := env.cache( - "QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache" -): - CACHES["rq"] = queue_store - USE_REDIS_QUEUE = True - RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"} - +if TEST: + TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}} +else: + TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}} # Internationalization # https://docs.djangoproject.com/en/4.0/topics/i18n/ @@ -448,12 +442,11 @@ if sentry_dsn := env("SENTRY_DSN"): import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.redis import RedisIntegration - from sentry_sdk.integrations.rq import RqIntegration from sentry_sdk.utils import get_default_release sentry_kwargs = { "dsn": sentry_dsn, - "integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()], + "integrations": [DjangoIntegration(), RedisIntegration()], "release": get_default_release(), } diff --git a/website/spotify/management/commands/refresh_spotify_playlists.py b/website/spotify/management/commands/refresh_spotify_playlists.py index 5b54aff..df189a1 100644 --- a/website/spotify/management/commands/refresh_spotify_playlists.py +++ b/website/spotify/management/commands/refresh_spotify_playlists.py @@ -1,10 +1,11 @@ from django.core.cache import cache from django.core.management.base import BaseCommand +from django_tasks import task from website.spotify.models import SpotifyPlaylistPage -from website.utils.queue import enqueue_or_sync +@task() def refresh_cache(page_id: int) -> None: page = SpotifyPlaylistPage.objects.get(id=page_id) cache.delete(page.playlist_cache_key) @@ -15,5 +16,7 @@ def refresh_cache(page_id: int) -> None: class Command(BaseCommand): def handle(self, *args: list, **options: dict) -> None: - for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator(): - enqueue_or_sync(refresh_cache, args=[page.id]) + for page_id in ( + SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator() + ): + refresh_cache.enqueue(page_id) diff --git a/website/utils/queue.py b/website/utils/queue.py deleted file mode 100644 index 1b9c5f1..0000000 --- a/website/utils/queue.py +++ /dev/null @@ -1,21 +0,0 @@ -from typing import Callable - -from django.conf import settings -from django_rq import get_queue - - -def enqueue_or_sync( - job_func: Callable, args: list | None = None, kwargs: dict | None = None -) -> None: - """ - Run a task now, or put in RQ - """ - if args is None: - args = [] - if kwargs is None: - kwargs = {} - - if settings.USE_REDIS_QUEUE: - get_queue().enqueue(job_func, args=args, kwargs=kwargs) - else: - job_func(*args, **kwargs)