Compare commits
39 commits
renovate/w
...
master
Author | SHA1 | Date | |
---|---|---|---|
b08c4a69f6 | |||
62e3cd13da | |||
d211e2b27b | |||
ea73189b9c | |||
0bd43a4bee | |||
9788b8e4dd | |||
7d4a9647d4 | |||
4cfb0a896e | |||
09dbbc407b | |||
040647b506 | |||
4134426823 | |||
451c2d4cf9 | |||
5e34485fec | |||
91de1fcf19 | |||
009b4926a4 | |||
1822f1c4d5 | |||
a8fe244ab4 | |||
df84b28114 | |||
a25e83a0df | |||
46669c9b17 | |||
41a04af8dc | |||
d242f94024 | |||
1421d19fdc | |||
ef7dd5f860 | |||
872fd4fc82 | |||
fffd41dc82 | |||
df3dff0708 | |||
af379243ad | |||
f18801abba | |||
2a6c3146f1 | |||
5d31c16a3c | |||
a7ec3a8f8e | |||
26d9e256ea | |||
f427fc7bb5 | |||
d199c3a681 | |||
36211e88f2 | |||
2639d6eb1c | |||
43503921db | |||
54aaca3087 |
44 changed files with 742 additions and 1260 deletions
|
@ -109,12 +109,6 @@ djlint:
|
||||||
script:
|
script:
|
||||||
- djlint website/ --lint --check
|
- djlint website/ --lint --check
|
||||||
|
|
||||||
gunicorn_check:
|
|
||||||
extends: .python_test_template
|
|
||||||
script:
|
|
||||||
- apt-get update && apt-get install --yes libpq-dev
|
|
||||||
- gunicorn --check-config --config etc/gunicorn.conf.py
|
|
||||||
|
|
||||||
npm_lint:
|
npm_lint:
|
||||||
image: node:20-slim
|
image: node:20-slim
|
||||||
stage: test
|
stage: test
|
||||||
|
|
|
@ -39,7 +39,14 @@ ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLA
|
||||||
RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz
|
RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz && tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz
|
||||||
|
|
||||||
ENV PATH=$VIRTUAL_ENV/bin:$PATH \
|
ENV PATH=$VIRTUAL_ENV/bin:$PATH \
|
||||||
PYTHONUNBUFFERED=1
|
PYTHONUNBUFFERED=1 \
|
||||||
|
GRANIAN_INTERFACE=wsgi \
|
||||||
|
GRANIAN_HOST=0.0.0.0 \
|
||||||
|
GRANIAN_PORT=8080 \
|
||||||
|
GRANIAN_WORKERS_LIFETIME=1800 \
|
||||||
|
GRANIAN_RESPAWN_INTERVAL=10 \
|
||||||
|
GRANIAN_PROCESS_NAME=website \
|
||||||
|
GRANIAN_RESPAWN_FAILED_WORKERS=1
|
||||||
|
|
||||||
EXPOSE 8000
|
EXPOSE 8000
|
||||||
|
|
||||||
|
|
2
Procfile
2
Procfile
|
@ -2,4 +2,4 @@ web: ./manage.py runserver 0.0.0.0:8080
|
||||||
watch-js: npm run build:js -- --watch
|
watch-js: npm run build:js -- --watch
|
||||||
watch-css: npm run build:css -- --watch
|
watch-css: npm run build:css -- --watch
|
||||||
watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
|
watch-contrib: ./scripts/copy-npm-contrib.sh; while inotifywait -e modify ./scripts/copy-npm-contrib.sh; do ./scripts/copy-npm-contrib.sh; done
|
||||||
rqworker: ./manage.py rqworker --with-scheduler
|
worker: ./manage.py db_worker --interval 5
|
||||||
|
|
|
@ -1,11 +1,9 @@
|
||||||
version: '3.7'
|
|
||||||
services:
|
services:
|
||||||
web:
|
web:
|
||||||
build:
|
build:
|
||||||
context: ../../
|
context: ../../
|
||||||
target: dev
|
target: dev
|
||||||
environment:
|
environment:
|
||||||
- QUEUE_STORE_URL=redis://redis/0
|
|
||||||
- DEBUG=true
|
- DEBUG=true
|
||||||
- SECRET_KEY=super-secret-key
|
- SECRET_KEY=super-secret-key
|
||||||
- DATABASE_URL=postgres://website:website@db/website
|
- DATABASE_URL=postgres://website:website@db/website
|
||||||
|
@ -14,15 +12,11 @@ services:
|
||||||
tmpfs:
|
tmpfs:
|
||||||
- /tmp
|
- /tmp
|
||||||
depends_on:
|
depends_on:
|
||||||
- redis
|
|
||||||
- db
|
- db
|
||||||
ports:
|
ports:
|
||||||
- 127.0.0.1:8000:8000
|
- 127.0.0.1:8000:8000
|
||||||
- 127.0.0.1:8080:8080
|
- 127.0.0.1:8080:8080
|
||||||
|
|
||||||
redis:
|
|
||||||
image: redis:6-alpine
|
|
||||||
|
|
||||||
db:
|
db:
|
||||||
image: postgres:14-alpine
|
image: postgres:14-alpine
|
||||||
environment:
|
environment:
|
||||||
|
|
|
@ -3,4 +3,5 @@
|
||||||
@daily ./manage.py update_unsplash_photos
|
@daily ./manage.py update_unsplash_photos
|
||||||
@daily ./manage.py rebuild_references_index
|
@daily ./manage.py rebuild_references_index
|
||||||
@weekly ./manage.py refresh_spotify_playlists
|
@weekly ./manage.py refresh_spotify_playlists
|
||||||
|
@weekly ./manage.py prune_db_task_results
|
||||||
*/10 * * * * ./manage.py publish_scheduled_pages
|
*/10 * * * * ./manage.py publish_scheduled_pages
|
||||||
|
|
|
@ -1,10 +0,0 @@
|
||||||
wsgi_app = "website.wsgi:application"
|
|
||||||
disable_redirect_access_to_syslog = True
|
|
||||||
preload_app = True
|
|
||||||
bind = "127.0.0.1:8080"
|
|
||||||
max_requests = 1200
|
|
||||||
max_requests_jitter = 50
|
|
||||||
forwarded_allow_ips = "*"
|
|
||||||
|
|
||||||
# Run additional threads so the GIL isn't sitting completely idle
|
|
||||||
threads = 4
|
|
|
@ -4,4 +4,4 @@ set -e
|
||||||
|
|
||||||
cd /app
|
cd /app
|
||||||
|
|
||||||
exec gunicorn -c etc/gunicorn.conf.py
|
exec granian website.wsgi:application
|
||||||
|
|
|
@ -4,4 +4,4 @@ set -e
|
||||||
|
|
||||||
cd /app
|
cd /app
|
||||||
|
|
||||||
exec python manage.py rqworker --with-scheduler
|
exec python manage.py db_worker -v3 --interval 10
|
||||||
|
|
1349
package-lock.json
generated
1349
package-lock.json
generated
File diff suppressed because it is too large
Load diff
|
@ -28,19 +28,19 @@
|
||||||
"stylelint-config-standard-scss": "6.1.0"
|
"stylelint-config-standard-scss": "6.1.0"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@fontsource/fira-code": "5.0.2",
|
"@fontsource/fira-code": "5.0.18",
|
||||||
"@fortawesome/fontawesome-free": "6.5.2",
|
"@fortawesome/fontawesome-free": "6.5.2",
|
||||||
"@ledge/is-ie-11": "7.0.0",
|
"@ledge/is-ie-11": "7.0.0",
|
||||||
"bulma": "0.9.4",
|
"bulma": "0.9.4",
|
||||||
"elevator.js": "1.0.1",
|
"elevator.js": "1.0.1",
|
||||||
"esbuild": "0.20.2",
|
"esbuild": "0.20.2",
|
||||||
"glightbox": "3.3.0",
|
"glightbox": "3.3.0",
|
||||||
"htmx.org": "1.9.2",
|
"htmx.org": "2.0.1",
|
||||||
"lite-youtube-embed": "0.3.0",
|
"lite-youtube-embed": "0.3.2",
|
||||||
"lodash.clamp": "4.0.3",
|
"lodash.clamp": "4.0.3",
|
||||||
"lodash.debounce": "4.0.8",
|
"lodash.debounce": "4.0.8",
|
||||||
"lodash.throttle": "4.1.1",
|
"lodash.throttle": "4.1.1",
|
||||||
"npm-run-all": "4.1.5",
|
"npm-run-all2": "5.0.0",
|
||||||
"sass": "1.75.0",
|
"sass": "1.75.0",
|
||||||
"shareon": "2.4.0"
|
"shareon": "2.4.0"
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,16 +1,11 @@
|
||||||
{
|
{
|
||||||
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
"$schema": "https://docs.renovatebot.com/renovate-schema.json",
|
||||||
"extends": [
|
"extends": [
|
||||||
"config:base"
|
"config:base",
|
||||||
|
"replacements:all",
|
||||||
|
"workarounds:all"
|
||||||
],
|
],
|
||||||
"prConcurrentLimit": 0,
|
"prConcurrentLimit": 0,
|
||||||
"packageRules": [
|
|
||||||
{
|
|
||||||
"matchUpdateTypes": ["patch"],
|
|
||||||
"schedule": ["every weekend"],
|
|
||||||
"enabled": false
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"regexManagers": [
|
"regexManagers": [
|
||||||
{
|
{
|
||||||
"fileMatch": ["^Dockerfile$"],
|
"fileMatch": ["^Dockerfile$"],
|
||||||
|
@ -18,5 +13,6 @@
|
||||||
"depNameTemplate": "just-containers/s6-overlay",
|
"depNameTemplate": "just-containers/s6-overlay",
|
||||||
"datasourceTemplate": "github-releases"
|
"datasourceTemplate": "github-releases"
|
||||||
}
|
}
|
||||||
]
|
],
|
||||||
|
"dependencyDashboardApproval": true
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,15 +1,13 @@
|
||||||
Django==5.0.4
|
Django==4.2.15
|
||||||
wagtail==5.2.5
|
wagtail==5.2.6
|
||||||
django-environ==0.11.2
|
django-environ==0.11.2
|
||||||
whitenoise[brotli]==6.6.0
|
whitenoise[brotli]==6.6.0
|
||||||
Pygments==2.17.2
|
Pygments==2.18.0
|
||||||
beautifulsoup4
|
beautifulsoup4[lxml]
|
||||||
lxml==5.2.1
|
|
||||||
requests
|
requests
|
||||||
wagtail-generic-chooser==0.6
|
wagtail-generic-chooser==0.6
|
||||||
django-rq==2.10.1
|
|
||||||
django-redis==5.4.0
|
django-redis==5.4.0
|
||||||
gunicorn==22.0.0
|
granian[pname]==1.6.0
|
||||||
psycopg==3.1.18
|
psycopg==3.1.18
|
||||||
djangorestframework
|
djangorestframework
|
||||||
django-htmx==1.17.2
|
django-htmx==1.17.2
|
||||||
|
@ -29,6 +27,9 @@ django-enforce-host==1.1.0
|
||||||
django-proxy==1.2.2
|
django-proxy==1.2.2
|
||||||
wagtail-lite-youtube-embed==0.1.0
|
wagtail-lite-youtube-embed==0.1.0
|
||||||
django-minify-html==1.7.1
|
django-minify-html==1.7.1
|
||||||
|
metadata-parser==0.12.1
|
||||||
|
django-tasks==0.5.0
|
||||||
|
lightningcss==0.2.0
|
||||||
|
|
||||||
# DRF OpenAPI dependencies
|
# DRF OpenAPI dependencies
|
||||||
uritemplate
|
uritemplate
|
||||||
|
|
|
@ -3,6 +3,7 @@ const STORAGE_KEY = "dark-mode";
|
||||||
|
|
||||||
const htmlTag = document.getElementsByTagName("html")[0];
|
const htmlTag = document.getElementsByTagName("html")[0];
|
||||||
const darkModeToggle = document.getElementById("dark-mode-toggle");
|
const darkModeToggle = document.getElementById("dark-mode-toggle");
|
||||||
|
const comentarioComments = document.getElementsByTagName("comentario-comments");
|
||||||
|
|
||||||
const matchesDarkMode = window.matchMedia("(prefers-color-scheme: dark)");
|
const matchesDarkMode = window.matchMedia("(prefers-color-scheme: dark)");
|
||||||
|
|
||||||
|
@ -14,6 +15,10 @@ function handleDarkMode(darkMode) {
|
||||||
} else {
|
} else {
|
||||||
htmlTag.classList.remove(DARK_MODE_CLASS);
|
htmlTag.classList.remove(DARK_MODE_CLASS);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for (const commentElement of comentarioComments) {
|
||||||
|
commentElement.setAttribute("theme", darkMode ? "dark" : "light");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if (window.localStorage.getItem(STORAGE_KEY) === null) {
|
if (window.localStorage.getItem(STORAGE_KEY) === null) {
|
||||||
|
|
|
@ -1,35 +0,0 @@
|
||||||
#commento {
|
|
||||||
.commento-profile-button {
|
|
||||||
@include dark-mode {
|
|
||||||
fill: $dark-mode-text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
.commento-name,
|
|
||||||
#commento-mod-tools-lock-button,
|
|
||||||
.commento-login-text,
|
|
||||||
.commento-anonymous-checkbox-container label {
|
|
||||||
@include dark-mode {
|
|
||||||
color: $dark-mode-text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#commento-textarea-root,
|
|
||||||
#commento-guest-details-input-root,
|
|
||||||
.commento-textarea-container textarea {
|
|
||||||
@include dark-mode {
|
|
||||||
background-color: $black;
|
|
||||||
color: $dark-mode-text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
.commento-card {
|
|
||||||
@include dark-mode {
|
|
||||||
border-top: 1px solid color.adjust($black, $alpha: -0.4);
|
|
||||||
|
|
||||||
p {
|
|
||||||
color: $dark-mode-text;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -67,5 +67,6 @@ section.content {
|
||||||
}
|
}
|
||||||
|
|
||||||
#comments {
|
#comments {
|
||||||
|
margin-top: 2rem;
|
||||||
scroll-margin-top: var(--hero-height); // hero height (ish)
|
scroll-margin-top: var(--hero-height); // hero height (ish)
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,6 +18,17 @@
|
||||||
|
|
||||||
.title {
|
.title {
|
||||||
margin-bottom: 0;
|
margin-bottom: 0;
|
||||||
|
|
||||||
|
a {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: row;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
|
||||||
|
i {
|
||||||
|
font-size: 50%;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
.content-details {
|
.content-details {
|
||||||
|
|
|
@ -15,8 +15,8 @@ section#similar-content {
|
||||||
}
|
}
|
||||||
|
|
||||||
.media {
|
.media {
|
||||||
@include desktop {
|
transform: scale(85%);
|
||||||
transform: scale(85%);
|
margin-top: 0;
|
||||||
}
|
padding-top: 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -19,7 +19,6 @@
|
||||||
@import "spotify";
|
@import "spotify";
|
||||||
@import "404";
|
@import "404";
|
||||||
@import "password_required";
|
@import "password_required";
|
||||||
@import "commento";
|
|
||||||
@import "similar_content";
|
@import "similar_content";
|
||||||
@import "support_pill";
|
@import "support_pill";
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
import factory
|
||||||
|
|
||||||
from website.common.factories import BaseContentFactory, BaseListingFactory
|
from website.common.factories import BaseContentFactory, BaseListingFactory
|
||||||
|
|
||||||
from . import models
|
from . import models
|
||||||
|
@ -11,3 +13,11 @@ class BlogPostListPageFactory(BaseListingFactory):
|
||||||
class BlogPostPageFactory(BaseContentFactory):
|
class BlogPostPageFactory(BaseContentFactory):
|
||||||
class Meta:
|
class Meta:
|
||||||
model = models.BlogPostPage
|
model = models.BlogPostPage
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalBlogPostPageFactory(BaseContentFactory):
|
||||||
|
external_url = factory.Faker("url")
|
||||||
|
|
||||||
|
class Meta:
|
||||||
|
model = models.ExternalBlogPostPage
|
||||||
|
exclude = ["subtitle"]
|
||||||
|
|
45
website/blog/migrations/0006_externalblogpostpage.py
Normal file
45
website/blog/migrations/0006_externalblogpostpage.py
Normal file
|
@ -0,0 +1,45 @@
|
||||||
|
# Generated by Django 5.0.4 on 2024-05-29 21:10
|
||||||
|
|
||||||
|
import django.db.models.deletion
|
||||||
|
import django.utils.timezone
|
||||||
|
import modelcluster.fields
|
||||||
|
import wagtailmetadata.models
|
||||||
|
from django.db import migrations, models
|
||||||
|
|
||||||
|
|
||||||
|
class Migration(migrations.Migration):
|
||||||
|
dependencies = [
|
||||||
|
("blog", "0005_auto_20230602_1236"),
|
||||||
|
("wagtailcore", "0089_log_entry_data_json_null_to_object"),
|
||||||
|
]
|
||||||
|
|
||||||
|
operations = [
|
||||||
|
migrations.CreateModel(
|
||||||
|
name="ExternalBlogPostPage",
|
||||||
|
fields=[
|
||||||
|
(
|
||||||
|
"page_ptr",
|
||||||
|
models.OneToOneField(
|
||||||
|
auto_created=True,
|
||||||
|
on_delete=django.db.models.deletion.CASCADE,
|
||||||
|
parent_link=True,
|
||||||
|
primary_key=True,
|
||||||
|
serialize=False,
|
||||||
|
to="wagtailcore.page",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("external_url", models.URLField()),
|
||||||
|
("date", models.DateField(default=django.utils.timezone.now)),
|
||||||
|
(
|
||||||
|
"tags",
|
||||||
|
modelcluster.fields.ParentalManyToManyField(
|
||||||
|
blank=True, to="blog.blogposttagpage"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
],
|
||||||
|
options={
|
||||||
|
"abstract": False,
|
||||||
|
},
|
||||||
|
bases=("wagtailcore.page", wagtailmetadata.models.MetadataMixin),
|
||||||
|
),
|
||||||
|
]
|
|
@ -1,18 +1,26 @@
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
from django.contrib.postgres.search import TrigramSimilarity
|
from django.contrib.postgres.search import TrigramSimilarity
|
||||||
from django.db import models
|
from django.db import models
|
||||||
from django.db.models.functions import Cast
|
from django.db.models.functions import Cast, Coalesce
|
||||||
|
from django.http import HttpRequest, HttpResponse, HttpResponsePermanentRedirect
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
from django.utils.functional import cached_property
|
from django.utils.functional import cached_property
|
||||||
|
from metadata_parser import ParsedResult
|
||||||
from modelcluster.fields import ParentalManyToManyField
|
from modelcluster.fields import ParentalManyToManyField
|
||||||
from wagtail.admin.panels import FieldPanel
|
from wagtail.admin.panels import FieldPanel
|
||||||
from wagtail.models import PageQuerySet
|
from wagtail.models import Page, PageQuerySet, Site
|
||||||
from wagtail.search import index
|
from wagtail.search import index
|
||||||
from wagtailautocomplete.edit_handlers import AutocompletePanel
|
from wagtailautocomplete.edit_handlers import AutocompletePanel
|
||||||
|
|
||||||
from website.common.models import BaseContentPage, BaseListingPage
|
from website.common.models import BaseContentPage, BaseListingPage, BasePage
|
||||||
from website.common.utils import TocEntry
|
from website.common.utils import (
|
||||||
|
TocEntry,
|
||||||
|
extend_query_params,
|
||||||
|
get_page_metadata,
|
||||||
|
get_url_mime_type,
|
||||||
|
)
|
||||||
from website.contrib.singleton_page.utils import SingletonPageCache
|
from website.contrib.singleton_page.utils import SingletonPageCache
|
||||||
|
|
||||||
|
|
||||||
|
@ -23,6 +31,8 @@ class BlogPostListPage(BaseListingPage):
|
||||||
"blog.BlogPostTagListPage",
|
"blog.BlogPostTagListPage",
|
||||||
"blog.BlogPostCollectionListPage",
|
"blog.BlogPostCollectionListPage",
|
||||||
"blog.BlogPostCollectionPage",
|
"blog.BlogPostCollectionPage",
|
||||||
|
"blog.BlogPostCollectionPage",
|
||||||
|
"blog.ExternalBlogPostPage",
|
||||||
]
|
]
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
|
@ -31,9 +41,12 @@ class BlogPostListPage(BaseListingPage):
|
||||||
|
|
||||||
def get_listing_pages(self) -> models.QuerySet:
|
def get_listing_pages(self) -> models.QuerySet:
|
||||||
return (
|
return (
|
||||||
BlogPostPage.objects.descendant_of(self)
|
Page.objects.live()
|
||||||
.live()
|
|
||||||
.public()
|
.public()
|
||||||
|
.annotate(date=Coalesce("blogpostpage__date", "externalblogpostpage__date"))
|
||||||
|
.descendant_of(self)
|
||||||
|
.type(BlogPostPage, ExternalBlogPostPage)
|
||||||
|
.specific()
|
||||||
.order_by("-date", "title")
|
.order_by("-date", "title")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -80,36 +93,40 @@ class BlogPostPage(BaseContentPage):
|
||||||
return SingletonPageCache.get_url(BlogPostListPage)
|
return SingletonPageCache.get_url(BlogPostListPage)
|
||||||
|
|
||||||
def get_similar_posts(self) -> models.QuerySet:
|
def get_similar_posts(self) -> models.QuerySet:
|
||||||
try:
|
listing_pages = BlogPostListPage.objects.get().get_listing_pages()
|
||||||
listing_pages = BlogPostListPage.objects.get().get_listing_pages()
|
|
||||||
except BlogPostListPage.DoesNotExist:
|
|
||||||
return BlogPostPage.objects.none()
|
|
||||||
|
|
||||||
similar_posts = listing_pages.exclude(id=self.id).alias(
|
similar_posts = listing_pages.exclude(id=self.id).alias(
|
||||||
title_similarity=TrigramSimilarity("title", self.title),
|
title_similarity=TrigramSimilarity("title", self.title),
|
||||||
# If this page has no subtitle, ignore it as part of similarity
|
|
||||||
subtitle_similarity=TrigramSimilarity("subtitle", self.subtitle)
|
|
||||||
if self.subtitle
|
|
||||||
else models.Value(1),
|
|
||||||
)
|
)
|
||||||
|
|
||||||
page_tags = list(self.tags.public().live().values_list("id", flat=True))
|
page_tags = list(self.tags.public().live().values_list("id", flat=True))
|
||||||
|
# If this page has no tags, ignore it as part of similarity
|
||||||
|
divisor = len(page_tags) if page_tags else models.Value(1)
|
||||||
similar_posts = similar_posts.alias(
|
similar_posts = similar_posts.alias(
|
||||||
# If this page has no tags, ignore it as part of similarity
|
|
||||||
# NB: Cast to a float, because `COUNT` returns a `bigint`.
|
# NB: Cast to a float, because `COUNT` returns a `bigint`.
|
||||||
tag_similarity=Cast(
|
_blog_tag_similarity=Cast(
|
||||||
models.Count("tags", filter=models.Q(tags__in=page_tags)),
|
models.Count(
|
||||||
|
"blogpostpage__tags",
|
||||||
|
filter=models.Q(blogpostpage__tags__in=page_tags),
|
||||||
|
),
|
||||||
output_field=models.FloatField(),
|
output_field=models.FloatField(),
|
||||||
)
|
)
|
||||||
/ len(page_tags)
|
/ divisor,
|
||||||
if page_tags
|
_external_tag_similarity=Cast(
|
||||||
else models.Value(1)
|
models.Count(
|
||||||
|
"externalblogpostpage__tags",
|
||||||
|
filter=models.Q(externalblogpostpage__tags__in=page_tags),
|
||||||
|
),
|
||||||
|
output_field=models.FloatField(),
|
||||||
|
)
|
||||||
|
/ divisor,
|
||||||
|
tag_similarity=models.F("_blog_tag_similarity")
|
||||||
|
+ models.F("_external_tag_similarity"),
|
||||||
)
|
)
|
||||||
|
|
||||||
similar_posts = similar_posts.annotate(
|
similar_posts = similar_posts.annotate(
|
||||||
similarity=(models.F("tag_similarity") * 2)
|
similarity=(models.F("tag_similarity") * 2)
|
||||||
+ (models.F("title_similarity") * 10)
|
+ (models.F("title_similarity") * 10)
|
||||||
+ (models.F("subtitle_similarity"))
|
|
||||||
).order_by("-similarity")[:3]
|
).order_by("-similarity")[:3]
|
||||||
|
|
||||||
return similar_posts
|
return similar_posts
|
||||||
|
@ -137,7 +154,12 @@ class BlogPostTagPage(BaseListingPage):
|
||||||
|
|
||||||
def get_listing_pages(self) -> models.QuerySet:
|
def get_listing_pages(self) -> models.QuerySet:
|
||||||
blog_list_page = BlogPostListPage.objects.get()
|
blog_list_page = BlogPostListPage.objects.get()
|
||||||
return blog_list_page.get_listing_pages().filter(tags=self)
|
listing_pages = blog_list_page.get_listing_pages()
|
||||||
|
|
||||||
|
return listing_pages.filter(
|
||||||
|
models.Q(blogpostpage__tags=self)
|
||||||
|
| models.Q(externalblogpostpage__tags=self)
|
||||||
|
).distinct()
|
||||||
|
|
||||||
|
|
||||||
class BlogPostCollectionListPage(BaseListingPage):
|
class BlogPostCollectionListPage(BaseListingPage):
|
||||||
|
@ -167,3 +189,101 @@ class BlogPostCollectionPage(BaseListingPage):
|
||||||
.public()
|
.public()
|
||||||
.order_by("-date", "title")
|
.order_by("-date", "title")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalBlogPostPage(BaseContentPage):
|
||||||
|
subpage_types: list[Any] = []
|
||||||
|
parent_page_types = [BlogPostListPage]
|
||||||
|
preview_modes: list[Any] = []
|
||||||
|
|
||||||
|
is_external = True
|
||||||
|
|
||||||
|
# Some `BaseContentPage` fields aren't relevant
|
||||||
|
body = None
|
||||||
|
subtitle = None
|
||||||
|
hero_image = None
|
||||||
|
hero_unsplash_photo = None
|
||||||
|
|
||||||
|
external_url = models.URLField()
|
||||||
|
|
||||||
|
tags = ParentalManyToManyField("blog.BlogPostTagPage", blank=True)
|
||||||
|
date = models.DateField(default=timezone.now)
|
||||||
|
|
||||||
|
content_panels = BasePage.content_panels + [FieldPanel("external_url")]
|
||||||
|
|
||||||
|
promote_panels = BaseContentPage.promote_panels + [
|
||||||
|
FieldPanel("date"),
|
||||||
|
AutocompletePanel("tags"),
|
||||||
|
]
|
||||||
|
|
||||||
|
search_fields = BaseContentPage.search_fields + [
|
||||||
|
index.RelatedFields("tags", [index.SearchField("title", boost=1)]),
|
||||||
|
index.SearchField("external_url"),
|
||||||
|
]
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def tag_list_page_url(self) -> Optional[str]:
|
||||||
|
return SingletonPageCache.get_url(BlogPostTagListPage)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def tags_list(self) -> models.QuerySet:
|
||||||
|
"""
|
||||||
|
Use this to get a page's tags.
|
||||||
|
"""
|
||||||
|
tags = self.tags.order_by("slug")
|
||||||
|
|
||||||
|
# In drafts, `django-modelcluster` doesn't support these filters
|
||||||
|
if isinstance(tags, PageQuerySet):
|
||||||
|
return tags.public().live()
|
||||||
|
|
||||||
|
return tags
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def metadata(self) -> ParsedResult:
|
||||||
|
return get_page_metadata(self.external_url)
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def _body_html(self) -> str:
|
||||||
|
try:
|
||||||
|
return self.metadata.get_metadatas("description")[0]
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
return ""
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def plain_text(self) -> str:
|
||||||
|
# The metadata is already just text
|
||||||
|
return self._body_html
|
||||||
|
|
||||||
|
def hero_url(
|
||||||
|
self, image_size: str, wagtail_image_spec_extra: Optional[str] = None
|
||||||
|
) -> Optional[str]:
|
||||||
|
try:
|
||||||
|
return self.metadata.get_metadatas("image")[0]
|
||||||
|
except (KeyError, IndexError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def hero_image_url(self) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
@cached_property
|
||||||
|
def hero_image_alt(self) -> str:
|
||||||
|
return ""
|
||||||
|
|
||||||
|
def get_meta_image_mime(self) -> Optional[str]:
|
||||||
|
return get_url_mime_type(self.hero_url(""))
|
||||||
|
|
||||||
|
def get_url(
|
||||||
|
self, request: HttpRequest | None = None, current_site: Site | None = None
|
||||||
|
) -> str:
|
||||||
|
return self.get_full_url(request)
|
||||||
|
|
||||||
|
def get_full_url(self, request: HttpRequest | None = None) -> str:
|
||||||
|
full_url = urlsplit(super().get_full_url(request))
|
||||||
|
return extend_query_params(self.external_url, {"utm_source": full_url.netloc})
|
||||||
|
|
||||||
|
def serve(self, request: HttpRequest, *args: tuple, **kwargs: dict) -> HttpResponse:
|
||||||
|
"""
|
||||||
|
Send the user directly to the external page
|
||||||
|
"""
|
||||||
|
return HttpResponsePermanentRedirect(self.get_full_url(request))
|
||||||
|
|
|
@ -2,6 +2,16 @@
|
||||||
|
|
||||||
{% load wagtail_cache navbar_tags %}
|
{% load wagtail_cache navbar_tags %}
|
||||||
|
|
||||||
|
{% block post_toc %}
|
||||||
|
<hr class="dropdown-divider" />
|
||||||
|
<li>
|
||||||
|
<a href="#similar-content">Similar content</a>
|
||||||
|
</li>
|
||||||
|
<li>
|
||||||
|
<a href="#comments">Comments</a>
|
||||||
|
</li>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
{% block post_content %}
|
{% block post_content %}
|
||||||
{% if not request.is_preview %}
|
{% if not request.is_preview %}
|
||||||
{% include "common/shareon.html" %}
|
{% include "common/shareon.html" %}
|
||||||
|
|
16
website/blog/templates/blog/external_blog_post_page.html
Normal file
16
website/blog/templates/blog/external_blog_post_page.html
Normal file
|
@ -0,0 +1,16 @@
|
||||||
|
{% comment %}
|
||||||
|
This template is never used, but exists just in case.
|
||||||
|
{% endcomment %}
|
||||||
|
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html lang="en-GB">
|
||||||
|
<head>
|
||||||
|
<title>Redirecting...</title>
|
||||||
|
<link rel="canonical" href="{{ page.external_url }}" />
|
||||||
|
<meta charset="utf-8" />
|
||||||
|
<meta http-equiv="refresh" content="0; url={{ page.external_url }}" />
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<p>Redirecting...</p>
|
||||||
|
</body>
|
||||||
|
</html>
|
|
@ -1,9 +1,15 @@
|
||||||
|
import pickle
|
||||||
|
|
||||||
from django.test import TestCase
|
from django.test import TestCase
|
||||||
from django.urls import reverse
|
from django.urls import reverse
|
||||||
|
|
||||||
from website.home.models import HomePage
|
from website.home.models import HomePage
|
||||||
|
|
||||||
from .factories import BlogPostListPageFactory, BlogPostPageFactory
|
from .factories import (
|
||||||
|
BlogPostListPageFactory,
|
||||||
|
BlogPostPageFactory,
|
||||||
|
ExternalBlogPostPageFactory,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class BlogPostPageTestCase(TestCase):
|
class BlogPostPageTestCase(TestCase):
|
||||||
|
@ -69,14 +75,15 @@ class BlogPostListPageTestCase(TestCase):
|
||||||
|
|
||||||
BlogPostPageFactory(parent=cls.page)
|
BlogPostPageFactory(parent=cls.page)
|
||||||
BlogPostPageFactory(parent=cls.page)
|
BlogPostPageFactory(parent=cls.page)
|
||||||
|
ExternalBlogPostPageFactory(parent=cls.page, external_url="https://example.com")
|
||||||
|
|
||||||
def test_accessible(self) -> None:
|
def test_accessible(self) -> None:
|
||||||
response = self.client.get(self.page.url)
|
response = self.client.get(self.page.url)
|
||||||
self.assertEqual(response.status_code, 200)
|
self.assertEqual(response.status_code, 200)
|
||||||
self.assertEqual(len(response.context["listing_pages"]), 2)
|
self.assertEqual(len(response.context["listing_pages"]), 3)
|
||||||
|
|
||||||
def test_queries(self) -> None:
|
def test_queries(self) -> None:
|
||||||
with self.assertNumQueries(39):
|
with self.assertNumQueries(43):
|
||||||
self.client.get(self.page.url)
|
self.client.get(self.page.url)
|
||||||
|
|
||||||
def test_feed_accessible(self) -> None:
|
def test_feed_accessible(self) -> None:
|
||||||
|
@ -84,3 +91,31 @@ class BlogPostListPageTestCase(TestCase):
|
||||||
self.assertRedirects(
|
self.assertRedirects(
|
||||||
response, reverse("feed"), status_code=301, fetch_redirect_response=True
|
response, reverse("feed"), status_code=301, fetch_redirect_response=True
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ExternalBlogPostPageTestCase(TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpTestData(cls) -> None:
|
||||||
|
cls.home_page = HomePage.objects.get()
|
||||||
|
cls.blog_post_list_page = BlogPostListPageFactory(parent=cls.home_page)
|
||||||
|
cls.page = ExternalBlogPostPageFactory(
|
||||||
|
parent=cls.blog_post_list_page, external_url="https://example.com"
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_redirects(self) -> None:
|
||||||
|
with self.assertNumQueries(10):
|
||||||
|
response = self.client.get(self.page.url)
|
||||||
|
self.assertRedirects(
|
||||||
|
response,
|
||||||
|
self.page.external_url + "?utm_source=localhost",
|
||||||
|
status_code=301,
|
||||||
|
fetch_redirect_response=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_metadata(self) -> None:
|
||||||
|
metadata = self.page.metadata
|
||||||
|
|
||||||
|
self.assertIsNone(metadata.soup)
|
||||||
|
|
||||||
|
# Confirm it can pickle
|
||||||
|
pickle.dumps(metadata)
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
from datetime import timedelta
|
from datetime import timedelta
|
||||||
from math import ceil
|
from math import ceil
|
||||||
from typing import Any, Optional
|
from typing import Any, Optional
|
||||||
from urllib.parse import urlencode
|
|
||||||
|
|
||||||
from django.core.paginator import EmptyPage, Paginator
|
from django.core.paginator import EmptyPage, Paginator
|
||||||
from django.core.paginator import Page as PaginatorPage
|
from django.core.paginator import Page as PaginatorPage
|
||||||
|
@ -31,6 +30,7 @@ from .serializers import PaginationSerializer
|
||||||
from .streamfield import add_heading_anchors, get_blocks, get_content_html
|
from .streamfield import add_heading_anchors, get_blocks, get_content_html
|
||||||
from .utils import (
|
from .utils import (
|
||||||
TocEntry,
|
TocEntry,
|
||||||
|
extend_query_params,
|
||||||
extract_text,
|
extract_text,
|
||||||
get_site_title,
|
get_site_title,
|
||||||
get_table_of_contents,
|
get_table_of_contents,
|
||||||
|
@ -181,6 +181,7 @@ class BaseContentPage(BasePage, MetadataMixin):
|
||||||
for size, width in UNSPLASH_SIZES.items()
|
for size, width in UNSPLASH_SIZES.items()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@cached_property
|
||||||
def hero_image_url(self) -> Optional[str]:
|
def hero_image_url(self) -> Optional[str]:
|
||||||
return self.hero_url("regular")
|
return self.hero_url("regular")
|
||||||
|
|
||||||
|
@ -286,10 +287,7 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
|
||||||
|
|
||||||
url = super().get_meta_url()
|
url = super().get_meta_url()
|
||||||
|
|
||||||
if not query_data:
|
return extend_query_params(url, query_data)
|
||||||
return url
|
|
||||||
|
|
||||||
return url + "?" + urlencode(query_data)
|
|
||||||
|
|
||||||
@route(r"^feed/$")
|
@route(r"^feed/$")
|
||||||
def feed(self, request: HttpRequest) -> HttpResponse:
|
def feed(self, request: HttpRequest) -> HttpResponse:
|
||||||
|
@ -299,8 +297,11 @@ class BaseListingPage(RoutablePageMixin, BaseContentPage):
|
||||||
def random(self, request: HttpRequest) -> HttpResponse:
|
def random(self, request: HttpRequest) -> HttpResponse:
|
||||||
page = self.get_listing_pages().order_by("?").first()
|
page = self.get_listing_pages().order_by("?").first()
|
||||||
if page is None:
|
if page is None:
|
||||||
return redirect(self.get_url(request=request), permanent=False)
|
response = redirect(self.get_url(request=request), permanent=False)
|
||||||
return redirect(page.get_url(request=request), permanent=False)
|
else:
|
||||||
|
response = redirect(page.get_url(request=request), permanent=False)
|
||||||
|
response.headers["X-Robots-Tag"] = "noindex"
|
||||||
|
return response
|
||||||
|
|
||||||
|
|
||||||
class ListingPage(BaseListingPage):
|
class ListingPage(BaseListingPage):
|
||||||
|
|
|
@ -1,5 +1,5 @@
|
||||||
<section class="container" id="comments">
|
<section class="container" id="comments">
|
||||||
<div id="commento"></div>
|
<comentario-comments></comentario-comments>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
<script async defer src="https://commento.theorangeone.net/js/commento.js"></script>
|
<script async defer src="https://comentario.theorangeone.net/comentario.js"></script>
|
||||||
|
|
|
@ -10,6 +10,13 @@
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
|
{% block post_toc %}
|
||||||
|
<hr class="dropdown-divider" />
|
||||||
|
<li>
|
||||||
|
<a href="#comments">Comments</a>
|
||||||
|
</li>
|
||||||
|
{% endblock %}
|
||||||
|
|
||||||
{% block post_content %}
|
{% block post_content %}
|
||||||
{% if not request.is_preview %}
|
{% if not request.is_preview %}
|
||||||
{% include "common/shareon.html" %}
|
{% include "common/shareon.html" %}
|
||||||
|
|
|
@ -16,7 +16,10 @@
|
||||||
{% include "common/breadcrumbs.html" with parents=page.get_parent_pages %}
|
{% include "common/breadcrumbs.html" with parents=page.get_parent_pages %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
<h2 class="title is-3">
|
<h2 class="title is-3">
|
||||||
<a href="{% pageurl page %}">{{ page.title }}</a>
|
<a href="{% pageurl page %}">
|
||||||
|
{{ page.title }}
|
||||||
|
{% if page.is_external %}<i class="fa-solid fa-arrow-up-right-from-square" title="This page is from a external source"></i>{% endif %}
|
||||||
|
</a>
|
||||||
</h2>
|
</h2>
|
||||||
{% include "common/content-details.html" %}
|
{% include "common/content-details.html" %}
|
||||||
<p>{{ page.summary }}</p>
|
<p>{{ page.summary }}</p>
|
||||||
|
|
|
@ -1,11 +1,7 @@
|
||||||
{% if SEO_INDEX %}
|
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Allow: /
|
{% if SEO_INDEX %}Allow: /{% else %}Disallow: /{% endif %}
|
||||||
{% else %}
|
|
||||||
User-agent: *
|
# https://github.com/ai-robots-txt/ai.robots.txt
|
||||||
Disallow: /
|
{{ ai_robots_txt }}
|
||||||
{% endif %}
|
|
||||||
|
|
||||||
Disallow: {% url "wagtailadmin_home" %}
|
|
||||||
Disallow: {% url "api:index" %}
|
|
||||||
Sitemap: {{ sitemap }}
|
Sitemap: {{ sitemap }}
|
||||||
|
|
|
@ -56,6 +56,7 @@
|
||||||
{% for toc_item in page.table_of_contents %}
|
{% for toc_item in page.table_of_contents %}
|
||||||
{% include "common/toc-item.html" %}
|
{% include "common/toc-item.html" %}
|
||||||
{% endfor %}
|
{% endfor %}
|
||||||
|
{% block post_toc %}{% endblock %}
|
||||||
</ul>
|
</ul>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -13,6 +13,8 @@
|
||||||
|
|
||||||
<meta name="article:modified_time" content="{{ object.last_published_at|date:'c' }}" />
|
<meta name="article:modified_time" content="{{ object.last_published_at|date:'c' }}" />
|
||||||
{% if object.date %}<meta name="article:published_time" content="{{ object.date|date:'c' }}" />{% endif %}
|
{% if object.date %}<meta name="article:published_time" content="{{ object.date|date:'c' }}" />{% endif %}
|
||||||
|
|
||||||
|
<meta name="fediverse:creator" content="@jake@theorangeone.net" />
|
||||||
{% endblock %}
|
{% endblock %}
|
||||||
|
|
||||||
{% block meta %}
|
{% block meta %}
|
||||||
|
|
|
@ -3,6 +3,7 @@ from django.test import SimpleTestCase
|
||||||
from wagtail.rich_text import features as richtext_feature_registry
|
from wagtail.rich_text import features as richtext_feature_registry
|
||||||
|
|
||||||
from website.common.utils import (
|
from website.common.utils import (
|
||||||
|
extend_query_params,
|
||||||
extract_text,
|
extract_text,
|
||||||
get_table_of_contents,
|
get_table_of_contents,
|
||||||
heading_id,
|
heading_id,
|
||||||
|
@ -111,3 +112,25 @@ class HeadingIDTestCase(SimpleTestCase):
|
||||||
self.assertEqual(heading_id("123"), "ref-123")
|
self.assertEqual(heading_id("123"), "ref-123")
|
||||||
self.assertEqual(heading_id("test"), "test")
|
self.assertEqual(heading_id("test"), "test")
|
||||||
self.assertEqual(heading_id("Look, a title!"), "look-a-title")
|
self.assertEqual(heading_id("Look, a title!"), "look-a-title")
|
||||||
|
|
||||||
|
|
||||||
|
class ExtendQueryParamsTestCase(SimpleTestCase):
|
||||||
|
def test_params(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
extend_query_params("https://example.com", {"foo": "bar"}),
|
||||||
|
"https://example.com?foo=bar",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
extend_query_params("https://example.com?foo=bar", {"bar": "foo"}),
|
||||||
|
"https://example.com?foo=bar&bar=foo",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
extend_query_params("https://example.com?foo=baz", {"foo": "baz"}),
|
||||||
|
"https://example.com?foo=baz",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_removes_param(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
extend_query_params("https://example.com?foo=bar", {"foo": None}),
|
||||||
|
"https://example.com",
|
||||||
|
)
|
||||||
|
|
|
@ -1,14 +1,17 @@
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
from itertools import pairwise
|
from itertools import pairwise
|
||||||
from typing import Optional, Type
|
from typing import Any, Optional, Type
|
||||||
|
from urllib.parse import urlsplit, urlunsplit
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
from bs4 import BeautifulSoup, SoupStrainer
|
from bs4 import BeautifulSoup, SoupStrainer
|
||||||
from django.conf import settings
|
from django.conf import settings
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
from django.http import QueryDict
|
||||||
from django.http.request import HttpRequest
|
from django.http.request import HttpRequest
|
||||||
from django.utils.text import slugify
|
from django.utils.text import slugify
|
||||||
from django_cache_decorator import django_cache_decorator
|
from django_cache_decorator import django_cache_decorator
|
||||||
|
from metadata_parser import MetadataParser, ParsedResult
|
||||||
from wagtail.models import Page, Site
|
from wagtail.models import Page, Site
|
||||||
from wagtail.models import get_page_models as get_wagtail_page_models
|
from wagtail.models import get_page_models as get_wagtail_page_models
|
||||||
|
|
||||||
|
@ -112,3 +115,36 @@ def get_or_none(queryset: models.QuerySet) -> models.Model:
|
||||||
return queryset.get()
|
return queryset.get()
|
||||||
except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
|
except (queryset.model.DoesNotExist, queryset.model.MultipleObjectsReturned):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
@django_cache_decorator(time=21600)
|
||||||
|
def get_ai_robots_txt() -> str:
|
||||||
|
"""
|
||||||
|
https://github.com/ai-robots-txt/ai.robots.txt
|
||||||
|
"""
|
||||||
|
return requests_session.get(
|
||||||
|
"https://raw.githubusercontent.com/ai-robots-txt/ai.robots.txt/main/robots.txt"
|
||||||
|
).content.decode()
|
||||||
|
|
||||||
|
|
||||||
|
@django_cache_decorator(time=21600)
|
||||||
|
def get_page_metadata(url: str) -> ParsedResult:
|
||||||
|
metadata = MetadataParser(url=url, search_head_only=True).parsed_result
|
||||||
|
|
||||||
|
# HACK: BeautifulSoup doesn't pickle nicely, and so can't be cached
|
||||||
|
metadata.soup = None
|
||||||
|
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
def extend_query_params(url: str, params: dict[str, Any]) -> str:
|
||||||
|
scheme, netloc, path, query, fragment = urlsplit(url)
|
||||||
|
query_dict = QueryDict(query, mutable=True)
|
||||||
|
|
||||||
|
for k, v in params.items():
|
||||||
|
if v is None:
|
||||||
|
del query_dict[k]
|
||||||
|
else:
|
||||||
|
query_dict[k] = v
|
||||||
|
|
||||||
|
return urlunsplit((scheme, netloc, path, query_dict.urlencode(), fragment))
|
||||||
|
|
|
@ -23,6 +23,7 @@ from website.search.models import SearchPage
|
||||||
|
|
||||||
from .feed_generators import CustomFeed
|
from .feed_generators import CustomFeed
|
||||||
from .models import BaseListingPage, BasePage
|
from .models import BaseListingPage, BasePage
|
||||||
|
from .utils import extend_query_params, get_ai_robots_txt
|
||||||
|
|
||||||
|
|
||||||
class Error404View(TemplateView):
|
class Error404View(TemplateView):
|
||||||
|
@ -52,6 +53,7 @@ class RobotsView(TemplateView):
|
||||||
def get_context_data(self, **kwargs: dict) -> dict:
|
def get_context_data(self, **kwargs: dict) -> dict:
|
||||||
context = super().get_context_data(**kwargs)
|
context = super().get_context_data(**kwargs)
|
||||||
context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
|
context["sitemap"] = self.request.build_absolute_uri(reverse("sitemap"))
|
||||||
|
context["ai_robots_txt"] = get_ai_robots_txt()
|
||||||
return context
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
@ -114,7 +116,9 @@ class AllPagesFeed(Feed):
|
||||||
return item.title
|
return item.title
|
||||||
|
|
||||||
def item_link(self, item: BasePage) -> str:
|
def item_link(self, item: BasePage) -> str:
|
||||||
return item.get_full_url(request=self.request) + "?utm_medium=rss"
|
return extend_query_params(
|
||||||
|
item.get_full_url(request=self.request), {"utm_medium": "rss"}
|
||||||
|
)
|
||||||
|
|
||||||
def item_pubdate(self, item: BasePage) -> datetime:
|
def item_pubdate(self, item: BasePage) -> datetime:
|
||||||
if item_date := getattr(item, "date", None):
|
if item_date := getattr(item, "date", None):
|
||||||
|
|
|
@ -1,21 +1,16 @@
|
||||||
|
import lightningcss
|
||||||
from django.http import HttpRequest, HttpResponse
|
from django.http import HttpRequest, HttpResponse
|
||||||
from django.utils.datastructures import OrderedSet
|
|
||||||
from django.views.decorators.cache import cache_control
|
from django.views.decorators.cache import cache_control
|
||||||
from pygments.formatters.html import HtmlFormatter
|
from pygments.formatters.html import HtmlFormatter
|
||||||
|
|
||||||
|
|
||||||
@cache_control(max_age=3600)
|
@cache_control(max_age=3600)
|
||||||
def pygments_styles(request: HttpRequest) -> HttpResponse:
|
def pygments_styles(request: HttpRequest) -> HttpResponse:
|
||||||
default_styles = (
|
default_styles = HtmlFormatter(style="default").get_style_defs(
|
||||||
HtmlFormatter(style="default")
|
"html:not(.dark-mode) .highlight"
|
||||||
.get_style_defs("html:not(.dark-mode) .highlight")
|
|
||||||
.split("\n")
|
|
||||||
)
|
)
|
||||||
dark_styles = (
|
dark_styles = HtmlFormatter(style="monokai").get_style_defs(
|
||||||
HtmlFormatter(style="monokai")
|
"html.dark-mode .highlight"
|
||||||
.get_style_defs("html.dark-mode .highlight")
|
|
||||||
.split("\n")
|
|
||||||
)
|
|
||||||
return HttpResponse(
|
|
||||||
"".join(OrderedSet(default_styles + dark_styles)), content_type="text/css"
|
|
||||||
)
|
)
|
||||||
|
compressed = lightningcss.process_stylesheet(default_styles + dark_styles)
|
||||||
|
return HttpResponse(compressed, content_type="text/css")
|
||||||
|
|
|
@ -3,13 +3,15 @@ from datetime import timedelta
|
||||||
|
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
from django.utils import timezone
|
from django.utils import timezone
|
||||||
|
from django_tasks import task
|
||||||
|
|
||||||
from website.contrib.unsplash.models import UnsplashPhoto
|
from website.contrib.unsplash.models import UnsplashPhoto
|
||||||
from website.contrib.unsplash.utils import get_unsplash_photo
|
from website.contrib.unsplash.utils import get_unsplash_photo
|
||||||
from website.utils.queue import enqueue_or_sync
|
|
||||||
|
|
||||||
|
|
||||||
def update_photo(photo: UnsplashPhoto) -> None:
|
@task()
|
||||||
|
def update_photo(photo_id: int) -> None:
|
||||||
|
photo = UnsplashPhoto.objects.get(id=photo_id)
|
||||||
photo.data = get_unsplash_photo(photo.unsplash_id)
|
photo.data = get_unsplash_photo(photo.unsplash_id)
|
||||||
photo.data_last_updated = timezone.now()
|
photo.data_last_updated = timezone.now()
|
||||||
photo.save()
|
photo.save()
|
||||||
|
@ -25,6 +27,6 @@ class Command(BaseCommand):
|
||||||
photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
|
photos = UnsplashPhoto.objects.filter(data_last_updated__lte=max_age)
|
||||||
self.stdout.write(f"Found {photos.count()} photos to update.")
|
self.stdout.write(f"Found {photos.count()} photos to update.")
|
||||||
|
|
||||||
for photo in photos:
|
for photo_id, unsplash_id in photos.values_list("id", "unsplash_id"):
|
||||||
self.stdout.write(f"Updating {photo.unsplash_id}")
|
self.stdout.write(f"Updating {unsplash_id}")
|
||||||
enqueue_or_sync(update_photo, args=[photo])
|
update_photo.enqueue(photo_id)
|
||||||
|
|
|
@ -7,7 +7,7 @@ from wagtail.search.utils import parse_query_string
|
||||||
from wagtail_favicon.models import FaviconSettings
|
from wagtail_favicon.models import FaviconSettings
|
||||||
from wagtail_favicon.utils import get_rendition_url
|
from wagtail_favicon.utils import get_rendition_url
|
||||||
|
|
||||||
from website.common.utils import get_or_none, get_site_title
|
from website.common.utils import extend_query_params, get_or_none, get_site_title
|
||||||
from website.contrib.singleton_page.utils import SingletonPageCache
|
from website.contrib.singleton_page.utils import SingletonPageCache
|
||||||
|
|
||||||
from .models import SearchPage
|
from .models import SearchPage
|
||||||
|
@ -87,4 +87,4 @@ class GoView(RedirectView):
|
||||||
if slug_match := get_or_none(pages.filter(slug__iexact=query)):
|
if slug_match := get_or_none(pages.filter(slug__iexact=query)):
|
||||||
return slug_match.get_url(request=self.request)
|
return slug_match.get_url(request=self.request)
|
||||||
|
|
||||||
return f"{search_page_url}?{self.request.GET.urlencode()}"
|
return extend_query_params(search_page_url, self.request.GET)
|
||||||
|
|
|
@ -69,7 +69,6 @@ INSTALLED_APPS = [
|
||||||
"generic_chooser",
|
"generic_chooser",
|
||||||
"wagtail_draftail_snippet",
|
"wagtail_draftail_snippet",
|
||||||
"wagtailautocomplete",
|
"wagtailautocomplete",
|
||||||
"django_rq",
|
|
||||||
"rest_framework",
|
"rest_framework",
|
||||||
"corsheaders",
|
"corsheaders",
|
||||||
"wagtail_favicon",
|
"wagtail_favicon",
|
||||||
|
@ -80,6 +79,7 @@ INSTALLED_APPS = [
|
||||||
"django_otp",
|
"django_otp",
|
||||||
"django_otp.plugins.otp_totp",
|
"django_otp.plugins.otp_totp",
|
||||||
"django_minify_html",
|
"django_minify_html",
|
||||||
|
"django_tasks.backends.database",
|
||||||
"health_check",
|
"health_check",
|
||||||
"health_check.db",
|
"health_check.db",
|
||||||
"health_check.cache",
|
"health_check.cache",
|
||||||
|
@ -146,16 +146,10 @@ CACHES = {
|
||||||
# https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
|
# https://docs.wagtail.io/en/v2.13/reference/settings.html#redirects
|
||||||
WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
|
WAGTAIL_REDIRECTS_FILE_STORAGE = "cache"
|
||||||
|
|
||||||
RQ_QUEUES = {}
|
if TEST:
|
||||||
|
TASKS = {"default": {"BACKEND": "django_tasks.backends.immediate.ImmediateBackend"}}
|
||||||
USE_REDIS_QUEUE = False
|
else:
|
||||||
if queue_store := env.cache(
|
TASKS = {"default": {"BACKEND": "django_tasks.backends.database.DatabaseBackend"}}
|
||||||
"QUEUE_STORE_URL", default=None, backend="django_redis.cache.RedisCache"
|
|
||||||
):
|
|
||||||
CACHES["rq"] = queue_store
|
|
||||||
USE_REDIS_QUEUE = True
|
|
||||||
RQ_QUEUES["default"] = {"USE_REDIS_CACHE": "rq"}
|
|
||||||
|
|
||||||
|
|
||||||
# Internationalization
|
# Internationalization
|
||||||
# https://docs.djangoproject.com/en/4.0/topics/i18n/
|
# https://docs.djangoproject.com/en/4.0/topics/i18n/
|
||||||
|
@ -320,14 +314,15 @@ if DEBUG:
|
||||||
INSTALLED_APPS.append("django_browser_reload")
|
INSTALLED_APPS.append("django_browser_reload")
|
||||||
MIDDLEWARE.append("django_browser_reload.middleware.BrowserReloadMiddleware")
|
MIDDLEWARE.append("django_browser_reload.middleware.BrowserReloadMiddleware")
|
||||||
|
|
||||||
# Add django-debug-toolbar
|
if not TEST:
|
||||||
INSTALLED_APPS.append("debug_toolbar")
|
# Add django-debug-toolbar
|
||||||
MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
|
INSTALLED_APPS.append("debug_toolbar")
|
||||||
DEBUG_TOOLBAR_CONFIG = {
|
MIDDLEWARE.append("debug_toolbar.middleware.DebugToolbarMiddleware")
|
||||||
"SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
|
DEBUG_TOOLBAR_CONFIG = {
|
||||||
"RESULTS_CACHE_SIZE": 5,
|
"SHOW_TOOLBAR_CALLBACK": "website.common.utils.show_toolbar_callback",
|
||||||
"SHOW_COLLAPSED": True,
|
"RESULTS_CACHE_SIZE": 5,
|
||||||
}
|
"SHOW_COLLAPSED": True,
|
||||||
|
}
|
||||||
|
|
||||||
# Add Wagtail styleguide
|
# Add Wagtail styleguide
|
||||||
INSTALLED_APPS.append("wagtail.contrib.styleguide")
|
INSTALLED_APPS.append("wagtail.contrib.styleguide")
|
||||||
|
@ -391,6 +386,11 @@ LOGGING = {
|
||||||
"level": "WARNING",
|
"level": "WARNING",
|
||||||
"propagate": False,
|
"propagate": False,
|
||||||
},
|
},
|
||||||
|
"metadata_parser": {
|
||||||
|
"handlers": ["console"],
|
||||||
|
"level": "CRITICAL",
|
||||||
|
"propagate": False,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -442,12 +442,11 @@ if sentry_dsn := env("SENTRY_DSN"):
|
||||||
import sentry_sdk
|
import sentry_sdk
|
||||||
from sentry_sdk.integrations.django import DjangoIntegration
|
from sentry_sdk.integrations.django import DjangoIntegration
|
||||||
from sentry_sdk.integrations.redis import RedisIntegration
|
from sentry_sdk.integrations.redis import RedisIntegration
|
||||||
from sentry_sdk.integrations.rq import RqIntegration
|
|
||||||
from sentry_sdk.utils import get_default_release
|
from sentry_sdk.utils import get_default_release
|
||||||
|
|
||||||
sentry_kwargs = {
|
sentry_kwargs = {
|
||||||
"dsn": sentry_dsn,
|
"dsn": sentry_dsn,
|
||||||
"integrations": [DjangoIntegration(), RqIntegration(), RedisIntegration()],
|
"integrations": [DjangoIntegration(), RedisIntegration()],
|
||||||
"release": get_default_release(),
|
"release": get_default_release(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -1,10 +1,11 @@
|
||||||
from django.core.cache import cache
|
from django.core.cache import cache
|
||||||
from django.core.management.base import BaseCommand
|
from django.core.management.base import BaseCommand
|
||||||
|
from django_tasks import task
|
||||||
|
|
||||||
from website.spotify.models import SpotifyPlaylistPage
|
from website.spotify.models import SpotifyPlaylistPage
|
||||||
from website.utils.queue import enqueue_or_sync
|
|
||||||
|
|
||||||
|
|
||||||
|
@task()
|
||||||
def refresh_cache(page_id: int) -> None:
|
def refresh_cache(page_id: int) -> None:
|
||||||
page = SpotifyPlaylistPage.objects.get(id=page_id)
|
page = SpotifyPlaylistPage.objects.get(id=page_id)
|
||||||
cache.delete(page.playlist_cache_key)
|
cache.delete(page.playlist_cache_key)
|
||||||
|
@ -15,5 +16,7 @@ def refresh_cache(page_id: int) -> None:
|
||||||
|
|
||||||
class Command(BaseCommand):
|
class Command(BaseCommand):
|
||||||
def handle(self, *args: list, **options: dict) -> None:
|
def handle(self, *args: list, **options: dict) -> None:
|
||||||
for page in SpotifyPlaylistPage.objects.all().defer_streamfields().iterator():
|
for page_id in (
|
||||||
enqueue_or_sync(refresh_cache, args=[page.id])
|
SpotifyPlaylistPage.objects.all().values_list("id", flat=True).iterator()
|
||||||
|
):
|
||||||
|
refresh_cache.enqueue(page_id)
|
||||||
|
|
|
@ -12,6 +12,14 @@ class TalksListPage(BaseListingPage):
|
||||||
max_count = 1
|
max_count = 1
|
||||||
subpage_types = ["talks.TalkPage"]
|
subpage_types = ["talks.TalkPage"]
|
||||||
|
|
||||||
|
def get_listing_pages(self) -> models.QuerySet:
|
||||||
|
return (
|
||||||
|
TalkPage.objects.live()
|
||||||
|
.public()
|
||||||
|
.descendant_of(self)
|
||||||
|
.order_by("-date", "title")
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TalkPage(BaseContentPage):
|
class TalkPage(BaseContentPage):
|
||||||
subpage_types: list[Any] = []
|
subpage_types: list[Any] = []
|
||||||
|
|
|
@ -37,7 +37,7 @@ class TalksListPageTestCase(TestCase):
|
||||||
self.assertEqual(len(response.context["listing_pages"]), 2)
|
self.assertEqual(len(response.context["listing_pages"]), 2)
|
||||||
|
|
||||||
def test_queries(self) -> None:
|
def test_queries(self) -> None:
|
||||||
with self.assertNumQueries(35):
|
with self.assertNumQueries(34):
|
||||||
self.client.get(self.page.url)
|
self.client.get(self.page.url)
|
||||||
|
|
||||||
def test_feed_accessible(self) -> None:
|
def test_feed_accessible(self) -> None:
|
||||||
|
|
|
@ -1,21 +0,0 @@
|
||||||
from typing import Callable
|
|
||||||
|
|
||||||
from django.conf import settings
|
|
||||||
from django_rq import get_queue
|
|
||||||
|
|
||||||
|
|
||||||
def enqueue_or_sync(
|
|
||||||
job_func: Callable, args: list | None = None, kwargs: dict | None = None
|
|
||||||
) -> None:
|
|
||||||
"""
|
|
||||||
Run a task now, or put in RQ
|
|
||||||
"""
|
|
||||||
if args is None:
|
|
||||||
args = []
|
|
||||||
if kwargs is None:
|
|
||||||
kwargs = {}
|
|
||||||
|
|
||||||
if settings.USE_REDIS_QUEUE:
|
|
||||||
get_queue().enqueue(job_func, args=args, kwargs=kwargs)
|
|
||||||
else:
|
|
||||||
job_func(*args, **kwargs)
|
|
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"m.homeserver": {
|
"m.homeserver": {
|
||||||
"base_url": "https://matrix.jakehoward.tech"
|
"base_url": "https://matrix.theorangeone.net"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
{"m.server": "matrix.jakehoward.tech:443"}
|
{"m.server": "matrix.theorangeone.net:443"}
|
||||||
|
|
Loading…
Reference in a new issue