Add the basics of some edge caching

ansible/host_vars/casey/main.yml

@@ -7,6 +7,11 @@ nginx_https_redirect: true
 certbot_certs:
   - domains:
       - headscale.jakehoward.tech
+  - domains:
+      - whoami-cdn.theorangeone.net
+
+cdn_domains:
+  - whoami-cdn.theorangeone.net
 
 restic_backup_locations:
   - /var/lib/headscale/

ansible/roles/forrest/files/prometheus/prometheus.yml

@@ -70,6 +70,8 @@ scrape_configs:
           - https://theorangeone.net
           - https://tt-rss.jakehoward.tech
           - https://vaultwarden.jakehoward.tech/alive
+          - https://whoami-cdn.theorangeone.net
+          - https://whoami.theorangeone.net
     relabel_configs:
       - source_labels: [__address__]
         target_label: __param_target

ansible/roles/gateway/files/nginx-cdn.conf

@@ -0,0 +1,30 @@
+# {{ ansible_managed }}
+
+proxy_cache_path /var/lib/nginx/cache levels=1:2 keys_zone=cdncache:20m max_size=1g inactive=48h;
+
+{% for domain in cdn_domains %}
+server {
+    listen 8800 ssl proxy_protocol;
+    http2 on;
+
+    server_name {{ domain }};
+
+    ssl_certificate /etc/letsencrypt/live/{{ domain }}/fullchain.pem;
+    ssl_certificate_key /etc/letsencrypt/live/{{ domain }}/privkey.pem;
+    ssl_trusted_certificate /etc/letsencrypt/live/{{ domain }}/chain.pem;
+
+    include includes/ssl.conf;
+
+    real_ip_header proxy_protocol;
+
+    set_real_ip_from 127.0.0.1;
+
+    proxy_cache_use_stale error timeout http_500 http_502 http_503 http_504;
+
+    location / {
+        proxy_cache cdncache;
+        add_header X-Cache-Status $upstream_cache_status;
+        proxy_pass https://{{ wireguard.clients.ingress.ip }}:443;
+    }
+}
+{% endfor %}

ansible/roles/gateway/files/nginx.conf

@@ -9,8 +9,13 @@ access_log /var/log/nginx/gateway.log gateway;
 access_log /var/log/nginx/ips.log ips;
 
 map $ssl_preread_server_name $gateway_destination {
-    headscale.jakehoward.tech 127.0.0.1:8888;
     default                      {{ wireguard.clients.ingress.ip }}:8443;
+
+    headscale.jakehoward.tech    127.0.0.1:8888;
+
+    {% for domain in cdn_domains %}
+    {{ domain }}  127.0.0.1:8800;
+    {% endfor %}
 }
 
 server {

ansible/roles/gateway/tasks/nginx.yml

@@ -6,6 +6,14 @@
   become: true
   register: nginx_config
 
+- name: Install CDN config
+  template:
+    src: files/nginx-cdn.conf
+    dest: /etc/nginx/http.d/cdn.conf
+    mode: "0644"
+  become: true
+  register: nginx_config
+
 - name: Reload Nginx
   service:
     name: nginx

ansible/roles/nginx/files/includes/proxy.conf

@@ -21,7 +21,7 @@ proxy_set_header Early-Data $ssl_early_data;
 proxy_set_header Host $host;
 proxy_set_header Proxy "";
 proxy_set_header Upgrade $http_upgrade;
-proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+proxy_set_header X-Forwarded-For $remote_addr;
 proxy_set_header X-Forwarded-Host $host;
 proxy_set_header X-Forwarded-Method $request_method;
 proxy_set_header X-Forwarded-Port $server_port;
@@ -32,9 +32,3 @@ proxy_set_header X-Forwarded-Uri $request_uri;
 proxy_set_header X-Original-Method $request_method;
 proxy_set_header X-Original-URL $scheme://$http_host$request_uri;
 proxy_set_header X-Real-IP $remote_addr;
-
-# Helper variable for proxying websockets.
-map $http_upgrade $connection_upgrade {
-    default upgrade;
-    '' close;
-}

ansible/roles/nginx/files/nginx.conf

@@ -29,6 +29,12 @@ http {
 
     include includes/proxy.conf;
 
+    # Helper variable for proxying websockets.
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
     log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for"';

ansible/roles/pve_docker/files/whoami/docker-compose.yml

@@ -6,7 +6,7 @@ services:
     restart: unless-stopped
     labels:
       - traefik.enable=true
-      - traefik.http.routers.whoami.rule=Host(`whoami.theorangeone.net`)
+      - traefik.http.routers.whoami.rule=Host(`whoami.theorangeone.net`) || Host(`whoami-cdn.theorangeone.net`)
     networks:
       - default
       - traefik

ansible/roles/traefik/files/traefik.yml

@@ -31,10 +31,10 @@ entryPoints:
           {% endif %}
     proxyProtocol:
       trustedIPs:
-        - "{{ wireguard.cidr }}"
-        - "{{ pve_hosts.internal_cidr }}"
-        - "{{ nebula.cidr }}"
-        - "{{ tailscale_cidr }}"
+        - "{{ pve_hosts.ingress.ip }}/32"
+    forwardedHeaders:
+      trustedIPs:
+        - "{{ wireguard.server.ip }}/32"  # This is obtained from the connecting `proxy_protocol`
   traefik:
     address: :8080
 

terraform/theorangeone.net.tf

@@ -18,6 +18,14 @@ resource "cloudflare_record" "theorangeonenet_whoami" {
   ttl     = 1
 }
 
+resource "cloudflare_record" "theorangeonenet_whoami_cdn" {
+  zone_id = cloudflare_zone.theorangeonenet.id
+  name    = "whoami-cdn"
+  value   = cloudflare_record.sys_domain_casey.hostname
+  type    = "CNAME"
+  ttl     = 1
+}
+
 resource "cloudflare_record" "theorangeonenet_mx1" {
   zone_id  = cloudflare_zone.theorangeonenet.id
   name     = "@"