Setup blackbox for some HTTP monitoring

ansible/roles/forrest/files/prometheus/blackbox.yml

@@ -0,0 +1,23 @@
+modules:
+  http:
+    prober: http
+    timeout: 10s
+
+  http_external:
+    prober: http
+    timeout: 10s
+    http:
+      proxy_url: http://{{ pve_hosts.qbittorrent.ip }}:3128
+      skip_resolve_phase_with_proxy: true
+
+  https_redir:
+    prober: http
+    timeout: 10s
+    http:
+      method: GET
+      valid_status_codes: [301, 302, 307, 308]
+      no_follow_redirects: true
+      fail_if_ssl: true
+      fail_if_header_not_matches:
+        - header: Location
+          regexp: ^https

ansible/roles/forrest/files/prometheus/docker-compose.yml

@@ -14,6 +14,12 @@ services:
     ports:
       - "{{ pve_hosts.forrest.ip }}:9090:9090"
 
+  blackbox:
+    image: prom/blackbox-exporter:latest
+    restart: unless-stopped
+    volumes:
+      - ./blackbox.yml:/etc/blackbox_exporter/config.yml
+
 networks:
   prometheus:
     external: true

ansible/roles/forrest/files/prometheus/prometheus.yml

@@ -4,7 +4,6 @@ scrape_configs:
       - source_labels: [__name__]
         regex: go_.+
         action: drop
-
     static_configs:
       - targets:
           - "{{ pve_hosts.pve.ip }}:9273"
@@ -14,7 +13,6 @@ scrape_configs:
       - source_labels: [__name__]
         regex: go_.+
         action: drop
-
     static_configs:
       - targets:
           - "{{ nebula.clients.walker.ip }}:8080"
@@ -23,14 +21,57 @@ scrape_configs:
 
   - job_name: homeassistant
     metrics_path: /api/prometheus
-
     authorization:
       credentials: "{{ homeassistant_token }}"
-
     metric_relabel_configs:
       - source_labels: [__name__]
         regex: go_.+
         action: drop
-
     static_configs:
       - targets: ["{{ pve_hosts.homeassistant.ip }}:8123"]
+
+  - job_name: blackbox_http_external
+    scrape_interval: 1m
+    metrics_path: /probe
+    params:
+      module: [http_external]
+    static_configs:
+      - targets:
+          - https://bin.theorangeone.net
+          - https://git.theorangeone.net/-/liveness?token={{ gitlab_liveness_access_token }}
+          - https://grafana.jakehoward.tech/api/health
+          - https://homeassistant.jakehoward.tech
+          - https://intersect.jakehoward.tech
+          - https://mastodon.theorangeone.net/health
+          - https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version
+          - https://matrix.jakehoward.tech/_matrix/federation/v1/version
+          - https://media.jakehoward.tech
+          - https://notes.theorangeone.net
+          - https://plausible.theorangeone.net
+          - https://recipes.jakehoward.tech
+          - https://theorangeone.net
+          - https://tt-rss.jakehoward.tech
+          - https://vaultwarden.jakehoward.tech/alive
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: blackbox_https_redir
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [https_redir]
+    static_configs:
+      - targets:
+          - http://{{ hosts.casey_ip }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115

ansible/roles/forrest/tasks/prometheus.yml

@@ -28,3 +28,12 @@
     validate: docker-compose -f %s config
   notify: restart prometheus
   become: true
+
+- name: Install blackbox config
+  template:
+    src: files/prometheus/blackbox.yml
+    dest: /opt/prometheus/blackbox.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: restart prometheus
+  become: true

ansible/roles/forrest/vars/main.yml

@@ -2,3 +2,4 @@ grafana_smtp_password: "{{ vault_grafana_smtp_password }}"
 grafana_smtp_user: "{{ vault_grafana_smtp_user }}"
 grafana_from_email: "{{ vault_grafana_from_email }}"
 homeassistant_token: "{{ vault_homeassistant_token }}"
+gitlab_liveness_access_token: "{{ vault_gitlab_liveness_access_token }}"

ansible/roles/forrest/vars/vault.yml

@@ -1,25 +1,28 @@
 $ANSIBLE_VAULT;1.1;AES256
-65646538666533343137303366373335663635303538626561393235633839663431316630656336
+32646438376436643962653461396539363034323036336232636639626666643734643264353931
-3836623632353036626337653263323461313133363839320a633536373265623838333938326337
+6162306233363232306161653864613637633638376561650a326130393064646338303736613434
-63623566363764363135656137386466396566366566353935373361386637303335393934666362
+65343435643861396630303036643261656364343963366138653230326563613636373635636433
-3536363465386136310a353562373535313961613031653266636131336162366434393161333932
+3662623563653931310a376366656637316164653263353265643062306361383938663665613433
-33303035613763306138356535633432636133313432353238646463616463313833613730336664
+31336435613463333736363738636266643262346336393366373736303639333265623431653266
-64326433623039656431303663353062313534353564393338666662346539633538663662353938
+36313537393931343761313336316233643664383764373264393366643034303734653533313361
-37346235353663666239323032396238366536396264626530646535646464366439643438356333
+36353738346230336233383132396531326661393861336266373363323537396566663862656537
-61383661653837316335393435623866343230653134383565656361343761633433323863306630
+61636134363861623463623166383065353837333865366130353864626130366138353039346336
-65613461663662393632666239383362316662373361316663353265333031613132613964333630
+64646631366364626263303836383738303630616362353436666435613061326163616633656331
-32376465306464653133353132376562346363303566393338386361353038333338393261323631
+38343166633630303465613466643139306336326265653838353531373735393439376637366137
-61323562346161356366623038383364643262306566663535336266353961353461363630333735
+61623566633530323563313662393061643232333834646433666163653839656239633832646439
-34636432303031663932333534373238366665316135633933386436353138316162656563303863
+38663733633832343465313738616163303230336139373764306639303739353931396266613333
-32656461626634383961353665376133373966656332383566643432313334643836646538376262
+64333336343535396662356133353635613164316132346138316361613431313064666337303233
-33633761363964363265356533643330323062653531633361356362333533616334343265313537
+32353634623232666335613838323763313334323863633539343337386631663864666562363165
-33303537613136623033386430613434316162613034356166396338353638366532336461653966
+31626563383261333630343735636235393064396263643239356466616135643139666434626435
-66343664626233633166323038303837633265313662313461303134396665613530643037656632
+30313831306266656539323239396132663234316139613934303166306136663832373265616331
-38336363336362636632353530356662653435323635343938613531316463346236373639353965
+36333935656138636439653335346637656434633935663731383833656164363831353862383737
-38343965613537633836666332613335646262336666656239316335663830643839346132613561
+64313565353364306533333838646164623461653635623961653532353835643262633936356361
-34376364316465343331653638316337613936353666633839383931316131303836333161306566
+66653339636565643466623666616135383430366535323363616261323665336365373365376534
-39333930363937373235653535656134336333326239386332633835666530376364663662306662
+37623630356361346636643666643534626239616235343730313532343736303734373765323939
-65393666376636646436353032326331383766666237656633636562623635393333646136643737
+30643339346363316465636331626230626364353062643032643931333233303533633639346632
-32613230363539393832393766306230636465386164323537643761663836353935313031333939
+37373137323165346339346239396134363863353238626638366539343364613964336536623939
-63333561643364326639633764386264346361363461343864636362633463343838643965666265
+65333438306436613032366431646166373261393637366437353530383661623036326163393230
-61323563613639356564
+64656431326164643933363230666461336465323231356138633333373661313733353634393663
+36306161306332303436343433393033316533666663613964396162353765383737346234616435
+66353935626632656531336665623432623139646538343664323066306362666336623165633037
+353539636130316161303539376164613964