From bb4361e25ff48b3e145d13d0f3ce797a3fcd30a8 Mon Sep 17 00:00:00 2001
From: Jake Howard <git@theorangeone.net>
Date: Tue, 6 Dec 2022 10:04:23 +0000
Subject: [PATCH] Use a blackbox with custom DNS rather than proxy

This makes timing information more relevant and should improve reliability
---
 .../forrest/files/prometheus/blackbox.yml     |  7 -------
 .../files/prometheus/docker-compose.yml       | 20 ++++++++++++++-----
 .../forrest/files/prometheus/prometheus.yml   |  8 ++++----
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/ansible/roles/forrest/files/prometheus/blackbox.yml b/ansible/roles/forrest/files/prometheus/blackbox.yml
index dba6afb..6e40e5a 100644
--- a/ansible/roles/forrest/files/prometheus/blackbox.yml
+++ b/ansible/roles/forrest/files/prometheus/blackbox.yml
@@ -3,13 +3,6 @@ modules:
     prober: http
     timeout: 10s
 
-  http_external:
-    prober: http
-    timeout: 10s
-    http:
-      proxy_url: http://{{ pve_hosts.qbittorrent.ip }}:3128
-      skip_resolve_phase_with_proxy: true
-
   https_redir:
     prober: http
     timeout: 10s
diff --git a/ansible/roles/forrest/files/prometheus/docker-compose.yml b/ansible/roles/forrest/files/prometheus/docker-compose.yml
index 0457055..eaf1392 100644
--- a/ansible/roles/forrest/files/prometheus/docker-compose.yml
+++ b/ansible/roles/forrest/files/prometheus/docker-compose.yml
@@ -1,5 +1,12 @@
 version: "2.3"
 
+x-blackbox: &blackbox
+  image: prom/blackbox-exporter:latest
+  restart: unless-stopped
+  user: "{{ docker_user.id }}"
+  volumes:
+    - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+
 services:
   prometheus:
     image: prom/prometheus:latest
@@ -15,11 +22,14 @@ services:
       - "{{ pve_hosts.forrest.ip }}:9090:9090"
 
   blackbox:
-    image: prom/blackbox-exporter:latest
-    restart: unless-stopped
-    user: "{{ docker_user.id }}"
-    volumes:
-      - ./blackbox.yml:/etc/blackbox_exporter/config.yml:ro
+    <<: *blackbox
+
+  blackbox-external:
+    <<: *blackbox
+    # Don't use my internal DNS servers
+    dns:
+      - 9.9.9.9
+      - 149.112.112.112
 
   alertmanager:
     image: prom/alertmanager:latest
diff --git a/ansible/roles/forrest/files/prometheus/prometheus.yml b/ansible/roles/forrest/files/prometheus/prometheus.yml
index 2d6f4b3..c3f3e20 100644
--- a/ansible/roles/forrest/files/prometheus/prometheus.yml
+++ b/ansible/roles/forrest/files/prometheus/prometheus.yml
@@ -40,7 +40,7 @@ scrape_configs:
     scrape_interval: 1m
     metrics_path: /probe
     params:
-      module: [http_external]
+      module: [http]
     static_configs:
       - targets:
           - https://bin.theorangeone.net
@@ -64,7 +64,7 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: instance
       - target_label: __address__
-        replacement: blackbox:9115
+        replacement: blackbox-external:9115
 
   - job_name: blackbox_https_redir
     scrape_interval: 10m
@@ -80,13 +80,13 @@ scrape_configs:
       - source_labels: [__param_target]
         target_label: instance
       - target_label: __address__
-        replacement: blackbox:9115
+        replacement: blackbox-external:9115
 
   - job_name: blackbox_healthcheck
     scrape_interval: 10m
     metrics_path: /probe
     params:
-      module: [http_external]
+      module: [http]
     static_configs:
       - targets:
           - https://hc-ping.com/{{ prometheus_healthcheck_uuid }}