From 11eb833f44ee9e1898c90d9c3a99d0db8b6af849 Mon Sep 17 00:00:00 2001
From: Jake Howard <git@theorangeone.net>
Date: Sun, 19 Feb 2023 18:31:52 +0000
Subject: [PATCH] Add some prometheus alerts

---
 .../forrest/files/prometheus/alert-rules.d/blackbox.yml  | 6 ++++++
 .../roles/forrest/files/prometheus/docker-compose.yml    | 1 +
 ansible/roles/forrest/files/prometheus/prometheus.yml    | 3 +++
 ansible/roles/forrest/tasks/prometheus.yml               | 9 +++++++++
 4 files changed, 19 insertions(+)
 create mode 100644 ansible/roles/forrest/files/prometheus/alert-rules.d/blackbox.yml

diff --git a/ansible/roles/forrest/files/prometheus/alert-rules.d/blackbox.yml b/ansible/roles/forrest/files/prometheus/alert-rules.d/blackbox.yml
new file mode 100644
index 0000000..205eb98
--- /dev/null
+++ b/ansible/roles/forrest/files/prometheus/alert-rules.d/blackbox.yml
@@ -0,0 +1,6 @@
+groups:
+  - name: blackbox
+    rules:
+      - alert: HttpSuccess
+        expr: probe_success{job="blackbox_http_external"} == 0
+        for: 5m
diff --git a/ansible/roles/forrest/files/prometheus/docker-compose.yml b/ansible/roles/forrest/files/prometheus/docker-compose.yml
index d670712..cf372a8 100644
--- a/ansible/roles/forrest/files/prometheus/docker-compose.yml
+++ b/ansible/roles/forrest/files/prometheus/docker-compose.yml
@@ -14,6 +14,7 @@ services:
     user: "{{ docker_user.id }}"
     volumes:
       - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./alert-rules.d:/etc/prometheus/alert-rules.d:ro
       - /mnt/tank/dbs/prometheus/forrest/:/prometheus/
     networks:
       - default
diff --git a/ansible/roles/forrest/files/prometheus/prometheus.yml b/ansible/roles/forrest/files/prometheus/prometheus.yml
index 77baa3f..9e231a4 100644
--- a/ansible/roles/forrest/files/prometheus/prometheus.yml
+++ b/ansible/roles/forrest/files/prometheus/prometheus.yml
@@ -4,6 +4,9 @@ alerting:
         - targets:
             - alertmanager:9093
 
+rule_files:
+  - /etc/prometheus/alert-rules.d/*.yml
+
 scrape_configs:
   - job_name: pve
     static_configs:
diff --git a/ansible/roles/forrest/tasks/prometheus.yml b/ansible/roles/forrest/tasks/prometheus.yml
index ecdce77..35d1aba 100644
--- a/ansible/roles/forrest/tasks/prometheus.yml
+++ b/ansible/roles/forrest/tasks/prometheus.yml
@@ -42,3 +42,12 @@
     owner: "{{ docker_user.name }}"
   notify: restart prometheus
   become: true
+
+- name: Install prometheus alert rules
+  copy:
+    src: files/prometheus/alert-rules.d/
+    dest: /opt/prometheus/alert-rules.d/
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: reload prometheus
+  become: true