From 75862dbaf9651b1690a9406ea52403ccfd4df4aa Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Sun, 4 Dec 2022 22:11:49 +0000 Subject: [PATCH] Setup blackbox for some HTTP monitoring --- .../forrest/files/prometheus/blackbox.yml | 23 +++++++++ .../files/prometheus/docker-compose.yml | 6 +++ .../forrest/files/prometheus/prometheus.yml | 51 +++++++++++++++++-- ansible/roles/forrest/tasks/prometheus.yml | 9 ++++ ansible/roles/forrest/vars/main.yml | 1 + ansible/roles/forrest/vars/vault.yml | 51 ++++++++++--------- 6 files changed, 112 insertions(+), 29 deletions(-) create mode 100644 ansible/roles/forrest/files/prometheus/blackbox.yml diff --git a/ansible/roles/forrest/files/prometheus/blackbox.yml b/ansible/roles/forrest/files/prometheus/blackbox.yml new file mode 100644 index 0000000..dba6afb --- /dev/null +++ b/ansible/roles/forrest/files/prometheus/blackbox.yml @@ -0,0 +1,23 @@ +modules: + http: + prober: http + timeout: 10s + + http_external: + prober: http + timeout: 10s + http: + proxy_url: http://{{ pve_hosts.qbittorrent.ip }}:3128 + skip_resolve_phase_with_proxy: true + + https_redir: + prober: http + timeout: 10s + http: + method: GET + valid_status_codes: [301, 302, 307, 308] + no_follow_redirects: true + fail_if_ssl: true + fail_if_header_not_matches: + - header: Location + regexp: ^https diff --git a/ansible/roles/forrest/files/prometheus/docker-compose.yml b/ansible/roles/forrest/files/prometheus/docker-compose.yml index a57e3cf..935599f 100644 --- a/ansible/roles/forrest/files/prometheus/docker-compose.yml +++ b/ansible/roles/forrest/files/prometheus/docker-compose.yml @@ -14,6 +14,12 @@ services: ports: - "{{ pve_hosts.forrest.ip }}:9090:9090" + blackbox: + image: prom/blackbox-exporter:latest + restart: unless-stopped + volumes: + - ./blackbox.yml:/etc/blackbox_exporter/config.yml + networks: prometheus: external: true diff --git a/ansible/roles/forrest/files/prometheus/prometheus.yml b/ansible/roles/forrest/files/prometheus/prometheus.yml index a7c3107..bb8e59a 100644 --- a/ansible/roles/forrest/files/prometheus/prometheus.yml +++ b/ansible/roles/forrest/files/prometheus/prometheus.yml @@ -4,7 +4,6 @@ scrape_configs: - source_labels: [__name__] regex: go_.+ action: drop - static_configs: - targets: - "{{ pve_hosts.pve.ip }}:9273" @@ -14,7 +13,6 @@ scrape_configs: - source_labels: [__name__] regex: go_.+ action: drop - static_configs: - targets: - "{{ nebula.clients.walker.ip }}:8080" @@ -23,14 +21,57 @@ scrape_configs: - job_name: homeassistant metrics_path: /api/prometheus - authorization: credentials: "{{ homeassistant_token }}" - metric_relabel_configs: - source_labels: [__name__] regex: go_.+ action: drop - static_configs: - targets: ["{{ pve_hosts.homeassistant.ip }}:8123"] + + - job_name: blackbox_http_external + scrape_interval: 1m + metrics_path: /probe + params: + module: [http_external] + static_configs: + - targets: + - https://bin.theorangeone.net + - https://git.theorangeone.net/-/liveness?token={{ gitlab_liveness_access_token }} + - https://grafana.jakehoward.tech/api/health + - https://homeassistant.jakehoward.tech + - https://intersect.jakehoward.tech + - https://mastodon.theorangeone.net/health + - https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version + - https://matrix.jakehoward.tech/_matrix/federation/v1/version + - https://media.jakehoward.tech + - https://notes.theorangeone.net + - https://plausible.theorangeone.net + - https://recipes.jakehoward.tech + - https://theorangeone.net + - https://tt-rss.jakehoward.tech + - https://vaultwarden.jakehoward.tech/alive + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 + + - job_name: blackbox_https_redir + scrape_interval: 10m + metrics_path: /probe + params: + module: [https_redir] + static_configs: + - targets: + - http://{{ hosts.casey_ip }} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 diff --git a/ansible/roles/forrest/tasks/prometheus.yml b/ansible/roles/forrest/tasks/prometheus.yml index 596515a..ba7f339 100644 --- a/ansible/roles/forrest/tasks/prometheus.yml +++ b/ansible/roles/forrest/tasks/prometheus.yml @@ -28,3 +28,12 @@ validate: docker-compose -f %s config notify: restart prometheus become: true + +- name: Install blackbox config + template: + src: files/prometheus/blackbox.yml + dest: /opt/prometheus/blackbox.yml + mode: "{{ docker_compose_file_mask }}" + owner: "{{ docker_user.name }}" + notify: restart prometheus + become: true diff --git a/ansible/roles/forrest/vars/main.yml b/ansible/roles/forrest/vars/main.yml index eb2c4ae..642b5d8 100644 --- a/ansible/roles/forrest/vars/main.yml +++ b/ansible/roles/forrest/vars/main.yml @@ -2,3 +2,4 @@ grafana_smtp_password: "{{ vault_grafana_smtp_password }}" grafana_smtp_user: "{{ vault_grafana_smtp_user }}" grafana_from_email: "{{ vault_grafana_from_email }}" homeassistant_token: "{{ vault_homeassistant_token }}" +gitlab_liveness_access_token: "{{ vault_gitlab_liveness_access_token }}" diff --git a/ansible/roles/forrest/vars/vault.yml b/ansible/roles/forrest/vars/vault.yml index 92a84ca..dbd4cb5 100644 --- a/ansible/roles/forrest/vars/vault.yml +++ b/ansible/roles/forrest/vars/vault.yml @@ -1,25 +1,28 @@ $ANSIBLE_VAULT;1.1;AES256 -65646538666533343137303366373335663635303538626561393235633839663431316630656336 -3836623632353036626337653263323461313133363839320a633536373265623838333938326337 -63623566363764363135656137386466396566366566353935373361386637303335393934666362 -3536363465386136310a353562373535313961613031653266636131336162366434393161333932 -33303035613763306138356535633432636133313432353238646463616463313833613730336664 -64326433623039656431303663353062313534353564393338666662346539633538663662353938 -37346235353663666239323032396238366536396264626530646535646464366439643438356333 -61383661653837316335393435623866343230653134383565656361343761633433323863306630 -65613461663662393632666239383362316662373361316663353265333031613132613964333630 -32376465306464653133353132376562346363303566393338386361353038333338393261323631 -61323562346161356366623038383364643262306566663535336266353961353461363630333735 -34636432303031663932333534373238366665316135633933386436353138316162656563303863 -32656461626634383961353665376133373966656332383566643432313334643836646538376262 -33633761363964363265356533643330323062653531633361356362333533616334343265313537 -33303537613136623033386430613434316162613034356166396338353638366532336461653966 -66343664626233633166323038303837633265313662313461303134396665613530643037656632 -38336363336362636632353530356662653435323635343938613531316463346236373639353965 -38343965613537633836666332613335646262336666656239316335663830643839346132613561 -34376364316465343331653638316337613936353666633839383931316131303836333161306566 -39333930363937373235653535656134336333326239386332633835666530376364663662306662 -65393666376636646436353032326331383766666237656633636562623635393333646136643737 -32613230363539393832393766306230636465386164323537643761663836353935313031333939 -63333561643364326639633764386264346361363461343864636362633463343838643965666265 -61323563613639356564 +32646438376436643962653461396539363034323036336232636639626666643734643264353931 +6162306233363232306161653864613637633638376561650a326130393064646338303736613434 +65343435643861396630303036643261656364343963366138653230326563613636373635636433 +3662623563653931310a376366656637316164653263353265643062306361383938663665613433 +31336435613463333736363738636266643262346336393366373736303639333265623431653266 +36313537393931343761313336316233643664383764373264393366643034303734653533313361 +36353738346230336233383132396531326661393861336266373363323537396566663862656537 +61636134363861623463623166383065353837333865366130353864626130366138353039346336 +64646631366364626263303836383738303630616362353436666435613061326163616633656331 +38343166633630303465613466643139306336326265653838353531373735393439376637366137 +61623566633530323563313662393061643232333834646433666163653839656239633832646439 +38663733633832343465313738616163303230336139373764306639303739353931396266613333 +64333336343535396662356133353635613164316132346138316361613431313064666337303233 +32353634623232666335613838323763313334323863633539343337386631663864666562363165 +31626563383261333630343735636235393064396263643239356466616135643139666434626435 +30313831306266656539323239396132663234316139613934303166306136663832373265616331 +36333935656138636439653335346637656434633935663731383833656164363831353862383737 +64313565353364306533333838646164623461653635623961653532353835643262633936356361 +66653339636565643466623666616135383430366535323363616261323665336365373365376534 +37623630356361346636643666643534626239616235343730313532343736303734373765323939 +30643339346363316465636331626230626364353062643032643931333233303533633639346632 +37373137323165346339346239396134363863353238626638366539343364613964336536623939 +65333438306436613032366431646166373261393637366437353530383661623036326163393230 +64656431326164643933363230666461336465323231356138633333373661313733353634393663 +36306161306332303436343433393033316533666663613964396162353765383737346234616435 +66353935626632656531336665623432623139646538343664323066306362666336623165633037 +353539636130316161303539376164613964