Scrape healthchecks.io with prometheus and use it for outbound monitoring of prometheus scrapes

ansible/roles/forrest/files/prometheus/prometheus.yml

@@ -75,3 +75,25 @@ scrape_configs:
         target_label: instance
       - target_label: __address__
         replacement: blackbox:9115
+
+  - job_name: blackbox_healthcheck
+    scrape_interval: 10m
+    metrics_path: /probe
+    params:
+      module: [http_external]
+    static_configs:
+      - targets:
+          - https://hc-ping.com/{{ prometheus_healthcheck_uuid }}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+
+  - job_name: healthchecks
+    scheme: https
+    metrics_path: /projects/{{ healthchecks_project_uuid }}/metrics/{{ healthcheck_api_token }}
+    static_configs:
+      - targets: [healthchecks.io]

ansible/roles/forrest/handlers/main.yml

@@ -8,7 +8,7 @@
     chdir: /opt/prometheus
     cmd: "{{ docker_update_command }}"
 
-- name: restart telegraf
+- name: reload prometheus
   shell:
-    chdir: /opt/telegraf
+    chdir: /opt/prometheus
-    cmd: "{{ docker_update_command }}"
+    cmd: docker-compose exec prometheus kill -HUP 1

ansible/roles/forrest/tasks/prometheus.yml

@@ -12,7 +12,7 @@
     dest: /opt/prometheus/prometheus.yml
     mode: "{{ docker_compose_file_mask }}"
     owner: "{{ docker_user.name }}"
-  notify: restart prometheus
+  notify: reload prometheus
   become: true
 
 - name: Install prometheus compose file

ansible/roles/forrest/vars/main.yml

@@ -3,3 +3,6 @@ grafana_smtp_user: "{{ vault_grafana_smtp_user }}"
 grafana_from_email: "{{ vault_grafana_from_email }}"
 homeassistant_token: "{{ vault_homeassistant_token }}"
 gitlab_liveness_access_token: "{{ vault_gitlab_liveness_access_token }}"
+prometheus_healthcheck_uuid: "{{ vault_prometheus_healthcheck_uuid }}"
+healthchecks_project_uuid: "{{ vault_healthchecks_project_uuid }}"
+healthcheck_api_token: "{{ vault_healthcheck_api_token }}"

ansible/roles/forrest/vars/vault.yml

@@ -1,28 +1,38 @@
 $ANSIBLE_VAULT;1.1;AES256
-32646438376436643962653461396539363034323036336232636639626666643734643264353931
+37343335613562383564356561646365363139333631366661636536373331333335313666333432
-6162306233363232306161653864613637633638376561650a326130393064646338303736613434
+3032353032313634326466653435323937386339303831330a663066353862343831376332343863
-65343435643861396630303036643261656364343963366138653230326563613636373635636433
+65353137393733336366333264653461653035643339613332653764646231323336336134633466
-3662623563653931310a376366656637316164653263353265643062306361383938663665613433
+6235666363613639320a333262366261626437373032653937303937336138346165343231336238
-31336435613463333736363738636266643262346336393366373736303639333265623431653266
+37643465306237343039373435376438633264386337373132353039333033363133303766633236
-36313537393931343761313336316233643664383764373264393366643034303734653533313361
+35653333363865396164643162393536643533373634393064353738643165343838613536323036
-36353738346230336233383132396531326661393861336266373363323537396566663862656537
+35666464343934323862306634306465303861653032316135613865613265393038333337363536
-61636134363861623463623166383065353837333865366130353864626130366138353039346336
+39373433633964636461653438313932356139376266616533323239363961663035626462366366
-64646631366364626263303836383738303630616362353436666435613061326163616633656331
+35663532323964613166396164643830336264343766656630393761336663326234333738313239
-38343166633630303465613466643139306336326265653838353531373735393439376637366137
+36653832313939333937353630386530323333666664333437343365386464313861633433626439
-61623566633530323563313662393061643232333834646433666163653839656239633832646439
+33353033353432663633353762323934613563396230613439336338313430336534623534363764
-38663733633832343465313738616163303230336139373764306639303739353931396266613333
+36616436393231633932383430353034313631636431636536643964613133376332323330393666
-64333336343535396662356133353635613164316132346138316361613431313064666337303233
+65333264623037303764343439663934646637383538336637306433353538353363623330373163
-32353634623232666335613838323763313334323863633539343337386631663864666562363165
+32313637353634663837303863336139303361636438353335613666666130363735656565313034
-31626563383261333630343735636235393064396263643239356466616135643139666434626435
+65613863623430366635363465643738366364333465336331353162373061336334373531383738
-30313831306266656539323239396132663234316139613934303166306136663832373265616331
+34653439366435396664393234323837633039333462353830313961333736363039306231343766
-36333935656138636439653335346637656434633935663731383833656164363831353862383737
+30636566616561376664613235633631663837363830623037343538383333346336613636643064
-64313565353364306533333838646164623461653635623961653532353835643262633936356361
+39636466613336316638356662383839666661393634376431366533326135386334623864383632
-66653339636565643466623666616135383430366535323363616261323665336365373365376534
+31623537343835303436373331363264376561396535386464303732336432643336323362626161
-37623630356361346636643666643534626239616235343730313532343736303734373765323939
+61376530393863303965613338303132313262386436343866623163646238333639643035313137
-30643339346363316465636331626230626364353062643032643931333233303533633639346632
+36653632353033353462303236353139303734306461363937383331613733336539343730386435
-37373137323165346339346239396134363863353238626638366539343364613964336536623939
+36636636663965356636336663616561373965353363346238393064333237303338633763613362
-65333438306436613032366431646166373261393637366437353530383661623036326163393230
+38666135643732353039663030343336303536633662623965316639373161376162303539366635
-64656431326164643933363230666461336465323231356138633333373661313733353634393663
+65346566396366306665663530656230633832353066343431643437326635396134343132343738
-36306161306332303436343433393033316533666663613964396162353765383737346234616435
+66383563353562353532373639363466623937313066386533333366333030333534366539303431
-66353935626632656531336665623432623139646538343664323066306362666336623165633037
+63306564613332653136653431643732393632393633623133313162623061333833326233663833
-353539636130316161303539376164613964
+36346232623462646430633966323165393663323864616565343231663537346566663561326634
+33373436396538393136636165353030616537356634616662666362386565613263306135666164
+35633435313334353531323064633732656438346530383261323264626234393536323162393230
+36366664653233313936623363343834396366356162663036366438323663356661633938646437
+31383865313632343866666433313039303438393038323530636234616336626239373761623862
+33646431343236646465303032363932336331393261363666373331656166653439303936383861
+39613539613838626463376233316134653935376165303765313131393661623732613234623630
+66323137383038363737306230356161643262626162393964326166356630646537653930343463
+62326638623038626131323065323939343232636137633961656634353237306534346139393864
+37336138636262663463343638373763333539323364343735376232306466323864323431333863
+3562