From ed7faf46788561e2eb1dc34a08b6210e65345f57 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Mon, 5 Dec 2022 14:40:52 +0000 Subject: [PATCH] Scrape healthchecks.io with prometheus and use it for outbound monitoring of prometheus scrapes --- .../forrest/files/prometheus/prometheus.yml | 22 +++++++ ansible/roles/forrest/handlers/main.yml | 6 +- ansible/roles/forrest/tasks/prometheus.yml | 2 +- ansible/roles/forrest/vars/main.yml | 3 + ansible/roles/forrest/vars/vault.yml | 64 +++++++++++-------- 5 files changed, 66 insertions(+), 31 deletions(-) diff --git a/ansible/roles/forrest/files/prometheus/prometheus.yml b/ansible/roles/forrest/files/prometheus/prometheus.yml index bb8e59a..ec18168 100644 --- a/ansible/roles/forrest/files/prometheus/prometheus.yml +++ b/ansible/roles/forrest/files/prometheus/prometheus.yml @@ -75,3 +75,25 @@ scrape_configs: target_label: instance - target_label: __address__ replacement: blackbox:9115 + + - job_name: blackbox_healthcheck + scrape_interval: 10m + metrics_path: /probe + params: + module: [http_external] + static_configs: + - targets: + - https://hc-ping.com/{{ prometheus_healthcheck_uuid }} + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - target_label: __address__ + replacement: blackbox:9115 + + - job_name: healthchecks + scheme: https + metrics_path: /projects/{{ healthchecks_project_uuid }}/metrics/{{ healthcheck_api_token }} + static_configs: + - targets: [healthchecks.io] diff --git a/ansible/roles/forrest/handlers/main.yml b/ansible/roles/forrest/handlers/main.yml index e6a1083..2fc99ed 100644 --- a/ansible/roles/forrest/handlers/main.yml +++ b/ansible/roles/forrest/handlers/main.yml @@ -8,7 +8,7 @@ chdir: /opt/prometheus cmd: "{{ docker_update_command }}" -- name: restart telegraf +- name: reload prometheus shell: - chdir: /opt/telegraf - cmd: "{{ docker_update_command }}" + chdir: /opt/prometheus + cmd: docker-compose exec prometheus kill -HUP 1 diff --git a/ansible/roles/forrest/tasks/prometheus.yml b/ansible/roles/forrest/tasks/prometheus.yml index a754e92..7194e30 100644 --- a/ansible/roles/forrest/tasks/prometheus.yml +++ b/ansible/roles/forrest/tasks/prometheus.yml @@ -12,7 +12,7 @@ dest: /opt/prometheus/prometheus.yml mode: "{{ docker_compose_file_mask }}" owner: "{{ docker_user.name }}" - notify: restart prometheus + notify: reload prometheus become: true - name: Install prometheus compose file diff --git a/ansible/roles/forrest/vars/main.yml b/ansible/roles/forrest/vars/main.yml index 642b5d8..d34b06f 100644 --- a/ansible/roles/forrest/vars/main.yml +++ b/ansible/roles/forrest/vars/main.yml @@ -3,3 +3,6 @@ grafana_smtp_user: "{{ vault_grafana_smtp_user }}" grafana_from_email: "{{ vault_grafana_from_email }}" homeassistant_token: "{{ vault_homeassistant_token }}" gitlab_liveness_access_token: "{{ vault_gitlab_liveness_access_token }}" +prometheus_healthcheck_uuid: "{{ vault_prometheus_healthcheck_uuid }}" +healthchecks_project_uuid: "{{ vault_healthchecks_project_uuid }}" +healthcheck_api_token: "{{ vault_healthcheck_api_token }}" diff --git a/ansible/roles/forrest/vars/vault.yml b/ansible/roles/forrest/vars/vault.yml index dbd4cb5..82db81a 100644 --- a/ansible/roles/forrest/vars/vault.yml +++ b/ansible/roles/forrest/vars/vault.yml @@ -1,28 +1,38 @@ $ANSIBLE_VAULT;1.1;AES256 -32646438376436643962653461396539363034323036336232636639626666643734643264353931 -6162306233363232306161653864613637633638376561650a326130393064646338303736613434 -65343435643861396630303036643261656364343963366138653230326563613636373635636433 -3662623563653931310a376366656637316164653263353265643062306361383938663665613433 -31336435613463333736363738636266643262346336393366373736303639333265623431653266 -36313537393931343761313336316233643664383764373264393366643034303734653533313361 -36353738346230336233383132396531326661393861336266373363323537396566663862656537 -61636134363861623463623166383065353837333865366130353864626130366138353039346336 -64646631366364626263303836383738303630616362353436666435613061326163616633656331 -38343166633630303465613466643139306336326265653838353531373735393439376637366137 -61623566633530323563313662393061643232333834646433666163653839656239633832646439 -38663733633832343465313738616163303230336139373764306639303739353931396266613333 -64333336343535396662356133353635613164316132346138316361613431313064666337303233 -32353634623232666335613838323763313334323863633539343337386631663864666562363165 -31626563383261333630343735636235393064396263643239356466616135643139666434626435 -30313831306266656539323239396132663234316139613934303166306136663832373265616331 -36333935656138636439653335346637656434633935663731383833656164363831353862383737 -64313565353364306533333838646164623461653635623961653532353835643262633936356361 -66653339636565643466623666616135383430366535323363616261323665336365373365376534 -37623630356361346636643666643534626239616235343730313532343736303734373765323939 -30643339346363316465636331626230626364353062643032643931333233303533633639346632 -37373137323165346339346239396134363863353238626638366539343364613964336536623939 -65333438306436613032366431646166373261393637366437353530383661623036326163393230 -64656431326164643933363230666461336465323231356138633333373661313733353634393663 -36306161306332303436343433393033316533666663613964396162353765383737346234616435 -66353935626632656531336665623432623139646538343664323066306362666336623165633037 -353539636130316161303539376164613964 +37343335613562383564356561646365363139333631366661636536373331333335313666333432 +3032353032313634326466653435323937386339303831330a663066353862343831376332343863 +65353137393733336366333264653461653035643339613332653764646231323336336134633466 +6235666363613639320a333262366261626437373032653937303937336138346165343231336238 +37643465306237343039373435376438633264386337373132353039333033363133303766633236 +35653333363865396164643162393536643533373634393064353738643165343838613536323036 +35666464343934323862306634306465303861653032316135613865613265393038333337363536 +39373433633964636461653438313932356139376266616533323239363961663035626462366366 +35663532323964613166396164643830336264343766656630393761336663326234333738313239 +36653832313939333937353630386530323333666664333437343365386464313861633433626439 +33353033353432663633353762323934613563396230613439336338313430336534623534363764 +36616436393231633932383430353034313631636431636536643964613133376332323330393666 +65333264623037303764343439663934646637383538336637306433353538353363623330373163 +32313637353634663837303863336139303361636438353335613666666130363735656565313034 +65613863623430366635363465643738366364333465336331353162373061336334373531383738 +34653439366435396664393234323837633039333462353830313961333736363039306231343766 +30636566616561376664613235633631663837363830623037343538383333346336613636643064 +39636466613336316638356662383839666661393634376431366533326135386334623864383632 +31623537343835303436373331363264376561396535386464303732336432643336323362626161 +61376530393863303965613338303132313262386436343866623163646238333639643035313137 +36653632353033353462303236353139303734306461363937383331613733336539343730386435 +36636636663965356636336663616561373965353363346238393064333237303338633763613362 +38666135643732353039663030343336303536633662623965316639373161376162303539366635 +65346566396366306665663530656230633832353066343431643437326635396134343132343738 +66383563353562353532373639363466623937313066386533333366333030333534366539303431 +63306564613332653136653431643732393632393633623133313162623061333833326233663833 +36346232623462646430633966323165393663323864616565343231663537346566663561326634 +33373436396538393136636165353030616537356634616662666362386565613263306135666164 +35633435313334353531323064633732656438346530383261323264626234393536323162393230 +36366664653233313936623363343834396366356162663036366438323663356661633938646437 +31383865313632343866666433313039303438393038323530636234616336626239373761623862 +33646431343236646465303032363932336331393261363666373331656166653439303936383861 +39613539613838626463376233316134653935376165303765313131393661623732613234623630 +66323137383038363737306230356161643262626162393964326166356630646537653930343463 +62326638623038626131323065323939343232636137633961656634353237306534346139393864 +37336138636262663463343638373763333539323364343735376232306466323864323431333863 +3562