infrastructure/ansible/roles/forrest/files/prometheus/prometheus.yml

alerting:
  alertmanagers:
    - static_configs:
        - targets:
            - alertmanager:9093

rule_files:
  - /etc/prometheus/alert-rules.d/*.yml

scrape_configs:
  - job_name: pve
    static_configs:
      - targets:
          - "{{ pve_hosts.pve.ip }}"
    metrics_path: /pve
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: proxmox_exporter:9221

  - job_name: traefik
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: go_.+
        action: drop
    static_configs:
      - targets:
          - "{{ nebula.clients.walker.ip }}:8080"
          - "{{ pve_hosts.docker.ip }}:8080"

  - job_name: homeassistant
    metrics_path: /api/prometheus
    authorization:
      credentials: "{{ homeassistant_token }}"
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: python_.+
        action: drop
    static_configs:
      - targets: ["{{ pve_hosts.homeassistant.ip }}:8123"]

  - job_name: blackbox_http_external
    scrape_interval: 1m
    metrics_path: /probe
    params:
      module: [http]
    static_configs:
      - targets:
          - https://bin.theorangeone.net
          - https://git.theorangeone.net/api/healthz
          - https://grafana.jakehoward.tech/api/health
          - https://homeassistant.jakehoward.tech
          - https://intersect.jakehoward.tech
          - https://mastodon.theorangeone.net/health
          - https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version
          - https://matrix.jakehoward.tech/_matrix/federation/v1/version
          - https://media.jakehoward.tech
          - https://notes.theorangeone.net
          - https://plausible.theorangeone.net
          - https://recipes.jakehoward.tech
          - https://theorangeone.net
          - https://tt-rss.jakehoward.tech
          - https://vaultwarden.jakehoward.tech/alive
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-external:9115

  - job_name: blackbox_icmp
    scrape_interval: 10m
    metrics_path: /probe
    params:
      module: [icmp]
    static_configs:
      - targets:
          - "{{ pve_hosts.pve.ip }}"
          - "{{ pve_hosts.homeassistant.ip }}"
          - 9.9.9.9
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

  - job_name: blackbox_https_redir
    scrape_interval: 10m
    metrics_path: /probe
    params:
      module: [https_redir]
    static_configs:
      - targets:
          - http://{{ hosts.casey_ip }}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox-external:9115

  - job_name: blackbox_healthcheck
    scrape_interval: 10m
    metrics_path: /probe
    params:
      module: [http]
    static_configs:
      - targets:
          - https://hc-ping.com/{{ prometheus_healthcheck_uuid }}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115

  - job_name: healthchecks
    scheme: https
    metrics_path: /projects/{{ healthchecks_project_uuid }}/metrics/{{ healthcheck_api_token }}
    static_configs:
      - targets: [healthchecks.io]

  - job_name: node
    metrics_path: /metrics
    static_configs:
      - targets:
          - "{{ pve_hosts.pve.ip }}:9100"
          - pi.hole:9100
    metric_relabel_configs:
      - source_labels: [__name__]
        regex: go_.+
        action: drop

  - job_name: speedtest
    scrape_interval: 1h
    scrape_timeout: 2m
    metrics_path: /probe
    static_configs:
      - targets:
          - speedtest_exporter:9516
    metric_relabel_configs:
      - source_labels: [__name__]
        target_label: __name__
        replacement: speedtest_$1
Add alertmanager 2022-12-05 21:10:41 +00:00			`alerting:`
			`alertmanagers:`
			`- static_configs:`
			`- targets:`
			`- alertmanager:9093`

Add some prometheus alerts 2023-02-19 18:31:52 +00:00			`rule_files:`
			`- /etc/prometheus/alert-rules.d/*.yml`

Add telegraf And input to ping and output via prometheus 2021-03-04 15:16:54 +00:00			`scrape_configs:`
Replace telegraf with prometheus exporters Still missing SMART and ping, but those can come later. Swapped as the polling model for prometheus doesn't play especially well with telegraf, and leads to gaps in data 2023-01-31 09:11:09 +00:00			`- job_name: pve`
Add telegraf And input to ping and output via prometheus 2021-03-04 15:16:54 +00:00			`static_configs:`
			`- targets:`
Replace telegraf with prometheus exporters Still missing SMART and ping, but those can come later. Swapped as the polling model for prometheus doesn't play especially well with telegraf, and leads to gaps in data 2023-01-31 09:11:09 +00:00			`- "{{ pve_hosts.pve.ip }}"`
			`metrics_path: /pve`
			`relabel_configs:`
			`- source_labels: [__address__]`
			`target_label: __param_target`
			`- source_labels: [__param_target]`
			`target_label: instance`
			`- target_label: __address__`
			`replacement: proxmox_exporter:9221`
Scrape homeassistant metrics into prometheus 2022-09-06 21:29:11 +01:00
Monitor traefik with prometheus rather than influxdb 2021-03-04 16:26:55 +00:00			`- job_name: traefik`
Ignore go metrics 2021-03-05 14:27:33 +00:00			`metric_relabel_configs:`
			`- source_labels: [__name__]`
			`regex: go_.+`
			`action: drop`
Monitor traefik with prometheus rather than influxdb 2021-03-04 16:26:55 +00:00			`static_configs:`
			`- targets:`
Use variables in prometheus config Also replace grimes with walker 2021-05-05 18:12:42 +01:00			`- "{{ nebula.clients.walker.ip }}:8080"`
Correctly check hostname against PVE hosts Some of the hostnames have `-` in instead, which caused issues with the SSH config detecting which users to allow 2021-08-23 19:56:04 +01:00			`- "{{ pve_hosts.docker.ip }}:8080"`
Scrape homeassistant metrics into prometheus 2022-09-06 21:29:11 +01:00
			`- job_name: homeassistant`
			`metrics_path: /api/prometheus`
			`authorization:`
			`credentials: "{{ homeassistant_token }}"`
			`metric_relabel_configs:`
			`- source_labels: [__name__]`
Exclude python stats for HA 2022-12-05 20:36:57 +00:00			`regex: python_.+`
Scrape homeassistant metrics into prometheus 2022-09-06 21:29:11 +01:00			`action: drop`
			`static_configs:`
			`- targets: ["{{ pve_hosts.homeassistant.ip }}:8123"]`
Setup blackbox for some HTTP monitoring 2022-12-04 22:11:49 +00:00
			`- job_name: blackbox_http_external`
			`scrape_interval: 1m`
			`metrics_path: /probe`
			`params:`
Use a blackbox with custom DNS rather than proxy This makes timing information more relevant and should improve reliability 2022-12-06 10:04:23 +00:00			`module: [http]`
Setup blackbox for some HTTP monitoring 2022-12-04 22:11:49 +00:00			`static_configs:`
			`- targets:`
			`- https://bin.theorangeone.net`
Monitor gitea 2023-04-06 20:31:57 +01:00			`- https://git.theorangeone.net/api/healthz`
Setup blackbox for some HTTP monitoring 2022-12-04 22:11:49 +00:00			`- https://grafana.jakehoward.tech/api/health`
			`- https://homeassistant.jakehoward.tech`
			`- https://intersect.jakehoward.tech`
			`- https://mastodon.theorangeone.net/health`
			`- https://matrix.jakehoward.tech:8448/_matrix/federation/v1/version`
			`- https://matrix.jakehoward.tech/_matrix/federation/v1/version`
			`- https://media.jakehoward.tech`
			`- https://notes.theorangeone.net`
			`- https://plausible.theorangeone.net`
			`- https://recipes.jakehoward.tech`
			`- https://theorangeone.net`
			`- https://tt-rss.jakehoward.tech`
			`- https://vaultwarden.jakehoward.tech/alive`
			`relabel_configs:`
			`- source_labels: [__address__]`
			`target_label: __param_target`
			`- source_labels: [__param_target]`
			`target_label: instance`
			`- target_label: __address__`
Use a blackbox with custom DNS rather than proxy This makes timing information more relevant and should improve reliability 2022-12-06 10:04:23 +00:00			`replacement: blackbox-external:9115`
Setup blackbox for some HTTP monitoring 2022-12-04 22:11:49 +00:00
Add ICMP checks 2023-01-31 14:29:27 +00:00			`- job_name: blackbox_icmp`
			`scrape_interval: 10m`
			`metrics_path: /probe`
			`params:`
			`module: [icmp]`
			`static_configs:`
			`- targets:`
			`- "{{ pve_hosts.pve.ip }}"`
			`- "{{ pve_hosts.homeassistant.ip }}"`
			`- 9.9.9.9`
			`relabel_configs:`
			`- source_labels: [__address__]`
			`target_label: __param_target`
			`- source_labels: [__param_target]`
			`target_label: instance`
			`- target_label: __address__`
			`replacement: blackbox:9115`

Setup blackbox for some HTTP monitoring 2022-12-04 22:11:49 +00:00			`- job_name: blackbox_https_redir`
			`scrape_interval: 10m`
			`metrics_path: /probe`
			`params:`
			`module: [https_redir]`
			`static_configs:`
			`- targets:`
			`- http://{{ hosts.casey_ip }}`
			`relabel_configs:`
			`- source_labels: [__address__]`
			`target_label: __param_target`
			`- source_labels: [__param_target]`
			`target_label: instance`
			`- target_label: __address__`
Use a blackbox with custom DNS rather than proxy This makes timing information more relevant and should improve reliability 2022-12-06 10:04:23 +00:00			`replacement: blackbox-external:9115`
Scrape healthchecks.io with prometheus and use it for outbound monitoring of prometheus scrapes 2022-12-05 14:40:52 +00:00
			`- job_name: blackbox_healthcheck`
			`scrape_interval: 10m`
			`metrics_path: /probe`
			`params:`
Use a blackbox with custom DNS rather than proxy This makes timing information more relevant and should improve reliability 2022-12-06 10:04:23 +00:00			`module: [http]`
Scrape healthchecks.io with prometheus and use it for outbound monitoring of prometheus scrapes 2022-12-05 14:40:52 +00:00			`static_configs:`
			`- targets:`
			`- https://hc-ping.com/{{ prometheus_healthcheck_uuid }}`
			`relabel_configs:`
			`- source_labels: [__address__]`
			`target_label: __param_target`
			`- source_labels: [__param_target]`
			`target_label: instance`
			`- target_label: __address__`
			`replacement: blackbox:9115`

			`- job_name: healthchecks`
			`scheme: https`
			`metrics_path: /projects/{{ healthchecks_project_uuid }}/metrics/{{ healthcheck_api_token }}`
			`static_configs:`
			`- targets: [healthchecks.io]`
Replace telegraf with prometheus exporters Still missing SMART and ping, but those can come later. Swapped as the polling model for prometheus doesn't play especially well with telegraf, and leads to gaps in data 2023-01-31 09:11:09 +00:00
			`- job_name: node`
			`metrics_path: /metrics`
			`static_configs:`
			`- targets:`
			`- "{{ pve_hosts.pve.ip }}:9100"`
Install node-exporter on pi-hole 2023-01-31 23:08:07 +00:00			`- pi.hole:9100`
Replace telegraf with prometheus exporters Still missing SMART and ping, but those can come later. Swapped as the polling model for prometheus doesn't play especially well with telegraf, and leads to gaps in data 2023-01-31 09:11:09 +00:00			`metric_relabel_configs:`
			`- source_labels: [__name__]`
			`regex: go_.+`
			`action: drop`
Add speedtest exporter 2023-01-31 21:45:59 +00:00
			`- job_name: speedtest`
Reduce speedtest interval I don't need it to run that often 2023-02-28 08:40:26 +00:00			`scrape_interval: 1h`
Add speedtest exporter 2023-01-31 21:45:59 +00:00			`scrape_timeout: 2m`
			`metrics_path: /probe`
			`static_configs:`
			`- targets:`
			`- speedtest_exporter:9516`
			`metric_relabel_configs:`
			`- source_labels: [__name__]`
			`target_label: __name__`
			`replacement: speedtest_$1`