Add alertmanager

ansible/roles/forrest/files/prometheus/alertmanager.yml

@@ -0,0 +1,15 @@
+global:
+  resolve_timeout: 3m
+  smtp_smarthost: smtp.eu.mailgun.org:465
+  smtp_from: "{{ alertmanager_from_address }}"
+  smtp_auth_username: "{{ alertmanager_from_address }}"
+  smtp_auth_password: "{{ alertmanager_smtp_password }}"
+
+route:
+  receiver: default
+
+receivers:
+  - name: default
+    email_configs:
+      - to: "{{ alertmanager_to_address }}"
+        send_resolved: true

ansible/roles/forrest/files/prometheus/docker-compose.yml

@@ -19,6 +19,12 @@ services:
     volumes:
       - ./blackbox.yml:/etc/blackbox_exporter/config.yml
 
+  alertmanager:
+    image: prom/alertmanager:latest
+    restart: unless-stopped
+    volumes:
+      - ./alertmanager.yml:/etc/alertmanager/alertmanager.yml
+
 networks:
   grafana:
     external: true

ansible/roles/forrest/files/prometheus/prometheus.yml

@@ -1,3 +1,9 @@
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+            - alertmanager:9093
+
 scrape_configs:
   - job_name: telegraf
     metric_relabel_configs:

ansible/roles/forrest/tasks/prometheus.yml

@@ -33,3 +33,12 @@
     owner: "{{ docker_user.name }}"
   notify: restart prometheus
   become: true
+
+- name: Install alertmanager config
+  template:
+    src: files/prometheus/alertmanager.yml
+    dest: /opt/prometheus/alertmanager.yml
+    mode: "{{ docker_compose_file_mask }}"
+    owner: "{{ docker_user.name }}"
+  notify: restart prometheus
+  become: true

ansible/roles/forrest/vars/main.yml

@@ -6,3 +6,6 @@ gitlab_liveness_access_token: "{{ vault_gitlab_liveness_access_token }}"
 prometheus_healthcheck_uuid: "{{ vault_prometheus_healthcheck_uuid }}"
 healthchecks_project_uuid: "{{ vault_healthchecks_project_uuid }}"
 healthcheck_api_token: "{{ vault_healthcheck_api_token }}"
+alertmanager_from_address: "{{ vault_alertmanager_from_address }}"
+alertmanager_smtp_password: "{{ vault_alertmanager_smtp_password }}"
+alertmanager_to_address: "{{ vault_alertmanager_to_address }}"

ansible/roles/forrest/vars/vault.yml

@@ -1,38 +1,49 @@
 $ANSIBLE_VAULT;1.1;AES256
-37343335613562383564356561646365363139333631366661636536373331333335313666333432
+61666339653630373931366533656437386337663739313532663339356562656466373461303837
-3032353032313634326466653435323937386339303831330a663066353862343831376332343863
+3132343434633835333035303863346532363137656565380a646239326534346135333737653337
-65353137393733336366333264653461653035643339613332653764646231323336336134633466
+31626663636661396462353765653830396132306531393965306439316437623866306166656430
-6235666363613639320a333262366261626437373032653937303937336138346165343231336238
+3664373263643666350a353139653633663233623064353532313136393865333763353233343132
-37643465306237343039373435376438633264386337373132353039333033363133303766633236
+61363864666534636630623639346361656361633235313535323030633434656136626131656237
-35653333363865396164643162393536643533373634393064353738643165343838613536323036
+62633433613537363431336664313166306539616431626131653337363536633536386561373630
-35666464343934323862306634306465303861653032316135613865613265393038333337363536
+32323165393538636437653361363766636139646166646538666462333337333837383966643263
-39373433633964636461653438313932356139376266616533323239363961663035626462366366
+31646562316433323962353439353636393965393962633665306232653737353233346632326233
-35663532323964613166396164643830336264343766656630393761336663326234333738313239
+36333863623933313238663434643737363638666662323238643238326165396433653462663934
-36653832313939333937353630386530323333666664333437343365386464313861633433626439
+62663538366633386533316432323535656563346435353665333430623434616634393565633962
-33353033353432663633353762323934613563396230613439336338313430336534623534363764
+65383262356333643437636531663034303933313534393965633739633031393139386633376134
-36616436393231633932383430353034313631636431636536643964613133376332323330393666
+35653164313830643065613439663462306131383135366363653834386230363639366233643866
-65333264623037303764343439663934646637383538336637306433353538353363623330373163
+30626638306666653065353237663434633333376337343133656237666662343163656164353935
-32313637353634663837303863336139303361636438353335613666666130363735656565313034
+39386262303664653164356431643466306463343564633064376532663133393432626333393236
-65613863623430366635363465643738366364333465336331353162373061336334373531383738
+39306463663963383234363664643265393434363138373062643165313332643833663464343766
-34653439366435396664393234323837633039333462353830313961333736363039306231343766
+33393162336535303136653230633135636266653863646331346630386131303131333636613938
-30636566616561376664613235633631663837363830623037343538383333346336613636643064
+66386331643966313236356634623938363835323439343234613164616132616137353237356162
-39636466613336316638356662383839666661393634376431366533326135386334623864383632
+66333964363361353436396634353735663963643239356164333434373166623731643038656136
-31623537343835303436373331363264376561396535386464303732336432643336323362626161
+32376339316434373836333532373664323762636634336361616437306132356637616162303231
-61376530393863303965613338303132313262386436343866623163646238333639643035313137
+30356263366137363337316631643536643539653237636362386366656434353861343239306431
-36653632353033353462303236353139303734306461363937383331613733336539343730386435
+33326163336536636233643434633461626430313536376632623334303938316364316162386365
-36636636663965356636336663616561373965353363346238393064333237303338633763613362
+38393138356538623235663539393166306366396364306335656534366663616336666637323138
-38666135643732353039663030343336303536633662623965316639373161376162303539366635
+32343035373063306462636163616364616332333263663737653235613230313465363466616565
-65346566396366306665663530656230633832353066343431643437326635396134343132343738
+36373831383364333837303433663839353363663730333665663138386635626530323633626661
-66383563353562353532373639363466623937313066386533333366333030333534366539303431
+62633066633361366562383737646565613566653866663136326135383533303962326637316132
-63306564613332653136653431643732393632393633623133313162623061333833326233663833
+39313832303538363432646432616433303966626133653461363530623966363133646530353931
-36346232623462646430633966323165393663323864616565343231663537346566663561326634
+62646334646336633162633763613333663937636262633138323932313632653630393162363439
-33373436396538393136636165353030616537356634616662666362386565613263306135666164
+37343739313066343831623265353164386239373430623735343739373737373130643631343664
-35633435313334353531323064633732656438346530383261323264626234393536323162393230
+31626633303764633831646565653732326363336431353433353365633433626665353733663336
-36366664653233313936623363343834396366356162663036366438323663356661633938646437
+32643132666565373332323136386630633561656632613335613965343439633831336336306464
-31383865313632343866666433313039303438393038323530636234616336626239373761623862
+62303234646233633366383430366431336430356463643630623566373333646532333663623466
-33646431343236646465303032363932336331393261363666373331656166653439303936383861
+63326364393166393461653366373634626333623836303466636136663331383239336432363661
-39613539613838626463376233316134653935376165303765313131393661623732613234623630
+62383836653562633765333034306161326539396330663831363135326235343535393663363638
-66323137383038363737306230356161643262626162393964326166356630646537653930343463
+36616130663538376534616161393930633462316539643561323665313339646234306232656662
-62326638623038626131323065323939343232636137633961656634353237306534346139393864
+30356139303465313561636433646162346136303632653230353437383063396438616536376533
-37336138636262663463343638373763333539323364343735376232306466323864323431333863
+30363362613635336162336537333932306666666632336664613631623830616538633131663365
-3562
+36656164623139643333333237376333656632306436313865656337616439343434383231616561
+32323232306437353034643735356532653233306131356632393963616337303835663632653837
+63656433633061313034656565373061643132623163626230386561323466353534323734326437
+34336439363563373637336139376261376439306166656337663366383463313938333731306664
+34393062653536346464373637343232666465336564613235323931373465323864386362336561
+37633837336136373030346533633562343439316264333462623763666162373938326437353638
+63343062313732356338623731626232626265366466623437383734313630393030366330353864
+63613766373438366564663761623766353636303066343665353437643830626364663031336537
+36663739303232363162646464663339623939363733393835383366636463643633396362363137
+62613138656563666661363630633935346433356235303936356431343637663938646161343266
+61313861346562396439383262396435313064643264353536326238623461653965626334653866
+36646239313666396338