From 5cc552d0ebb5391ef6121ebc248ebb5a4458c768 Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Sun, 16 Jan 2022 16:55:40 +0000 Subject: [PATCH] Add container to automatically backup DBs --- ansible/host_vars/decker/vault.yml | 19 ++++---- ansible/host_vars/forrest/vault.yml | 9 ++++ ansible/host_vars/grimes/vault.yml | 19 ++++---- .../{pve-docker.yml => pve-docker/main.yml} | 2 + ansible/host_vars/pve-docker/vault.yml | 9 ++++ ansible/host_vars/walker/vault.yml | 19 ++++---- ansible/main.yml | 9 ++++ .../roles/db_auto_backup/defaults/main.yml | 1 + .../db_auto_backup/files/docker-compose.yml | 11 +++++ .../roles/db_auto_backup/handlers/main.yml | 4 ++ ansible/roles/db_auto_backup/tasks/main.yml | 17 +++++++ ansible/roles/db_auto_backup/vars/main.yml | 1 + .../files/docker-utils/db-backup | 45 ------------------- .../roles/traefik/files/docker-compose.yml | 2 +- 14 files changed, 97 insertions(+), 70 deletions(-) create mode 100644 ansible/host_vars/forrest/vault.yml rename ansible/host_vars/{pve-docker.yml => pve-docker/main.yml} (83%) create mode 100644 ansible/host_vars/pve-docker/vault.yml create mode 100644 ansible/roles/db_auto_backup/defaults/main.yml create mode 100644 ansible/roles/db_auto_backup/files/docker-compose.yml create mode 100644 ansible/roles/db_auto_backup/handlers/main.yml create mode 100644 ansible/roles/db_auto_backup/tasks/main.yml create mode 100644 ansible/roles/db_auto_backup/vars/main.yml delete mode 100755 ansible/roles/docker_cleanup/files/docker-utils/db-backup diff --git a/ansible/host_vars/decker/vault.yml b/ansible/host_vars/decker/vault.yml index 653f1bf..ce7903a 100644 --- a/ansible/host_vars/decker/vault.yml +++ b/ansible/host_vars/decker/vault.yml @@ -1,9 +1,12 @@ $ANSIBLE_VAULT;1.1;AES256 -64386132336631373533383835363066313631666162666662376665643434333935666334393633 -6662663138396139626663313961303265633535653439330a393732323931653137626638313765 -34343931396166363338346431616632326263653663326537386561646466633835343663323534 -3833653734373962610a383238623138636164623732336165613930323364346333646338383566 -62633532343063653665363663356461383134333439636230333839646331626239346438306636 -62373262663730343963643061383262356437346535323031326539663637636432376463643666 -33616463326261326336316331373331613635613036636235643934646466306530653363303266 -33393864386538656234 +37326662353562626466613939643162346663306230333066323231346233633561363932313364 +6636326134326435356161653231643666343432373133380a623161326465613235626236623062 +63303436626538646432323337343062376235363734623935663135666531306562616630343835 +6537356330336261360a666166366663633937326534616534316531366136613237633035383738 +38333832653935623637333437386531353831616130656532356662363765306439633464626661 +66386538336266353538356431393162373763383734633638323866396434363465303866303163 +31366566316338636239313539343465343336376435633834396239643535663563373832303331 +35643966653666653538626236663437616164653764323562346238663538396233636233326165 +62373633383539353237376130363334373936623532653538326366366261613833383734376330 +34393234393461346137336561363264613139616161333239363334346465323234376661616166 +656331326539323739626633376662613564 diff --git a/ansible/host_vars/forrest/vault.yml b/ansible/host_vars/forrest/vault.yml new file mode 100644 index 0000000..eacb481 --- /dev/null +++ b/ansible/host_vars/forrest/vault.yml @@ -0,0 +1,9 @@ +$ANSIBLE_VAULT;1.1;AES256 +36376462326539663933303664633661303163333865656435356465373264626366336137303563 +6239643535636538636434313739303030333162613635610a643831613934643631306232613130 +65386166663136646161643133643238643033363533616664653565313463396138663839353131 +3637333263663333610a653361336264313835383239396662626462353239616165626134666663 +36386234633039653431343564653463376561306430663939663338646665616532393364363363 +38613034393265376133366232386662373634623662613762653439633931323634613838656262 +30623763366362653834636161646339393933346134613132623365656363373165323633663432 +37636538383734646363 diff --git a/ansible/host_vars/grimes/vault.yml b/ansible/host_vars/grimes/vault.yml index 4866140..22839fb 100644 --- a/ansible/host_vars/grimes/vault.yml +++ b/ansible/host_vars/grimes/vault.yml @@ -1,9 +1,12 @@ $ANSIBLE_VAULT;1.1;AES256 -61636635633634366161363765363961396430313436353337616466653964373464633236663631 -3066653963336137343065343631623730653536343934660a666662306464313738636163316131 -66386565303630376663643330396630303832323839366164303061303331636362306236396131 -3136326432323939380a373764616161623333343834623566663139396139323561323463376330 -39386531373266353063316566366636363538663865373638643736366135373937313030373630 -36303166643533653038323466353230383464353130323233333838656432343931643035663535 -66383332363762353832316535663234373066386662656135343564353363303232613766313563 -32336561313639366461 +35343036383263323932663736373236313935646135656437646566373637373933643631663466 +3234633065393161663761323330626230383633643865610a663064313938353131663833633534 +63353431633763313731316564363863343232623663383366386133383035343465383935626464 +3661373034663330360a653734363033663531383338343239636263626162353036333964383862 +38316636653961643638386162323466643032646663383866306565636234333431366538613930 +65376137353932393931333366373962663939656664373536653063666534653631663964366466 +61316232663430346237343165363461396661343836316137326238313437356562333038306235 +38613732356434326637383832303636666162316333366564346562656530343461326662666230 +63663535616461646539623863373631383630313533623138613530383334333939366638653131 +61666539316263396666616264636533633035393937623332653632663130326630303337643439 +336466346361336239333938636239306563 diff --git a/ansible/host_vars/pve-docker.yml b/ansible/host_vars/pve-docker/main.yml similarity index 83% rename from ansible/host_vars/pve-docker.yml rename to ansible/host_vars/pve-docker/main.yml index 50265cb..3da7c38 100644 --- a/ansible/host_vars/pve-docker.yml +++ b/ansible/host_vars/pve-docker/main.yml @@ -6,3 +6,5 @@ traefik_provider_grafana: true traefik_provider_gitlab: true with_fail2ban: true + +db_backups_dir: /mnt/tank/dbs/backups diff --git a/ansible/host_vars/pve-docker/vault.yml b/ansible/host_vars/pve-docker/vault.yml new file mode 100644 index 0000000..b7c98ce --- /dev/null +++ b/ansible/host_vars/pve-docker/vault.yml @@ -0,0 +1,9 @@ +$ANSIBLE_VAULT;1.1;AES256 +35383562343262633962376665646331613539666465663661376361306439366662646439376561 +6139303637323938303537313331353937636631396537630a626362383465336661636431373163 +36666665373636353263636366303064386262653038396338396532376363616236623430363431 +3965653231323338360a396635666137343865373063376639333735323434346136663636396533 +65616465633839663335666236383039356334353561343830363264353532326530326565323339 +61643637663966626264626166663639666465383063333266353064396565653564623735663939 +35646461393163633639326563353835313762353166346237383430336632353761623438353930 +61333536343662396331 diff --git a/ansible/host_vars/walker/vault.yml b/ansible/host_vars/walker/vault.yml index 0f34a25..90dcecb 100644 --- a/ansible/host_vars/walker/vault.yml +++ b/ansible/host_vars/walker/vault.yml @@ -1,9 +1,12 @@ $ANSIBLE_VAULT;1.1;AES256 -63343332346238306230643233623336383766656433366339346331653036633636666238613764 -3431336432616166386462346532633664616562636136630a613836643565633962656432653333 -65356132316139363261373961663930383131393535633861343734393666326665653931663036 -3632613637663132360a373266303662623739633831613764313061616239303135386630616638 -62323930366166326433363835316536646363616431653566306363323736343761643038346262 -39316564333435663539653563653737333730616131393766643964303536373235323430616261 -39306535356562313133653337383762373636373234363732636266613165333439356334383661 -39343333303337363766 +65616232306563653238306536316238353432656365303665343830323833376436303231646230 +6633613632646639326266333639663734326135373165660a616534353763643737646363363635 +35316462343935666362313735376164343238313564366232346330313565613039643735626535 +3335366566303730640a656665323266386430383263326161376435663062353763396264316462 +62663166326262633437643065396132326366646331323330316565626637656632643162636563 +63623563386164333638633638633061616266316333336133313166373639643633643631386136 +39633565343862333134323737393761323365636534303863646233646639636437656335633836 +66356237386162316365376238343430373866623463633635383634383336393264363364663139 +32613761643030343764396339386538333663376633646332613330373838343137373833643235 +61303762336132326339363366623231366565316139383561656364376564336230346533323638 +626365336439666234343531666266646437 diff --git a/ansible/main.yml b/ansible/main.yml index 402db44..2ac5f98 100644 --- a/ansible/main.yml +++ b/ansible/main.yml @@ -43,6 +43,15 @@ - "{{ user }}" - docker_cleanup +- hosts: + - pve-docker + - forrest + - walker + - grimes + - decker + roles: + - db_auto_backup + - hosts: - pve-docker - walker diff --git a/ansible/roles/db_auto_backup/defaults/main.yml b/ansible/roles/db_auto_backup/defaults/main.yml new file mode 100644 index 0000000..a766fd8 --- /dev/null +++ b/ansible/roles/db_auto_backup/defaults/main.yml @@ -0,0 +1 @@ +db_backups_dir: ./backups diff --git a/ansible/roles/db_auto_backup/files/docker-compose.yml b/ansible/roles/db_auto_backup/files/docker-compose.yml new file mode 100644 index 0000000..bdd7f47 --- /dev/null +++ b/ansible/roles/db_auto_backup/files/docker-compose.yml @@ -0,0 +1,11 @@ +version: "2.3" + +services: + backup: + image: ghcr.io/realorangeone/db-auto-backup:latest + restart: unless-stopped + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + - "{{ db_backups_dir }}:/var/backups" + environment: + - HEALTHCHECKS_ID={{ db_auto_backup_healthchecks_id }} diff --git a/ansible/roles/db_auto_backup/handlers/main.yml b/ansible/roles/db_auto_backup/handlers/main.yml new file mode 100644 index 0000000..e1be2cc --- /dev/null +++ b/ansible/roles/db_auto_backup/handlers/main.yml @@ -0,0 +1,4 @@ +- name: restart db-auto-backup + shell: + chdir: /opt/db-auto-backup + cmd: "{{ docker_update_command }}" diff --git a/ansible/roles/db_auto_backup/tasks/main.yml b/ansible/roles/db_auto_backup/tasks/main.yml new file mode 100644 index 0000000..cc6fd8c --- /dev/null +++ b/ansible/roles/db_auto_backup/tasks/main.yml @@ -0,0 +1,17 @@ +- name: Create install directory + file: + path: /opt/db-auto-backup + state: directory + owner: "{{ docker_user.name }}" + mode: "{{ docker_compose_directory_mask }}" + become: true + +- name: Install compose file + template: + src: files/docker-compose.yml + dest: /opt/db-auto-backup/docker-compose.yml + mode: "{{ docker_compose_file_mask }}" + owner: "{{ docker_user.name }}" + validate: docker-compose -f %s config + notify: restart db-auto-backup + become: true diff --git a/ansible/roles/db_auto_backup/vars/main.yml b/ansible/roles/db_auto_backup/vars/main.yml new file mode 100644 index 0000000..afa6846 --- /dev/null +++ b/ansible/roles/db_auto_backup/vars/main.yml @@ -0,0 +1 @@ +db_auto_backup_healthchecks_id: "{{ vault_db_auto_backup_healthchecks_id }}" diff --git a/ansible/roles/docker_cleanup/files/docker-utils/db-backup b/ansible/roles/docker_cleanup/files/docker-utils/db-backup deleted file mode 100755 index 11ac011..0000000 --- a/ansible/roles/docker_cleanup/files/docker-utils/db-backup +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -BACKUP_DIR=$1 - -if [ -z "$BACKUP_DIR" ] - then - echo "No backup dir" -fi - -all_containers=$(docker ps --format "{{.ID}}:{{ .Image }}") - -for line in $all_containers -do - IFS=':' read -a container_details <<< $line - - container_name=${container_details[1]} - container_id=${container_details[0]} - - case "$container_name" in - "mariadb") - db_name=$(docker exec $container_id bash -c 'echo $MYSQL_USER') - echo Backing up mariadb $db_name - docker exec $container_id bash -c 'mysqldump -u $MYSQL_USER -p$MYSQL_PASSWORD --all-databases' | pv > $BACKUP_DIR/$db_name.sql - ;; - - "postgres") - db_name=$(docker exec $container_id bash -c 'echo $POSTGRES_USER') - echo Backing up postgres $db_name - docker exec $container_id bash -c 'PGPASSWORD=$POSTGRES_PASSWORD pg_dumpall -U $POSTGRES_USER' | pv > $BACKUP_DIR/$db_name.sql - ;; - "yandex/clickhouse-server") - # Hardcode for plausible - tables=$(docker exec $container_id clickhouse-client --query "SELECT name FROM system.tables where database == 'plausible';") - for table in $tables - do - echo Backing up clickhouse table $table - docker exec $container_id clickhouse-client --query "SELECT * FROM plausible.$table" --format CSVWithNames | pv > $BACKUP_DIR/plausible-$table.csv - done - ;; - - esac -done - -echo "Setting user permissions..." -chown -R root:root $BACKUP_DIR diff --git a/ansible/roles/traefik/files/docker-compose.yml b/ansible/roles/traefik/files/docker-compose.yml index 30e3186..2786a7a 100644 --- a/ansible/roles/traefik/files/docker-compose.yml +++ b/ansible/roles/traefik/files/docker-compose.yml @@ -17,7 +17,7 @@ services: - "{{ private_ip }}:8080:8080" depends_on: - docker_proxy - - nginx + - shenanigans networks: - default - traefik