From 55df3e8fa340bb539b4c717bf64b29776e302cec Mon Sep 17 00:00:00 2001 From: Jake Howard Date: Sun, 1 Dec 2024 20:58:34 +0000 Subject: [PATCH] Separate restic repositories Smaller VMs no longer need to load the full index --- ansible/host_vars/casey/main.yml | 1 + ansible/host_vars/casey/vault.yml | 21 +++++++++------ ansible/host_vars/restic/main.yml | 2 +- ansible/host_vars/restic/vault.yml | 29 +++++++++------------ ansible/host_vars/tang/main.yml | 1 + ansible/host_vars/tang/vault.yml | 21 +++++++++------ ansible/host_vars/walker/main.yml | 1 + ansible/host_vars/walker/vault.yml | 28 ++++++++++++-------- ansible/roles/restic/defaults/main.yml | 27 ++++++++++++++++++- ansible/roles/restic/files/backrest.sh | 2 +- ansible/roles/restic/files/restic-backup.sh | 2 ++ ansible/roles/restic/files/restic-forget.sh | 5 ---- ansible/roles/restic/tasks/main.yml | 11 -------- ansible/roles/restic/vars/main.yml | 25 ------------------ 14 files changed, 89 insertions(+), 87 deletions(-) delete mode 100644 ansible/roles/restic/files/restic-forget.sh diff --git a/ansible/host_vars/casey/main.yml b/ansible/host_vars/casey/main.yml index f5c42cb..04fb2a9 100644 --- a/ansible/host_vars/casey/main.yml +++ b/ansible/host_vars/casey/main.yml @@ -12,3 +12,4 @@ cdn_domains: restic_backup_locations: - /var/lib/headscale/ +restic_key: "{{ vault_restic_key }}" diff --git a/ansible/host_vars/casey/vault.yml b/ansible/host_vars/casey/vault.yml index 510f68e..284dd99 100644 --- a/ansible/host_vars/casey/vault.yml +++ b/ansible/host_vars/casey/vault.yml @@ -1,9 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -30643138356634323666316163396138663836316261363966636335366534336330616635383663 -6461393538346263363164613930396266323930626335370a306165306663336538316163666364 -65383835386635336433393162613031386334646632666638613162623434646531356533346132 -3162373933336365660a353163316338303630633761336238363966376336643838616135303231 -32646530376561326635633563393066656232363734653464326665396236656232613362333461 -39393134626466656561346138633362653732333639333765303961383365623737666164326532 -66356263326366323435623834306439633061386364633132613362386663633733386637363266 -31393438326531353265 +36373834643031616365396635633037316166323436353966386636346635666364356438313961 +3933656632393237353465313163326636316661626230300a326231383439323964306239366432 +30383063323065303638353064653532393463666361303962353132303762376435613264353038 +3338316334666637350a363963313730663633633036313737356263626233313766363230323564 +62646438663039343433316436386164646237663535366366656261333539366437393965613932 +37356661313130376163303339663265313666333438623962643231643961396264366331316130 +30346633363562333531373635303937383436303837396561623439346663666132313434373035 +35383534316362333439363365353836316334363239383832646264366339336664373032313733 +65353264643135356666396165316333666539663735303661353764323761633536316565653734 +39366236373463653166656466343239313238303561313131626136383739306132323538663665 +61373736656533396337656534393563316133383838666434343266636362336436633062373730 +36303139666132663834363031666233373435373233333361633863633530346263636236356238 +33353264656636626130666136663430356237336232366131336337306162313364 diff --git a/ansible/host_vars/restic/main.yml b/ansible/host_vars/restic/main.yml index 04e2379..40b8392 100644 --- a/ansible/host_vars/restic/main.yml +++ b/ansible/host_vars/restic/main.yml @@ -5,4 +5,4 @@ restic_backup_locations: - /mnt/home-assistant - /home/rclone/sync -restic_forget: true +restic_dir: "" diff --git a/ansible/host_vars/restic/vault.yml b/ansible/host_vars/restic/vault.yml index 85ec3ef..7fadbfd 100644 --- a/ansible/host_vars/restic/vault.yml +++ b/ansible/host_vars/restic/vault.yml @@ -1,17 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -32353739643531336665636334646135323336353562316362333266316263653364656132643661 -3736386461316563376134326638376261323734663032630a306530636166666561343264393266 -62326437343637363038646632396461303365646466666666386432306134313562356538623133 -6561323739386337630a623835656239633866666333616664366339333232303031343561633239 -62636636623462316536333334306562626637643936623963376663326164333962646134376566 -62646336353937316238333036376232323834346530626136316233626166326231633330646266 -36653263636266626233313263346263633734386339386664323331363263306465626165336337 -38653766366530373230623334386234303461336133323663626439383530373966363830633364 -37336635356334633338633161356161353133656633386563393363303064613761306137323261 -34626164663936306665613861343039666330613263303932333766306663616134316566313963 -66653263643134343363353637343636633936343165363934376537343538643434376434336633 -31613339613035633335643034336265376630326662393865626336303261363130333637643162 -32383863313139663066363766613865653966613430616631346432623164366663313838363164 -37613863326433653531656139633533353539366563653532626534346165626535643434333861 -34306433373134376137633836666162663130623130353062316439303466393035633636386234 -38333132376361376363 +32383134663132313532643364346432373163316165653830383962646532623961393364613630 +6533396333313566653132373539306232333762363432630a373635353338663538336431346563 +65653630646433633464373635623035356433393961303237616338323762346365363664346666 +3864383762313635640a393764633736653864656361356133333566663738313064383037343830 +31626238303532326133383163346137316430326430613735656334333935336534383538346161 +66396564643665633536313437626537666263343337323539333962653939393230633437383934 +36343032656230393764386562613239326463343931393063313161666165626434373237333134 +66383439316439343466653063303034393761303461383933306665383837383833333732336237 +37363163623236613933313963373438313037643031653465386165373036336462623436643162 +64373861633635633964373630623833323765663461326666633039396334366564336663663661 +30323437323463353437323165313362376564646533363832653430326164393765343066306664 +31303131653865383738363963353232613933353638336336633465373335323031623931396635 +3066 diff --git a/ansible/host_vars/tang/main.yml b/ansible/host_vars/tang/main.yml index 9b4263e..296508a 100644 --- a/ansible/host_vars/tang/main.yml +++ b/ansible/host_vars/tang/main.yml @@ -3,3 +3,4 @@ private_ip: "{{ ansible_default_ipv4.address }}" restic_backup_locations: - /var/lib/private/adguardhome/ +restic_key: "{{ vault_restic_key }}" diff --git a/ansible/host_vars/tang/vault.yml b/ansible/host_vars/tang/vault.yml index ce5c4df..abc6c37 100644 --- a/ansible/host_vars/tang/vault.yml +++ b/ansible/host_vars/tang/vault.yml @@ -1,9 +1,14 @@ $ANSIBLE_VAULT;1.1;AES256 -62623062666237373636616333623434363662316639633962363833303663376331346338363365 -6633336638623237396134613033346665313964613538320a656134323135613834316462366161 -36633062623031306562313233356536643132346466376435303031333331643936613036616236 -3231613336396135340a376339396663343837353139393062353530626566626566366439353762 -37376236376437393863633730643531323762336536633034353132356266373361613434326333 -39663562353337666435653435623563383630383537663633336437613262323733363766666539 -66373538386163303731663331666138656435343436613633323766366261316337373830653837 -64313133396532376436 +35656531393530373937613161326336373836393466323139323264633139373030353234323638 +6664363939656563363935633061656230653031396263310a306138363230643562656438613763 +63393636646437323263353565396138633539306631626135343861313131653363643662336162 +3938386431633532630a396431666464646530616530666666633466343335616463643563643637 +30366239323637373532663730336338326561666332656631616630646430616631313166366339 +32663936356232313136666131356564383232336462646565316334373763633137353566313738 +36633262663466353935316163616130623063386435633238383862616430613334376635303534 +30336661353035353430363834633532336362323265626139333137343330656630666131353139 +37633139623963373861393233316436366531643330336362613963396365323434616133323261 +30333364303535386136653863656530323634373564666238643838323836356661303862393763 +39383434643562343963626165356665613131326432643566316662343734306138363462303766 +66386230636138646434643333326438613635306633393963623835613661303932346137303164 +38386133353037356564353565636432316363616139396566663361393036306538 diff --git a/ansible/host_vars/walker/main.yml b/ansible/host_vars/walker/main.yml index 3048056..1d13ac1 100644 --- a/ansible/host_vars/walker/main.yml +++ b/ansible/host_vars/walker/main.yml @@ -2,6 +2,7 @@ private_ip: "{{ ansible_tailscale0.ipv4.address }}" restic_backup_locations: - /opt +restic_key: "{{ vault_restic_key }}" nginx_https_redirect: true diff --git a/ansible/host_vars/walker/vault.yml b/ansible/host_vars/walker/vault.yml index 90dcecb..9d4bebc 100644 --- a/ansible/host_vars/walker/vault.yml +++ b/ansible/host_vars/walker/vault.yml @@ -1,12 +1,18 @@ $ANSIBLE_VAULT;1.1;AES256 -65616232306563653238306536316238353432656365303665343830323833376436303231646230 -6633613632646639326266333639663734326135373165660a616534353763643737646363363635 -35316462343935666362313735376164343238313564366232346330313565613039643735626535 -3335366566303730640a656665323266386430383263326161376435663062353763396264316462 -62663166326262633437643065396132326366646331323330316565626637656632643162636563 -63623563386164333638633638633061616266316333336133313166373639643633643631386136 -39633565343862333134323737393761323365636534303863646233646639636437656335633836 -66356237386162316365376238343430373866623463633635383634383336393264363364663139 -32613761643030343764396339386538333663376633646332613330373838343137373833643235 -61303762336132326339363366623231366565316139383561656364376564336230346533323638 -626365336439666234343531666266646437 +34613565386261353635353661303237356565326639653231353365383664656162323438663637 +3964636331333465393939353934363436636631396262620a613134393263373431306161313730 +62323666323064356335313062306133393839373636643137666264343535653062313661636634 +3935376666303364660a333532623636313765383134316561373938613237373737323366383264 +65346533616132303931393639313865313235613661333039303339326435346465346661386130 +33303736333337616430363030363866386561333434373038343462353430323933353037303236 +37346639666661393432323866653232303134343261646566323437613066323566313463656266 +32336466623065316432643839643534373738313538333363636636323036616362633631316365 +35663064313833373466643036393562343261663339303336663534663230343230313566383930 +62356636326266653961363062636635303738326664656535306566663239646332376261623764 +36323030636134656138646636336162313461386339636435623361333333613661366530363937 +36376261653061663230623765366330366361663362356266376632653133316437643961373862 +37356262383466326236393464666665626336616262306230663266303464376361396336373732 +64636535666332333035393931306466343366623737636562613662373933643532383863653462 +62333839613239613965303031613636373166306662663665383439383837326163376463363937 +61396161636565613437623061383834656435303634616335636166343139623662356662313831 +62343961313230323330393137646662376363616661633365303738373164306564 diff --git a/ansible/roles/restic/defaults/main.yml b/ansible/roles/restic/defaults/main.yml index 619e1ee..a237ac2 100644 --- a/ansible/roles/restic/defaults/main.yml +++ b/ansible/roles/restic/defaults/main.yml @@ -1,3 +1,28 @@ restic_backup_locations: [] restic_backup_excludes: [] -restic_forget: false +restic_dir: "{{ hostname_slug }}" +restic_b2_account_id: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 37653465353462333831333031373566343362316461643463303737356332363364666264666562 + 6436353430313239643831626537383664376264393430310a353434306465653934356164393265 + 65336534356636386634366533313039643438656439636434666464636338373130636661303536 + 6638356530373062640a376430663231396538386161353165363538346266663135363337303166 + 35613838363336393032643965373437663937626532356239336535663064363438 +restic_b2_account_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 31343939356266366364313631633836653364633331306134663163653939373663316632626161 + 3262376666323366313563313836643133393437663462320a653637633663636663353234323265 + 63633632383461616437636230643430613635626431383966326666383030393735623963333263 + 3433633830356530390a623436376364343630313066663965303238646564323266303865623965 + 66353963376434333030333332646464653637633737633832663762303339666336 +restic_key: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 65326130653631643138373231336439303962356666353737373832633666373965666466613034 + 3730333664373334316436396435356137313337633836330a653366343732656539663439313838 + 34326361353232663533663366393464643761343864623965386633313438303738343238623430 + 6661313730646166330a393336333635623832653232653863626135343164363831366134316465 + 30306238613865393664383865323233626636333063393936613864303365386431613838366363 + 39633865393761366134633933333237313362636234323330313135336637613235303764383938 + 34383938396366363937643864393838363638383066646663393363393562653431653764356537 + 61303164333662353732656265386430383566343462633435626334373331316663363563616662 + 66643135336539333738623231346331623464636637373639666435663961383936 diff --git a/ansible/roles/restic/files/backrest.sh b/ansible/roles/restic/files/backrest.sh index 3fcd4e4..8b587b8 100644 --- a/ansible/roles/restic/files/backrest.sh +++ b/ansible/roles/restic/files/backrest.sh @@ -5,7 +5,7 @@ set -e export AWS_ACCESS_KEY_ID="{{ restic_b2_account_id }}" export AWS_SECRET_ACCESS_KEY="{{ restic_b2_account_key }}" export RESTIC_PASSWORD="{{ restic_key }}" -export RESTIC_REPOSITORY="s3:{{ restic_b2_endpoint }}/{{ restic_b2_bucket }}" +export RESTIC_REPOSITORY="s3:{{ restic_b2_endpoint }}/{{ restic_b2_bucket }}/{{ restic_dir }}" export GOGC=20 # HACK: Work around for restic's high memory usage https://github.com/restic/restic/issues/1988 set -x diff --git a/ansible/roles/restic/files/restic-backup.sh b/ansible/roles/restic/files/restic-backup.sh index 115d52f..0e91590 100644 --- a/ansible/roles/restic/files/restic-backup.sh +++ b/ansible/roles/restic/files/restic-backup.sh @@ -3,3 +3,5 @@ set -e exec $HOME/backrest.sh --verbose backup --files-from=$HOME/restic-include.txt --exclude-file=$HOME/restic-excludes.txt + +exec $HOME/backrest.sh forget --prune --keep-daily 30 --keep-monthly 3 --group-by host diff --git a/ansible/roles/restic/files/restic-forget.sh b/ansible/roles/restic/files/restic-forget.sh deleted file mode 100644 index 27614ef..0000000 --- a/ansible/roles/restic/files/restic-forget.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env bash - -set -e - -exec $HOME/backrest.sh forget --prune --keep-daily 30 --keep-monthly 3 --group-by host diff --git a/ansible/roles/restic/tasks/main.yml b/ansible/roles/restic/tasks/main.yml index f1eda18..f22d2a0 100644 --- a/ansible/roles/restic/tasks/main.yml +++ b/ansible/roles/restic/tasks/main.yml @@ -21,7 +21,6 @@ loop: - backrest.sh - restic-backup.sh - - restic-forget.sh - name: Install includes files copy: @@ -57,16 +56,6 @@ job: CHECK_UUID={{ vault_restic_healthchecks_id }} /usr/bin/runitor -- /home/restic/restic-backup.sh user: restic -- name: Schedule forget - cron: - name: restic forget - hour: 2 - minute: 0 - weekday: 0 - job: CHECK_UUID={{ vault_restic_forget_healthchecks_id }} /usr/bin/runitor -- /home/restic/restic-forget.sh - user: restic - when: restic_forget - - name: Install pacman post script template: src: files/restic-post.sh diff --git a/ansible/roles/restic/vars/main.yml b/ansible/roles/restic/vars/main.yml index 9faa57d..1930043 100644 --- a/ansible/roles/restic/vars/main.yml +++ b/ansible/roles/restic/vars/main.yml @@ -1,28 +1,3 @@ -restic_b2_account_id: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 37653465353462333831333031373566343362316461643463303737356332363364666264666562 - 6436353430313239643831626537383664376264393430310a353434306465653934356164393265 - 65336534356636386634366533313039643438656439636434666464636338373130636661303536 - 6638356530373062640a376430663231396538386161353165363538346266663135363337303166 - 35613838363336393032643965373437663937626532356239336535663064363438 -restic_b2_account_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 31343939356266366364313631633836653364633331306134663163653939373663316632626161 - 3262376666323366313563313836643133393437663462320a653637633663636663353234323265 - 63633632383461616437636230643430613635626431383966326666383030393735623963333263 - 3433633830356530390a623436376364343630313066663965303238646564323266303865623965 - 66353963376434333030333332646464653637633737633832663762303339666336 -restic_key: !vault | - $ANSIBLE_VAULT;1.1;AES256 - 65326130653631643138373231336439303962356666353737373832633666373965666466613034 - 3730333664373334316436396435356137313337633836330a653366343732656539663439313838 - 34326361353232663533663366393464643761343864623965386633313438303738343238623430 - 6661313730646166330a393336333635623832653232653863626135343164363831366134316465 - 30306238613865393664383865323233626636333063393936613864303365386431613838366363 - 39633865393761366134633933333237313362636234323330313135336637613235303764383938 - 34383938396366363937643864393838363638383066646663393363393562653431653764356537 - 61303164333662353732656265386430383566343462633435626334373331316663363563616662 - 66643135336539333738623231346331623464636637373639666435663961383936 restic_b2_bucket: 0rng-restic restic_b2_endpoint: s3.eu-central-003.backblazeb2.com healthchecks_host: https://hc-ping.com