From 1a26cb442e5cc4341aab441f22da06130e8cb24e Mon Sep 17 00:00:00 2001 From: Thomas Schneider <thomas@fsmpi.rwth-aachen.de> Date: Fri, 14 Jul 2023 21:09:53 +0200 Subject: [PATCH] postgres: Add pgBackRest integration This serves to supersede rsnapshot, which has severe performance issues. --- postgres/defaults/main.yml | 6 ++ postgres/files/wal-archive-commvault.sh | 4 ++ postgres/files/wal-archive-pgbackrest.sh | 4 ++ postgres/tasks/main.yml | 46 +++++++++++--- postgres/tasks/pgbackrest.yml | 76 ++++++++++++++++++++++++ postgres/templates/pgbackrest.conf | 7 +++ 6 files changed, 134 insertions(+), 9 deletions(-) create mode 100755 postgres/files/wal-archive-commvault.sh create mode 100755 postgres/files/wal-archive-pgbackrest.sh create mode 100644 postgres/tasks/pgbackrest.yml create mode 100644 postgres/templates/pgbackrest.conf diff --git a/postgres/defaults/main.yml b/postgres/defaults/main.yml index 5818f96..7435ece 100644 --- a/postgres/defaults/main.yml +++ b/postgres/defaults/main.yml @@ -3,6 +3,12 @@ postgres_pgdg_repo: false postgres_rsnapshot: false postgres_commvault_compat: false +postgres_pgbackrest: false +postgres_wal_archive: "{{ postgres_commvault_compat or postgres_pgbackrest }}" + +pgbackrest_retention_diff: 7 +pgbackrest_retention_full: 2 +pgbackrest_frequently_minute: "*/15" ### Will be the version of the `postgresql` package if undefined # postgres_version: 15 diff --git a/postgres/files/wal-archive-commvault.sh b/postgres/files/wal-archive-commvault.sh new file mode 100755 index 0000000..3aba082 --- /dev/null +++ b/postgres/files/wal-archive-commvault.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +[ $# -lt 2 ] && exit 1 +cp --reflink=auto "$1" "/var/backups/pg_wal/$2" diff --git a/postgres/files/wal-archive-pgbackrest.sh b/postgres/files/wal-archive-pgbackrest.sh new file mode 100755 index 0000000..5f68615 --- /dev/null +++ b/postgres/files/wal-archive-pgbackrest.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +[ $# -lt 1 ] && exit 1 +pgbackrest --stanza=main archive-push "$1" diff --git a/postgres/tasks/main.yml b/postgres/tasks/main.yml index 60b1795..2718db7 100644 --- a/postgres/tasks/main.yml +++ b/postgres/tasks/main.yml @@ -75,28 +75,52 @@ path: /etc/cron.d/postgres-snapshot state: absent -- name: Configure Commvault backup compatibility - when: postgres_commvault_compat +- name: Configure Postgres WAL archive framework + when: postgres_wal_archive block: - - name: Create WAL backup directory + - name: Create WAL archiver drop-in directory file: - path: /var/backups/pg_wal + path: /etc/postgresql/wal-archive state: directory - owner: postgres + owner: root group: postgres - mode: '0750' + mode: "0750" - name: Configure Postgres WAL archive postgresql_set: name: "{{ item.name }}" value: "{{ item.value }}" + become: true + become_user: postgres loop: - name: archive_mode value: "on" - name: archive_command - value: 'cp %p /var/backups/pg_wal/%f' - notify: - - Restart Postgres + # pgBackRest checks the Postgres configuration, archive_command + # _needs_ to contain the string "pgbackrest", so add it as (unused) + # argument + value: >- + run-parts --report --arg=%p --arg=%f --arg=pgbackrest + /etc/postgresql/wal-archive + +- name: Configure Commvault backup compatibility + when: postgres_commvault_compat + block: + - name: Create WAL backup directory + file: + path: /var/backups/pg_wal + state: directory + owner: postgres + group: postgres + mode: '0750' + + - name: Configure Commvault Postgres WAL archive + copy: + src: wal-archive-commvault.sh + dest: /etc/postgresql/wal-archive/commvault + owner: root + group: postgres + mode: "0755" - name: Configure Postgres ident mappings blockinfile: @@ -123,3 +147,7 @@ options: map=postgres notify: - Restart Postgres + +- name: Configure pgBackRest + when: postgres_pgbackrest + import_tasks: pgbackrest.yml diff --git a/postgres/tasks/pgbackrest.yml b/postgres/tasks/pgbackrest.yml new file mode 100644 index 0000000..fb06c97 --- /dev/null +++ b/postgres/tasks/pgbackrest.yml @@ -0,0 +1,76 @@ +--- + +- name: Install pgBackRest + apt: + name: pgbackrest + state: present + +- name: Create pgBackRest repository + file: + path: /var/backups/pgbackrest + state: directory + owner: postgres + group: postgres + mode: "0750" + +- name: Configure pgBackRest + template: + src: pgbackrest.conf + dest: /etc/pgbackrest.conf + owner: root + group: postgres + mode: "0640" + +- name: Initialise pgBackRest + command: + cmd: pgbackrest stanza-create --stanza=main + creates: /var/backups/pgbackrest/backup/main + become: true + become_user: postgres + +- name: Configure pgBackRest Postgres WAL archive + copy: + src: wal-archive-pgbackrest.sh + dest: /etc/postgresql/wal-archive/pgbackrest + owner: root + group: postgres + mode: "0750" + +- name: Configure pgBackRest cron env + cron: + cron_file: pgbackrest + user: postgres + env: true + name: SHELL + value: /bin/bash + +- name: Configure pgBackRest cron jobs + cron: + cron_file: pgbackrest + weekday: "{{ item.weekday }}" + hour: "{{ item.hour }}" + minute: "{{ item.minute }}" + name: pgbackrest-{{ item.name }} + user: postgres + job: >- + {% if item.sleep %}sleep $(($RANDOM \% 3600)) &&{% endif %} + pgbackrest backup --stanza=main --type={{ item.type }} + loop: + - name: frequently + weekday: "*" + hour: "*" + minute: "{{ pgbackrest_frequently_minute }}" + sleep: false + type: incr + - name: daily + weekday: 1-6 + hour: 3 + minute: 5 + sleep: true + type: diff + - name: weekly + weekday: 0 + hour: 3 + minute: 5 + sleep: true + type: full diff --git a/postgres/templates/pgbackrest.conf b/postgres/templates/pgbackrest.conf new file mode 100644 index 0000000..a84affa --- /dev/null +++ b/postgres/templates/pgbackrest.conf @@ -0,0 +1,7 @@ +[global] +repo1-path=/var/backups/pgbackrest +repo1-retention-diff={{ pgbackrest_retention_diff }} +repo1-retention-full={{ pgbackrest_retention_full }} + +[main] +db-path=/var/lib/postgresql/{{ postgres_version }}/main -- GitLab