Automating backups to Hetzner Object Storage (S3 bucket) (#898)

vitabaks · Feb 20, 2025 · d1ac8a6 · d1ac8a6
1 parent 448207b
commit d1ac8a6
Show file tree

Hide file tree

Showing 4 changed files with 122 additions and 0 deletions.
diff --git a/automation/roles/cloud-resources/defaults/main.yml b/automation/roles/cloud-resources/defaults/main.yml
@@ -66,4 +66,12 @@ digital_ocean_spaces_name: "{{ patroni_cluster_name }}-backup"  # Name of the Sp
 digital_ocean_spaces_region: "nyc3"  # The region to create the Space in.
 digital_ocean_spaces_absent: false  # Allow to delete Spaces Object Storage when deleting a cluster servers using the 'state=absent' variable.
 
+hetzner_object_storage_create: true  # if 'cloud_provider=hetzner'
+hetzner_object_storage_name: "{{ patroni_cluster_name }}-backup"  # Name of the Object Storage (S3 bucket).
+hetzner_object_storage_region: "{{ server_location }}"  # The region where the Object Storage (S3 bucket) will be created.
+hetzner_object_storage_endpoint: "https://{{ hetzner_object_storage_region }}.your-objectstorage.com"
+hetzner_object_storage_access_key: ""  # (required) Object Storage ACCESS KEY
+hetzner_object_storage_secret_key: ""  # (required) Object Storage SECRET KEY
+hetzner_object_storage_absent: false  # Allow to delete Object Storage when deleting a cluster servers using the 'state=absent' variable.
+
 ...
diff --git a/automation/roles/cloud-resources/tasks/hetzner.yml b/automation/roles/cloud-resources/tasks/hetzner.yml
@@ -50,6 +50,21 @@
       environment:
         PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin"
         PIP_BREAK_SYSTEM_PACKAGES: "1"
+
+    - name: Ensure that 'boto3' dependency is present on controlling host
+      ansible.builtin.pip:
+        name: boto3
+        executable: pip3
+        extra_args: --user
+      become: false
+      vars:
+        ansible_become: false
+      environment:
+        PATH: "{{ ansible_env.PATH }}:/usr/local/bin:/usr/bin"
+        PIP_BREAK_SYSTEM_PACKAGES: "1"
+      when:
+        - (pgbackrest_install | bool or wal_g_install | bool)
+        - hetzner_object_storage_create | bool
   delegate_to: 127.0.0.1
   run_once: true
 
@@ -428,6 +443,26 @@
       when:
         - cloud_firewall | bool
 
+    # Object Storage (S3 bucket for backups)
+    - name: "Hetzner Cloud: Create Object Storage (S3 bucket) '{{ hetzner_object_storage_name }}'"
+      amazon.aws.s3_bucket:
+        endpoint_url: "{{ hetzner_object_storage_endpoint }}"
+        ceph: true
+        aws_access_key: "{{ hetzner_object_storage_access_key }}"
+        aws_secret_key: "{{ hetzner_object_storage_secret_key }}"
+        name: "{{ hetzner_object_storage_name }}"
+        region: "{{ hetzner_object_storage_region }}"
+        requester_pays: false
+        state: present
+      register: s3_bucket_result
+      failed_when: s3_bucket_result.failed and not "GetBucketRequestPayment" in s3_bucket_result.msg
+      # TODO: https://github.com/ansible-collections/amazon.aws/issues/2447
+      when:
+        - (pgbackrest_install | bool or wal_g_install | bool)
+        - hetzner_object_storage_create | bool
+        - hetzner_object_storage_access_key | length > 0
+        - hetzner_object_storage_secret_key | length > 0
+
     # Server and volume
     - name: "Hetzner Cloud: Create or modify server"
       hetzner.hcloud.server:
@@ -758,6 +793,23 @@
         api_token: "{{ lookup('ansible.builtin.env', 'HCLOUD_API_TOKEN') }}"
         state: "absent"
         name: "{{ patroni_cluster_name }}-firewall"
+
+    - name: "Hetzner Cloud: Delete Object Storage (S3 bucket) '{{ hetzner_object_storage_name }}'"
+      amazon.aws.s3_bucket:
+        endpoint_url: "{{ hetzner_object_storage_endpoint }}"
+        ceph: true
+        access_key: "{{ hetzner_object_storage_access_key }}"
+        secret_key: "{{ hetzner_object_storage_secret_key }}"
+        name: "{{ hetzner_object_storage_name }}"
+        region: "{{ hetzner_object_storage_region }}"
+        requester_pays: false
+        state: absent
+        force: true
+      when:
+        - (pgbackrest_install | bool or wal_g_install | bool)
+        - hetzner_object_storage_absent | bool
+        - hetzner_object_storage_access_key | length > 0
+        - hetzner_object_storage_secret_key | length > 0
   when: state == 'absent'
 
 ...
diff --git a/automation/roles/pgbackrest/tasks/auto_conf.yml b/automation/roles/pgbackrest/tasks/auto_conf.yml
@@ -172,4 +172,42 @@
   no_log: true  # do not output contents to the ansible log
   when: cloud_provider | default('') | lower == 'digitalocean'
 
+# Hetzner Object Storage (if 'cloud_provider=hetzner')
+- name: "Set variable 'pgbackrest_conf' for backup in Hetzner Object Storage (S3 bucket)"
+  ansible.builtin.set_fact:
+    pgbackrest_conf:
+      global:
+        - { option: "log-level-file", value: "detail" }
+        - { option: "log-path", value: "/var/log/pgbackrest" }
+        - { option: "repo1-type", value: "s3" }
+        - { option: "repo1-path", value: "{{ PGBACKREST_REPO_PATH | default('/pgbackrest') }}" }
+        - { option: "repo1-s3-key", value: "{{ PGBACKREST_S3_KEY | default(hetzner_object_storage_access_key | default('')) }}" }
+        - { option: "repo1-s3-key-secret", value: "{{ PGBACKREST_S3_KEY_SECRET | default(hetzner_object_storage_secret_key | default('')) }}" }
+        - { option: "repo1-s3-bucket", value: "{{ PGBACKREST_S3_BUCKET | default(hetzner_object_storage_name | default(patroni_cluster_name + '-backup')) }}" }
+        - { option: "repo1-s3-endpoint", value: "{{ PGBACKREST_S3_ENDPOINT | default(hetzner_object_storage_endpoint | default('https://' + (hetzner_object_storage_region | default(server_location)) + '.your-objectstorage.com')) }}" }
+        - { option: "repo1-s3-region", value: "{{ PGBACKREST_S3_REGION | default(hetzner_object_storage_region | default(server_location)) }}" }
+        - { option: "repo1-s3-uri-style", value: "{{ PGBACKREST_S3_URI_STYLE | default('path') }}" }
+        - { option: "repo1-retention-full", value: "{{ PGBACKREST_RETENTION_FULL | default('4') }}" }
+        - { option: "repo1-retention-archive", value: "{{ PGBACKREST_RETENTION_ARCHIVE | default('4') }}" }
+        - { option: "repo1-retention-archive-type", value: "{{ PGBACKREST_RETENTION_ARCHIVE_TYPE | default('full') }}" }
+        - { option: "repo1-bundle", value: "y" }
+        - { option: "repo1-block", value: "y" }
+        - { option: "start-fast", value: "y" }
+        - { option: "stop-auto", value: "y" }
+        - { option: "link-all", value: "y" }
+        - { option: "resume", value: "n" }
+        - { option: "archive-async", value: "y" }
+        - { option: "archive-get-queue-max", value: "1GiB" }
+        - { option: "spool-path", value: "/var/spool/pgbackrest" }
+        - { option: "process-max", value: "{{ PGBACKREST_PROCESS_MAX | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+        - { option: "backup-standby", value: "{{ 'y' if groups['postgres_cluster'] | length > 1 else 'n' }}" }
+      stanza:
+        - { option: "log-level-console", value: "info" }
+        - { option: "recovery-option", value: "recovery_target_action=promote" }
+        - { option: "pg1-path", value: "{{ postgresql_data_dir }}" }
+  delegate_to: localhost
+  run_once: true # noqa run-once
+  no_log: true  # do not output contents to the ansible log
+  when: cloud_provider | default('') | lower == 'hetzner'
+
 ...
diff --git a/automation/roles/wal-g/tasks/auto_conf.yml b/automation/roles/wal-g/tasks/auto_conf.yml
@@ -114,4 +114,28 @@
   no_log: true  # do not output contents to the ansible log
   when: cloud_provider | default('') | lower == 'digitalocean'
 
+# Hetzner Object Storage (if 'cloud_provider=hetzner')
+- name: "Set variable 'wal_g_json' for backup in AWS S3 bucket"
+  ansible.builtin.set_fact:
+    wal_g_json:
+      - { option: "AWS_ACCESS_KEY_ID", value: "{{ WALG_AWS_ACCESS_KEY_ID | default(hetzner_object_storage_access_key | default('')) }}" }
+      - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ WALG_AWS_SECRET_ACCESS_KEY | default(hetzner_object_storage_secret_key | default('')) }}" }
+      - { option: "AWS_ENDPOINT", value: "{{ WALG_S3_ENDPOINT | default(hetzner_object_storage_endpoint | default('https://' + (hetzner_object_storage_region | default(server_location)) + '.your-objectstorage.com')) }}" }
+      - { option: "AWS_S3_FORCE_PATH_STYLE", value: "{{ AWS_S3_FORCE_PATH_STYLE | default(true) }}" }
+      - { option: "AWS_REGION", value: "{{ WALG_S3_REGION | default(hetzner_object_storage_region | default(server_location)) }}" }
+      - { option: "WALG_S3_PREFIX", value: "{{ WALG_S3_PREFIX | default('s3://' + (hetzner_object_storage_name | default(patroni_cluster_name + '-backup'))) }}" }
+      - { option: "WALG_COMPRESSION_METHOD", value: "{{ WALG_COMPRESSION_METHOD | default('brotli') }}" }
+      - { option: "WALG_DELTA_MAX_STEPS", value: "{{ WALG_DELTA_MAX_STEPS | default('6') }}" }
+      - { option: "WALG_DOWNLOAD_CONCURRENCY", value: "{{ WALG_DOWNLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+      - { option: "WALG_UPLOAD_CONCURRENCY", value: "{{ WALG_UPLOAD_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+      - { option: "WALG_UPLOAD_DISK_CONCURRENCY", value: "{{ WALG_UPLOAD_DISK_CONCURRENCY | default([ansible_processor_vcpus | int // 2, 1] | max) }}" }
+      - { option: "PGDATA", value: "{{ postgresql_data_dir }}" }
+      - { option: "PGHOST", value: "{{ postgresql_unix_socket_dir | default('/var/run/postgresql') }}" }
+      - { option: "PGPORT", value: "{{ postgresql_port | default('5432') }}" }
+      - { option: "PGUSER", value: "{{ patroni_superuser_username | default('postgres') }}" }
+  delegate_to: localhost
+  run_once: true # noqa run-once
+  no_log: true  # do not output contents to the ansible log
+  when: cloud_provider | default('') | lower == 'hetzner'
+
 ...