From ea8e3f9112ad5aa50d5b891eb638f0d466902dd8 Mon Sep 17 00:00:00 2001
From: Piotr Dobrowolski <informatic@hackerspace.pl>
Date: Tue, 10 Oct 2023 00:41:50 +0200
Subject: [PATCH] kube/postgres: pgupgrade automation

Change-Id: Ibcbddf57b8cdcac75ce366a95db63817bec42a22
Reviewed-on: https://gerrit.hackerspace.pl/c/hscloud/+/1698
Reviewed-by: q3k <q3k@hackerspace.pl>
---
 kube/postgres.libsonnet | 80 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)

diff --git a/kube/postgres.libsonnet b/kube/postgres.libsonnet
index edb1cae3..2935213a 100644
--- a/kube/postgres.libsonnet
+++ b/kube/postgres.libsonnet
@@ -29,6 +29,30 @@ local kube = import "kube.libsonnet";
         # string values.
         # Example: { max_connections: "300" }
         opts: {},
+
+        # Postgres cluster upgrade automation. In order to update running
+        # postgres version:
+        # * set image to target postgres version
+        # * pgupgrade.from to previous/current major version
+        # * pgupgrade.to to target major version number (optional, this should
+        #   be figured out from image version)
+        # * switch pgupgrade.enable to true.
+        #
+        # While we do have some countermeasures to prevent stupid typos, you
+        # should still probably make a database backup, eg. using:
+        #    kubectl exec deploy/postgres -- pg_dumpall > dump.sql
+        #
+        # After succesful upgrade /var/lib/postgresql/data/pgdata-old directory
+        # needs to be removed by hand. In case a rollback is needed pgdata needs
+        # to be swapped with the pgdata-old directory and the postgres image
+        # needs to be adjusted accordingly.
+        pgupgrade: {
+            enable: false,
+            from: "10",
+            # Extract target version from image name, supported:
+            #   postgres:1.2-suffix, postgres:1-suffix, postgres:1.2, postgres:1
+            to: std.native('regexSubst')("^[^:]+:([^.]+).*$", cfg.image, "${1}"),
+        },
     },
 
     makeName(suffix):: cfg.prefix + suffix,
@@ -88,6 +112,62 @@ local kube = import "kube.libsonnet";
                             },
                         },
                     },
+
+                    initContainers_: if cfg.pgupgrade.enable then {
+                        pgupgrade: kube.Container(postgres.makeName("pgupgrade")) {
+                            image: "tianon/postgres-upgrade:%s-to-%s" % [cfg.pgupgrade.from, cfg.pgupgrade.to],
+                            command: [
+                                "bash", "-c", |||
+                                set -e -x -o pipefail
+
+                                CURRENT_VERSION="$(cat "$PGDATA/PG_VERSION")"
+                                if [[ "$CURRENT_VERSION" == "$VERSION_TO" ]]; then
+                                    echo "Already running target version ($VERSION_TO)"
+                                    exit 0
+                                fi
+
+                                if [[ "$CURRENT_VERSION" != "$VERSION_FROM" ]]; then
+                                    echo "Running unexpected source version, wanted $VERSION_FROM, got $CURRENT_VERSION"
+                                    exit 1
+                                fi
+
+                                rm -rf $PGDATANEXT || true
+
+                                if [ ! -s "$PGDATANEXT/PG_VERSION" ]; then
+                                    echo "Initializing new database..."
+                                    PGDATA="$PGDATANEXT" eval "initdb $POSTGRES_INITDB_ARGS"
+                                fi
+
+                                chmod 700 $PGDATA $PGDATANEXT
+
+                                echo "Running upgrade..."
+                                pg_upgrade --link --old-datadir $PGDATA --new-datadir $PGDATANEXT || (sleep 3600 ; exit 1)
+
+                                echo "Copying pg_hba.conf"
+                                cp $PGDATA/pg_hba.conf $PGDATANEXT/pg_hba.conf
+
+                                echo "Done, swapping..."
+                                mv $PGDATA $PGDATAOLD
+                                mv $PGDATANEXT $PGDATA
+                            |||
+                            ],
+                            env_: postgres.deployment.spec.template.spec.containers_.postgres.env_ + {
+                                VERSION_TO: cfg.pgupgrade.to,
+                                VERSION_FROM: cfg.pgupgrade.from,
+
+                                # pg_upgrade target directory, swapped with
+                                # PGDATA after cluster data upgrade is finished
+                                PGDATANEXT: "/var/lib/postgresql/data/pgdata-next",
+
+                                # Directory used to stash previous pgdata
+                                # version
+                                PGDATAOLD: "/var/lib/postgresql/data/pgdata-old",
+                            },
+                            volumeMounts_: {
+                                data: { mountPath: "/var/lib/postgresql/data" },
+                            },
+                        },
+                    } else {},
                     securityContext: {
                         runAsUser: 999,
                     },