Merge branch 'master' of https://github.com/zalando/spilo

postgres-appliance/Dockerfile

@@ -1,6 +1,6 @@
 ARG BASE_IMAGE=ubuntu:22.04
 ARG PGVERSION=16
-ARG TIMESCALEDB="2.3.1 2.11.2 2.14.2"
+ARG TIMESCALEDB="2.11.2 2.14.2"
 ARG DEMO=false
 ARG COMPRESS=false
 ARG ADDITIONAL_LOCALES=
@@ -48,14 +48,13 @@ ARG TIMESCALEDB
 ARG TIMESCALEDB_APACHE_ONLY=false
 ARG TIMESCALEDB_TOOLKIT=true
 ARG COMPRESS
-ARG PGOLDVERSIONS="11 12 13 14 15"
+ARG PGOLDVERSIONS="12 13 14 15"
 ARG WITH_PERL=false
 
 ARG DEB_PG_SUPPORTED_VERSIONS="$PGOLDVERSIONS $PGVERSION"
 
 # Install PostgreSQL, extensions and contribs
 ENV POSTGIS_VERSION=3.4 \
-    POSTGIS_LEGACY=3.3 \
     BG_MON_COMMIT=ef60961eff92672b1e21f5260dc1211367da6f1f \
     PG_AUTH_MON_COMMIT=3d010e5959285c32b155e8064c9c9b57869aeca7 \
     PG_MON_COMMIT=a6c5982368edd876edbee01e51b91e7387071e21 \
@@ -74,7 +73,7 @@ COPY --from=dependencies-builder /builddeps/wal-g /usr/local/bin/
 COPY build_scripts/patroni_wale.sh build_scripts/compress_build.sh /builddeps/
 
 # Install patroni and wal-e
-ENV PATRONIVERSION=3.2.2
+ENV PATRONIVERSION=3.3.1
 ENV WALE_VERSION=1.1.1
 
 WORKDIR /

postgres-appliance/build_scripts/base.sh

@@ -266,7 +266,6 @@ if [ "$DEMO" != "true" ]; then
                     d2="$d1"
                     d1="../../${v1##*/}/$d1"
                     if [ "${d2%-*}" = "contrib/postgis" ]; then
-                        if [ "${v2##*/}" = "11" ]; then d2="${d2%-*}-$POSTGIS_LEGACY"; fi
                         d1="../$d1"
                     fi
                     d2="$v2/$d2"

postgres-appliance/major_upgrade/inplace_upgrade.py

@@ -14,6 +14,7 @@
 from collections import defaultdict
 from threading import Thread
 from multiprocessing.pool import ThreadPool
+from patroni import global_config
 
 logger = logging.getLogger(__name__)
 
@@ -130,12 +131,11 @@ def check_patroni_api(self, member):
             return logger.error('API request to %s name failed: %r', member.name, e)
 
     def toggle_pause(self, paused):
-        from patroni.config import get_global_config
         from patroni.utils import polling_loop
 
         cluster = self.dcs.get_cluster()
         config = cluster.config.data.copy()
-        if get_global_config(cluster).is_paused == paused:
+        if global_config.from_cluster(cluster).is_paused == paused:
             return logger.error('Cluster is %spaused, can not continue', ('' if paused else 'not '))
 
         config['pause'] = paused
@@ -203,15 +203,14 @@ def ensure_replica_state(member):
         return all(ensure_replica_state(member) for member in cluster.members if member.name != self.postgresql.name)
 
     def sanity_checks(self, cluster):
-        from patroni.config import get_global_config
 
         if not cluster.initialize:
             return logger.error('Upgrade can not be triggered because the cluster is not initialized')
 
         if len(cluster.members) != self.replica_count:
             return logger.error('Upgrade can not be triggered because the number of replicas does not match (%s != %s)',
                                 len(cluster.members), self.replica_count)
-        if get_global_config(cluster).is_paused:
+        if global_config.from_cluster(cluster).is_paused:
             return logger.error('Upgrade can not be triggered because Patroni is in maintenance mode')
 
         lock_owner = cluster.leader and cluster.leader.name
@@ -319,7 +318,7 @@ def stop_rsyncd(self):
                 shutil.rmtree(self.rsyncd_conf_dir)
                 self.rsyncd_configs_created = False
             except Exception as e:
-                logger.error('Failed to remove %s: %r', self.rsync_conf_dir, e)
+                logger.error('Failed to remove %s: %r', self.rsyncd_conf_dir, e)
 
     def checkpoint(self, member):
         name, (_, cur) = member

postgres-appliance/major_upgrade/pg_upgrade.py

@@ -5,6 +5,7 @@
 import psutil
 
 from patroni.postgresql import Postgresql
+from patroni.postgresql.mpp import get_mpp
 
 logger = logging.getLogger(__name__)
 
@@ -302,6 +303,6 @@ def PostgresqlUpgrade(config):
     is_running = _PostgresqlUpgrade.is_running
     _PostgresqlUpgrade.is_running = lambda s: False
     try:
-        return _PostgresqlUpgrade(config['postgresql'])
+        return _PostgresqlUpgrade(config['postgresql'], get_mpp(config))
     finally:
         _PostgresqlUpgrade.is_running = is_running

postgres-appliance/scripts/_zmon_schema.dump

@@ -45,29 +45,16 @@ DECLARE
 BEGIN
   server_version_num := current_setting('server_version_num')::integer;
   SELECT pg_is_in_recovery() INTO in_recovery;
-  IF server_version_num >= 100000
-  THEN
-    SELECT CASE WHEN in_recovery THEN pg_last_wal_replay_lsn()
-           ELSE pg_current_wal_lsn()
-           END INTO xlog_location_string;
-    SELECT pg_last_wal_receive_lsn() INTO receive_location;
-  ELSE
-    SELECT CASE WHEN in_recovery THEN pg_last_xlog_replay_location()
-           ELSE pg_current_xlog_location()
-           END INTO xlog_location_string;
-    SELECT pg_last_xlog_receive_location() INTO receive_location;
-  END IF;
+  SELECT CASE WHEN in_recovery THEN pg_last_wal_replay_lsn()
+        ELSE pg_current_wal_lsn()
+        END INTO xlog_location_string;
+  SELECT pg_last_wal_receive_lsn() INTO receive_location;
   SELECT ('x'||lpad(split_part(xlog_location_string, '/', 1), 16, '0'))::bit(64)::bigint INTO wal_segment;
   SELECT ('x'||lpad(split_part(xlog_location_string, '/', 2), 16, '0'))::bit(64)::bigint INTO wal_offset;
   SELECT 1 FROM pg_ls_dir('.') as t(name) WHERE name = 'dontsync' INTO nosync;
-  IF server_version_num >= 90300
-  THEN
-    wal_multiplier = CAST(x'FFFFFFFF' as bigint);
-  ELSE
-    wal_multiplier = CAST(x'FF000000' as bigint);
-  END IF;
+  wal_multiplier = CAST(x'FFFFFFFF' as bigint);
 
-  IF server_version_num >= 90100 AND in_recovery
+  IF in_recovery
   THEN
     wal_delay_seconds := extract(epoch from now() - pg_last_xact_replay_timestamp())::bigint;
   END IF;
@@ -100,7 +87,7 @@ BEGIN
                   'synchronous_commit'
                  )
    UNION ALL
-  SELECT 'cluster_name', COALESCE(CASE WHEN server_version_num >= 90500 THEN current_setting('cluster_name') ELSE NULL END, substring(s.setting from E'/pgsql_([^/]+)/[^/]+/data$'))
+  SELECT 'cluster_name', COALESCE(current_setting('cluster_name'), substring(s.setting from E'/pgsql_([^/]+)/[^/]+/data$'))
     FROM pg_settings as s
    WHERE s.name = 'data_directory'
    UNION ALL
@@ -119,11 +106,9 @@ BEGIN
   SELECT 'is_streaming' as name, CAST(receive_location IS NOT NULL AND in_recovery AS TEXT) as setting
     UNION ALL
   SELECT 'archive_nosync' as name, CAST(nosync IS NOT NULL AS TEXT) as setting;
-  -- pg_stat_activity column names differ depending on server version
-  IF server_version_num >= 90600 THEN
-    RETURN QUERY
-    SELECT a.name, a.setting
-      FROM unnest((select array[('active_connections'::text,
+  RETURN QUERY
+  SELECT a.name, a.setting
+    FROM unnest((select array[('active_connections'::text,
                                  count(CASE WHEN state = 'active' THEN 1 END)::text),
                                 ('idle_in_transaction_connections'::text,
                                  count(CASE WHEN state = 'idle in transaction' THEN 1 END)::text),
@@ -137,28 +122,8 @@ BEGIN
                                  -- exclude autovacuum transactions
                                  coalesce(extract(epoch from statement_timestamp() - min(CASE WHEN query like 'autovacuum:%' THEN NULL ELSE xact_start END)),0)::text)
                                ]
-                     from pg_stat_activity
-                 )) AS a (name text, setting text);
-  ELSE
-    RETURN QUERY
-    SELECT a.name, a.setting
-      FROM unnest((select array[('active_connections'::text,
-                                 count(CASE WHEN state = 'active' THEN 1 END)::text),
-                                ('idle_in_transaction_connections'::text,
-                                 count(CASE WHEN state = 'idle in transaction' THEN 1 END)::text),
-                                ('idle_in_transaction_max_age'::text,
-                                 coalesce(max(CASE WHEN state = 'idle in transaction' THEN extract(epoch from statement_timestamp() - state_change) END), 0)::text),
-                                ('locked_connections'::text,
-                                 count(CASE WHEN waiting THEN 1 END)::text),
-                                ('current_connections'::text,
-                                 count(1)::text),
-                                ('transaction_max_age'::text,
-                                 -- exclude autovacuum transactions
-                                 coalesce(extract(epoch from statement_timestamp() - min(CASE WHEN query like 'autovacuum:%' THEN NULL ELSE xact_start END)),0)::text)
-                               ]
-                     from pg_stat_activity
-                 )) AS a (name text, setting text);
-  END IF;
+                    from pg_stat_activity
+                )) AS a (name text, setting text);
 END
 $BODY$
 LANGUAGE plpgsql
@@ -412,7 +377,7 @@ for r in rv:
 data_directory = s["data_directory"]
 log_directory = s["log_directory"] = os.path.join(data_directory, s["log_directory"])
 pg_tblspc = os.path.join(data_directory, "pg_tblspc")
-pg_xlog = os.path.join(data_directory, "pg_xlog" if int(s["server_version_num"]) < 100000 else "pg_wal")
+pg_xlog = os.path.join(data_directory, "pg_wal")
 
 result = {}
 

postgres-appliance/scripts/pg_partman/after-create.sql

@@ -8,9 +8,7 @@ BEGIN
         CREATE ROLE part_man ADMIN admin;
     END IF;
 
-    IF current_setting('server_version_num')::integer >= 110000 THEN
-        EXECUTE 'GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA @extschema@ TO part_man';
-    END IF;
+    EXECUTE 'GRANT EXECUTE ON ALL PROCEDURES IN SCHEMA @extschema@ TO part_man';
 END;$$;
 GRANT USAGE ON SCHEMA @extschema@ TO part_man;
 GRANT ALL ON ALL TABLES IN SCHEMA @extschema@ TO part_man;

postgres-appliance/scripts/post_init.sh

@@ -5,7 +5,7 @@ cd "$(dirname "${BASH_SOURCE[0]}")" || exit 1
 export PGOPTIONS="-c synchronous_commit=local -c search_path=pg_catalog"
 
 PGVER=$(psql -d "$2" -XtAc "SELECT pg_catalog.current_setting('server_version_num')::int/10000")
-if [ "$PGVER" -ge 12 ]; then RESET_ARGS="oid, oid, bigint"; fi
+RESET_ARGS="oid, oid, bigint"
 
 (echo "\set ON_ERROR_STOP on"
 echo "DO \$\$
@@ -192,12 +192,8 @@ CREATE EXTENSION IF NOT EXISTS set_user SCHEMA public;
 ALTER EXTENSION set_user UPDATE;
 GRANT EXECUTE ON FUNCTION public.set_user(text) TO admin;
 GRANT EXECUTE ON FUNCTION public.pg_stat_statements_reset($RESET_ARGS) TO admin;"
-    if [ "$PGVER" -lt 10 ]; then
-        echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO admin;"
-    else
-        echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;"
-    fi
-    if [ "$ENABLE_PG_MON" = "true" ] && [ "$PGVER" -ge 11 ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi
+    echo "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO admin;"
+    if [ "$ENABLE_PG_MON" = "true" ]; then echo "CREATE EXTENSION IF NOT EXISTS pg_mon SCHEMA public;"; fi
     cat metric_helpers.sql
 done < <(psql -d "$2" -tAc 'select pg_catalog.quote_ident(datname) from pg_catalog.pg_database where datallowconn')
 ) | psql -Xd "$2"

postgres-appliance/scripts/wale_restore.sh

@@ -43,7 +43,7 @@ ATTEMPT=0
 server_version="-1"
 while true; do
     [[ -z $wal_segment_backup_start ]] && wal_segment_backup_start=$($WAL_E backup-list 2> /dev/null \
-        | sed '0,/^(backup_\)?name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//')
+        | sed '0,/^\(backup_\)\?name\s*\(last_\)\?modified\s*/d' | sort -bk2 | tail -n1 | awk '{print $3;}' | sed 's/_.*$//')
 
     [[ -n "$CONNSTR" && $server_version == "-1" ]] && server_version=$(psql -d "$CONNSTR" -tAc 'show server_version_num' 2> /dev/null || echo "-1")
 
@@ -61,11 +61,7 @@ if [[ $server_version != "-1" ]]; then
     readonly lsn_offset=$((16#${wal_segment_backup_start:16:8}))
     printf -v backup_start_lsn "%X/%X" $lsn_segment $((lsn_offset << 24))
 
-    if [[ $server_version -ge 100000 ]]; then
-        readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_wal_lsn_diff(COALESCE(pg_last_wal_receive_lsn(), '0/0'), '$backup_start_lsn')::bigint, pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '$backup_start_lsn')::bigint) ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), '$backup_start_lsn')::bigint END"
-    else
-        readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_xlog_location_diff(COALESCE(pg_last_xlog_receive_location(), '0/0'), '$backup_start_lsn')::bigint, pg_xlog_location_diff(pg_last_xlog_replay_location(), '$backup_start_lsn')::bigint) ELSE pg_xlog_location_diff(pg_current_xlog_location(), '$backup_start_lsn')::bigint END"
-    fi
+    readonly query="SELECT CASE WHEN pg_is_in_recovery() THEN GREATEST(pg_wal_lsn_diff(COALESCE(pg_last_wal_receive_lsn(), '0/0'), '$backup_start_lsn')::bigint, pg_wal_lsn_diff(pg_last_wal_replay_lsn(), '$backup_start_lsn')::bigint) ELSE pg_wal_lsn_diff(pg_current_wal_lsn(), '$backup_start_lsn')::bigint END"
 
     ATTEMPT=0
     while true; do

postgres-appliance/tests/docker-compose.yml

@@ -51,7 +51,7 @@ services:
               postgresql:
                 parameters:
                   shared_buffers: 32MB
-            PGVERSION: '11'
+            PGVERSION: '12'
             # Just to test upgrade with clone. Without CLONE_SCOPE they don't work
             CLONE_WAL_S3_BUCKET: *bucket
             CLONE_AWS_ACCESS_KEY_ID: *access_key