revert: revert PR 1956, 1965, 1971

Dockerfile-15

@@ -214,13 +214,16 @@ COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout
 COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
 COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
 COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf
 COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
 COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
 
 RUN sed -i \
+    -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
     -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
     -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
-    /etc/postgresql/postgresql.conf && \
+    -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
     echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \    
     echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     usermod -aG postgres wal-g && \

Dockerfile-17

@@ -218,13 +218,16 @@ COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout
 COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
 COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
 COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf
 COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
 COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
 
 RUN sed -i \
+    -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
     -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
     -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
-    /etc/postgresql/postgresql.conf && \
+    -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
     echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     usermod -aG postgres wal-g && \

Dockerfile-orioledb-17

@@ -218,22 +218,25 @@ COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout
 COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf
 COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts
 COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf
+COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf
 COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh
 COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh
 
 RUN sed -i \
+    -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \
     -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \
     -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \
-    /etc/postgresql/postgresql.conf && \
+    -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \
     echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \
     usermod -aG postgres wal-g && \
     chown -R postgres:postgres /etc/postgresql-custom
 
     # Remove items from postgresql.conf
-RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf"
-    # As of pg 16.4+ db_user_namespace is totally deprecated and will break the server if setting is present
-RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf"
+RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" 
+    #as of pg 16.4 + this db_user_namespace totally deprecated and will break the server if setting is present
+RUN sed -i 's/db_user_namespace = off/#db_user_namespace = off/g;' "/etc/postgresql/postgresql.conf" 
 RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' "/etc/postgresql-custom/supautils.conf"
 RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf"
 RUN echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf"

ansible/files/database_optimizations.service → ansible/files/database-optimizations.service

@@ -4,7 +4,7 @@ Description=Postgresql optimizations
 [Service]
 Type=oneshot
 # we do not want failures from these commands to cause downstream service startup to fail
-ExecStart=-/opt/supabase-admin-api optimize db --destination-config-file-path /etc/postgresql-custom/conf.d/generated_optimizations.conf
+ExecStart=-/opt/supabase-admin-api optimize db --destination-config-file-path /etc/postgresql-custom/conf.d/generated-optimizations.conf
 ExecStart=-/opt/supabase-admin-api optimize pgbouncer --destination-config-file-path /etc/pgbouncer-custom/generated-optimizations.ini
 User=adminapi
 

ansible/files/pgbouncer_config/pgbouncer.service.j2

@@ -4,8 +4,8 @@ Documentation=man:pgbouncer(1)
 Documentation=https://www.pgbouncer.org/
 After=network.target
 {% if supabase_internal is defined %}
-Requires=database_optimizations.service
-After=database_optimizations.service
+Requires=database-optimizations.service
+After=database-optimizations.service
 {% endif %}
 
 [Service]

ansible/files/postgres_prestart.sh.j2

@@ -15,7 +15,7 @@ check_orioledb_enabled() {
 }
 
 get_shared_buffers() {
-    local opt_conf="/etc/postgresql-custom/conf.d/generated_optimizations.conf"
+    local opt_conf="/etc/postgresql-custom/generated-optimizations.conf"
     if [ ! -f "$opt_conf" ]; then
         return 0
     fi

ansible/files/postgresql_config/conf.d/wal-g.conf → ansible/files/postgresql_config/custom_walg.conf

@@ -1,5 +1,6 @@
 # - Archiving -
 
+#archive_mode = on
 #archive_command = '/usr/bin/admin-mgr wal-push %p >> /var/log/wal-g/wal-push.log 2>&1'
 #archive_timeout = 120
 

ansible/files/postgresql_config/postgresql.conf.j2

@@ -64,7 +64,7 @@ listen_addresses = '*'		# what IP address(es) to listen on;
 #port = 5432				# (change requires restart)
 #max_connections = 100			# (change requires restart)
 #superuser_reserved_connections = 3	# (change requires restart)
-unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
+#unix_socket_directories = '/tmp'	# comma-separated list of directories
 					# (change requires restart)
 #unix_socket_group = ''			# (change requires restart)
 #unix_socket_permissions = 0777		# begin with 0 to use octal notation
@@ -242,8 +242,8 @@ checkpoint_flush_after = 256kB		# measured in pages, 0 disables
 
 # - Archiving -
 
-archive_mode = on  # enables archiving; off, on, or always (change requires restart)
-archive_command = '/bin/true'   # command to use to archive a logfile segment
+#archive_mode = off  # enables archiving; off, on, or always (change requires restart)
+#archive_command = ''   # command to use to archive a logfile segment
                                 # placeholders: %p = path of file to archive
                                 # %f = file name only
                                 # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
@@ -752,6 +752,17 @@ jit_provider = 'llvmjit'		# JIT library to use
 #include_if_exists = '...'		# include file only if it exists
 #include = '...'			# include file
 
+# Automatically generated optimizations
+#include = '/etc/postgresql-custom/generated-optimizations.conf'
+# User-supplied custom parameters, override any automatically generated ones
+#include = '/etc/postgresql-custom/custom-overrides.conf'
+
+# WAL-G specific configurations
+#include = '/etc/postgresql-custom/wal-g.conf'
+
+# read replica specific configurations
+include = '/etc/postgresql-custom/read-replica.conf'
+
 # supautils specific configurations
 #include = '/etc/postgresql-custom/supautils.conf'
 

ansible/files/postgresql_config/postgresql.service

@@ -2,8 +2,8 @@
 Description=PostgreSQL database server
 Documentation=man:postgres(1)
 {% if supabase_internal is defined %}
-Requires=database_optimizations.service
-After=database_optimizations.service
+Requires=database-optimizations.service
+After=database-optimizations.service
 {% endif %}
 
 [Service]

ansible/tasks/setup-postgres.yml

@@ -147,11 +147,11 @@
         group: 'postgres'
         mode: '0664'
         owner: 'postgres'
-        path: "/etc/postgresql-custom/conf.d/{{ pg_config_item }}"
+        path: "/etc/postgresql-custom/{{ pg_config_item }}"
         state: 'touch'
       loop:
-        - 'custom_overrides.conf'
-        - 'generated_optimizations.conf'
+        - 'custom-overrides.conf'
+        - 'generated-optimizations.conf'
       loop_control:
         loop_var: 'pg_config_item'
 
@@ -167,13 +167,13 @@
       loop_control:
         loop_var: 'pg_config_item'
 
-    - name: Move read-replica.conf file to /etc/postgresql-custom/conf.d/read-replica.conf
+    - name: Move read-replica.conf file to /etc/postgresql-custom/read-replica.conf
       ansible.builtin.copy:
-        dest: '/etc/postgresql-custom/conf.d/read_replica.conf'
+        dest: '/etc/postgresql-custom/read-replica.conf'
         mode: '0664'
         owner: 'postgres'
         group: 'postgres'
-        src: 'files/postgresql_config/conf.d/read_replica.conf'
+        src: 'files/postgresql_config/custom_read_replica.conf'
 
     - name: configure pam
       block:
@@ -291,7 +291,7 @@
     dest: "/etc/systemd/system/{{ systemd_svc_item | basename }}"
     src: "files/{{ systemd_svc_item }}"
   loop:
-    - 'database_optimizations.service'
+    - 'database-optimizations.service'
     - 'postgresql_config/postgresql.service'
   loop_control:
     loop_var: 'systemd_svc_item'

ansible/tasks/setup-supabase-internal.yml

@@ -75,6 +75,20 @@
     state: 'directory'
   become: true
 
+- name: Include file for generated optimizations in postgresql.conf
+  ansible.builtin.replace:
+    path: '/etc/postgresql/postgresql.conf'
+    regexp: "#include = '/etc/postgresql-custom/generated-optimizations.conf'"
+    replace: "include = '/etc/postgresql-custom/generated-optimizations.conf'"
+  become: true
+
+- name: Include file for custom overrides in postgresql.conf
+  ansible.builtin.replace:
+    path: '/etc/postgresql/postgresql.conf'
+    regexp: "#include = '/etc/postgresql-custom/custom-overrides.conf'"
+    replace: "include = '/etc/postgresql-custom/custom-overrides.conf'"
+  become: true
+
 - name: Install Postgres exporter
   ansible.builtin.import_tasks:
     file: internal/postgres-exporter.yml

ansible/tasks/setup-wal-g.yml

@@ -54,6 +54,14 @@
         path: '/etc/wal-g/config.json'
         state: 'touch'
 
+    - name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf
+      ansible.builtin.template:
+        dest: '/etc/postgresql-custom/wal-g.conf'
+        group: 'postgres'
+        mode: '0664'
+        owner: 'postgres'
+        src: 'files/postgresql_config/custom_walg.conf'
+
     - name: Add script to be run for restore_command
       ansible.builtin.template:
         dest: '/home/postgres/wal_fetch.sh'
@@ -69,18 +77,25 @@
         owner: 'root'
         src: 'files/walg_helper_scripts/wal_change_ownership.sh'
 
-    - name: Move custom wal-g.conf file to /etc/postgresql-custom/conf.d/wal-g.conf
+    - name: Include /etc/postgresql-custom/wal-g.conf in postgresql.conf
+      ansible.builtin.replace:
+        path: '/etc/postgresql/postgresql.conf'
+        regexp: "#include = '/etc/postgresql-custom/wal-g.conf'"
+        replace: "include = '/etc/postgresql-custom/wal-g.conf'"
+      become: true
+
+    - name: Move custom wal-g.conf file to /etc/postgresql-custom/wal-g.conf
       ansible.builtin.copy:
-        dest: '/etc/postgresql-custom/conf.d/wal-g.conf'
+        dest: '/etc/postgresql-custom/wal-g.conf'
         group: 'postgres'
         mode: '0664'
         owner: 'postgres'
-        src: 'files/postgresql_config/conf.d/wal-g.conf'
+        src: 'files/postgresql_config/custom_walg.conf'
 
-    - name: Create symlink for admin-mgr compatibility
-      ansible.builtin.file:
-        src: '/etc/postgresql-custom/conf.d/wal-g.conf'
-        dest: '/etc/postgresql-custom/wal-g.conf'
-        state: 'link'
-        owner: 'postgres'
-        group: 'postgres'
+    # - name: Create symlink for admin-mgr compatibility
+    #   ansible.builtin.file:
+    #     src: '/etc/postgresql-custom/conf.d/wal-g.conf'
+    #     dest: '/etc/postgresql-custom/wal-g.conf'
+    #     state: 'link'
+    #     owner: 'postgres'
+    #     group: 'postgres'

audit-specs/baselines/ami-build/files-postgres-config.yml

@@ -55,13 +55,13 @@ file:
     owner: postgres
     group: postgres
     mode: '0775'
-  /etc/postgresql-custom/conf.d/wal-g.conf:
+  /etc/postgresql-custom/wal-g.conf:
     exists: true
     filetype: file
     owner: postgres
     group: postgres
     mode: '0664'
-  /etc/postgresql-custom/conf.d/read_replica.conf:
+  /etc/postgresql-custom/read-replica.conf:
     exists: true
     filetype: file
     owner: postgres

audit-specs/baselines/baseline.yml

@@ -5461,7 +5461,7 @@ file:
     owner: "0"
     group: "0"
     filetype: file
-  /etc/systemd/system/database_optimizations.service:
+  /etc/systemd/system/database-optimizations.service:
     exists: true
     mode: "0644"
     owner: "0"

audit-specs/baselines/prod-deployed/files-systemd-deployed.yml

@@ -55,7 +55,7 @@ file:
     group: '0'
     mode: '0644'
     owner: '0'
-  /etc/systemd/system/database_optimizations.service:
+  /etc/systemd/system/database-optimizations.service:
     exists: true
     filetype: file
     group: '0'

nix/checks.nix

@@ -253,10 +253,10 @@
                 }
 
                 # Start HTTP mock server for http extension tests
-                HTTP_MOCK_PORT_FILE="$BUILD_TMP/http-mock-port"
+                export HTTP_MOCK_PORT_FILE="$BUILD_TMP/http-mock-port"
 
                 log info "Starting HTTP mock server (will find free port)..."
-                HTTP_MOCK_PORT_FILE="$HTTP_MOCK_PORT_FILE" log_cmd ${pkgs.python3}/bin/python3 ${./tests/http-mock-server.py} &
+                log_cmd ${pkgs.python3}/bin/python3 ${./tests/http-mock-server.py} &
                 HTTP_MOCK_PID=$!
 
                 # Clean up on exit

nix/packages/lib.nix

@@ -48,6 +48,10 @@
           name = "logging.conf";
           path = ../../ansible/files/postgresql_config/postgresql-csvlog.conf;
         };
+        readReplicaConfigFile = builtins.path {
+          name = "readreplica.conf";
+          path = ../../ansible/files/postgresql_config/custom_read_replica.conf;
+        };
         pgHbaConfigFile = builtins.path {
           name = "pg_hba.conf";
           path = ../../ansible/files/postgresql_config/pg_hba.conf.j2;
@@ -87,6 +91,7 @@
         POSTGRESQL_CONFIG_DIR = "${postgresqlConfigBaseDir}";
         PSQLORIOLEDB17_BINDIR = "${psql_orioledb-17}";
         PGSODIUM_GETKEY = "${paths.getkeyScript}";
+        READREPL_CONF_FILE = "${paths.readReplicaConfigFile}";
         LOGGING_CONF_FILE = "${paths.loggingConfigFile}";
         SUPAUTILS_CONF_FILE = "${paths.supautilsConfigFile}";
         PG_HBA = "${paths.pgHbaConfigFile}";
@@ -119,6 +124,7 @@
         cp ${paths.supautilsConfigFile} $out/etc/postgresql-custom/supautils.conf || { echo "Failed to copy supautils.conf"; exit 1; }
         cp ${paths.pgconfigFile} $out/etc/postgresql/postgresql.conf || { echo "Failed to copy postgresql.conf"; exit 1; }
         cp ${paths.loggingConfigFile} $out/etc/postgresql-custom/logging.conf || { echo "Failed to copy logging.conf"; exit 1; }
+        cp ${paths.readReplicaConfigFile} $out/etc/postgresql-custom/read-replica.conf || { echo "Failed to copy read-replica.conf"; exit 1; }
         cp ${paths.pgHbaConfigFile} $out/etc/postgresql/pg_hba.conf || { echo "Failed to copy pg_hba.conf"; exit 1; }
         cp ${paths.pgIdentConfigFile} $out/etc/postgresql/pg_ident.conf || { echo "Failed to copy pg_ident.conf"; exit 1; }
         cp -r ${paths.postgresqlExtensionCustomScriptsPath}/* $out/extension-custom-scripts/ || { echo "Failed to copy custom scripts"; exit 1; }

nix/tests/postgresql.conf.in

@@ -62,7 +62,7 @@ listen_addresses = '*'		# what IP address(es) to listen on;
 #port = @PGSQL_DEFAULT_PORT@				# (change requires restart)
 max_connections = 100			# (change requires restart)
 #superuser_reserved_connections = 3	# (change requires restart)
-unix_socket_directories = '/var/run/postgresql'	# comma-separated list of directories
+unix_socket_directories = '/tmp'	# comma-separated list of directories
 					# (change requires restart)
 #unix_socket_group = ''			# (change requires restart)
 #unix_socket_permissions = 0777		# begin with 0 to use octal notation

nix/tools/run-server.sh.in

@@ -240,6 +240,7 @@ vault.getkey_script = '$PGSODIUM_GETKEY_SCRIPT'" \
 -e "s|hba_file = '/etc/postgresql/pg_hba.conf'|hba_file = '$DATDIR/pg_hba.conf'|" \
 -e "s|ident_file = '/etc/postgresql/pg_ident.conf'|ident_file = '$DATDIR/pg_ident.conf'|" \
 -e "s|include = '/etc/postgresql/logging.conf'|#&|" \
+-e "s|include = '/etc/postgresql-custom/read-replica.conf'|include = '$DATDIR/read-replica.conf'|" \
 -e "\$a\\
 session_preload_libraries = 'supautils'" \
 -e "s|include_dir = '/etc/postgresql-custom/conf.d'|include_dir = '$DATDIR/conf.d'|" \