feat(tuned): migrate sysctl tunings to tuned

ansible/files/gotrue.service.j2

@@ -16,6 +16,7 @@ After=apparmor.service
 
 # We want sysctl's to be applied
 After=systemd-sysctl.service
+After=tuned.service
 
 # UFW Is modified by cloud init, but started non-blocking, so configuration
 # could be in-flight while gotrue is starting. I want to ensure future rules

ansible/tasks/setup-postgres.yml

@@ -77,11 +77,12 @@
     # for the nix built version
     - name: create ssl-cert and postgres groups
       ansible.builtin.group:
-        name: "{{ group_item }}"
+        gid: "{{ group_item['gid'] }}"
+        name: "{{ group_item['grp'] }}"
         state: 'present'
       loop:
-        - 'ssl-cert'
-        - 'postgres'
+        - { grp: 'ssl-cert', gid: '1001' }
+        - { grp: 'postgres', gid: '1002' }
       loop_control:
         loop_var: 'group_item'
 

ansible/tasks/setup-system.yml

@@ -153,46 +153,6 @@
         owner: 'root'
         group: 'root'
 
-    # Set Sysctl params specific to keepalives
-    - name: Set net.ipv4.tcp_keepalive_time=1800
-      ansible.builtin.sysctl:
-        name: 'net.ipv4.tcp_keepalive_time'
-        value: 1800
-        state: 'present'
-
-    - name: Set net.ipv4.tcp_keepalive_intvl=60
-      ansible.builtin.sysctl:
-        name: 'net.ipv4.tcp_keepalive_intvl'
-        value: 60
-        state: 'present'
-
-    # postgres_exporter runs on port 9187 and postgresT occasionlly chooses it as random srcport
-    # adminapi for 8085
-    - name: Set net.ipv4.ip_local_reserved_ports
-      ansible.builtin.sysctl:
-        name: 'net.ipv4.ip_local_reserved_ports'
-        value: '9187,8085'
-        state: 'present'
-
-- name: Execute tasks when (debpkg_mode or nixpkg_mode)
-  when:
-    - (debpkg_mode or nixpkg_mode)
-  block:
-    # Set Sysctl params for restarting the OS on OOM after 10
-    - name: Set vm.panic_on_oom=1
-      ansible.builtin.sysctl:
-        name: 'vm.panic_on_oom'
-        reload: true
-        state: 'present'
-        value: '1'
-
-    - name: Set kernel.panic=10
-      ansible.builtin.sysctl:
-        name: 'kernel.panic'
-        reload: true
-        state: 'present'
-        value: '10'
-
 - name: set hosts file
   ansible.builtin.copy:
     content: |
@@ -204,13 +164,3 @@
     group: 'root'
   when:
     - (debpkg_mode or stage2_nix)
-
-- name: configure system
-  ansible.posix.sysctl:
-    name: 'net.core.somaxconn'
-    value: 16834
-
-- name: configure system
-  ansible.posix.sysctl:
-    name: 'net.ipv4.ip_local_port_range'
-    value: '1025 65000'

ansible/tasks/setup-tuned.yml

@@ -1,8 +1,8 @@
-- name: Install and configure tuned when stage2_nix
+- name: tuned - Install and configure tuned when stage2_nix
   when:
     - (stage2_nix or nixpkg_mode)
   block:
-    - name: Install tuned
+    - name: tuned - Install tuned
       ansible.builtin.apt:
         force_apt_get: true
         name: 'tuned'
@@ -11,7 +11,7 @@
         update_cache: true
       become: true
 
-    - name: Create a tuned profile directory
+    - name: tuned - Create a tuned profile directory
       ansible.builtin.file:
         group: 'root'
         mode: '0755'
@@ -20,7 +20,7 @@
         state: 'directory'
       become: true
 
-    - name: Create a profile symlink for older tuned versions
+    - name: tuned - Create a profile symlink for older tuned versions
       ansible.builtin.file:
         force: true
         group: 'root'
@@ -31,7 +31,7 @@
         state: 'link'
       become: true
 
-    - name: Create a tuned profile
+    - name: tuned - Create a tuned profile
       become: true
       community.general.ini_file:
         create: true
@@ -103,19 +103,6 @@
       when:
         - ansible_facts['swaptotal_mb'] > 0
       block:
-        - name: tuned - Decrease the kernel swappiness
-          become: true
-          community.general.ini_file:
-            create: true
-            group: 'root'
-            mode: '0644'
-            no_extra_spaces: true
-            option: 'vm.swappiness'
-            path: '/etc/tuned/profiles/postgresql/tuned.conf'
-            section: 'sysctl'
-            state: 'present'
-            value: 10
-
         - name: tuned - Load zstd compressor module
           become: true
           community.general.modprobe:
@@ -147,15 +134,110 @@
             state: 'present'
             value: 'zswap.enabled=1 zswap.zpool=zsmalloc zswap.compressor=zstd zswap.max_pool_percent=10'
 
-    - name: Activate the tuned service
+    - name: tuned - Inherit the throughput-performance profile
+      become: true
+      community.general.ini_file:
+        create: true
+        group: 'root'
+        mode: '0644'
+        no_extra_spaces: true
+        option: 'include'
+        path: '/etc/tuned/profiles/postgresql/tuned.conf'
+        section: 'main'
+        state: 'present'
+        value: 'throughput-performance'
+
+    - name: tuned - Add Supabase-specific tunings and overrides
+      become: true
+      community.general.ini_file:
+        create: true
+        group: 'root'
+        mode: '0644'
+        no_extra_spaces: true
+        option: "{{ supa_item['option'] }}"
+        path: '/etc/tuned/profiles/postgresql/tuned.conf'
+        section: 'sysctl'
+        state: 'present'
+        value: "{{ supa_item['value'] }}"
+      loop:
+        - { option: 'fs.file-max',                      value: '312139770' }
+        - { option: 'fs.aio-max-nr',                    value: '1048576' }
+        - { option: 'kernel.numa_balancing',            value: '0' }
+        - { option: 'kernel.shmmax',                    value: '18446744073692700000' }
+        - { option: 'kernel.shmall',                    value: '18446744073692700000' }
+        - { option: 'kernel.shmmni',                    value: '4096' }
+        - { option: 'kernel.sem',                       value: '250 512000 100 2048' }
+        - { option: 'kernel.panic',                     value: '10' }
+        - { option: 'kernel.panic_on_oom',              value: '1' }
+        - { option: 'kernel.panic_on_oops',             value: '1' }
+        - { option: 'net.core.netdev_budget',           value: '1024' }
+        - { option: 'net.core.netdev_max_backlog',      value: '10000' }
+        - { option: 'net.core.rmem_default',            value: '262144' }
+        - { option: 'net.core.rmem_max',                value: '104857600' }
+        - { option: 'net.core.somaxconn',               value: '16384' }
+        - { option: 'net.core.wmem_default',            value: '262144' }
+        - { option: 'net.core.wmem_max',                value: '104857600' }
+        - { option: 'net.ipv4.ip_local_port_range',     value: '1025 65499' }
+        - { option: 'net.ipv4.ip_local_reserved_ports', value: '3000,3001,8085,9122,9187,9999' }
+        - { option: 'net.ipv4.tcp_keepalive_intvl',     value: '60' }
+        - { option: 'net.ipv4.tcp_keepalive_time',      value: '1800' }
+        - { option: 'net.ipv4.tcp_max_syn_backlog',     value: '4096' }
+        - { option: 'net.ipv4.tcp_rmem',                value: '4096 87380 16777216' }
+        - { option: 'net.ipv4.tcp_timestamps',          value: '0' }
+        - { option: 'net.ipv4.tcp_tw_reuse',            value: '1' }
+        - { option: 'net.ipv4.tcp_window_scaling',      value: '1' }
+        - { option: 'net.ipv4.tcp_wmem',                value: '4096 65536 16777216' }
+        - { option: 'net.netfilter.nf_conntrack_max',   value: '250000' }
+        - { option: 'vm.dirty_background_ratio',        value: '10' }
+        - { option: 'vm.dirty_expire_centisecs',        value: '3000' }
+        - { option: 'vm.dirty_ratio',                   value: '40' }
+        - { option: 'vm.dirty_writeback_centisecs',     value: '500' }
+#        - { option: 'vm.overcommit_memory',             value: '2' }
+        - { option: 'vm.panic_on_oom',                  value: '1' }
+        - { option: 'vm.swappiness',                    value: '10' }
+      loop_control:
+        loop_var: 'supa_item'
+
+    - name: tuned - Configure explicit HugePages for Postgresql
+      become: true
+      block:
+        - name: tuned - Compute the necessary HugePages
+          become: true
+          community.general.ini_file:
+            create: true
+            group: 'root'
+            mode: '0644'
+            option: 'vm.nr_hugepages'
+            path: '/etc/tuned/profiles/postgresql/tuned.conf'
+            section: 'sysctl'
+            state: 'present'
+            value: "{{ (shared_buffers * overhead) / hugepagesize | round | int }}"
+          vars:
+            hugepagesize: 2048  # assumes a 2MB page
+            shared_buffers: 131072  # we use 128MB (128 * 1024) of shared_buffers by default
+            overhead: 1.05
+
+        - name: tuned - Add the postgres group to the HugePages access
+          become: true
+          community.general.ini_file:
+            create: true
+            group: 'root'
+            mode: '0644'
+            option: 'vm.hugetlb_shm_group'
+            path: '/etc/tuned/profiles/postgresql/tuned.conf'
+            section: 'sysctl'
+            state: 'present'
+            value: '1002'
+
+    - name: tuned - Activate the tuned service
       ansible.builtin.systemd_service:
         daemon_reload: true
         enabled: true
         name: 'tuned'
         state: 'restarted'
       become: true
 
-    - name: Activate the PostgreSQL tuned profile
+    - name: tuned - Activate the PostgreSQL tuned profile
       ansible.builtin.command:
         cmd: tuned-adm profile postgresql
       become: true

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.053-orioledb"
-  postgres17: "17.6.1.096"
-  postgres15: "15.14.1.096"
+  postgresorioledb-17: "17.6.0.053-orioledb-tuned-1"
+  postgres17: "17.6.1.096-tuned-1"
+  postgres15: "15.14.1.096-tuned-1"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.25.1