diff --git a/deployment-configuration/helm/templates/auto-database-postgres-operator.yaml b/deployment-configuration/helm/templates/auto-database-postgres-operator.yaml index 25d380b0..c8022c17 100644 --- a/deployment-configuration/helm/templates/auto-database-postgres-operator.yaml +++ b/deployment-configuration/helm/templates/auto-database-postgres-operator.yaml @@ -24,6 +24,12 @@ metadata: spec: instances: {{ .app.harness.database.postgres.instances | default 1 }} + {{- with .app.harness.database.postgres.parameters }} + postgresql: + parameters: +{{ toYaml . | nindent 6 }} + {{- end }} + inheritedMetadata: labels: app: {{ .app.harness.database.name | quote }} diff --git a/deployment-configuration/value-template.yaml b/deployment-configuration/value-template.yaml index e700cfb7..e5e8aa13 100644 --- a/deployment-configuration/value-template.yaml +++ b/deployment-configuration/value-template.yaml @@ -115,6 +115,8 @@ harness: # -- CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443). # -- Resolved automatically at deploy time via cluster lookup. Set explicitly only as a fallback for helm-template or air-gapped use. apiServerCidr: [] + # -- PostgreSQL configuration parameters for CloudNative-PG clusters (operator: true). Values must be strings. + parameters: {} ports: - name: http port: 5432 diff --git a/docs/applications/databases.md b/docs/applications/databases.md index afb6258d..a5275597 100644 --- a/docs/applications/databases.md +++ b/docs/applications/databases.md @@ -87,6 +87,7 @@ harness operator: false instances: 1 apiServerCidr: [] + parameters: {} ports: - name: http port: 5432 @@ -109,6 +110,8 @@ helm install cnpg cloudnative-pg/cloudnative-pg `apiServerCidr`: List of CIDRs allowed for CNPG database pods to reach the Kubernetes API server on port 443. **Resolved automatically at deploy time** by looking up the `kubernetes` Service and Endpoints in the `default` namespace. The explicit list is only used as a fallback when lookup returns nothing (e.g. `helm template` dry-run). Leave empty (`[]`) for auto-detection; set explicitly only for air-gapped or restricted environments. +`parameters`: Optional map of PostgreSQL configuration parameters rendered to CloudNative-PG as `spec.postgresql.parameters` when `operator: true`. Values must be strings, for example `max_connections: "200"`. CloudNative-PG rejects parameters that are fixed or managed by the operator. + #### Neo4j @@ -209,4 +212,3 @@ Further reading: [MongoDB archiving & compression](https://www.mongodb.com/blog/ Further reading: [pg_dumpall docs](https://www.postgresql.org/docs/10/app-pg-dumpall.html) - diff --git a/docs/model/DatabaseConfig.md b/docs/model/DatabaseConfig.md index 5440bec9..fb5ea661 100644 --- a/docs/model/DatabaseConfig.md +++ b/docs/model/DatabaseConfig.md @@ -12,6 +12,7 @@ Name | Type | Description | Notes **operator** | **bool** | Use the CloudNative-PG operator instead of a plain Deployment (postgres only) | [optional] **instances** | **int** | Number of PostgreSQL instances managed by the CNPG operator (only used when operator is true) | [optional] **api_server_cidr** | **List[str]** | CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443). Override with your cluster API-server or service CIDR. | [optional] +**parameters** | **Dict[str, str]** | PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings. | [optional] **initialdb** | **str** | Initial database name (postgres only) | [optional] **args** | **List[str]** | Additional command-line arguments passed to the database server process (postgres only) | [optional] diff --git a/libraries/models/api/openapi.yaml b/libraries/models/api/openapi.yaml index d740a5a4..59f71e80 100644 --- a/libraries/models/api/openapi.yaml +++ b/libraries/models/api/openapi.yaml @@ -962,6 +962,11 @@ components: type: array items: type: string + parameters: + description: 'PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings.' + type: object + additionalProperties: + type: string initialdb: description: 'Initial database name (postgres only)' type: string diff --git a/libraries/models/cloudharness_model/models/database_config.py b/libraries/models/cloudharness_model/models/database_config.py index 0b69962e..a1382b03 100644 --- a/libraries/models/cloudharness_model/models/database_config.py +++ b/libraries/models/cloudharness_model/models/database_config.py @@ -37,10 +37,11 @@ class DatabaseConfig(CloudHarnessBaseModel): operator: Optional[StrictBool] = Field(default=None, description="Use the CloudNative-PG operator instead of a plain Deployment (postgres only)") instances: Optional[Annotated[int, Field(strict=True, ge=1)]] = Field(default=None, description="Number of PostgreSQL instances managed by the CNPG operator (only used when operator is true)") api_server_cidr: Optional[List[StrictStr]] = Field(default=None, description="CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443). Override with your cluster API-server or service CIDR.", alias="apiServerCidr") + parameters: Optional[Dict[str, StrictStr]] = Field(default=None, description="PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings.") initialdb: Optional[StrictStr] = Field(default=None, description="Initial database name (postgres only)") args: Optional[List[StrictStr]] = Field(default=None, description="Additional command-line arguments passed to the database server process (postgres only)") additional_properties: Dict[str, Any] = {} - __properties: ClassVar[List[str]] = ["image", "name", "ports", "operator", "instances", "apiServerCidr", "initialdb", "args"] + __properties: ClassVar[List[str]] = ["image", "name", "ports", "operator", "instances", "apiServerCidr", "parameters", "initialdb", "args"] def to_dict(self) -> Dict[str, Any]: """Return the dictionary representation of the model using alias. @@ -92,6 +93,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "operator": obj.get("operator"), "instances": obj.get("instances"), "apiServerCidr": obj.get("apiServerCidr"), + "parameters": obj.get("parameters"), "initialdb": obj.get("initialdb"), "args": obj.get("args") }) diff --git a/libraries/models/docs/DatabaseConfig.md b/libraries/models/docs/DatabaseConfig.md index 5440bec9..fb5ea661 100644 --- a/libraries/models/docs/DatabaseConfig.md +++ b/libraries/models/docs/DatabaseConfig.md @@ -12,6 +12,7 @@ Name | Type | Description | Notes **operator** | **bool** | Use the CloudNative-PG operator instead of a plain Deployment (postgres only) | [optional] **instances** | **int** | Number of PostgreSQL instances managed by the CNPG operator (only used when operator is true) | [optional] **api_server_cidr** | **List[str]** | CIDR(s) allowed for CNPG pods to reach the Kubernetes API server (port 443). Override with your cluster API-server or service CIDR. | [optional] +**parameters** | **Dict[str, str]** | PostgreSQL configuration parameters passed to CloudNative-PG as spec.postgresql.parameters (postgres operator only). Values must be strings. | [optional] **initialdb** | **str** | Initial database name (postgres only) | [optional] **args** | **List[str]** | Additional command-line arguments passed to the database server process (postgres only) | [optional] diff --git a/libraries/models/test/test_deserialize.py b/libraries/models/test/test_deserialize.py index 4a856df8..e7c97f1a 100644 --- a/libraries/models/test/test_deserialize.py +++ b/libraries/models/test/test_deserialize.py @@ -2,7 +2,7 @@ from os.path import join, dirname as dn, realpath import oyaml as yaml -from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent, ApplicationTestConfig +from cloudharness_model import HarnessMainConfig, ApplicationConfig, User, ApplicationHarnessConfig, CDCEvent, ApplicationTestConfig, DatabaseConfig HERE = dn(realpath(__file__)) @@ -37,6 +37,27 @@ def test_camelcase(): assert u.last_name == "a" assert u["lastName"] == "a" + +def test_database_config_parameters_round_trip(): + config = DatabaseConfig.from_dict({ + "image": "postgres:17", + "operator": True, + "parameters": { + "max_connections": "200", + "shared_buffers": "1GB", + }, + }) + + assert config.parameters == { + "max_connections": "200", + "shared_buffers": "1GB", + } + assert config.to_dict()["parameters"] == { + "max_connections": "200", + "shared_buffers": "1GB", + } + + def test_robustness(): d = {'aliases': [], 'database': {'auto': True, 'mongo': {'image': 'mongo:5', 'ports': [{'name': 'http', 'port': 27017}]}, 'name': 'keycloak-postgres', 'neo4j': {'dbms_security_auth_enabled': 'false', 'image': 'neo4j:4.1.9', 'memory': {'heap': {'initial': '64M', 'max': '128M'}, 'pagecache': {'size': '64M'}, 'size': '256M'}, 'ports': [{'name': 'http', 'port': 7474}, {'name': 'bolt', 'port': 7687}]}, 'pass': 'password', 'postgres': {'image': 'postgres:10.4', 'initialdb': 'auth_db', 'ports': [{'name': 'http', 'port': 5432}]}, 'resources': {'limits': {'cpu': '1000m', 'memory': '2Gi'}, 'requests': {'cpu': '100m', 'memory': '512Mi'}}, 'size': '2Gi', 'type': 'postgres', 'user': 'user'}, 'dependencies': {'build': [], 'hard': [], 'soft': []}, 'deployment': {'auto': True, 'image': 'osb/accounts:3e02a15477b4696ed554e08cedf4109c67908cbe6b03331072b5b73e83b4fc2b', 'name': 'accounts', 'port': 8080, 'replicas': 1, 'resources': {'limits': {'cpu': '500m', 'memory': '1024Mi'}, 'requests': {'cpu': '10m', 'memory': '512Mi'}}}, 'domain': None, 'env': [{'name': 'KEYCLOAK_IMPORT', 'value': '/tmp/realm.json'}, {'name': 'KEYCLOAK_USER', 'value': 'admin'}, {'name': 'KEYCLOAK_PASSWORD', 'value': 'metacell'}, {'name': 'PROXY_ADDRESS_FORWARDING', 'value': 'true'}, {'name': 'DB_VENDOR', 'value': 'POSTGRES'}, {'name': 'DB_ADDR', 'value': 'keycloak-postgres'}, {'name': 'DB_DATABASE', 'value': 'auth_db'}, {'name': 'DB_USER', 'value': 'user'}, {'name': 'DB_PASSWORD', 'value': 'password'}, {'name': 'JAVA_OPTS', 'value': '-server -Xms64m -Xmx896m -XX:MetaspaceSize=96M -XX:MaxMetaspaceSize=256m -Djava.net.preferIPv4Stack=true -Djboss.modules.system.pkgs=org.jboss.byteman -Djava.awt.headless=true --add-exports=java.base/sun.nio.ch=ALL-UNNAMED --add-exports=jdk.unsupported/sun.misc=ALL-UNNAMED --add-exports=jdk.unsupported/sun.reflect=ALL-UNNAMED'}], 'name': 'accounts', 'readinessProbe': {'path': '/realms/master'}, 'resources': [{'dst': '/tmp/realm.json', 'name': 'realm-config', 'src': 'realm.json'}], 'secrets': {}, 'secured': False, 'service': {'auto': True, 'name': 'accounts', 'port': 8080}, 'subdomain': 'accounts', 'uri_role_mapping': [{'roles': ['administrator'], 'uri': '/*'}], 'use_services': []} @@ -161,4 +182,4 @@ def internal_data(self, value): if __name__ == "__main__": - test_property_access() \ No newline at end of file + test_property_access() diff --git a/tools/deployment-cli-tools/tests/test_helm.py b/tools/deployment-cli-tools/tests/test_helm.py index b424db24..511a6a3f 100644 --- a/tools/deployment-cli-tools/tests/test_helm.py +++ b/tools/deployment-cli-tools/tests/test_helm.py @@ -2,6 +2,8 @@ from ch_cli_tools.configurationgenerator import * from ch_cli_tools.preprocessing import preprocess_build_overrides, generate_hash_based_image_tags import pytest +import shutil +import subprocess HERE = os.path.dirname(os.path.realpath(__file__)) RESOURCES = os.path.join(HERE, 'resources') @@ -12,6 +14,24 @@ def exists(path): return path.exists() +def render_helm_chart(chart_path): + completed = subprocess.run( + ["helm", "template", str(chart_path)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + return [manifest for manifest in yaml.safe_load_all(completed.stdout) if manifest] + + +def find_manifest(manifests, kind, name): + for manifest in manifests: + if manifest.get("kind") == kind and manifest.get("metadata", {}).get("name") == name: + return manifest + raise AssertionError(f"Could not find {kind}/{name}") + + def test_collect_helm_values(tmp_path): out_folder = tmp_path / 'test_collect_helm_values' values = create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, include=['samples', 'myapp'], @@ -268,6 +288,50 @@ def test_clear_unused_dbconfig(tmp_path): assert db_config['postgres'] is None +def test_cnpg_postgres_parameters_render_only_when_set(tmp_path): + out_folder = tmp_path / 'test_cnpg_postgres_parameters_render_only_when_set' + create_helm_chart([CLOUDHARNESS_ROOT, RESOURCES], output_path=out_folder, domain="my.local", + env='withpostgres', local=False, include=["myapp"], exclude=["legacy"]) + + helm_path = out_folder / HELM_CHART_PATH + shutil.rmtree(helm_path / 'charts') + values_path = helm_path / 'values.yaml' + with open(values_path, 'r') as values_file: + values = yaml.safe_load(values_file) + postgres = values['apps']['myapp']['harness']['database']['postgres'] + postgres['operator'] = True + postgres['parameters'] = { + 'max_connections': '200', + 'shared_buffers': '1GB', + } + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + db_name = values['apps']['myapp']['harness']['database']['name'] + cluster = find_manifest(manifests, 'Cluster', db_name) + assert cluster['spec']['postgresql']['parameters'] == { + 'max_connections': '200', + 'shared_buffers': '1GB', + } + + postgres['parameters'] = {} + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + cluster = find_manifest(manifests, 'Cluster', db_name) + assert 'postgresql' not in cluster['spec'] + + postgres.pop('parameters') + with open(values_path, 'w') as values_file: + yaml.safe_dump(values, values_file) + + manifests = render_helm_chart(helm_path) + cluster = find_manifest(manifests, 'Cluster', db_name) + assert 'postgresql' not in cluster['spec'] + + def test_clear_all_dbconfig_if_nodb(tmp_path): out_folder = tmp_path / 'test_clear_all_dbconfig_if_nodb'