Skip to content

Commit 27e1cb5

Browse files
tychtjantychtjan
committed
feat: auto-load staging secrets from .env and fix DS_PASSWORD plumbing
- Add `-include .env` to Makefile for STAGING_ADMIN_TOKEN and STAGING_DS_PASSWORD - Pass DS_PASSWORD through Makefile → tox → pytest for staging tests - Override ds_password from DS_PASSWORD env var in upload_demo_layout.py - Patch data_sources_credentials.yaml at test session start with real password - Update staging config to use correct ds_url and empty ds_password default - Add isNullable to LDM reference sources and PDM fixture FK columns to match staging server's generateLogicalModel response jira: DX-341 risk: low
1 parent 4303cb0 commit 27e1cb5

File tree

10 files changed

+113
-65
lines changed

10 files changed

+113
-65
lines changed

‎.env‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# (C) 2026 GoodData Corporation
2+
# Staging environment secrets (copy to .env and fill in)
3+
STAGING_ADMIN_TOKEN=
4+
STAGING_DS_PASSWORD=

‎Makefile‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,19 @@
11
# (C) 2021 GoodData Corporation
2+
# Load .env if it exists (staging secrets, gitignored)
3+
-include .env
4+
25
# list all full paths to files and directories in CWD containing "gooddata", filter out ones ending by "client"
36
NO_CLIENT_GD_PROJECTS_ABS = $(filter-out %client, $(wildcard $(CURDIR)/packages/*gooddata*))
47
# for each path, take only the base name of the path
58
NO_CLIENT_GD_PROJECTS_DIRS = $(foreach dir, $(NO_CLIENT_GD_PROJECTS_ABS), $(notdir $(dir)))
69
# TODO: replace API_VERSION in the future by call to API
710
API_VERSION="v1"
8-
# Generate from localhost
11+
# Default: generate from localhost; use `make api-client STAGING=1` to download from remote
12+
ifdef STAGING
13+
BASE_URL="https://demo-cicd.cloud.gooddata.com"
14+
else
915
BASE_URL="http://localhost:3000"
10-
# Generate from PROD
11-
# BASE_URL="https://demo-cicd.cloud.gooddata.com"
16+
endif
1217
URL="${BASE_URL}/api/${API_VERSION}/schemas"
1318

1419
include ci_tests.mk
@@ -90,18 +95,21 @@ test:
9095

9196
.PHONY: test-staging
9297
test-staging:
93-
@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make test-staging TOKEN=<api-token>" && exit 1)
94-
$(MAKE) -C packages/gooddata-sdk test-staging TOKEN=$(TOKEN)
98+
@test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1)
99+
@test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1)
100+
$(MAKE) -C packages/gooddata-sdk test-staging TOKEN=$(STAGING_ADMIN_TOKEN) DS_PASSWORD=$(STAGING_DS_PASSWORD)
95101

96102
.PHONY: clean-staging
97103
clean-staging:
98-
@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make clean-staging TOKEN=<api-token>" && exit 1)
99-
cd packages/tests-support && STAGING=1 TOKEN="$(TOKEN)" python clean_staging.py
104+
@test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1)
105+
@test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1)
106+
cd packages/tests-support && STAGING=1 TOKEN="$(STAGING_ADMIN_TOKEN)" DS_PASSWORD="$(STAGING_DS_PASSWORD)" python clean_staging.py
100107

101108
.PHONY: load-staging
102109
load-staging:
103-
@test -n "$(TOKEN)" || (echo "ERROR: TOKEN is required. Usage: make load-staging TOKEN=<api-token>" && exit 1)
104-
cd packages/tests-support && STAGING=1 TOKEN="$(TOKEN)" python upload_demo_layout.py
110+
@test -n "$(STAGING_ADMIN_TOKEN)" || (echo "ERROR: STAGING_ADMIN_TOKEN is required. Set it in .env or pass on CLI." && exit 1)
111+
@test -n "$(STAGING_DS_PASSWORD)" || (echo "ERROR: STAGING_DS_PASSWORD is required. Set it in .env or pass on CLI." && exit 1)
112+
cd packages/tests-support && STAGING=1 TOKEN="$(STAGING_ADMIN_TOKEN)" DS_PASSWORD="$(STAGING_DS_PASSWORD)" python upload_demo_layout.py
105113

106114
.PHONY: release
107115
release:

‎packages/gooddata-sdk/tests/catalog/expected/declarative_ldm_with_sql_dataset.json‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@
2323
"id": "campaigns.campaign_id",
2424
"type": "attribute"
2525
},
26-
"dataType": "INT"
26+
"dataType": "INT",
27+
"isNullable": true
2728
}
2829
]
2930
}
@@ -266,7 +267,8 @@
266267
"id": "campaigns.campaign_id",
267268
"type": "attribute"
268269
},
269-
"dataType": "INT"
270+
"dataType": "INT",
271+
"isNullable": true
270272
}
271273
]
272274
},
@@ -283,7 +285,8 @@
283285
"id": "customers.customer_id",
284286
"type": "attribute"
285287
},
286-
"dataType": "INT"
288+
"dataType": "INT",
289+
"isNullable": true
287290
}
288291
]
289292
},
@@ -300,7 +303,8 @@
300303
"id": "date",
301304
"type": "date"
302305
},
303-
"dataType": "DATE"
306+
"dataType": "DATE",
307+
"isNullable": true
304308
}
305309
]
306310
},
@@ -317,7 +321,8 @@
317321
"id": "products.product_id",
318322
"type": "attribute"
319323
},
320-
"dataType": "INT"
324+
"dataType": "INT",
325+
"isNullable": true
321326
}
322327
]
323328
}

‎packages/gooddata-sdk/tests/catalog/expected/declarative_pdm_ldm_request.json‎

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
"isPrimaryKey": false,
2020
"name": "campaign_id",
2121
"referencedTableColumn": "campaign_id",
22-
"referencedTableId": "campaigns"
22+
"referencedTableId": "campaigns",
23+
"isNullable": true
2324
},
2425
{
2526
"data_type": "STRING",
@@ -116,19 +117,22 @@
116117
"isPrimaryKey": false,
117118
"name": "campaign_id",
118119
"referencedTableColumn": "campaign_id",
119-
"referencedTableId": "campaigns"
120+
"referencedTableId": "campaigns",
121+
"isNullable": true
120122
},
121123
{
122124
"data_type": "INT",
123125
"isPrimaryKey": false,
124126
"name": "customer_id",
125127
"referencedTableColumn": "customer_id",
126-
"referencedTableId": "customers"
128+
"referencedTableId": "customers",
129+
"isNullable": true
127130
},
128131
{
129132
"data_type": "DATE",
130133
"isPrimaryKey": false,
131-
"name": "date"
134+
"name": "date",
135+
"isNullable": true
132136
},
133137
{
134138
"data_type": "STRING",
@@ -159,7 +163,8 @@
159163
"isPrimaryKey": false,
160164
"name": "product_id",
161165
"referencedTableColumn": "product_id",
162-
"referencedTableId": "products"
166+
"referencedTableId": "products",
167+
"isNullable": true
163168
},
164169
{
165170
"data_type": "NUMERIC",
@@ -170,12 +175,14 @@
170175
{
171176
"data_type": "STRING",
172177
"isPrimaryKey": false,
173-
"name": "wdf__region"
178+
"name": "wdf__region",
179+
"isNullable": true
174180
},
175181
{
176182
"data_type": "STRING",
177183
"isPrimaryKey": false,
178-
"name": "wdf__state"
184+
"name": "wdf__state",
185+
"isNullable": true
179186
}
180187
],
181188
"id": "order_lines",

‎packages/gooddata-sdk/tests/conftest.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,32 @@ def test_config(request):
5858
return config
5959

6060

61+
@pytest.fixture(scope="session", autouse=True)
62+
def _patch_ds_credentials(test_config):
63+
"""Rewrite data_sources_credentials.yaml with the real DS_PASSWORD at session start.
64+
65+
The credentials file is used by many tests and snapshot/restore fixtures.
66+
On staging the password differs from the hardcoded default, so we patch the
67+
file in-place and restore the original content on teardown.
68+
"""
69+
env_ds_password = os.environ.get("DS_PASSWORD")
70+
if not env_ds_password:
71+
yield
72+
return
73+
74+
original_content = _credentials_path.read_text(encoding="utf-8")
75+
credentials = yaml.safe_load(original_content)
76+
credentials["data_sources"]["demo-test-ds"] = env_ds_password
77+
78+
_credentials_path.write_text(yaml.dump(credentials, default_flow_style=False), encoding="utf-8")
79+
logger.info("Patched data_sources_credentials.yaml with DS_PASSWORD env var")
80+
81+
yield
82+
83+
_credentials_path.write_text(original_content, encoding="utf-8")
84+
logger.info("Restored original data_sources_credentials.yaml")
85+
86+
6187
@pytest.fixture()
6288
def setenvvar(monkeypatch):
6389
with mock.patch.dict(os.environ, clear=True):
@@ -73,7 +99,7 @@ def setenvvar(monkeypatch):
7399

74100

75101
@pytest.fixture(scope="session", autouse=True)
76-
def staging_preflight(test_config):
102+
def staging_preflight(test_config, _patch_ds_credentials):
77103
"""When staging: true, verify connectivity and that required resources exist.
78104
79105
Calls pytest.exit() on failure so the session stops immediately with a clear

‎packages/gooddata-sdk/tests/gd_test_config.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,6 @@ environments:
5151
org_id: "python-sdk-dex"
5252
org_name: "Python SDK Dex"
5353
org_hostname: "python-sdk-dex.dev-latest.stg11.panther.intgdc.com"
54-
ds_url: "jdbc:postgresql://cnpg-cluster-pooler:5432/tiger?sslmode=prefer"
55-
ds_username: "tiger"
56-
ds_password: "passw0rd" # overridden from DS_PASSWORD env var
54+
ds_url: "jdbc:postgresql://postgresql-ha-pgpool.postgresql-ha/tiger_tests?sslmode=prefer"
55+
ds_username: "postgres"
56+
ds_password: "" # overridden from DS_PASSWORD env var

‎packages/gooddata-sdk/tox.ini‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ pass_env =
1212
OVERWRITE
1313
TOKEN
1414
GD_TEST_ENV
15+
DS_PASSWORD
1516
setenv =
1617
COVERAGE_CORE=sysmon
1718
commands =

0 commit comments

Comments
 (0)