CCM-12963 E2E Tests

.env.template

@@ -3,3 +3,26 @@ API_KEY=
 HEADERAUTH=
 PR_NUMBER=prxx # remove if needs to run against main
 NHSD_APIM_TOKEN=
+PROXY_NAME=
+# * nhs-notify-supplier--internal-dev--nhs-notify-supplier
+# * nhs-notify-supplier--internal-dev--nhs-notify-supplier-PR-XX
+# * nhs-notify-supplier--ref--nhs-notify-supplier -- ref env
+
+# API Keys
+# ========
+# In order to find out the value of an environments given API key, follow these steps
+# 1. Log in to Non-Prod/ Prod APIGEE
+# 2. Navigate to 'Publish' > 'Apps' and search for the app linked to authentication
+# 3. Copy the "key" from the Credentials related to the app
+export NON_PROD_API_KEY=xxx
+export INTEGRATION_API_KEY=xxx
+export PRODUCTION_API_KEY=xxx
+
+# Private Keys
+# ============
+# private key used to generate authentication for tests ran against the internal-dev and internal-qa
+export NON_PROD_PRIVATE_KEY=xxx # path to the private key file
+# private key used to generate authentication for tests ran against the int environment
+export INTEGRATION_PRIVATE_KEY=xxx
+# private key used to generate authentication for tests ran against the prod environment
+export PRODUCTION_PRIVATE_KEY=xxx

.gitignore

@@ -32,6 +32,13 @@ dist
 /specification/api/components/security/security.yml
 /specification/api/components/parameters/authorization/authorization.yml
 /scripts/JWT/*.pem
+/tests/e2e-tests/test-report.xml
 
 # ignore PACTS
 .pacts
+
+# python venvs
+.venv/
+venv/
+env/
+ENV/

Makefile

@@ -115,3 +115,54 @@ ${VERBOSE}.SILENT: \
 	config \
 	dependencies \
 	deploy \
+
+#####################
+# E2E Test commands #
+#####################
+
+TEST_CMD := APIGEE_ACCESS_TOKEN="$(APIGEE_ACCESS_TOKEN)" \
+	PYTHONPATH=. poetry run pytest --disable-warnings -vv \
+	--color=yes \
+	-n 4 \
+	--api-name=nhs-notify-supplier \
+	--proxy-name="$(PROXY_NAME)" \
+	-s \
+	--reruns 5 \
+	--reruns-delay 5 \
+	--only-rerun 'AssertionError: Unexpected 429' \
+	--only-rerun 'AssertionError: Unexpected 504' \
+	--only-rerun 'AssertionError: Unexpected 502' \
+	--junitxml=test-report.xml
+
+
+.internal-dev-test:
+	@cd tests/e2e-tests && \
+	$(TEST_CMD) \
+	api \
+	-m devtest
+
+.integration-test:
+	$(TEST_CMD) \
+	tests/api \
+	-m inttest
+
+
+PROD_CMD := APIGEE_ACCESS_TOKEN="$(APIGEE_ACCESS_TOKEN)" \
+	PYTHONPATH=. poetry run pytest --disable-warnings -vv \
+	--color=yes \
+	-n 4 \
+	--api-name=nhs-notify-supplier \
+	--proxy-name="$(PROXY_NAME)" \
+	-s \
+	--reruns 5 \
+	--reruns-delay 5 \
+	--only-rerun 'AssertionError: Unexpected 429' \
+	--only-rerun 'AssertionError: Unexpected 504' \
+	--only-rerun 'AssertionError: Unexpected 502' \
+	--junitxml=test-report.xml
+
+.prod-test:
+	@cd tests/e2e-tests && \
+	$(PROD_CMD) \
+	tests/api \
+	-m prodtest

pytest.ini

@@ -0,0 +1,14 @@
+[pytest]
+pythonpath =
+    tests/e2e-tests/api
+python_files = *_tests.py test_*.py
+addopts = --strict-markers
+markers =
+    smoketest: suitable to run against all environments even production
+    mtlstest: suitable to run against periodically against environments
+    sandboxtest: suitable to run against sandbox environment
+    devtest: suitable to run against internal-dev environments
+    inttest: suitable to run against integration environment
+    prodtest: suitable to run against production environment
+    uattest: suitable to run against uat environment
+    test: suitable to run against all environments

tests/.gitignore

@@ -9,3 +9,10 @@ node_modules/
 /target
 /playwright/.auth/
 /allure-report
+
+*/__pycache__/
+**/__pycache__/
+__pycache__/
+*/__pycache__/
+**/__pycache__/
+*.pyc

tests/e2e-tests/README.md

@@ -0,0 +1,33 @@
+# E2E Tests
+
+## Generate An Apigee Access Token
+
+To generate authentication using Apigee, you must have access to an Apigee account and use `get_token` via the command line and generate an Apigee access token.
+
+**Tokens expire once per day and require refreshing.**
+
+* Install [`get_token`](https://docs.apigee.com/api-platform/system-administration/auth-tools#install)
+* Run the following command and log in with your Apigee credentials when prompted:
+
+```shell
+export APIGEE_ACCESS_TOKEN=$(SSO_LOGIN_URL=https://login.apigee.com get_token)
+```
+
+* If your token does not refresh, try clearing the cache:
+
+```shell
+export APIGEE_ACCESS_TOKEN=$(SSO_LOGIN_URL=https://login.apigee.com get_token --clear-sso-cache)
+```
+
+### Set Proxy Name
+
+Set the `PROXY_NAME` environment variable to specify the environment for test execution. You can find the proxy name by logging into [Apigee](https://apigee.com/edge), navigating to 'API Proxies' and searching for 'supplier-api'.
+
+```shell
+export PROXY_NAME=nhs-notify-supplier--internal-dev--nhs-notify-supplier
+```
+
+Available values for `PROXY_NAME` include:
+
+* `nhs-notify-supplier--internal-dev--nhs-notify-supplier`
+* `nhs-notify-supplier--internal-dev--nhs-notify-supplier-pr<num>`

tests/e2e-tests/api/authentication/test_401_errors.py

@@ -0,0 +1,34 @@
+import requests
+import pytest
+from lib.constants import METHODS, VALID_ENDPOINT_LETTERS
+from lib.fixtures import *
+from lib.errorhandler import ErrorHandler
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+@pytest.mark.parametrize("method", METHODS)
+@pytest.mark.parametrize("endpoints", VALID_ENDPOINT_LETTERS)
+def test_401_invalid(url, method, endpoints):
+
+    resp = getattr(requests, method)(f"{url}{endpoints}", headers={
+        "Authorization": "invalid",
+    })
+
+    ErrorHandler.handle_retry(resp)
+    assert resp.status_code == 401, f"Response: {resp.status_code}: {resp.text}"
+
+@pytest.mark.test
+@pytest.mark.nhsd_apim_authorization({"access": "application", "level": "level0"})
+@pytest.mark.parametrize("method", METHODS)
+@pytest.mark.parametrize("endpoints", VALID_ENDPOINT_LETTERS)
+def test_401_invalid_level(nhsd_apim_proxy_url, nhsd_apim_auth_headers, method, endpoints):
+    print(nhsd_apim_proxy_url)
+
+    resp = getattr(requests, method)(f"{nhsd_apim_proxy_url}{endpoints}", headers={
+        **nhsd_apim_auth_headers
+    })
+
+    ErrorHandler.handle_retry(resp)
+    assert resp.status_code == 401, f"Response: {resp.status_code}: {resp.text}"

tests/e2e-tests/api/data/test_get_letter_data.py

@@ -0,0 +1,65 @@
+import uuid
+import requests
+import pytest
+from lib.fixtures import *  # NOSONAR
+from lib.constants import LETTERS_ENDPOINT
+from lib.generators import Generators
+from lib.errorhandler import ErrorHandler
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+
+def test_200_get_letter_status(url, bearer_token):
+
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_letter_id = requests.get(f"{url}/{LETTERS_ENDPOINT}/", headers=headers)
+
+    letter_id = get_letter_id.json().get("data")[0].get("id")
+    get_letter_data = requests.get(f"{url}/{LETTERS_ENDPOINT}/{letter_id}/data", headers=headers)
+
+    ErrorHandler.handle_retry(get_letter_data)
+    assert get_letter_data.status_code == 200, f"Response: {get_letter_data.status_code}: {get_letter_data.text}"
+    assert get_letter_data.headers.get("Content-Type") == "application/pdf"
+
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_404_letter_does_not_exist(url, bearer_token):
+
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}/xx", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 404
+    assert get_message_response.json().get("errors")[0].get("detail") == "No resource found with that ID"
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_404_letter_does_not_exist(url, bearer_token):
+
+    letter_id = uuid.uuid4().hex
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}/{letter_id}/data", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 404
+    assert get_message_response.json().get("errors")[0].get("detail") == "No resource found with that ID"
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_500_letter_does_not_exist(url, bearer_token):
+
+    letter_id = "00000000-0000-0000-0000-000000000000"
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}/{letter_id}/data", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 500

tests/e2e-tests/api/headers/test_x_request_id.py

@@ -0,0 +1,43 @@
+import requests
+import pytest
+from lib.fixtures import *  # NOSONAR
+from lib.constants import VALID_ENDPOINT_LETTERS, MI_ENDPOINT
+from lib.generators import Generators
+
+METHODS = ["get", "post"]
+
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+@pytest.mark.parametrize("method", METHODS)
+@pytest.mark.parametrize("endpoints", VALID_ENDPOINT_LETTERS)
+def test_header_letters_endpoint(
+    url,
+    method,
+    endpoints,
+    bearer_token
+):
+    resp = getattr(requests, method)(f"{url}/{endpoints}", headers={
+        "Authorization": bearer_token.value,
+        "X-Request-ID": None
+    })
+
+    assert resp.status_code == 500
+
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_header_mi_endpoint(
+    url,
+    bearer_token
+):
+    resp = getattr(requests, "post")(f"{url}/{MI_ENDPOINT}", headers={
+        "Authorization": bearer_token.value,
+        "X-Request-ID": ""
+    })
+
+    assert resp.status_code == 500

tests/e2e-tests/api/letters/test_get_letter_status.py

@@ -0,0 +1,36 @@
+import requests
+import pytest
+from lib.fixtures import *  # NOSONAR
+from lib.constants import LETTERS_ENDPOINT
+from lib.generators import Generators
+from lib.errorhandler import ErrorHandler
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+
+def test_200_get_letter_status(url, bearer_token):
+
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_letters = requests.get(f"{url}/{LETTERS_ENDPOINT}/", headers=headers)
+
+    letter_id = get_letters.json().get("data")[0].get("id")
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}/{letter_id}", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 200, f"Response: {get_message_response.status_code}: {get_message_response.text}"
+
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_404_letter_does_not_exist(url, bearer_token):
+
+    headers = Generators.generate_valid_headers(bearer_token.value)
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}/xx", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 404, f"Response: {get_message_response.status_code}: {get_message_response.text}"
+    assert get_message_response.json().get("errors")[0].get("detail") == "No resource found with that ID"

tests/e2e-tests/api/letters/test_get_list_of_letters.py

@@ -0,0 +1,20 @@
+import requests
+import pytest
+from lib.fixtures import *  # NOSONAR
+from lib.constants import LETTERS_ENDPOINT
+from lib.generators import Generators
+from lib.errorhandler import ErrorHandler
+
+@pytest.mark.test
+@pytest.mark.devtest
+@pytest.mark.inttest
+@pytest.mark.prodtest
+def test_200_get_letters(url, bearer_token):
+
+    headers = Generators.generate_valid_headers(bearer_token.value)
+
+    get_message_response = requests.get(f"{url}/{LETTERS_ENDPOINT}?limit=1", headers=headers)
+
+    ErrorHandler.handle_retry(get_message_response)
+    assert get_message_response.status_code == 200, f"Response: {get_message_response.status_code}: {get_message_response.text}"
+    assert get_message_response.json().get("data")[0].get("attributes").get("status") == "PENDING"