Feat/prevent repeated ssns (#1187)

landonshumway-ia · web-flow · commit 137ac542b92e · 2025-11-03T14:34:50.000-05:00
diff --git a/.github/workflows/check-compact-connect.yml b/.github/workflows/check-compact-connect.yml
@@ -101,9 +101,11 @@ jobs:
 
       - name: Install dev dependencies
         run: "pip install -r backend/compact-connect/requirements-dev.txt"
-
+      # Note we are currently pinning the pip version to deal with compatibility issues released with pip 25.3
+      # see https://stackoverflow.com/a/79802727 If this issues is addressed in a later version, we can remove the
+      # extra pip install command so we stop pinning the pip version
       - name: Install all Python dependencies
-        run: "cd backend/compact-connect; bin/sync_deps.sh"
+        run: "cd backend/compact-connect; pip install -U 'pip<25.3'; bin/sync_deps.sh"
 
       - name: Test backend
         run: "cd backend/compact-connect; bin/run_tests.sh -l all -no"
diff --git a/backend/common-cdk/common_constructs/stack.py b/backend/common-cdk/common_constructs/stack.py
@@ -97,6 +97,10 @@ def __init__(self, *args, environment_context: dict, environment_name: str, **kw
 
         self.environment_context = environment_context
         self.environment_name = environment_name
+        # We only set the API_BASE_URL common env var if the API_DOMAIN_NAME is set
+        # The API_BASE_URL is used by the feature flag client to call the flag check endpoint
+        if self.api_domain_name:
+            self.common_env_vars.update({'API_BASE_URL': f'https://{self.api_domain_name}'})
 
     @cached_property
     def hosted_zone(self) -> IHostedZone | None:
diff --git a/backend/compact-connect/docs/README.md b/backend/compact-connect/docs/README.md
@@ -31,6 +31,7 @@ Export your license data to a CSV file, formatted as follows:
    - String lengths are enforced - exceeding them will cause validation errors
    - Some fields have a set list of allowed values. For those fields, make sure to enter the value exactly, including
      spacing and capitalization
+   - SSNs must be unique within a single CSV upload file. Do not include multiple rows with the same `ssn` in one file. If duplicate SSNs are sent within the same file, the first row will be processed, but all other duplicate rows will be rejected.
 
 #### Field Descriptions
 
diff --git a/backend/compact-connect/docs/it_staff_onboarding_instructions.md b/backend/compact-connect/docs/it_staff_onboarding_instructions.md
@@ -240,6 +240,8 @@ For your convenience, use of this feature is included in the [Postman Collection
 - `licenseType` must match exactly with one of the valid types for the specified compact
 - All date fields must use the `YYYY-MM-DD` format
 - The API does not accept `null` values. For optional fields with no value, omit the field or leave it empty in CSV.
+- For CSV uploads, SSNs must be unique within a single file. Do not include multiple rows with the same `ssn` in one upload. If duplicate SSNs are sent within the same file, the first row will be processed, but all other duplicate rows will be rejected.
+- For JSON uploads, SSNs must be unique within a single request payload (array). Do not include duplicate `ssn` values in the same batch. Attempting to do so will cause the entire request to be rejected.
 
 ## Common Upload Strategies: JSON vs CSV
 
diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/data_client.py
@@ -1428,9 +1428,9 @@ def encumber_privilege(self, adverse_action: AdverseActionData) -> None:
 
             now = config.current_standard_datetime
             # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002
-            from cc_common.feature_flag_client import is_feature_enabled
+            from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled
 
-            if is_feature_enabled('encumbrance-multi-category-flag'):
+            if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG):
                 encumbrance_details = {
                     'clinicalPrivilegeActionCategories': adverse_action.clinicalPrivilegeActionCategories,
                     'adverseActionId': adverse_action.adverseActionId,
@@ -2674,9 +2674,9 @@ def encumber_home_jurisdiction_license_privileges(
             'Found privileges to encumber', privilege_count=len(unencumbered_privileges_associated_with_license)
         )
         # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002
-        from cc_common.feature_flag_client import is_feature_enabled
+        from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled
 
-        if is_feature_enabled('encumbrance-multi-category-flag'):
+        if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG):
             encumbrance_details = {
                 'clinicalPrivilegeActionCategories': adverse_action.clinicalPrivilegeActionCategories,
                 'licenseJurisdiction': jurisdiction,
diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/provider_record_util.py
@@ -375,9 +375,9 @@ def construct_simplified_privilege_history_object(
             ):
                 # TODO - remove the flag as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002
                 # as well as check for deprecated field
-                from cc_common.feature_flag_client import is_feature_enabled
+                from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled
 
-                if is_feature_enabled('encumbrance-multi-category-flag'):
+                if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG):
                     if 'clinicalPrivilegeActionCategory' in event['encumbranceDetails']:
                         event['note'] = event['encumbranceDetails'].get('clinicalPrivilegeActionCategory')
                     else:
diff --git a/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py b/backend/compact-connect/lambdas/python/common/cc_common/data_model/schema/common.py
@@ -288,7 +288,6 @@ class UpdateCategory(CCEnum):
     DEACTIVATION = 'deactivation'
     EXPIRATION = 'expiration'
     ISSUANCE = 'issuance'
-    OTHER = 'other'
     RENEWAL = 'renewal'
     ENCUMBRANCE = 'encumbrance'
     HOME_JURISDICTION_CHANGE = 'homeJurisdictionChange'
@@ -297,6 +296,9 @@ class UpdateCategory(CCEnum):
     # this is specific to privileges that are deactivated due to a state license deactivation
     LICENSE_DEACTIVATION = 'licenseDeactivation'
     EMAIL_CHANGE = 'emailChange'
+    # NOTE: this value should explicitly be used for license upload updates, not anywhere else
+    # it is referenced in the event that an invalid license upload needs to be reverted.
+    LICENSE_UPLOAD_UPDATE_OTHER = 'other'
 
 
 class ActiveInactiveStatus(CCEnum):
diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_client.py
@@ -11,6 +11,7 @@
 import requests
 
 from cc_common.config import config, logger
+from cc_common.feature_flag_enum import FeatureFlagEnum
 
 
 @dataclass
@@ -43,14 +44,16 @@ def to_dict(self) -> dict[str, Any]:
         return result
 
 
-def is_feature_enabled(flag_name: str, context: FeatureFlagContext | None = None, fail_default: bool = False) -> bool:
+def is_feature_enabled(
+    flag_name: FeatureFlagEnum, context: FeatureFlagContext | None = None, fail_default: bool = False
+) -> bool:
     """
     Check if a feature flag is enabled.
 
     This function calls the internal feature flag API endpoint to determine
     if a feature flag is enabled for the given context.
 
-    :param flag_name: The name of the feature flag to check
+    :param flag_name: The name of the feature flag to check.
     :param context: Optional FeatureFlagContext for feature flag evaluation
     :param fail_default: If True, return True on errors; if False, return False on errors (default: False)
     :return: True if the feature flag is enabled, False otherwise (or fail_default value on error)
@@ -76,6 +79,7 @@ def is_feature_enabled(flag_name: str, context: FeatureFlagContext | None = None
         ):
     """
     try:
+        logger.info("checking status of feature flag", flag_name=flag_name)
         api_base_url = _get_api_base_url()
         endpoint_url = f'{api_base_url}/v1/flags/{flag_name}/check'
 
@@ -103,7 +107,8 @@ def is_feature_enabled(flag_name: str, context: FeatureFlagContext | None = None
             # Invalid response format - return fail_default value
             return fail_default
 
-        return bool(response_data['enabled'])
+        logger.info('Checked flag status successfully', flag_name=flag_name, enabled=response_data['enabled'])
+        return response_data['enabled']
 
     # We catch all exceptions to prevent a feature flag issue causing the system from operating
     except Exception as e:  # noqa: BLE001
diff --git a/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py b/backend/compact-connect/lambdas/python/common/cc_common/feature_flag_enum.py
@@ -0,0 +1,15 @@
+from enum import StrEnum
+
+
+class FeatureFlagEnum(StrEnum):
+    """
+    Central source for all feature flags currently referenced in the python code of the project.
+    Flags should be defined here when first added, and removed when the flag
+    is no longer in use.
+    """
+
+    # flag used by internal testing
+    TEST_FLAG = 'test-flag'
+    # runtime flags
+    ENCUMBRANCE_MULTI_CATEGORY_FLAG = 'encumbrance-multi-category-flag'
+    DUPLICATE_SSN_UPLOAD_CHECK_FLAG = 'duplicate-ssn-upload-check-flag'
diff --git a/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py b/backend/compact-connect/lambdas/python/common/tests/unit/test_feature_flag_client.py
@@ -1,5 +1,7 @@
 from unittest.mock import MagicMock, patch
 
+from cc_common.feature_flag_enum import FeatureFlagEnum
+
 from tests import TstLambdas
 
 
@@ -13,7 +15,7 @@ def test_is_feature_enabled_returns_true_when_flag_enabled(self):
         mock_response.json.return_value = {'enabled': True}
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post:
-            result = is_feature_enabled('test-flag')
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG)
 
             # Verify the result
             self.assertTrue(result)
@@ -35,7 +37,7 @@ def test_is_feature_enabled_returns_false_when_flag_disabled(self):
         mock_response.json.return_value = {'enabled': False}
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag')
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG)
 
             # Verify the result
             self.assertFalse(result)
@@ -51,7 +53,7 @@ def test_is_feature_enabled_with_context(self):
         context = FeatureFlagContext(user_id='user123', custom_attributes={'licenseType': 'lpc'})
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post:
-            result = is_feature_enabled('test-flag', context=context)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, context=context)
 
             # Verify the result
             self.assertTrue(result)
@@ -71,7 +73,7 @@ def test_is_feature_enabled_fail_closed_on_timeout(self):
         from cc_common.feature_flag_client import is_feature_enabled
 
         with patch('cc_common.feature_flag_client.requests.post', side_effect=Exception('Timeout')):
-            result = is_feature_enabled('test-flag', fail_default=False)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False)
 
             # Verify it fails closed (returns False)
             self.assertFalse(result)
@@ -81,7 +83,7 @@ def test_is_feature_enabled_fail_open_on_timeout(self):
         from cc_common.feature_flag_client import is_feature_enabled
 
         with patch('cc_common.feature_flag_client.requests.post', side_effect=Exception('Timeout')):
-            result = is_feature_enabled('test-flag', fail_default=True)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True)
 
             # Verify it fails open (returns True)
             self.assertTrue(result)
@@ -95,7 +97,7 @@ def test_is_feature_enabled_fail_closed_on_http_error(self):
         mock_response.raise_for_status.side_effect = Exception('500 Server Error')
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=False)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False)
 
             # Verify it fails closed (returns False)
             self.assertFalse(result)
@@ -109,7 +111,7 @@ def test_is_feature_enabled_fail_open_on_http_error(self):
         mock_response.raise_for_status.side_effect = Exception('500 Server Error')
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=True)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True)
 
             # Verify it fails open (returns True)
             self.assertTrue(result)
@@ -124,7 +126,7 @@ def test_is_feature_enabled_fail_closed_on_invalid_response(self):
         mock_response.raise_for_status = MagicMock()
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=False)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False)
 
             # Verify it fails closed (returns False)
             self.assertFalse(result)
@@ -139,7 +141,7 @@ def test_is_feature_enabled_fail_open_on_invalid_response(self):
         mock_response.raise_for_status = MagicMock()
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=True)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True)
 
             # Verify it fails open (returns True)
             self.assertTrue(result)
@@ -154,7 +156,7 @@ def test_is_feature_enabled_fail_closed_on_json_parse_error(self):
         mock_response.raise_for_status = MagicMock()
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=False)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=False)
 
             # Verify it fails closed (returns False)
             self.assertFalse(result)
@@ -168,7 +170,7 @@ def test_is_feature_enabled_fail_open_on_json_parse_error(self):
         mock_response.json.side_effect = ValueError('Invalid JSON')
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response):
-            result = is_feature_enabled('test-flag', fail_default=True)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, fail_default=True)
 
             # Verify it fails open (returns True)
             self.assertTrue(result)
@@ -220,7 +222,7 @@ def test_is_feature_enabled_with_context_user_id_only(self):
         context = FeatureFlagContext(user_id='user789')
 
         with patch('cc_common.feature_flag_client.requests.post', return_value=mock_response) as mock_post:
-            result = is_feature_enabled('test-flag', context=context)
+            result = is_feature_enabled(FeatureFlagEnum.TEST_FLAG, context=context)
 
             # Verify the result
             self.assertTrue(result)
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/bulk_upload.py
@@ -13,6 +13,9 @@
 )
 from cc_common.event_batch_writer import EventBatchWriter
 from cc_common.exceptions import CCInternalException
+
+# initialize flag outside of handler so the flag is cached for the lifecycle of the lambda execution environment
+from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled  # noqa: E402
 from cc_common.utils import (
     ResponseEncoder,
     api_handler,
@@ -22,6 +25,10 @@
 from license_csv_reader import LicenseCSVReader
 from marshmallow import ValidationError
 
+duplicate_ssn_check_flag_enabled = is_feature_enabled(
+    FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG, fail_default=True
+)
+
 
 @api_handler
 @authorize_compact_jurisdiction(action='write')
@@ -139,6 +146,9 @@ def process_bulk_upload_file(
     current_batch = []
     total_processed = 0
     failed_validation_count = 0
+    # track which ssns were included in this file to detect duplicates,
+    # which are not allowed within the same file upload
+    ssns_in_file_upload = {}
 
     with EventBatchWriter(config.events_client) as event_writer:
         for i, raw_license in enumerate(reader.licenses(stream)):
@@ -148,6 +158,17 @@ def process_bulk_upload_file(
                     # dict() here, because it prevents `compact` and `jurisdiction` from being allowed in the
                     # raw_license
                     validated_license = schema.load(dict(compact=compact, jurisdiction=jurisdiction, **raw_license))
+                    # verify that this ssn has not been used previously in the same batch
+                    license_ssn = validated_license['ssn']
+                    if duplicate_ssn_check_flag_enabled:
+                        matched_ssn_index = ssns_in_file_upload.get(license_ssn)
+                        if matched_ssn_index:
+                                raise ValidationError(
+                                    message=f'Duplicate License SSN detected. SSN matches with record '
+                                            f'{matched_ssn_index}. Every record must have a unique SSN within the same '
+                                            f'file.'
+                                )
+                        ssns_in_file_upload.update({license_ssn: i + 1})
                 except TypeError as e:
                     # This will be raised, if `raw_license` includes compact and/or jurisdiction fields
                     logger.error('License contains unsupported fields', fields=list(raw_license.keys()), exc_info=e)
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/encumbrance.py
@@ -103,9 +103,9 @@ def _generate_adverse_action_for_record_type(
     adverse_action.actionAgainst = adverse_action_against_record_type
     adverse_action.encumbranceType = EncumbranceType(adverse_action_request['encumbranceType'])
     # TODO - remove the flag conditions as part of https://github.com/csg-org/CompactConnect/issues/1136 # noqa: FIX002
-    from cc_common.feature_flag_client import is_feature_enabled
+    from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled
 
-    if is_feature_enabled('encumbrance-multi-category-flag'):
+    if is_feature_enabled(FeatureFlagEnum.ENCUMBRANCE_MULTI_CATEGORY_FLAG):
         if 'clinicalPrivilegeActionCategory' in adverse_action_request:
             # replicate data to both the deprecated and new fields
             adverse_action.clinicalPrivilegeActionCategory = ClinicalPrivilegeActionCategory(
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/ingest.py
@@ -297,7 +297,7 @@ def _populate_update_record(*, existing_license: dict, updated_values: dict, rem
         update_type = UpdateCategory.DEACTIVATION
         logger.info('License deactivation detected')
     if update_type is None:
-        update_type = UpdateCategory.OTHER
+        update_type = UpdateCategory.LICENSE_UPLOAD_UPDATE_OTHER
         logger.info('License update detected')
 
     now = config.current_standard_datetime
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/handlers/licenses.py
@@ -10,6 +10,14 @@
 
 schema = LicensePostRequestSchema()
 
+# initialize flag outside of handler so the flag is cached for the lifecycle of the execution environment
+from cc_common.feature_flag_client import FeatureFlagEnum, is_feature_enabled  # noqa: E402
+
+# low risk flag, so we default to enabled if failure detected
+duplicate_ssn_check_flag_enabled = is_feature_enabled(
+    FeatureFlagEnum.DUPLICATE_SSN_UPLOAD_CHECK_FLAG, fail_default=True
+)
+
 
 @api_handler
 @optional_signature_auth
@@ -63,6 +71,19 @@ def post_licenses(event: dict, context: LambdaContext):  # noqa: ARG001 unused-a
                 'errors': invalid_records,
             }
         )
+    if duplicate_ssn_check_flag_enabled:
+        # verify that none of the SSNs are repeats within the same batch
+        license_ssns = [license_record['ssn'] for license_record in licenses]
+        if len(set(license_ssns)) < len(license_ssns):
+            raise CCInvalidRequestCustomResponseException(
+                response_body={
+                    'message': 'Invalid license records in request. See errors for more detail.',
+                    'errors': {
+                        'SSN': 'Same SSN detected on multiple rows. '
+                        'Every record must have a unique SSN within the same request.'
+                    },
+                }
+            )
 
     event_time = config.current_standard_datetime
 
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_bulk_upload.py
diff --git a/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py b/backend/compact-connect/lambdas/python/provider-data-v1/tests/function/test_handlers/test_licenses.py
diff --git a/backend/compact-connect/stacks/feature_flag_stack/__init__.py b/backend/compact-connect/stacks/feature_flag_stack/__init__.py
