diff --git a/sagemaker-core/src/sagemaker/core/helper/iam_policies.py b/sagemaker-core/src/sagemaker/core/helper/iam_policies.py index 2613418ea8..86c8c79f5a 100644 --- a/sagemaker-core/src/sagemaker/core/helper/iam_policies.py +++ b/sagemaker-core/src/sagemaker/core/helper/iam_policies.py @@ -812,6 +812,148 @@ }, }, }, + "evaluation": { + "role_name": "SageMaker-AutoRole-Evaluation", + "trust_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "sagemaker.amazonaws.com" + }, + "Action": "sts:AssumeRole", + "Condition": { + "StringEquals": {"aws:SourceAccount": "ACCOUNT_PLACEHOLDER"} + }, + } + ], + }, + "policies": { + # --- Training permissions (superset of "training" role type) --- + "s3_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket", + "s3:GetBucketLocation", + ], + "Resource": "S3_PLACEHOLDER", + } + ], + }, + "ecr_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["ecr:GetAuthorizationToken"], + "Resource": "*", + }, + { + "Effect": "Allow", + "Action": [ + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage", + "ecr:BatchCheckLayerAvailability", + ], + "Resource": "arn:aws:ecr:*:*:repository/*", + }, + ], + }, + "cloudwatch_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["cloudwatch:PutMetricData"], + "Resource": "*", + }, + { + "Effect": "Allow", + "Action": [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents", + "logs:DescribeLogStreams", + ], + "Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/TrainingJobs*", + }, + ], + }, + "kms_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"], + "Resource": "KMS_PLACEHOLDER", + } + ], + }, + # --- Evaluation-specific permissions --- + "bedrock_evaluation_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "bedrock:CreateEvaluationJob", + "bedrock:GetEvaluationJob", + "bedrock:InvokeModel", + "bedrock:InvokeModelWithResponseStream", + ], + "Resource": "*", + } + ], + }, + "mlflow_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sagemaker-mlflow:GetExperimentByName", + "sagemaker-mlflow:CreateExperiment", + "sagemaker-mlflow:CreateRun", + "sagemaker-mlflow:LogBatch", + "sagemaker-mlflow:LogMetric", + "sagemaker-mlflow:LogParam", + "sagemaker-mlflow:SetTag", + "sagemaker-mlflow:UpdateRun", + ], + "Resource": "arn:aws:sagemaker:*:*:mlflow-app/*", + } + ], + }, + "sagemaker_evaluation_policy": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "sagemaker:CreateTrainingJob", + "sagemaker:DescribeTrainingJob", + "sagemaker:StopTrainingJob", + "sagemaker:CreatePipeline", + "sagemaker:DescribePipeline", + "sagemaker:StartPipelineExecution", + "sagemaker:DescribePipelineExecution", + "sagemaker:AddTags", + ], + "Resource": [ + "arn:aws:sagemaker:*:*:training-job/*", + "arn:aws:sagemaker:*:*:pipeline/*", + ], + } + ], + }, + }, + }, } # Actions the *caller* must have to orchestrate Pipeline-based evaluations diff --git a/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py b/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py index 3baa1205da..e7d8f1f1d3 100644 --- a/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py +++ b/sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py @@ -25,7 +25,7 @@ logger = logging.getLogger(__name__) -ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod") +ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod", "evaluation") # Permissions the HyperPod CLI flow needs on the *caller* identity — the local # principal that runs `hyperpod connect-cluster` and `hyperpod start-job`. The CLI diff --git a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py index 21bed9e84b..83bfb24736 100644 --- a/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py +++ b/sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py @@ -757,10 +757,10 @@ def _get_aws_execution_context(self) -> Dict[str, str]: # This is the job execution role for the # serverless / SMTJ evaluation backends. The HyperPod backend submits via # the CLI under the caller's own credentials (see _submit_hyperpod_eval_job) - # and does not resolve a role here, so "training" is always correct here. + # and does not resolve a role here, so "evaluation" is correct here. role_arn = resolve_and_validate_role( provided_role=self.role, - role_type="training", + role_type="evaluation", sagemaker_session=self.sagemaker_session, ) diff --git a/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py b/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py index 9b785561c2..70307c7522 100644 --- a/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py +++ b/sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py @@ -734,7 +734,7 @@ def test_get_aws_execution_context(self, mock_resolve, mock_role, mock_session, assert context['account_id'] == '123456789012' mock_role.assert_called_once_with( provided_role=None, - role_type="training", + role_type="evaluation", sagemaker_session=mock_session, ) @@ -761,7 +761,7 @@ def test_get_aws_execution_context_with_explicit_role(self, mock_resolve, mock_r assert context['role_arn'] == explicit_role mock_role.assert_called_once_with( provided_role=explicit_role, - role_type="training", + role_type="evaluation", sagemaker_session=mock_session, ) diff --git a/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py b/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py new file mode 100644 index 0000000000..39e560ab7a --- /dev/null +++ b/sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py @@ -0,0 +1,185 @@ +"""Unit tests for the 'evaluation' role type in iam_role_resolver.""" + +import pytest +from unittest.mock import MagicMock, patch +from botocore.exceptions import ClientError + +from sagemaker.core.helper.iam_role_resolver import ( + resolve_and_validate_role, + RoleValidationError, + _evaluate_permissions, + _role_trusts_service, + _get_smoke_test_actions, + _expected_trust_services, +) + + +class TestEvaluationRoleType: + """Tests for the 'evaluation' role type configuration.""" + + def test_evaluation_role_type_exists(self): + """The 'evaluation' role type should be recognized.""" + from sagemaker.core.helper.iam_policies import IAM_POLICY_CONFIG + assert "evaluation" in IAM_POLICY_CONFIG + + def test_evaluation_trust_includes_sagemaker(self): + """Evaluation role type should require sagemaker.amazonaws.com trust.""" + expected = _expected_trust_services("evaluation") + assert "sagemaker.amazonaws.com" in expected + + def test_evaluation_trust_does_not_require_bedrock(self): + """Evaluation role type should NOT require bedrock.amazonaws.com trust. + + The serverless evaluation backend runs as the SageMaker execution role + and calls Bedrock APIs using the role's own credentials. Bedrock does + not need to assume the role, so trust is not required. + """ + expected = _expected_trust_services("evaluation") + assert "bedrock.amazonaws.com" not in expected + + def test_evaluation_smoke_actions_include_bedrock(self): + """Smoke test actions should include Bedrock evaluation actions.""" + actions = _get_smoke_test_actions("evaluation") + assert "bedrock:CreateEvaluationJob" in actions + assert "bedrock:GetEvaluationJob" in actions + + def test_evaluation_smoke_actions_include_bedrock_invoke(self): + """Smoke test actions should include Bedrock invoke actions.""" + actions = _get_smoke_test_actions("evaluation") + assert "bedrock:InvokeModel" in actions + + def test_resolve_raises_when_bedrock_permissions_denied(self): + """Should raise RoleValidationError when Bedrock permissions are denied.""" + mock_iam = MagicMock() + + # Simulate: bedrock:CreateEvaluationJob denied + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "EvaluationResults": [ + {"EvalActionName": "bedrock:CreateEvaluationJob", "EvalDecision": "implicitDeny"}, + {"EvalActionName": "bedrock:GetEvaluationJob", "EvalDecision": "allowed"}, + {"EvalActionName": "bedrock:InvokeModel", "EvalDecision": "allowed"}, + {"EvalActionName": "bedrock:InvokeModelWithResponseStream", "EvalDecision": "allowed"}, + ] + } + ] + mock_iam.get_paginator.return_value = paginator + + verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation") + assert verdict is False + assert "bedrock:CreateEvaluationJob" in denied + + def test_resolve_passes_when_all_allowed(self): + """Should pass when all evaluation permissions are allowed.""" + mock_iam = MagicMock() + + actions = _get_smoke_test_actions("evaluation") + paginator = MagicMock() + paginator.paginate.return_value = [ + { + "EvaluationResults": [ + {"EvalActionName": a, "EvalDecision": "allowed"} for a in actions + ] + } + ] + mock_iam.get_paginator.return_value = paginator + + verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation") + assert verdict is True + assert denied == [] + + def test_trust_check_passes_with_sagemaker(self): + """Should pass when trust policy includes sagemaker.amazonaws.com.""" + mock_iam = MagicMock() + mock_iam.get_role.return_value = { + "Role": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "sagemaker.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + } + } + } + + result = _role_trusts_service(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation") + assert result is True + + def test_resolve_and_validate_passes_with_sagemaker_trust(self): + """Full resolve_and_validate_role should pass with sagemaker trust only.""" + with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session: + mock_boto = MagicMock() + mock_session.return_value = mock_boto + + mock_iam = MagicMock() + mock_boto.client.return_value = mock_iam + + # Role exists with sagemaker trust only (no bedrock needed) + mock_iam.get_role.return_value = { + "Role": { + "Arn": "arn:aws:iam::123456789012:role/MyRole", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "sagemaker.amazonaws.com"}, + "Action": "sts:AssumeRole", + }], + } + } + } + + # All permissions allowed + actions = _get_smoke_test_actions("evaluation") + paginator = MagicMock() + paginator.paginate.return_value = [ + {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]} + ] + mock_iam.get_paginator.return_value = paginator + + result = resolve_and_validate_role( + provided_role="arn:aws:iam::123456789012:role/MyRole", + role_type="evaluation", + ) + assert result == "arn:aws:iam::123456789012:role/MyRole" + + def test_resolve_and_validate_passes_with_correct_role(self): + """Full resolve_and_validate_role should pass with correct permissions and trust.""" + with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session: + mock_boto = MagicMock() + mock_session.return_value = mock_boto + + mock_iam = MagicMock() + mock_boto.client.return_value = mock_iam + + # Role exists with correct trust + mock_iam.get_role.return_value = { + "Role": { + "Arn": "arn:aws:iam::123456789012:role/MyRole", + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": ["sagemaker.amazonaws.com", "bedrock.amazonaws.com"]}, + "Action": "sts:AssumeRole", + }], + } + } + } + + # All permissions allowed + actions = _get_smoke_test_actions("evaluation") + paginator = MagicMock() + paginator.paginate.return_value = [ + {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]} + ] + mock_iam.get_paginator.return_value = paginator + + result = resolve_and_validate_role( + provided_role="arn:aws:iam::123456789012:role/MyRole", + role_type="evaluation", + ) + assert result == "arn:aws:iam::123456789012:role/MyRole"