fix: Add pre-validation for Bedrock and MLflow role permissions

sagemaker-core/src/sagemaker/core/helper/iam_policies.py

@@ -812,6 +812,148 @@
             },
         },
     },
+    "evaluation": {
+        "role_name": "SageMaker-AutoRole-Evaluation",
+        "trust_policy": {
+            "Version": "2012-10-17",
+            "Statement": [
+                {
+                    "Effect": "Allow",
+                    "Principal": {
+                        "Service": "sagemaker.amazonaws.com"
+                    },
+                    "Action": "sts:AssumeRole",
+                    "Condition": {
+                        "StringEquals": {"aws:SourceAccount": "ACCOUNT_PLACEHOLDER"}
+                    },
+                }
+            ],
+        },
+        "policies": {
+            # --- Training permissions (superset of "training" role type) ---
+            "s3_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "s3:GetObject",
+                            "s3:PutObject",
+                            "s3:ListBucket",
+                            "s3:GetBucketLocation",
+                        ],
+                        "Resource": "S3_PLACEHOLDER",
+                    }
+                ],
+            },
+            "ecr_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["ecr:GetAuthorizationToken"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "ecr:GetDownloadUrlForLayer",
+                            "ecr:BatchGetImage",
+                            "ecr:BatchCheckLayerAvailability",
+                        ],
+                        "Resource": "arn:aws:ecr:*:*:repository/*",
+                    },
+                ],
+            },
+            "cloudwatch_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["cloudwatch:PutMetricData"],
+                        "Resource": "*",
+                    },
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "logs:CreateLogGroup",
+                            "logs:CreateLogStream",
+                            "logs:PutLogEvents",
+                            "logs:DescribeLogStreams",
+                        ],
+                        "Resource": "arn:aws:logs:*:*:log-group:/aws/sagemaker/TrainingJobs*",
+                    },
+                ],
+            },
+            "kms_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": ["kms:Encrypt", "kms:Decrypt", "kms:GenerateDataKey"],
+                        "Resource": "KMS_PLACEHOLDER",
+                    }
+                ],
+            },
+            # --- Evaluation-specific permissions ---
+            "bedrock_evaluation_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "bedrock:CreateEvaluationJob",
+                            "bedrock:GetEvaluationJob",
+                            "bedrock:InvokeModel",
+                            "bedrock:InvokeModelWithResponseStream",
+                        ],
+                        "Resource": "*",
+                    }
+                ],
+            },
+            "mlflow_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "sagemaker-mlflow:GetExperimentByName",
+                            "sagemaker-mlflow:CreateExperiment",
+                            "sagemaker-mlflow:CreateRun",
+                            "sagemaker-mlflow:LogBatch",
+                            "sagemaker-mlflow:LogMetric",
+                            "sagemaker-mlflow:LogParam",
+                            "sagemaker-mlflow:SetTag",
+                            "sagemaker-mlflow:UpdateRun",
+                        ],
+                        "Resource": "arn:aws:sagemaker:*:*:mlflow-app/*",
+                    }
+                ],
+            },
+            "sagemaker_evaluation_policy": {
+                "Version": "2012-10-17",
+                "Statement": [
+                    {
+                        "Effect": "Allow",
+                        "Action": [
+                            "sagemaker:CreateTrainingJob",
+                            "sagemaker:DescribeTrainingJob",
+                            "sagemaker:StopTrainingJob",
+                            "sagemaker:CreatePipeline",
+                            "sagemaker:DescribePipeline",
+                            "sagemaker:StartPipelineExecution",
+                            "sagemaker:DescribePipelineExecution",
+                            "sagemaker:AddTags",
+                        ],
+                        "Resource": [
+                            "arn:aws:sagemaker:*:*:training-job/*",
+                            "arn:aws:sagemaker:*:*:pipeline/*",
+                        ],
+                    }
+                ],
+            },
+        },
+    },
 }
 
 # Actions the *caller* must have to orchestrate Pipeline-based evaluations

sagemaker-core/src/sagemaker/core/helper/iam_role_resolver.py

@@ -25,7 +25,7 @@
 
 logger = logging.getLogger(__name__)
 
-ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod")
+ROLE_TYPES = ("training", "serving", "pipeline", "feature_store", "bedrock", "hyperpod", "evaluation")
 
 # Permissions the HyperPod CLI flow needs on the *caller* identity — the local
 # principal that runs `hyperpod connect-cluster` and `hyperpod start-job`. The CLI

sagemaker-train/src/sagemaker/train/evaluate/base_evaluator.py

@@ -757,10 +757,10 @@ def _get_aws_execution_context(self) -> Dict[str, str]:
         # This is the job execution role for the
         # serverless / SMTJ evaluation backends. The HyperPod backend submits via
         # the CLI under the caller's own credentials (see _submit_hyperpod_eval_job)
-        # and does not resolve a role here, so "training" is always correct here.
+        # and does not resolve a role here, so "evaluation" is correct here.
         role_arn = resolve_and_validate_role(
             provided_role=self.role,
-            role_type="training",
+            role_type="evaluation",
             sagemaker_session=self.sagemaker_session,
         )
 

sagemaker-train/tests/unit/train/evaluate/test_base_evaluator.py

@@ -734,7 +734,7 @@ def test_get_aws_execution_context(self, mock_resolve, mock_role, mock_session,
         assert context['account_id'] == '123456789012'
         mock_role.assert_called_once_with(
             provided_role=None,
-            role_type="training",
+            role_type="evaluation",
             sagemaker_session=mock_session,
         )
 
@@ -761,7 +761,7 @@ def test_get_aws_execution_context_with_explicit_role(self, mock_resolve, mock_r
         assert context['role_arn'] == explicit_role
         mock_role.assert_called_once_with(
             provided_role=explicit_role,
-            role_type="training",
+            role_type="evaluation",
             sagemaker_session=mock_session,
         )
 

sagemaker-train/tests/unit/train/evaluate/test_bedrock_role_validation.py

@@ -0,0 +1,185 @@
+"""Unit tests for the 'evaluation' role type in iam_role_resolver."""
+
+import pytest
+from unittest.mock import MagicMock, patch
+from botocore.exceptions import ClientError
+
+from sagemaker.core.helper.iam_role_resolver import (
+    resolve_and_validate_role,
+    RoleValidationError,
+    _evaluate_permissions,
+    _role_trusts_service,
+    _get_smoke_test_actions,
+    _expected_trust_services,
+)
+
+
+class TestEvaluationRoleType:
+    """Tests for the 'evaluation' role type configuration."""
+
+    def test_evaluation_role_type_exists(self):
+        """The 'evaluation' role type should be recognized."""
+        from sagemaker.core.helper.iam_policies import IAM_POLICY_CONFIG
+        assert "evaluation" in IAM_POLICY_CONFIG
+
+    def test_evaluation_trust_includes_sagemaker(self):
+        """Evaluation role type should require sagemaker.amazonaws.com trust."""
+        expected = _expected_trust_services("evaluation")
+        assert "sagemaker.amazonaws.com" in expected
+
+    def test_evaluation_trust_does_not_require_bedrock(self):
+        """Evaluation role type should NOT require bedrock.amazonaws.com trust.
+
+        The serverless evaluation backend runs as the SageMaker execution role
+        and calls Bedrock APIs using the role's own credentials. Bedrock does
+        not need to assume the role, so trust is not required.
+        """
+        expected = _expected_trust_services("evaluation")
+        assert "bedrock.amazonaws.com" not in expected
+
+    def test_evaluation_smoke_actions_include_bedrock(self):
+        """Smoke test actions should include Bedrock evaluation actions."""
+        actions = _get_smoke_test_actions("evaluation")
+        assert "bedrock:CreateEvaluationJob" in actions
+        assert "bedrock:GetEvaluationJob" in actions
+
+    def test_evaluation_smoke_actions_include_bedrock_invoke(self):
+        """Smoke test actions should include Bedrock invoke actions."""
+        actions = _get_smoke_test_actions("evaluation")
+        assert "bedrock:InvokeModel" in actions
+
+    def test_resolve_raises_when_bedrock_permissions_denied(self):
+        """Should raise RoleValidationError when Bedrock permissions are denied."""
+        mock_iam = MagicMock()
+
+        # Simulate: bedrock:CreateEvaluationJob denied
+        paginator = MagicMock()
+        paginator.paginate.return_value = [
+            {
+                "EvaluationResults": [
+                    {"EvalActionName": "bedrock:CreateEvaluationJob", "EvalDecision": "implicitDeny"},
+                    {"EvalActionName": "bedrock:GetEvaluationJob", "EvalDecision": "allowed"},
+                    {"EvalActionName": "bedrock:InvokeModel", "EvalDecision": "allowed"},
+                    {"EvalActionName": "bedrock:InvokeModelWithResponseStream", "EvalDecision": "allowed"},
+                ]
+            }
+        ]
+        mock_iam.get_paginator.return_value = paginator
+
+        verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation")
+        assert verdict is False
+        assert "bedrock:CreateEvaluationJob" in denied
+
+    def test_resolve_passes_when_all_allowed(self):
+        """Should pass when all evaluation permissions are allowed."""
+        mock_iam = MagicMock()
+
+        actions = _get_smoke_test_actions("evaluation")
+        paginator = MagicMock()
+        paginator.paginate.return_value = [
+            {
+                "EvaluationResults": [
+                    {"EvalActionName": a, "EvalDecision": "allowed"} for a in actions
+                ]
+            }
+        ]
+        mock_iam.get_paginator.return_value = paginator
+
+        verdict, denied = _evaluate_permissions(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation")
+        assert verdict is True
+        assert denied == []
+
+    def test_trust_check_passes_with_sagemaker(self):
+        """Should pass when trust policy includes sagemaker.amazonaws.com."""
+        mock_iam = MagicMock()
+        mock_iam.get_role.return_value = {
+            "Role": {
+                "AssumeRolePolicyDocument": {
+                    "Version": "2012-10-17",
+                    "Statement": [{
+                        "Effect": "Allow",
+                        "Principal": {"Service": "sagemaker.amazonaws.com"},
+                        "Action": "sts:AssumeRole",
+                    }],
+                }
+            }
+        }
+
+        result = _role_trusts_service(mock_iam, "arn:aws:iam::123456789012:role/MyRole", "evaluation")
+        assert result is True
+
+    def test_resolve_and_validate_passes_with_sagemaker_trust(self):
+        """Full resolve_and_validate_role should pass with sagemaker trust only."""
+        with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session:
+            mock_boto = MagicMock()
+            mock_session.return_value = mock_boto
+
+            mock_iam = MagicMock()
+            mock_boto.client.return_value = mock_iam
+
+            # Role exists with sagemaker trust only (no bedrock needed)
+            mock_iam.get_role.return_value = {
+                "Role": {
+                    "Arn": "arn:aws:iam::123456789012:role/MyRole",
+                    "AssumeRolePolicyDocument": {
+                        "Version": "2012-10-17",
+                        "Statement": [{
+                            "Effect": "Allow",
+                            "Principal": {"Service": "sagemaker.amazonaws.com"},
+                            "Action": "sts:AssumeRole",
+                        }],
+                    }
+                }
+            }
+
+            # All permissions allowed
+            actions = _get_smoke_test_actions("evaluation")
+            paginator = MagicMock()
+            paginator.paginate.return_value = [
+                {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]}
+            ]
+            mock_iam.get_paginator.return_value = paginator
+
+            result = resolve_and_validate_role(
+                provided_role="arn:aws:iam::123456789012:role/MyRole",
+                role_type="evaluation",
+            )
+            assert result == "arn:aws:iam::123456789012:role/MyRole"
+
+    def test_resolve_and_validate_passes_with_correct_role(self):
+        """Full resolve_and_validate_role should pass with correct permissions and trust."""
+        with patch("sagemaker.core.helper.iam_role_resolver._get_boto_session") as mock_session:
+            mock_boto = MagicMock()
+            mock_session.return_value = mock_boto
+
+            mock_iam = MagicMock()
+            mock_boto.client.return_value = mock_iam
+
+            # Role exists with correct trust
+            mock_iam.get_role.return_value = {
+                "Role": {
+                    "Arn": "arn:aws:iam::123456789012:role/MyRole",
+                    "AssumeRolePolicyDocument": {
+                        "Version": "2012-10-17",
+                        "Statement": [{
+                            "Effect": "Allow",
+                            "Principal": {"Service": ["sagemaker.amazonaws.com", "bedrock.amazonaws.com"]},
+                            "Action": "sts:AssumeRole",
+                        }],
+                    }
+                }
+            }
+
+            # All permissions allowed
+            actions = _get_smoke_test_actions("evaluation")
+            paginator = MagicMock()
+            paginator.paginate.return_value = [
+                {"EvaluationResults": [{"EvalActionName": a, "EvalDecision": "allowed"} for a in actions]}
+            ]
+            mock_iam.get_paginator.return_value = paginator
+
+            result = resolve_and_validate_role(
+                provided_role="arn:aws:iam::123456789012:role/MyRole",
+                role_type="evaluation",
+            )
+            assert result == "arn:aws:iam::123456789012:role/MyRole"