Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 31 additions & 2 deletions src/harness_sdk/instrumentation/anthropic/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
Coverage:
- Messages.create / AsyncMessages.create (non-streaming and stream=True)
- Messages.stream / AsyncMessages.stream (MessageStream API)
- AnthropicVertex / AsyncAnthropicVertex, including LangChain ChatAnthropicVertex

The OTEL contrib ``AnthropicInstrumentor.instrument()`` only handles non-streaming
``Messages.create``; we call its inner ``messages_create(handler)`` wrapper directly
Expand Down Expand Up @@ -45,7 +46,32 @@
logger = get_custom_logger(__name__)

_ANTHROPIC_MESSAGES_MODULE = "anthropic.resources.messages"
_ANTHROPIC = "anthropic"
_PROVIDER_ANTHROPIC = "anthropic"
_PROVIDER_VERTEX = "gcp.vertex_ai"


def _resolve_provider(messages_instance: Any) -> str:
"""Detect AnthropicVertex vs direct Anthropic API from Messages._client."""
client = getattr(messages_instance, "_client", None)
if client is None:
return _PROVIDER_ANTHROPIC
cls_name = type(client).__name__
if cls_name in ("AnthropicVertex", "AsyncAnthropicVertex"):
return _PROVIDER_VERTEX
return _PROVIDER_ANTHROPIC


def _apply_vertex_client_attributes(attributes: dict[str, Any], messages_instance: Any) -> None:
"""Best-effort gcp.project / gcp.location from AnthropicVertex client."""
client = getattr(messages_instance, "_client", None)
if client is None:
return
project = getattr(client, "project_id", None)
region = getattr(client, "region", None)
if project:
attributes["gcp.project"] = project
if region:
attributes["gcp.location"] = region


def _get_handler() -> TelemetryHandler:
Expand Down Expand Up @@ -85,11 +111,14 @@ def _build_invocation(
capture_content: bool,
) -> LLMInvocation:
attributes = get_llm_request_attributes(params, instance)
provider = _resolve_provider(instance)
if provider == _PROVIDER_VERTEX:
_apply_vertex_client_attributes(attributes, instance)
model_attr = attributes.get("gen_ai.request.model")
request_model = model_attr if isinstance(model_attr, str) else params.model
invocation = LLMInvocation(
request_model=request_model,
provider=_ANTHROPIC,
provider=provider,
input_messages=get_input_messages(params.messages) if capture_content else [],
system_instruction=get_system_instruction(params.system) if capture_content else [],
attributes=attributes,
Expand Down
53 changes: 52 additions & 1 deletion test/instrumentation/anthropic/test_anthropic.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@

pytest.importorskip("anthropic")

from anthropic import Anthropic, AsyncAnthropic
from anthropic import Anthropic, AnthropicVertex, AsyncAnthropic
from anthropic.resources.messages import AsyncMessages, Messages

from harness_sdk.plugins.control import ControlResult, get_control_registry
from harness_sdk.gen_ai.exceptions import ControlEvaluationBlocked
from harness_sdk.instrumentation import anthropic as anthropic_instrumentation
from harness_sdk.instrumentation.anthropic import AnthropicInstrumentorWrapper


Expand Down Expand Up @@ -48,6 +49,34 @@ def _fake_messages_create(_self, *_args, **_kwargs):
return _FakeMessage()


def test_resolve_provider_detects_anthropic_vertex():
vertex_client = type("AnthropicVertex", (SimpleNamespace,), {})()
messages = SimpleNamespace(_client=vertex_client)

assert anthropic_instrumentation._resolve_provider(messages) == "gcp.vertex_ai"


def test_resolve_provider_defaults_to_anthropic():
anthropic_client = type("Anthropic", (SimpleNamespace,), {})()
messages = SimpleNamespace(_client=anthropic_client)

assert anthropic_instrumentation._resolve_provider(messages) == "anthropic"


def test_apply_vertex_client_attributes():
vertex_client = type("AnthropicVertex", (SimpleNamespace,), {})(
project_id="my-proj",
region="us-east5",
)
messages = SimpleNamespace(_client=vertex_client)
attributes = {}

anthropic_instrumentation._apply_vertex_client_attributes(attributes, messages)

assert attributes["gcp.project"] == "my-proj"
assert attributes["gcp.location"] == "us-east5"


def test_anthropic_span_has_gen_ai_attributes(agent, exporter, anthropic_instrumentor):
with patch.object(Messages, "create", new=_fake_messages_create):
anthropic_instrumentor.instrument()
Expand All @@ -63,13 +92,35 @@ def test_anthropic_span_has_gen_ai_attributes(agent, exporter, anthropic_instrum
assert len(spans) == 1
attrs = spans[0].attributes
assert attrs.get("gen_ai.system") == "anthropic"
assert attrs.get("gen_ai.provider.name") == "anthropic"
assert attrs.get("gen_ai.operation.name") == "chat"
assert attrs.get("gen_ai.request.model") == "claude-3-haiku-20240307"
assert attrs.get("gen_ai.response.id") == "msg_test"
assert attrs.get("gen_ai.usage.input_tokens") == 2
assert attrs.get("gen_ai.usage.output_tokens") == 4


def test_anthropic_vertex_span_has_provider_and_gcp_attributes(
agent, exporter, anthropic_instrumentor
):
with patch.object(Messages, "create", new=_fake_messages_create):
anthropic_instrumentor.instrument()
client = AnthropicVertex(project_id="my-proj", region="us-east5")
client.messages.create(
model="claude-3-5-sonnet@20240620",
max_tokens=10,
messages=[{"role": "user", "content": "hello"}],
)

spans = exporter.get_finished_spans()
exporter.clear()
assert len(spans) == 1
attrs = spans[0].attributes
assert attrs.get("gen_ai.provider.name") == "gcp.vertex_ai"
assert attrs.get("gcp.project") == "my-proj"
assert attrs.get("gcp.location") == "us-east5"


async def test_async_messages_create_span_has_gen_ai_attributes(agent, exporter, anthropic_instrumentor):
async def _fake(_self, *_args, **_kwargs):
return _FakeMessage()
Expand Down
Loading