AttributeError: 'NoneType' object has no attribute 'verdict'

[nicolablago]

Lately I've seen more and more of these errors when running parallel eval of many test cases.

I am using multi-turn metrics.

self.score = self._calculate_score()
             ^^^^^^^^^^^^^^^^^^^^^^^
File "/app/.venv/lib/python3.12/site-packages/deepeval/metrics/conversation_completeness/conversation_completeness.py", line 304, in _calculate_score
if verdict.verdict.strip().lower() != "no":
           ^^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'verdict'

[trevor-cai]

@nicolablago Thank you for reporting this issue. I put this issue in my queue and will be investigating.

[trevor-cai]

@nicolablago a fix with tests is on its way.

[nicolablago]

@trevor-cai Just wanted to tell you that this sometimes it also happens:

AttributeError: 'NoneType' object has no attribute 'intentions'

it's in the same file (/deepeval/metrics/conversation_completeness/conversation_completeness.py", line 266, in _a_extract_user_intentions) , probably similar fix

[trevor-cai]

@nicolablago, Could you share:

• Which version of DeepEval you are running?
• How you construct your evaluation model (code snippet)?
• Are you using a built-in model or a custom DeepEvalBaseLLM?
• A full stack trace including any inner exceptions (look for aiohttp.client_exceptions.ServerDisconnectedError or similar in your logs)?

Any relevant DeepEval settings you have tweaked.

[nicolablago]

Which version of DeepEval you are running?

3.7.4

How you construct your evaluation model (code snippet)?
Are you using a built-in model or a custom DeepEvalBaseLLM?

    model_name = "gemini-2.5-flash"
    model = GeminiModel(
        model_name,
        project="...",
        location="europe-west4",
        generation_kwargs={"thinkingConfig": {"thinkingBudget": 0}},
    )

    test_cases: list[ConversationalTestCase] = []

    for id_, conversation in conversation_dict.items():
        turns: list[Turn] = []
        for exchange in conversation:
            m = Turn(role="user", content=exchange.message)
            turns.append(m)
            r = Turn(role="assistant", content=exchange.reply)
            turns.append(r)

        test_cases.append(
            ConversationalTestCase(turns=turns, name=str(id_))
        )

    # Create evaluation metrics
    # complete an end-to-end conversation by satisfying user needs
    cc = ConversationCompletenessMetric(model=model)
    # consistently generate relevant responses
    tr = TurnRelevancyMetric(model=model)
    # retain factual information presented
    kr = KnowledgeRetentionMetric(model=model)

    evaluation_result = evaluate(
        test_cases=test_cases,
        metrics=[cc, tr, kr],
        display_config=DisplayConfig(
            show_indicator=False, print_results=False, verbose_mode=False
        ),
    )

A full stack trace including any inner exceptions

ERROR:    Exception in ASGI application
'Traceback (most recent call last):   File "/app/.venv/lib/python3.12/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi     result = await app(  # type: ignore[func-returns-value]
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
'    return await self.app(scope, receive, send)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/fastapi/applications.py", line 1139, in __call__
'    await super().__call__(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/applications.py", line 107, in __call__
'    await self.middleware_stack(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
    raise exc
'  File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
'    await self.app(scope, receive, _send)
'  File "/app/.venv/lib/python3.12/site-packages/opentelemetry/instrumentation/asgi/__init__.py", line 805, in __call__
'    await self.app(scope, otel_receive, otel_send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 186, in __call__
    raise exc
'  File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/errors.py", line 164, in __call__
'    await self.app(scope, receive, _send)
'  File "/app/.venv/lib/python3.12/site-packages/opentelemetry/instrumentation/fastapi/__init__.py", line 307, in __call__
'    await self.app(scope, receive, send)
'  File "/app/cmd/app/src/app/internal/middleware/enrich_logs.py", line 33, in __call__
'    await self.app(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
'    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
'  File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
'    await app(scope, receive, sender)
'  File "/app/.venv/lib/python3.12/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
'    await self.app(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 716, in __call__
'    await self.middleware_stack(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 736, in app
'    await route.handle(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/routing.py", line 290, in handle
'    await self.app(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 120, in app
'    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
'  File "/app/.venv/lib/python3.12/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
'    await app(scope, receive, sender)
'  File "/app/.venv/lib/python3.12/site-packages/fastapi/routing.py", line 107, in app
'    await response(scope, receive, send)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/responses.py", line 167, in __call__
    await self.background()
'  File "/app/.venv/lib/python3.12/site-packages/starlette/background.py", line 36, in __call__
    await task()
'  File "/app/.venv/lib/python3.12/site-packages/starlette/background.py", line 23, in __call__
'    await run_in_threadpool(self.func, *self.args, **self.kwargs)
'  File "/app/.venv/lib/python3.12/site-packages/starlette/concurrency.py", line 32, in run_in_threadpool
    return await anyio.to_thread.run_sync(func)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/anyio/to_thread.py", line 61, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 2525, in run_sync_in_worker_thread
    return await future
           ^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/anyio/_backends/_asyncio.py", line 986, in run
'    result = context.run(func, *args)
             ^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/cmd/app/src/app/routers/evaluation_router.py", line 332, in _run_background_task
'    for result in evaluate_conversations(exchanges=exchanges):
                  ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/cmd/app/src/app/internal/evaluation_helpers.py", line 232, in evaluate_conversations
'    evaluation_result = evaluate(
                        ^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/evaluate/evaluate.py", line 236, in evaluate
'    test_results = loop.run_until_complete(
                   ^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/python/cpython-3.12.12-linux-x86_64-gnu/lib/python3.12/asyncio/base_events.py", line 691, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/evaluate/execute.py", line 798, in a_execute_test_cases
    await asyncio.wait_for(
'  File "/python/cpython-3.12.12-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for
    return await fut
           ^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/evaluate/execute.py", line 585, in execute_with_semaphore
    return await _await_with_outer_deadline(
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/evaluate/execute.py", line 255, in _await_with_outer_deadline
'    return await asyncio.wait_for(coro, timeout=timeout)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/python/cpython-3.12.12-linux-x86_64-gnu/lib/python3.12/asyncio/tasks.py", line 520, in wait_for
    return await fut
           ^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/evaluate/execute.py", line 1044, in _a_execute_conversational_test_cases
    await measure_metrics_with_indicator(
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/metrics/indicator.py", line 237, in measure_metrics_with_indicator
    await asyncio.gather(*tasks)
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/metrics/indicator.py", line 250, in safe_a_measure
    await metric.a_measure(
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/metrics/conversation_completeness/conversation_completeness.py", line 116, in a_measure
'    self.user_intentions = await self._a_extract_user_intentions(
                           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
'  File "/app/.venv/lib/python3.12/site-packages/deepeval/metrics/conversation_completeness/conversation_completeness.py", line 266, in _a_extract_user_intentions
    return res.intentions
           ^^^^^^^^^^^^^^
AttributeError: 'NoneType' object has no attribute 'intentions'