Treating OpenAI responses exclusively as either content or function calls, but they can be both

[siavashg]

OpenAIAgentModel.request_stream makes an assumption about the response being exclusively text or tool calls, but a single OpenAI response can be both. A response that contains both triggers an uncaught exception in OpenAIStreamTextResponse (see below for examples).

Here we're assuming that a response exclusively contains content or tool_calls:

pydantic-ai/pydantic_ai_slim/pydantic_ai/models/openai.py

Lines 199 to 217 in d595c08

	# the first chunk may only contain `role`, so we iterate until we get either `tool_calls` or `content`
	while delta.tool_calls is None and delta.content is None:
	try:
	next_chunk = await response.__anext__()
	except StopAsyncIteration as e:
	raise UnexpectedModelBehavior('Streamed response ended without content or tool calls') from e
	delta = next_chunk.choices[0].delta
	start_cost += _map_cost(next_chunk)
	if delta.content is not None:
	return OpenAIStreamTextResponse(delta.content, response, timestamp, start_cost)
	else:
	assert delta.tool_calls is not None, f'Expected delta with tool_calls, got {delta}'
	return OpenAIStreamStructuredResponse(
	response,
	{c.index: c for c in delta.tool_calls},
	timestamp,
	start_cost,
	)

An OpenAI response can contain both. Here's a demonstration using OpenAI's client:

from devtools import debug
from openai import OpenAI

client = OpenAI()

tools = [
    {
        "type": "function",
        "function": {
            "name": "get_weather",
            "parameters": {
                "type": "object",
                "properties": {
                    "location": {"type": "string"},
                    "unit": {"type": "string", "enum": ["c", "f"]},
                },
                "required": ["location", "unit"],
                "additionalProperties": False,
            },
        },
    }
]

completion = client.chat.completions.create(
    model="gpt-4o",
    messages=[
        {
            "role": "system",
            "content": "Always tell the user what you are about to do",
        },
        {
            "role": "user",
            "content": "What 1+1 and what's the weather like in Paris today?",
        },
    ],
    tools=tools,
    tool_choice="auto",
)
debug(completion)

Output:

 completion: ChatCompletion(
        id='chatcmpl-Ab71YFy1O6SNnnV6uDIBov3Rn8Sk0',
        choices=[
            Choice(
                finish_reason='tool_calls',
                index=0,
                logprobs=None,
                message=ChatCompletionMessage(
                    content=(
                        '1 + 1 equals 2. \n'
                        '\n'
                        'Now, I will look up the current weather in Paris.'
                    ),
                    refusal=None,
                    role='assistant',
                    audio=None,
                    function_call=None,
                    tool_calls=[
                        ChatCompletionMessageToolCall(
                            id='call_Ubvr33St36ChbOUbLMQNy2Ot',
                            function=Function(
                                arguments='{"location":"Paris","unit":"c"}',
                                name='get_weather',
                            ),
                            type='function',
                        ),
                    ],
                ),
            ),
        ],
        created=1733408500,
        model='gpt-4o-2024-08-06',
        object='chat.completion',
        service_tier=None,
        system_fingerprint='fp_7f6be3efb0',
        usage=CompletionUsage(
            completion_tokens=40,
            prompt_tokens=72,
            total_tokens=112,
            completion_tokens_details=CompletionTokensDetails(
                accepted_prediction_tokens=0,
                audio_tokens=0,
                reasoning_tokens=0,
                rejected_prediction_tokens=0,
            ),
            prompt_tokens_details=PromptTokensDetails(
                audio_tokens=0,
                cached_tokens=0,
            ),
        ),
    ) (ChatCompletion)

The equivalent Pydantic AI example will raise an exception:

from __future__ import annotations as _annotations

import asyncio
from typing import Any

from devtools import debug
from pydantic_ai import Agent

weather_agent = Agent(
    "openai:gpt-4o",
    system_prompt="Always tell the user what you are about to do",
)


@weather_agent.tool_plain
async def get_weather(location: str, unit: str) -> dict[str, Any]:
    debug(location, unit)
    return {"result": "123"}


async def main():
    prompt = "What 1+1 and what's the weather like in Paris today?"
    async with weather_agent.run_stream(prompt) as result:
        async for text in result.stream(debounce_by=0.01):
            print(text)
        debug(result)


if __name__ == "__main__":
    asyncio.run(main())

Output:

1
1 +
1 + 1 equals
1 + 1 equals 2
1 + 1 equals 2.

Now,
1 + 1 equals 2.

Now, I will
1 + 1 equals 2.

Now, I will get the
1 + 1 equals 2.

Now, I will get the weather information
1 + 1 equals 2.

Now, I will get the weather information for Paris
1 + 1 equals 2.

Now, I will get the weather information for Paris today.
Traceback (most recent call last):
  File "/tmp/pydantic-ai/weather.py", line 30, in <module>
    asyncio.run(main())
  File "/Users/siavash/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/runners.py", line 190, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/Users/siavash/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/siavash/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/tmp/pydantic-ai/weather.py", line 24, in main
    async for text in result.stream(debounce_by=0.01):
  File "/tmp/pydantic-ai/.venv/lib/python3.11/site-packages/pydantic_ai/result.py", line 152, in stream
    async for text in self.stream_text(debounce_by=debounce_by):
  File "/tmp/pydantic-ai/.venv/lib/python3.11/site-packages/pydantic_ai/result.py", line 191, in stream_text
    async for _ in group_iter:
  File "/tmp/pydantic-ai/.venv/lib/python3.11/site-packages/pydantic_ai/_utils.py", line 198, in async_iter_groups
    item = done.pop().result()
           ^^^^^^^^^^^^^^^^^^^
  File "/tmp/pydantic-ai/.venv/lib/python3.11/site-packages/pydantic_ai/models/openai.py", line 286, in __anext__
    assert choice.delta.content is not None, f'Expected delta with content, invalid chunk: {chunk!r}'
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Expected delta with content, invalid chunk: ChatCompletionChunk(id='chatcmpl-Ab7Am8lOPLxVN1pySDvqpH16EHO38', choices=[Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=[ChoiceDeltaToolCall(index=0, id='call_SObzzp2KWfVzNUruHE5f7e1T', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]), finish_reason=None, index=0, logprobs=None)], created=1733409072, model='gpt-4o-2024-08-06', object='chat.completion.chunk', service_tier=None, system_fingerprint='fp_7f6be3efb0', usage=None)

[samuelcolvin]

What should we do about this?

There's an equivalent case where the AI returns multiple tools calls, one of which is a result tool, currently the other tool calls are ignored.

[siavashg]

What should we do about this?

I'm afraid I don't have enough context of the codebase yet to know what the implications really would be but as the LLMs can and will produce output like this, we should handle it in more expected ways.

Using the weather example above I expected it to stream the text content (as it did), and eventually invoke the get_weather tool. Following the existing structure for tool calls, the results would then be passed back to the LLM and it would react to them in a second step/iteration.

What's the rational to split into two distinct stream response classes, instead of just one that can handle both delta.content and delta.tool_calls? It seems the way it is structured now with it would be quite complicated to change course somewhere deep down in the StreamTextResponse to go back up and continue as a StreamStructuredResponse.

There's an equivalent case where the AI returns multiple tools calls, one of which is a result tool, currently the other tool calls are ignored.

I'm unable to reproduce that. In the examples below it does multi tool calls just fine, both for streamed text and structured responses?

Streaming text:

from __future__ import annotations as _annotations

import asyncio

from devtools import debug
from pydantic_ai import Agent

agent = Agent("openai:gpt-4o")


@agent.tool_plain
async def name_to_id(name: str) -> int:
    debug("Name to ID", name)
    return 1234


@agent.tool_plain
async def id_to_name(id: int) -> str:
    debug("ID to Name", id)
    return "Santa"


async def main():
    prompt = "What's Bobs's ID? And who is ID 3?"
    debug(prompt)
    async with agent.run_stream(prompt) as result:
        async for text in result.stream(debounce_by=0.01):
            print(text)


if __name__ == "__main__":
    asyncio.run(main())

Output:

multi.py:25 main
    prompt: "What's Bobs's ID? And who is ID 3?" (str) len=34
multi.py:13 name_to_id
    'Name to ID' (str) len=10
    name: 'Bobs' (str) len=4
multi.py:19 id_to_name
    'ID to Name' (str) len=10
    id: 3 (int)

Bob's
Bob's ID is
Bob's ID is 123
Bob's ID is 1234,
Bob's ID is 1234, and the
Bob's ID is 1234, and the person with
Bob's ID is 1234, and the person with ID
Bob's ID is 1234, and the person with ID 3 is
Bob's ID is 1234, and the person with ID 3 is Santa.

Streaming structured responses:

from __future__ import annotations as _annotations

import asyncio

from devtools import debug
from pydantic import BaseModel, Field, ValidationError
from pydantic_ai import Agent


class ChatResult(BaseModel):
    text: str = Field(description="The text of the chat message")
    multi_ask: bool = Field(
        description="Whether the user asked more than one question",
        default=False,
    )


agent = Agent(
    "openai:gpt-4o",
    result_type=ChatResult,
)


@agent.tool_plain
async def name_to_id(name: str) -> int:
    debug("Name to ID", name)
    return 1234


@agent.tool_plain
async def id_to_name(id: int) -> str:
    debug("ID to Name", id)
    return "Santa"


async def main():
    prompt = "What's John's ID? And who is ID 3?"
    print(prompt)
    async with agent.run_stream(prompt) as result:
        async for message, last in result.stream_structured(debounce_by=0.01):
            try:
                message = await result.validate_structured_result(
                    message, allow_partial=not last
                )
                print(message)
            except ValidationError as exc:
                if all(
                    e["type"] == "missing" and e["loc"] == ("response",)
                    for e in exc.errors()
                ):
                    continue
                else:
                    raise


if __name__ == "__main__":
    asyncio.run(main())

Output:

What's John's ID? And who is ID 3?
multi_struct.py:26 name_to_id
    'Name to ID' (str) len=10
    name: 'John' (str) len=4
multi_struct.py:32 id_to_name
    'ID to Name' (str) len=10
    id: 3 (int)
text="John's ID is 1234, and" multi_ask=False
text="John's ID is 1234, and the person" multi_ask=False
text="John's ID is 1234, and the person with ID" multi_ask=False
text="John's ID is 1234, and the person with ID 3 is Santa." multi_ask=False
text="John's ID is 1234, and the person with ID 3 is Santa." multi_ask=False
text="John's ID is 1234, and the person with ID 3 is Santa." multi_ask=True
text="John's ID is 1234, and the person with ID 3 is Santa." multi_ask=True

[jlowin]

Ran into this error as well today. Chat-trained models are highly likely to produce both text and a function call in one message, both to explain their actions and to leverage a de facto light version of "chain of thought" to track the motivation for tool calls in a way that tool arguments might not reflect.

You can disable the possibility of producing a message with tool_choice="required" (in OpenAI API), which is sometimes useful, but conversely requiring an LLM to describe its actions before taking them is probably the most basic way to enhance quality over more than one call, so I think supporting combined messages is desirable.

[siavashg]

It's also the best way to reduce time to meaningful first token (TTFT) to the user when the action may involve tool use.

[sydney-runkle]

This is also an issue for anthropic 👍

[Etelis]

That's an overall issue with every structure parsing tools,
maybe we look at this problem in the wrong direction?
How about trying to parse the output greedily? trying to match every object returned, and maybe the unmatched sections might be just referenced? (instead of attempting to parse the whole output)

[gilada-shubham]

Any update on this ? Getting same issue.

[sydney-runkle]

We're actively working on supporting heterogenous responses 👍

[samihamine]

Just checking in to see if there’s been any progress on this issue. Is it still in the pipeline or being prioritized?
Thanks for your work, and looking forward to any updates!

[derickson]

I'm getting this issue or something similar when using LiteLLM Proxy

[sydney-runkle]

Hi!

This is still being worked on, though I think with the most recent release the behavior should be supported.

Can you please send an example of an error that you're running into?

[siavashg]

Thanks for actively looking into this! As of v0.0.18 the initial weather.py example I provided when opening the issue still illustrates the problem:

I can
I can tell you
I can tell you that
I can tell you that 1+
I can tell you that 1+1 equals
I can tell you that 1+1 equals 2
I can tell you that 1+1 equals 2. Now
I can tell you that 1+1 equals 2. Now, I
I can tell you that 1+1 equals 2. Now, I will check
I can tell you that 1+1 equals 2. Now, I will check the weather
I can tell you that 1+1 equals 2. Now, I will check the weather for you
I can tell you that 1+1 equals 2. Now, I will check the weather for you in Paris
I can tell you that 1+1 equals 2. Now, I will check the weather for you in Paris today.
Traceback (most recent call last):
  File "/src/github.com/siavashg/x/weather.py", line 30, in <module>
    asyncio.run(main())
  File "/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/runners.py", line 190, in run
    return runner.run(main)
           ^^^^^^^^^^^^^^^^
  File "/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/runners.py", line 118, in run
    return self._loop.run_until_complete(task)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/.local/share/uv/python/cpython-3.11.6-macos-aarch64-none/lib/python3.11/asyncio/base_events.py", line 653, in run_until_complete
    return future.result()
           ^^^^^^^^^^^^^^^
  File "/src/github.com/siavashg/x/weather.py", line 24, in main
    async for text in result.stream(debounce_by=0.01):
  File "/src/github.com/siavashg/x/.venv/lib/python3.11/site-packages/pydantic_ai/result.py", line 204, in stream
    async for text in self.stream_text(debounce_by=debounce_by):
  File "/src/github.com/siavashg/x/.venv/lib/python3.11/site-packages/pydantic_ai/result.py", line 247, in stream_text
    async for _ in group_iter:
  File "/src/github.com/siavashg/x/.venv/lib/python3.11/site-packages/pydantic_ai/_utils.py", line 204, in async_iter_groups
    item = done.pop().result()
           ^^^^^^^^^^^^^^^^^^^
  File "/src/github.com/siavashg/x/.venv/lib/python3.11/site-packages/pydantic_ai/models/openai.py", line 322, in __anext__
    assert choice.delta.content is not None, f'Expected delta with content, invalid chunk: {chunk!r}'
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
AssertionError: Expected delta with content, invalid chunk: ChatCompletionChunk(id='chatcmpl-AnPpNwp1JAHRAVeWeierBaUYADTHI', choices=[Choice(delta=ChoiceDelta(content=None, function_call=None, refusal=None, role=None, tool_calls=[ChoiceDeltaToolCall(index=0, id='call_Wcf1EeoUjusrmuN8BGHIieZ3', function=ChoiceDeltaToolCallFunction(arguments='', name='get_weather'), type='function')]), finish_reason=None, index=0, logprobs=None)], created=1736340717, model='gpt-4o-2024-08-06', object='chat.completion.chunk', service_tier=None, system_fingerprint='fp_d28bcae782', usage=None)

When the LLM starts by streaming text the subsequent tool calls will fail. Note that even in this little example the LLM may occasionally ignore the system prompt (Always tell the user what you are about to do) and instead make a tool call before it generates the text output. In that case it will execute successfully, but there are good reasons to want the LLM to start with a text response to provide an immediate feedback before calling long running tools.

Edit:
Adding this for reference: #468 might be addressing this.