-
Notifications
You must be signed in to change notification settings - Fork 1.7k
Open
Labels
bot triaged[Bot] This issue is triaged by ADK bot[Bot] This issue is triaged by ADK botcore[Component] This issue is related to the core interface and implementation[Component] This issue is related to the core interface and implementation
Description
According to Gemini documentation the thought signatures returned in case of the function call response must be returned with the function response.
Below is the llm_request.contents from a particular session (ADK 1.11.0). Notice the absence of the thought signatures in the "user" responses.
[Content(
parts=[
Part(
text='what is written on the handwritten note?'
),
],
role='user'
), Content(
parts=[
Part(
function_call=FunctionCall(
args={},
name='get_images_to_analyze'
),
thought_signature=b'\n\x91\x03\x01\xcb\xbd\xf0\xf0k\xd1\xa1\x16\x02+\x94\xb9\xabUyz|$\x1e\xb7\xe0\xfd\x9a\x8bP/\x84\x8c5R>\x19A UX\xc4O\x94+\x90\x8eF\x1b\x8e\x16\x00\xab\x95K\xaf@\xe6\xf00|\xee\xf8\xde`:\x86\x91J\xd4\xdc\t\xa3\xe7[\xf9\x87N+\xc5l\x92S9s\x0b\xec\xa0i\xb7t\x8a\x0b\xcf\x81t\xc2O...'
),
],
role='model'
), Content(
parts=[
Part(
function_response=FunctionResponse(
name='get_images_to_analyze',
response={
'result': [
ImageInfo(
cloud_storage_uri=<... Max depth ...>,
description=<... Max depth ...>,
mime_type=<... Max depth ...>,
public_url=<... Max depth ...>
),
ImageInfo(
cloud_storage_uri=<... Max depth ...>,
description=<... Max depth ...>,
mime_type=<... Max depth ...>,
public_url=<... Max depth ...>
),
]
}
)
),
],
role='user'
), Content(
parts=[
Part(
function_call=FunctionCall(
args={
'image_cloud_storage_uri': 'gs://cloud-samples-data/vision/handwritten.jpg',
'image_mime_type': 'image/png'
},
name='get_image_reference'
),
thought_signature=b'\n\xbc\x06\x01\xcb\xbd\xf0\xf0\x88\xb9tbV\xe7&\xb7\x16\x86S\xb4kq\x103\x86\x00\x82E7l\xb2\x80\xc1`V\xa9\xafcnN\x82\x0f\xb9"\xbb\xe5\r\x0e\xb9H\xbb\x14\xe4\x8es\x006v\xe1\x1c\xc7q\x9f\x95\x8a\x9b\xfa%\xca}R!\xec.\xae\xe9}o\xb8\x8fOP\xc4|\x9b\x03\xd4o\xabH+\x1d\x7f\xe0\x1fF\xe9...'
),
],
role='model'
), Content(
parts=[
Part(
function_response=FunctionResponse(
name='get_image_reference',
response={
'result': None
}
)
),
],
role='user'
), Content(
parts=[
Part(
text="""The handwritten note says:
"This is a handwritten message to use with Google Cloud Platform, To demonstrate the capabilities of the Cloud Vision API to detect handwritten text."""",
thought_signature=b'\n\xb9\x02\x01\xcb\xbd\xf0\xf0\x0b\x8a>LGA\xa6N\x16\x14\x85\xd8\x81\x9f\x14G\xee\xa8\xc8\x19\xe0EJz\xaa\xf1\xf7N\xe0:\x17G\x86\x01\xff\xdb\x80l\xdd\r\xce/\xe7X\x7f\x8c\xe7\xd2\x97\xfc\xa0\xa6l\xdb\xea]\xae\x97\xdeO\xa1$\x9a0j\xd6\x07\xec\xe4>]\x85\xf0[\t\xa2`8x\x1d\x98oiFN7\xfaDs...'
),
],
role='model'
), Content(
parts=[
Part(
text='show the images you have'
),
],
role='user'
), Content(
parts=[
Part(
function_call=FunctionCall(
args={},
name='get_images_to_analyze'
),
thought_signature=b'\n\xd9\x01\x01\xcb\xbd\xf0\xf0\xe8\x9c\xd6\x96\xff\xcf\xa29.nq\x7f<\xefXD7q\x96\xd7\xcd\xbe\xe4\xe6\x10(\x0e\x08\xaf\xa4\x9b\xacH?m#\x17\xcb\x89\xf1l7\xba]p\xd4g\xdd\xb0,\xfd\xf2\xc4^:\xae\xad Pgu\xaf\xd7l,z\x80P\xb80\xb0\x8c[\x95\xcdhN\x9f(\x13U3M\x95\xa1\xd7\x94\xf7\xee...'
),
],
role='model'
), Content(
parts=[
Part(
function_response=FunctionResponse(
name='get_images_to_analyze',
response={
'result': [
ImageInfo(
cloud_storage_uri=<... Max depth ...>,
description=<... Max depth ...>,
mime_type=<... Max depth ...>,
public_url=<... Max depth ...>
),
ImageInfo(
cloud_storage_uri=<... Max depth ...>,
description=<... Max depth ...>,
mime_type=<... Max depth ...>,
public_url=<... Max depth ...>
),
]
}
)
),
],
role='user'
), Content(
parts=[
Part(
text="""Here are the images I have:
* **Duck and truck**
Public URL: https://storage.googleapis.com/cloud-samples-data/vision/duck_and_truck.jpg
* **Handwritten note**
Public URL: https://storage.googleapis.com/cloud-samples-data/vision/handwritten.jpg""",
thought_signature=b'\n\x9e\x02\x01\xcb\xbd\xf0\xf0\xd8\x8b\x98$\x9a\x02\xb2\xbf\xa3\xf2\xe4\xaaD.\x19\xe2\xc1\x02\xbe9tq\xe3\xe7\xc3\xf3F\xd9\xf4M6A\xe46\xccO%\xc8\xa4S\x9a\xc2\xc4\xb4C\xb6S\xcb\x0cO\xbc\xc4\x12\xfb\x82;B\x1a-\xf8z\xca^)\xd3\x1a\x87\x8c\xcaN\xc2\xe8Qj\x88\n\xbf\xea\xd6c\xe4t\xec+\xe4&q6\x82...'
),
],
role='model'
), Content(
parts=[
Part(
text='What is the color of the duck?'
),
],
role='user'
), Content(
parts=[
Part(
function_call=FunctionCall(
args={
'image_cloud_storage_uri': 'gs://cloud-samples-data/vision/duck_and_truck.jpg',
'image_mime_type': 'image/png'
},
name='get_image_reference'
),
thought_signature=b'\n\x82\x04\x01\xcb\xbd\xf0\xf0F?\xc5F\xd3)\xc4\xc6\xa2\xb3\xb8\xdd\xda\xc3\xb0\xa9\xd7Y8\xc7\x99\x19\x93\xba\xf0\xa1G\x9a\xf7[\x1feT\x14\xdb\xb4\x07zjI\xdd\xfd\x10\x90\xe4N\xe7;\xfd\x183\xb7\xf7\xe9l\xf9\x9fd\xb0XH\x82\xa2\xd0n)\x00\xea\xbe\x06k7+\x1339\xe8\xee!z\x81\x9a\xf2V\xe6\x96o\xfb\x9a...'
),
],
role='model'
), Content(
parts=[
Part(
function_response=FunctionResponse(
name='get_image_reference',
response={
'result': None
}
)
),
],
role='user'
), Content(
parts=[
Part(
text='The duck is yellow with an orange beak.'
),
],
role='model'
), Content(
parts=[
Part(
text='What is the color of the truck?'
),
],
role='user'
), Content(
parts=[
Part(
text='The truck is red.',
thought_signature=b'\n\x9a\x03\x01\xcb\xbd\xf0\xf0\xf7i$\xfd\x90\x03\x17%\x1d\x87\xeeV\xb3\x14\xa5;:\xb6@yagB"\x88\x8bm\x8b\xdd\xaa\xfa\x83\xdb\x81#\xa2Q\x96\xbd\x0b\xd8D\xd0\xf0\xec\xe3a\xbcu\xfb2\x84\x85n\xbf\xf7\xc0E\x175\xf2\xed\xb3\xd3K\x88h\xf3\xf5T\xe9\x1b\tF\x1cF\x88\'\xd6\x94\xe2\x82\x11\xb9U\xb4|\xb5g...'
),
],
role='model'
), Content(
parts=[
Part(
text="What are the color of truck's tires?"
),
],
role='user'
), Content(
parts=[
Part(
text="The truck's tires are black.",
thought_signature=b'\n\xc3\x03\x01\xcb\xbd\xf0\xf0{\xbe#\x8e=5p\x04\x06\xe3\x19\xa84-|N\x079\x03\xd9&\xf7\xce\xbe\xc4\xccD\x08\x06\xf6\x81R\x99X_\x0f\xc9\xcf`n\xfe\xee\xef\x80{-&\xc2\xc2\xcdA^\x84\xa4-\x19\xa4\x9f\xda\xd2\xea\x12\x99.\xeb\xa8\xfa)M\x84\xf5/\x81\xd4\x98\xb5\xffr\x0bQ\xae\x12%\xda\xfe\xfd,\xa6\xca...'
),
],
role='model'
), Content(
parts=[
Part(
text='please analyze the picture - you are not giving a correct result'
),
],
role='user'
), Content(
parts=[
Part(
function_call=FunctionCall(
args={
'image_cloud_storage_uri': 'gs://cloud-samples-data/vision/duck_and_truck.jpg',
'image_mime_type': 'image/png'
},
name='get_image_reference'
),
thought_signature=b"\n\x9d\x08\x01\xcb\xbd\xf0\xf0\xba\x8e\xd6\x01}^A\xcb\x894\x99\x9b\xdc+g(r\x85mmh\xc9\x02\xfcRJ4\x9c\x1e'\xdf,\xaaP\xc5\x84\xf6\xdfl\tf\xce\x03Z\xb6*\x7fB\x11#\x14\x1e\xac\x92\xf5D\xdbx\x83>U\xce\xf9\x17]3\x93}\x08\xc3\xdf\x1d\x1e\xd3d/\xbcU\xa9\x1aB\x0cu\xf7\x18m\x83\xee\xda...'
),
],
role='model'
), Content(
parts=[
Part(
function_response=FunctionResponse(
name='get_image_reference',
response={
'result': None
}
)
),
],
role='user'
)]
doug7410
Metadata
Metadata
Assignees
Labels
bot triaged[Bot] This issue is triaged by ADK bot[Bot] This issue is triaged by ADK botcore[Component] This issue is related to the core interface and implementation[Component] This issue is related to the core interface and implementation