diff --git a/firebaseai/src/GenerateContentResponse.cs b/firebaseai/src/GenerateContentResponse.cs index 9c4e5e95..f7411e66 100644 --- a/firebaseai/src/GenerateContentResponse.cs +++ b/firebaseai/src/GenerateContentResponse.cs @@ -53,10 +53,26 @@ public IReadOnlyList Candidates { /// public string Text { get { - // Concatenate all of the text parts from the first candidate. + // Concatenate all of the text parts that aren't thoughts from the first candidate. return string.Join(" ", Candidates.FirstOrDefault().Content.Parts - .OfType().Select(tp => tp.Text)); + .OfType().Where(tp => !tp.IsThought).Select(tp => tp.Text)); + } + } + + /// + /// A summary of the model's thinking process, if available. + /// + /// Note that Thought Summaries are only available when `IncludeThoughts` is enabled + /// in the `ThinkingConfig`. For more information, see the + /// [Thinking](https://firebase.google.com/docs/ai-logic/thinking) documentation. + /// + public string ThoughtSummary { + get { + // Concatenate all of the text parts that are thoughts from the first candidate. + return string.Join(" ", + Candidates.FirstOrDefault().Content.Parts + .OfType().Where(tp => tp.IsThought).Select(tp => tp.Text)); } } @@ -65,7 +81,8 @@ public string Text { /// public IReadOnlyList FunctionCalls { get { - return Candidates.FirstOrDefault().Content.Parts.OfType().ToList(); + return Candidates.FirstOrDefault().Content.Parts + .OfType().Where(tp => !tp.IsThought).ToList(); } } diff --git a/firebaseai/src/GenerationConfig.cs b/firebaseai/src/GenerationConfig.cs index c3fa2ca5..e4a0ca97 100644 --- a/firebaseai/src/GenerationConfig.cs +++ b/firebaseai/src/GenerationConfig.cs @@ -216,14 +216,19 @@ internal Dictionary ToJson() { public readonly struct ThinkingConfig { #if !DOXYGEN public readonly int? ThinkingBudget { get; } + public readonly bool? IncludeThoughts { get; } #endif /// /// Initializes configuration options for Thinking features. /// /// The token budget for the model's thinking process. - public ThinkingConfig(int? thinkingBudget = null) { + /// + /// If true, summaries of the model's "thoughts" are included in responses. + /// + public ThinkingConfig(int? thinkingBudget = null, bool? includeThoughts = null) { ThinkingBudget = thinkingBudget; + IncludeThoughts = includeThoughts; } /// @@ -232,9 +237,8 @@ public ThinkingConfig(int? thinkingBudget = null) { /// internal Dictionary ToJson() { Dictionary jsonDict = new(); - if (ThinkingBudget.HasValue) { - jsonDict["thinkingBudget"] = ThinkingBudget.Value; - } + jsonDict.AddIfHasValue("thinkingBudget", ThinkingBudget); + jsonDict.AddIfHasValue("includeThoughts", IncludeThoughts); return jsonDict; } } diff --git a/firebaseai/src/Internal/InternalHelpers.cs b/firebaseai/src/Internal/InternalHelpers.cs index 4cd9fbda..7b9e9d75 100644 --- a/firebaseai/src/Internal/InternalHelpers.cs +++ b/firebaseai/src/Internal/InternalHelpers.cs @@ -234,6 +234,20 @@ public static ModelContent ConvertToModel(this ModelContent content) { public static ModelContent ConvertToSystem(this ModelContent content) { return content.ConvertRole("system"); } + + public static void AddIfHasValue(this JsonDict jsonDict, string key, + T? value) where T : struct { + if (value.HasValue) { + jsonDict.Add(key, value.Value); + } + } + + public static void AddIfHasValue(this JsonDict jsonDict, string key, + T value) where T : class { + if (value != null) { + jsonDict.Add(key, value); + } + } } } diff --git a/firebaseai/src/LiveSessionResponse.cs b/firebaseai/src/LiveSessionResponse.cs index 43903912..5751698e 100644 --- a/firebaseai/src/LiveSessionResponse.cs +++ b/firebaseai/src/LiveSessionResponse.cs @@ -199,7 +199,8 @@ private LiveSessionToolCall(List functionCalls) { /// internal static LiveSessionToolCall FromJson(Dictionary jsonDict) { return new LiveSessionToolCall( - jsonDict.ParseObjectList("functionCalls", ModelContentJsonParsers.FunctionCallPartFromJson)); + jsonDict.ParseObjectList("functionCalls", + innerDict => ModelContentJsonParsers.FunctionCallPartFromJson(innerDict, null, null))); } } diff --git a/firebaseai/src/ModelContent.cs b/firebaseai/src/ModelContent.cs index 4c152142..a21f7eff 100644 --- a/firebaseai/src/ModelContent.cs +++ b/firebaseai/src/ModelContent.cs @@ -119,6 +119,15 @@ public static ModelContent FunctionResponse( /// single value of `Part`, different data types may not mix. /// public interface Part { + /// + /// Indicates whether this `Part` is a summary of the model's internal thinking process. + /// + /// When `IncludeThoughts` is set to `true` in `ThinkingConfig`, the model may return one or + /// more "thought" parts that provide insight into how it reasoned through the prompt to arrive + /// at the final answer. These parts will have `IsThought` set to `true`. + /// + public bool IsThought { get; } + #if !DOXYGEN /// /// Intended for internal use only. @@ -136,15 +145,39 @@ public interface Part { /// Text value. /// public string Text { get; } + + private readonly bool? _isThought; + public bool IsThought { get { return _isThought ?? false; } } + + private readonly string _thoughtSignature; /// /// Creates a `TextPart` with the given text. /// /// The text value to use. - public TextPart(string text) { Text = text; } + public TextPart(string text) { + Text = text; + _isThought = null; + _thoughtSignature = null; + } + + /// + /// Intended for internal use only. + /// + internal TextPart(string text, bool? isThought, string thoughtSignature) { + Text = text; + _isThought = isThought; + _thoughtSignature = thoughtSignature; + } Dictionary Part.ToJson() { - return new Dictionary() { { "text", Text } }; + var jsonDict = new Dictionary() { + { "text", Text } + }; + + jsonDict.AddIfHasValue("thought", _isThought); + jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature); + return jsonDict; } } @@ -161,6 +194,11 @@ Dictionary Part.ToJson() { /// The data provided in the inline data part. /// public byte[] Data { get; } + + private readonly bool? _isThought; + public bool IsThought { get { return _isThought ?? false; } } + + private readonly string _thoughtSignature; /// /// Creates an `InlineDataPart` from data and a MIME type. @@ -176,16 +214,31 @@ Dictionary Part.ToJson() { /// The data representation of an image, video, audio or document; see [input files and /// requirements](https://firebase.google.com/docs/vertex-ai/input-file-requirements) for /// supported media types. - public InlineDataPart(string mimeType, byte[] data) { MimeType = mimeType; Data = data; } + public InlineDataPart(string mimeType, byte[] data) { + MimeType = mimeType; + Data = data; + _isThought = null; + _thoughtSignature = null; + } + + internal InlineDataPart(string mimeType, byte[] data, bool? isThought, string thoughtSignature) { + MimeType = mimeType; + Data = data; + _isThought = isThought; + _thoughtSignature = thoughtSignature; + } Dictionary Part.ToJson() { - return new Dictionary() { + var jsonDict = new Dictionary() { { "inlineData", new Dictionary() { { "mimeType", MimeType }, { "data", Convert.ToBase64String(Data) } } } }; + jsonDict.AddIfHasValue("thought", _isThought); + jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature); + return jsonDict; } } @@ -201,6 +254,9 @@ Dictionary Part.ToJson() { /// The URI of the file. /// public System.Uri Uri { get; } + + // This Part can only come from the user, and thus will never be a thought. + public bool IsThought { get { return false; } } /// /// Constructs a new file data part. @@ -241,27 +297,36 @@ Dictionary Part.ToJson() { /// public string Id { get; } + private readonly bool? _isThought; + public bool IsThought { get { return _isThought ?? false; } } + + private readonly string _thoughtSignature; + /// /// Intended for internal use only. /// - internal FunctionCallPart(string name, IDictionary args, string id) { + internal FunctionCallPart(string name, IDictionary args, string id, + bool? isThought, string thoughtSignature) { Name = name; Args = new Dictionary(args); Id = id; + _isThought = isThought; + _thoughtSignature = thoughtSignature; } Dictionary Part.ToJson() { - var jsonDict = new Dictionary() { + var innerDict = new Dictionary() { { "name", Name }, { "args", Args } }; - if (!string.IsNullOrEmpty(Id)) { - jsonDict["id"] = Id; - } + innerDict.AddIfHasValue("id", Id); - return new Dictionary() { - { "functionCall", jsonDict } + var jsonDict = new Dictionary() { + { "functionCall", innerDict } }; + jsonDict.AddIfHasValue("thought", _isThought); + jsonDict.AddIfHasValue("thoughtSignature", _thoughtSignature); + return jsonDict; } } @@ -285,6 +350,9 @@ Dictionary Part.ToJson() { /// The id from the FunctionCallPart this is in response to. /// public string Id { get; } + + // This Part can only come from the user, and thus will never be a thought. + public bool IsThought { get { return false; } } /// /// Constructs a new `FunctionResponsePart`. @@ -337,20 +405,27 @@ internal static ModelContent FromJson(Dictionary jsonDict) { jsonDict.ParseObjectList("parts", PartFromJson, JsonParseOptions.ThrowEverything).Where(p => p is not null)); } - private static InlineDataPart InlineDataPartFromJson(Dictionary jsonDict) { + private static InlineDataPart InlineDataPartFromJson(Dictionary jsonDict, + bool? isThought, string thoughtSignature) { return new InlineDataPart( jsonDict.ParseValue("mimeType", JsonParseOptions.ThrowEverything), - Convert.FromBase64String(jsonDict.ParseValue("data", JsonParseOptions.ThrowEverything))); + Convert.FromBase64String(jsonDict.ParseValue("data", JsonParseOptions.ThrowEverything)), + isThought, + thoughtSignature); } private static Part PartFromJson(Dictionary jsonDict) { + bool? isThought = jsonDict.ParseNullableValue("thought"); + string thoughtSignature = jsonDict.ParseValue("thoughtSignature"); if (jsonDict.TryParseValue("text", out string text)) { - return new TextPart(text); - } else if (jsonDict.TryParseObject("functionCall", ModelContentJsonParsers.FunctionCallPartFromJson, - out var fcPart)) { + return new TextPart(text, isThought, thoughtSignature); + } else if (jsonDict.TryParseObject("functionCall", + innerDict => ModelContentJsonParsers.FunctionCallPartFromJson(innerDict, isThought, thoughtSignature), + out var fcPart)) { return fcPart; - } else if (jsonDict.TryParseObject("inlineData", InlineDataPartFromJson, - out var inlineDataPart)) { + } else if (jsonDict.TryParseObject("inlineData", + innerDict => InlineDataPartFromJson(innerDict, isThought, thoughtSignature), + out var inlineDataPart)) { return inlineDataPart; } else { #if FIREBASEAI_DEBUG_LOGGING @@ -365,11 +440,14 @@ namespace Internal { // Class for parsing Parts that need to be called from other files as well. internal static class ModelContentJsonParsers { - internal static ModelContent.FunctionCallPart FunctionCallPartFromJson(Dictionary jsonDict) { + internal static ModelContent.FunctionCallPart FunctionCallPartFromJson(Dictionary jsonDict, + bool? isThought, string thoughtSignature) { return new ModelContent.FunctionCallPart( jsonDict.ParseValue("name", JsonParseOptions.ThrowEverything), jsonDict.ParseValue>("args", JsonParseOptions.ThrowEverything), - jsonDict.ParseValue("id")); + jsonDict.ParseValue("id"), + isThought, + thoughtSignature); } } diff --git a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs index b2b608f2..3851fe86 100644 --- a/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs +++ b/firebaseai/testapp/Assets/Firebase/Sample/FirebaseAI/UIHandlerAutomated.cs @@ -74,6 +74,7 @@ protected override void Start() { TestImagenGenerateImage, TestImagenGenerateImageOptions, TestThinkingBudget, + TestIncludeThoughts, }; // Set of tests that only run the single time. Func[] singleTests = { @@ -98,6 +99,7 @@ protected override void Start() { InternalTestGenerateImagesBase64, InternalTestGenerateImagesAllFiltered, InternalTestGenerateImagesBase64SomeFiltered, + InternalTestThoughtSummary, }; // Create the set of tests, combining the above lists. @@ -160,7 +162,8 @@ private void AssertFloatEq(string message, float value1, float value2) { private void AssertType(string message, object obj, out T output) { if (obj is T parsed) { output = parsed; - } else { + } + else { throw new Exception( $"Assertion failed ({testRunner.CurrentTestDescription}): {obj.GetType()} is wrong type ({message})"); } @@ -223,7 +226,8 @@ async Task TestBasicText(Backend backend) { Assert("Invalid CandidatesTokenCount", response.UsageMetadata?.CandidatesTokenCount > 0); Assert("Invalid PromptTokenCount", response.UsageMetadata?.PromptTokenCount > 0); Assert("Invalid TotalTokenCount", response.UsageMetadata?.TotalTokenCount > 0); - } else { + } + else { DebugLog("WARNING: UsageMetadata was missing from BasicText"); } @@ -458,7 +462,7 @@ async Task TestEnumSchemaResponse(Backend backend) { generationConfig: new GenerationConfig( responseMimeType: "text/x.enum", responseSchema: Schema.Enum(new string[] { enumValue }))); - + var response = await model.GenerateContentAsync( "Hello, I am testing setting the response schema to an enum."); @@ -474,7 +478,7 @@ async Task TestAnyOfSchemaResponse(Backend backend) { Schema.AnyOf(new[] { Schema.Int(), Schema.String() }), minItems: 2, maxItems: 6))); - + var response = await model.GenerateContentAsync( "Hello, I am testing setting the response schema with an array, cause you give me some random values."); @@ -721,7 +725,8 @@ async Task TestGenerateImage(Backend backend) { foreach (var part in candidate.Content.Parts) { if (part is ModelContent.TextPart) { foundText = true; - } else if (part is ModelContent.InlineDataPart dataPart) { + } + else if (part is ModelContent.InlineDataPart dataPart) { if (dataPart.MimeType.Contains("image")) { foundImage = true; } @@ -803,11 +808,42 @@ async Task TestThinkingBudget(Backend backend) { string result = response.Text; Assert("Response text was missing", !string.IsNullOrWhiteSpace(result)); + // ThoughtSummary should be null since we didn't set includeThoughts. + Assert("ThoughtSummary wasn't null", string.IsNullOrWhiteSpace(response.ThoughtSummary)); + Assert("UsageMetadata was missing", response.UsageMetadata != null); Assert("UsageMetadata.ThoughtsTokenCount was missing", response.UsageMetadata?.ThoughtsTokenCount > 0); } + // Test requesting thought summaries. + async Task TestIncludeThoughts(Backend backend) { + // Thinking Budget requires at least the 2.5 model. + var tool = new Tool(new FunctionDeclaration( + "GetKeyword", "Call to retrieve a special keyword.", + new Dictionary() { + { "input", Schema.String("Input string") }, + })); + var model = GetFirebaseAI(backend).GetGenerativeModel( + modelName: "gemini-2.5-flash", + generationConfig: new GenerationConfig( + thinkingConfig: new ThinkingConfig( + thinkingBudget: 1024, + includeThoughts: true + ) + ) + ); + + GenerateContentResponse response = await model.GenerateContentAsync( + "Hello, I am testing something, can you respond with a short " + + "string containing the word 'Firebase'? Don't call GetKeyword for this."); + + string result = response.Text; + Assert("Response text was missing", !string.IsNullOrWhiteSpace(result)); + + Assert("ThoughtSummary was missing", !string.IsNullOrWhiteSpace(response.ThoughtSummary)); + } + // Test providing a file from a GCS bucket (Firebase Storage) to the model. async Task TestReadFile() { // GCS is currently only supported with VertexAI. @@ -871,7 +907,8 @@ private Task LoadStreamingAsset(string fullPath) { request.SendWebRequest().completed += (_) => { if (request.result == UnityWebRequest.Result.Success) { tcs.SetResult(request.downloadHandler.text); - } else { + } + else { tcs.SetResult(null); } }; @@ -886,7 +923,8 @@ private async Task> GetJsonTestData(string filename) if (localPath.StartsWith("jar") || localPath.StartsWith("http")) { // Special case to access StreamingAsset content on Android jsonString = await LoadStreamingAsset(localPath); - } else if (File.Exists(localPath)) { + } + else if (File.Exists(localPath)) { jsonString = File.ReadAllText(localPath); } @@ -1255,7 +1293,7 @@ Task InternalTestGroundingMetadata_Empty() { return Task.CompletedTask; } - + // Test parsing an empty Segment object. Task InternalTestSegment_Empty() { var json = new Dictionary(); @@ -1466,5 +1504,20 @@ async Task InternalTestGenerateImagesBase64SomeFiltered() { Assert($"Failed to convert Image {i}", texture != null); } } + + async Task InternalTestThoughtSummary() { + Dictionary json = await GetVertexJsonTestData("unary-success-thinking-reply-thought-summary.json"); + GenerateContentResponse response = GenerateContentResponse.FromJson(json, FirebaseAI.Backend.InternalProvider.VertexAI); + + AssertEq("Response text", response.Text, "Mountain View"); + + AssertEq("Thought summary", response.ThoughtSummary, + "Right, someone needs the city where Google's headquarters are. " + + "I already know this. No need to overthink it, it's a straightforward request. " + + "Let me just pull up the city name from memory... " + + "Mountain View. That's it. Just the city, nothing else. Got it.\n"); + + ValidateUsageMetadata(response.UsageMetadata, 13, 2, 39, 54); + } } } diff --git a/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json new file mode 100644 index 00000000..36760129 --- /dev/null +++ b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json @@ -0,0 +1,42 @@ +{ + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "text": "Right, someone needs the city where Google's headquarters are. I already know this. No need to overthink it, it's a straightforward request. Let me just pull up the city name from memory... Mountain View. That's it. Just the city, nothing else. Got it.\n", + "thought": true + }, + { + "text": "Mountain View" + } + ] + }, + "finishReason": "STOP", + "avgLogprobs": -2.241081714630127 + } + ], + "usageMetadata": { + "promptTokenCount": 13, + "candidatesTokenCount": 2, + "totalTokenCount": 54, + "trafficType": "ON_DEMAND", + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 13 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 2 + } + ], + "thoughtsTokenCount": 39 + }, + "modelVersion": "gemini-2.5-flash", + "createTime": "2025-07-28T15:44:18.615090Z", + "responseId": "0pqHaLLFJf3sqsMPiLnviQw" +} \ No newline at end of file diff --git a/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json.meta b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json.meta new file mode 100644 index 00000000..a77a99f8 --- /dev/null +++ b/firebaseai/testapp/Assets/StreamingAssets/TestData/vertexai/unary-success-thinking-reply-thought-summary.json.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: d55d99a5733a7410596ff5fc381830a4 +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: