From cd8b333527c369f6e459525a1c8b195af8f06c6e Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 10 Feb 2026 11:26:12 +0000 Subject: [PATCH 1/6] [Opal ADK] Adds a session id to opal-adk-stream. This will allow passing off a session id to resume a previously started agent conversation in Opal ADK. --- packages/visual-editor/src/a2/a2/opal-adk-stream.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index 9640dc28b65..0e4536bab17 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -13,6 +13,7 @@ import { PidginTranslator } from "../agent/pidgin-translator.js"; import { AgentUI } from "../agent/ui.js"; import { AgentFileSystem } from "../agent/file-system.js"; import { MemoryManager } from "../agent/types.js"; +import { session } from "../../../eval/eval.js"; const OPAL_ADK_ENDPOINT = new URL( "v1beta1/executeAgentNodeStream", @@ -200,9 +201,7 @@ class OpalAdkStream { modelConstraint?: ModelConstraint, uiType?: string, uiPrompt?: LLMContent, - invocationId?: string, - sessionId?: string - ): Promise> { + invocationId?: string): Promise> { const ui = this.ui; if (!params || params.length === 0) { @@ -230,7 +229,6 @@ class OpalAdkStream { uiPrompt, nodeApi: opalAdkAgent, invocationId, - sessionId, }); ui.progress.sendOpalAdkRequest("", requestBody); From 85edf80a2dac8755f322163e130357007d2764a3 Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 10 Feb 2026 11:43:15 +0000 Subject: [PATCH 2/6] Removes bad import. --- packages/visual-editor/src/a2/a2/opal-adk-stream.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index 0e4536bab17..d82ffe63c6d 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -13,7 +13,6 @@ import { PidginTranslator } from "../agent/pidgin-translator.js"; import { AgentUI } from "../agent/ui.js"; import { AgentFileSystem } from "../agent/file-system.js"; import { MemoryManager } from "../agent/types.js"; -import { session } from "../../../eval/eval.js"; const OPAL_ADK_ENDPOINT = new URL( "v1beta1/executeAgentNodeStream", From d224974dee1733bc4af3686ad406c59d14e5ff51 Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 17 Feb 2026 16:48:07 +0000 Subject: [PATCH 3/6] [Opal ADK] Wires up the ability to send binary data to Opal ADK. This technically works. However, tuning will need to be done to the prompts provided by the planner and the substituting logic in Breadboard. The Opal ADK agent gets very confused when an input is referenced but it doesn't see it as part of its input. This is because binary input is stored in artifacts in ADK. I updated some prompts on the backend to indicate it should look in the artifacts to find the uploads and this helps but it has a way to go. This will provide a good place to start as it will still send the data to Opal ADK and you can see the agent bang its head against a wall. --- .../src/a2/a2/opal-adk-stream.ts | 124 +++++++++++------- .../visual-editor/src/a2/agent/agent-adk.ts | 18 ++- .../src/a2/deep-research/main.ts | 9 +- 3 files changed, 97 insertions(+), 54 deletions(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index d82ffe63c6d..4dc7df0eaf8 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -8,7 +8,6 @@ import { import { err, toLLMContent } from "./utils.js"; import { A2ModuleArgs } from "../runnable-module-factory.js"; import { iteratorFromStream } from "@breadboard-ai/utils"; -import { ModelConstraint } from "../agent/functions/generate.js"; import { PidginTranslator } from "../agent/pidgin-translator.js"; import { AgentUI } from "../agent/ui.js"; import { AgentFileSystem } from "../agent/file-system.js"; @@ -66,6 +65,7 @@ export type PlanStep = { export interface BuildStreamingRequestBodyOptions { completedPrompt: LLMContent; + executionInputs?: LLMContent[]; modelConstraint?: string; uiType?: string; uiPrompt?: LLMContent; @@ -76,15 +76,21 @@ export interface BuildStreamingRequestBodyOptions { type StreamingRequestPart = { text?: string; - partMetadata?: { input_name: string }; + inline_data?: { + mime_type: string; + data: string; + }; + file_data?: { + mime_type: string; + file_uri: string; + }; }; type StreamingRequestBody = { session_id?: string; objective?: LLMContent; - model_name?: string; invocation_id?: string; - contents?: Array<{ + execution_inputs?: Record; @@ -92,7 +98,6 @@ type StreamingRequestBody = { node_api?: string; }; agent_mode_node_config?: { - model_constraint?: string; ui_type?: string; ui_prompt?: LLMContent; }; @@ -129,43 +134,78 @@ class OpalAdkStream { } } - buildStreamingRequestBody( - options: BuildStreamingRequestBodyOptions - ): StreamingRequestBody { + buildStreamingRequestBody(options: BuildStreamingRequestBodyOptions): Outcome { const { completedPrompt, + executionInputs, uiType, uiPrompt, nodeApi, invocationId, sessionId, } = options; - console.log("uiType: ", uiType); - const contents: NonNullable = []; - - let textCount = 0; + const execution_inputs: NonNullable = {}; + let inputCount = 0; if (!completedPrompt.parts) { - console.error("opal-adk-stream: Missing required prompt."); - } - for (const part of completedPrompt.parts) { - if ("text" in part) { - textCount++; - contents.push({ - parts: [ - { - text: part.text, - partMetadata: { input_name: `text_${textCount}` }, - }, - ], + const error = err("opal-adk-stream: Missing required prompt."); + console.error(error); + return error; + }; + if (executionInputs) { + for (const content of executionInputs) { + if (!content.parts) { + const error = err("opal-adk-stream: Execution input has no parts."); + console.error(error); + return error; + } + if (content.parts.length > 1) { + const error = err("opal-adk-stream: Execution input has multiple part."); + console.error(error); + return error; + } + const part = content.parts[0]; + inputCount++; + let inputName = ""; + const requestPart: StreamingRequestPart = {}; + + if ("text" in part) { + inputName = "input_text"; + requestPart.text = part.text; + } else if ("inlineData" in part) { + const mime_type = part.inlineData.mimeType; + if (mime_type.startsWith("image/")) { + inputName = "input_image"; + } else if (mime_type.startsWith("audio/")) { + inputName = "input_audio"; + } else if (mime_type.startsWith("video/")) { + inputName = "input_video"; + } else { + inputName = "input_data"; + } + requestPart.inline_data = { + mime_type: part.inlineData.mimeType, + data: part.inlineData.data, + }; + } else if ("fileData" in part) { + inputName = "input_file"; + requestPart.file_data = { + mime_type: part.fileData.mimeType, + file_uri: part.fileData.fileUri, + }; + } else { + continue; + } + + execution_inputs[inputName] = { + parts: [requestPart], role: "user", - }); + }; } } const baseBody: StreamingRequestBody = { objective: completedPrompt, - model_name: undefined, - contents, + execution_inputs, }; // 'node-agent' is specifically for agent mode while any other @@ -195,25 +235,17 @@ class OpalAdkStream { } async executeOpalAdkStream( + objective: LLMContent, opalAdkAgent?: string, params?: LLMContent[], - modelConstraint?: ModelConstraint, uiType?: string, uiPrompt?: LLMContent, invocationId?: string): Promise> { const ui = this.ui; - if (!params || params.length === 0) { - return err("opal-adk-stream: No params provided"); - } - if (modelConstraint === undefined) { - modelConstraint = "none"; - } if (opalAdkAgent && !VALID_NODE_KEYS.includes(opalAdkAgent)) { - const error = err( - `opal-adk-stream: Invalid node key: ${opalAdkAgent}, ` + - `valid keys are: ${VALID_NODE_KEYS.join(", ")}` - ); + const error = err(`opal-adk-stream: Invalid node key: ${opalAdkAgent}, ` + + `valid keys are: ${VALID_NODE_KEYS.join(", ")}`); console.error(error); return error; } @@ -222,15 +254,21 @@ class OpalAdkStream { const baseUrl = OPAL_ADK_ENDPOINT; const url = new URL(baseUrl); url.searchParams.set("alt", "sse"); - const requestBody = this.buildStreamingRequestBody({ - completedPrompt: params[0], + const requestBodyOrError = this.buildStreamingRequestBody({ + completedPrompt: objective, + executionInputs: params, uiType, uiPrompt, nodeApi: opalAdkAgent, invocationId, + sessionId: '10' }); - ui.progress.sendOpalAdkRequest("", requestBody); + if (!ok(requestBodyOrError)) { + return requestBodyOrError; + } + const requestBody = requestBodyOrError; + ui.progress.sendOpalAdkRequest("", requestBody) const response = await this.moduleArgs.fetchWithCreds(url.toString(), { method: "POST", headers: { "Content-Type": "application/json" }, @@ -241,9 +279,7 @@ class OpalAdkStream { console.log("response: ", response); if (!response.ok) { const errorText = await response.text(); - const error = err( - `Streaming request failed: ${response.status} ${errorText}` - ); + const error = err(`Streaming request failed: ${response.status} ${errorText}`); console.error(error); return error; } diff --git a/packages/visual-editor/src/a2/agent/agent-adk.ts b/packages/visual-editor/src/a2/agent/agent-adk.ts index 3f9874d5e64..e5f688e972f 100644 --- a/packages/visual-editor/src/a2/agent/agent-adk.ts +++ b/packages/visual-editor/src/a2/agent/agent-adk.ts @@ -9,6 +9,7 @@ import { } from "@breadboard-ai/types"; import { ok } from "@breadboard-ai/utils"; import { Template } from "../a2/template.js"; +import { toLLMContent, isLLMContent } from "../a2/utils.js"; import { A2ModuleArgs } from "../runnable-module-factory.js"; import { OpalAdkStream, NODE_AGENT_KEY } from "../a2/opal-adk-stream.js"; import { AgentInputs, AgentOutputs, toAgentOutputs } from "./main.js" @@ -27,18 +28,25 @@ export async function invokeAgentAdk( const params = Object.fromEntries( Object.entries(rest).filter(([key]) => key.startsWith("p-z-")) ); - const opalAdkStream = new OpalAdkStream(moduleArgs); + + const opalAdkStream = new OpalAdkStream(caps, moduleArgs); const uiType = enableA2UI ? "a2ui" : "chat"; - const template = new Template(prompt_template, moduleArgs.context.currentGraph); + const template = new Template(caps, prompt_template); + // TODO: This is going to need to be updated for Opal ADK subtitution. const completed_prompt = await template.substitute(params); if (!ok(completed_prompt)) { return completed_prompt; } - console.log("substitutine: ", completed_prompt); + + const paramsArray = Object.entries(rest) + .filter(([key]) => key.startsWith("p-z-")) + .map(([_, value]) => + isLLMContent(value) ? value : toLLMContent(String(value)) + ); const results = await opalAdkStream.executeOpalAdkStream( + completed_prompt, NODE_AGENT_KEY, - [completed_prompt], - "none", + paramsArray, uiType, uiPrompt, invocation_id, diff --git a/packages/visual-editor/src/a2/deep-research/main.ts b/packages/visual-editor/src/a2/deep-research/main.ts index ac2e524aab1..76fd6d7c77b 100644 --- a/packages/visual-editor/src/a2/deep-research/main.ts +++ b/packages/visual-editor/src/a2/deep-research/main.ts @@ -159,11 +159,10 @@ async function invokeOpalAdk( if (!ok(substituting)) { return substituting; } - const opalAdkStream = new OpalAdkStream(moduleArgs); - const results = await opalAdkStream.executeOpalAdkStream(DEEP_RESEARCH_KEY, [ - substituting, - ]); - console.log("deep-research results", results); + const opalAdkStream = new OpalAdkStream(caps, moduleArgs); + const results = await opalAdkStream + .executeOpalAdkStream(substituting, DEEP_RESEARCH_KEY); + console.log("deep-research results", results) return { context: [...(context || []), results], }; From 9ebb36f3b30d25239a672e2a13b16f6412c29667 Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 17 Feb 2026 17:22:09 +0000 Subject: [PATCH 4/6] [Opal ADK] Fixes some bad merges. --- packages/visual-editor/src/a2/a2/opal-adk-stream.ts | 2 +- packages/visual-editor/src/a2/agent/agent-adk.ts | 4 ++-- packages/visual-editor/src/a2/deep-research/main.ts | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index 4dc7df0eaf8..1b92a6904a4 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -5,7 +5,7 @@ import { OPAL_BACKEND_API_PREFIX, Outcome, } from "@breadboard-ai/types"; -import { err, toLLMContent } from "./utils.js"; +import { err, ok, toLLMContent } from "./utils.js"; import { A2ModuleArgs } from "../runnable-module-factory.js"; import { iteratorFromStream } from "@breadboard-ai/utils"; import { PidginTranslator } from "../agent/pidgin-translator.js"; diff --git a/packages/visual-editor/src/a2/agent/agent-adk.ts b/packages/visual-editor/src/a2/agent/agent-adk.ts index e5f688e972f..92393cc2bae 100644 --- a/packages/visual-editor/src/a2/agent/agent-adk.ts +++ b/packages/visual-editor/src/a2/agent/agent-adk.ts @@ -29,9 +29,9 @@ export async function invokeAgentAdk( Object.entries(rest).filter(([key]) => key.startsWith("p-z-")) ); - const opalAdkStream = new OpalAdkStream(caps, moduleArgs); + const opalAdkStream = new OpalAdkStream(moduleArgs); const uiType = enableA2UI ? "a2ui" : "chat"; - const template = new Template(caps, prompt_template); + const template = new Template(prompt_template); // TODO: This is going to need to be updated for Opal ADK subtitution. const completed_prompt = await template.substitute(params); if (!ok(completed_prompt)) { diff --git a/packages/visual-editor/src/a2/deep-research/main.ts b/packages/visual-editor/src/a2/deep-research/main.ts index 76fd6d7c77b..b7ca51a78b6 100644 --- a/packages/visual-editor/src/a2/deep-research/main.ts +++ b/packages/visual-editor/src/a2/deep-research/main.ts @@ -159,7 +159,7 @@ async function invokeOpalAdk( if (!ok(substituting)) { return substituting; } - const opalAdkStream = new OpalAdkStream(caps, moduleArgs); + const opalAdkStream = new OpalAdkStream(moduleArgs); const results = await opalAdkStream .executeOpalAdkStream(substituting, DEEP_RESEARCH_KEY); console.log("deep-research results", results) From 6f515c5d3193d73c69e28b3784f5bb81f7e11033 Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 17 Feb 2026 17:23:53 +0000 Subject: [PATCH 5/6] [Opal ADK] Adds sessionId back to the executeOpalAdkStream. --- packages/visual-editor/src/a2/a2/opal-adk-stream.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index 1b92a6904a4..3bb7ff4c4c6 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -240,7 +240,8 @@ class OpalAdkStream { params?: LLMContent[], uiType?: string, uiPrompt?: LLMContent, - invocationId?: string): Promise> { + invocationId?: string, + sessionId?: string): Promise> { const ui = this.ui; if (opalAdkAgent && !VALID_NODE_KEYS.includes(opalAdkAgent)) { @@ -261,7 +262,7 @@ class OpalAdkStream { uiPrompt, nodeApi: opalAdkAgent, invocationId, - sessionId: '10' + sessionId }); if (!ok(requestBodyOrError)) { From 36200ba18a626f906d451d7276e1758d52a8edad Mon Sep 17 00:00:00 2001 From: walker7734 Date: Tue, 17 Feb 2026 17:34:00 +0000 Subject: [PATCH 6/6] [Opal ADK] Fixes linter issue. --- packages/visual-editor/src/a2/a2/opal-adk-stream.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts index 3bb7ff4c4c6..0be55275258 100644 --- a/packages/visual-editor/src/a2/a2/opal-adk-stream.ts +++ b/packages/visual-editor/src/a2/a2/opal-adk-stream.ts @@ -145,7 +145,6 @@ class OpalAdkStream { sessionId, } = options; const execution_inputs: NonNullable = {}; - let inputCount = 0; if (!completedPrompt.parts) { const error = err("opal-adk-stream: Missing required prompt."); console.error(error); @@ -164,7 +163,6 @@ class OpalAdkStream { return error; } const part = content.parts[0]; - inputCount++; let inputName = ""; const requestPart: StreamingRequestPart = {};