diff --git a/README.md b/README.md index 0c91c1a..b54a29f 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ This node pack currently includes the following nodes: - **Gemini 3 Pro (LLM) [Experimental]** - Text generation node using Gemini 3 Pro chat completions. - - Supports optional reasoning output and JSON payload inputs. + - Supports role selection, media inputs (images/video), optional reasoning output, and JSON payload inputs. - **Kling 2.6 Image-to-Video** - Generates video from a single input image. @@ -143,6 +143,7 @@ Feel free to adapt, simplify, or remix them to fit your own pipelines. ## Changelog - 2026-01-30: Added Flux 2 Image-to-Image node (Pro/Flex) with model dropdown. - 2026-01-30: Added Gemini 3 Pro LLM node (phase 1, experimental). +- 2026-01-30: Gemini 3 Pro LLM updated with role dropdown and media inputs (phase 1.5). ## About Kie.ai Kie.ai is a unified API and model marketplace for image, video, and audio generation. This project is community-maintained and not affiliated with Kie.ai. Learn more at [https://kie.ai](https://kie.ai?ref=e7565cf24a7fad4586341a87eaf21e42). diff --git a/kie_api/gemini3_pro_llm.py b/kie_api/gemini3_pro_llm.py index 8cae33f..1cd6ad7 100644 --- a/kie_api/gemini3_pro_llm.py +++ b/kie_api/gemini3_pro_llm.py @@ -6,9 +6,12 @@ from typing import Any from .auth import _load_api_key from .http import TransientKieError, requests from .log import _log +from .upload import _image_tensor_to_png_bytes, _truncate_url, _upload_image, _upload_video +from .video import _coerce_video_to_mp4_bytes CHAT_COMPLETIONS_URL = "https://api.kie.ai/gemini-3-pro/v1/chat/completions" REASONING_EFFORT_OPTIONS = ["low", "high"] +ROLE_OPTIONS = ["developer", "system", "user", "assistant", "tool"] def _parse_json_optional(raw: str | None, label: str) -> Any | None: @@ -23,20 +26,38 @@ def _parse_json_optional(raw: str | None, label: str) -> Any | None: raise RuntimeError(f"{label} is not valid JSON.") from exc -def _normalize_messages(prompt: str, messages_json: str | None) -> list[dict[str, Any]]: +def _normalize_messages( + prompt: str, + messages_json: str | None, + role: str, + image_urls: list[str], + video_urls: list[str], +) -> list[dict[str, Any]]: if messages_json: data = _parse_json_optional(messages_json, "messages_json") if not isinstance(data, list): raise RuntimeError("messages_json must be a JSON array of message objects.") return data + if role not in ROLE_OPTIONS: + raise RuntimeError("Invalid role. Use the pinned enum options.") + prompt_text = (prompt or "").strip() - if not prompt_text: - raise RuntimeError("prompt is required when messages_json is not provided.") + if not prompt_text and not image_urls and not video_urls: + raise RuntimeError("prompt or media input is required when messages_json is not provided.") + + content: list[dict[str, Any]] = [] + if prompt_text: + content.append({"type": "text", "text": prompt_text}) + for url in image_urls: + content.append({"type": "image_url", "image_url": {"url": url}}) + for url in video_urls: + content.append({"type": "image_url", "image_url": {"url": url}}) + return [ { - "role": "user", - "content": [{"type": "text", "text": prompt_text}], + "role": role, + "content": content, } ] @@ -65,6 +86,9 @@ def run_gemini3_pro_chat( *, prompt: str, messages_json: str | None = None, + role: str = "user", + images: Any | None = None, + video: Any | None = None, stream: bool = True, include_thoughts: bool = True, reasoning_effort: str = "high", @@ -86,7 +110,41 @@ def run_gemini3_pro_chat( raise RuntimeError("response_format cannot be used with tools.") _validate_tools_payload(tools_payload) - messages = _normalize_messages(prompt, messages_json) + api_key = _load_api_key() + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + + image_urls: list[str] = [] + video_urls: list[str] = [] + + if messages_json: + if images is not None or video is not None: + raise RuntimeError("images/video inputs cannot be used with messages_json.") + else: + if images is not None: + if not hasattr(images, "shape"): + raise RuntimeError("images input must be a tensor batch.") + if images.dim() != 4 or images.shape[-1] != 3: + raise RuntimeError("images input must have shape [B, H, W, 3].") + total_images = images.shape[0] + if total_images > 0: + _log(log, f"Uploading {total_images} image(s) for Gemini 3 Pro...") + for idx in range(total_images): + png_bytes = _image_tensor_to_png_bytes(images[idx]) + url = _upload_image(api_key, png_bytes) + image_urls.append(url) + _log(log, f"Image {idx + 1} upload success: {_truncate_url(url)}") + + if video is not None: + video_bytes, source = _coerce_video_to_mp4_bytes(video) + _log(log, f"Uploading video for Gemini 3 Pro ({source})...") + video_url = _upload_video(api_key, video_bytes) + video_urls.append(video_url) + _log(log, f"Video upload success: {_truncate_url(video_url)}") + + messages = _normalize_messages(prompt, messages_json, role, image_urls, video_urls) payload: dict[str, Any] = { "messages": messages, @@ -99,12 +157,6 @@ def run_gemini3_pro_chat( if response_format_payload is not None: payload["response_format"] = response_format_payload - api_key = _load_api_key() - headers = { - "Authorization": f"Bearer {api_key}", - "Content-Type": "application/json", - } - try: response = requests.post( CHAT_COMPLETIONS_URL, diff --git a/nodes.py b/nodes.py index 87dae1b..42794fd 100644 --- a/nodes.py +++ b/nodes.py @@ -43,6 +43,7 @@ from .kie_api.kling26_t2v import ( ) from .kie_api.gemini3_pro_llm import ( REASONING_EFFORT_OPTIONS as GEMINI3_REASONING_EFFORT_OPTIONS, + ROLE_OPTIONS as GEMINI3_ROLE_OPTIONS, run_gemini3_pro_chat, ) from .kie_api.flux2_i2i import ( @@ -640,7 +641,10 @@ Generate text using Gemini 3 Pro. Inputs: - prompt: Text prompt (required if messages_json is empty) -- messages_json: Optional JSON array of message objects (overrides prompt) +- role: Message role for the prompt (developer/system/user/assistant/tool) +- images: Optional image batch to include as media content +- video: Optional video input to include as media content +- messages_json: Optional JSON array of message objects (overrides prompt/role/media) - stream: Stream responses (SSE); output is returned after completion - include_thoughts: Include reasoning content in output - reasoning_effort: low or high @@ -659,8 +663,11 @@ Outputs: return { "required": { "prompt": ("STRING", {"multiline": True}), + "role": ("COMBO", {"options": GEMINI3_ROLE_OPTIONS, "default": "user"}), }, "optional": { + "images": ("IMAGE",), + "video": ("VIDEO",), "messages_json": ("STRING", {"multiline": True, "default": ""}), "stream": ("BOOLEAN", {"default": True}), "include_thoughts": ("BOOLEAN", {"default": True}), @@ -679,6 +686,9 @@ Outputs: def generate( self, prompt: str, + role: str = "user", + images: torch.Tensor | None = None, + video: object | None = None, messages_json: str = "", stream: bool = True, include_thoughts: bool = True, @@ -690,6 +700,9 @@ Outputs: content, reasoning, raw_json = run_gemini3_pro_chat( prompt=prompt, messages_json=messages_json, + role=role, + images=images, + video=video, stream=stream, include_thoughts=include_thoughts, reasoning_effort=reasoning_effort, diff --git a/web/docs/KIE_Gemini3Pro_LLM.md b/web/docs/KIE_Gemini3Pro_LLM.md index 7edca76..e35189e 100644 --- a/web/docs/KIE_Gemini3Pro_LLM.md +++ b/web/docs/KIE_Gemini3Pro_LLM.md @@ -3,14 +3,30 @@ Generate text using Gemini 3 Pro. This node is experimental. ## Inputs -- **prompt** (STRING, required): Text prompt. Used if `messages_json` is empty. -- **messages_json** (STRING, optional): JSON array of message objects (overrides `prompt`). -- **stream** (BOOLEAN, optional): Stream responses via SSE. Output is returned after completion. -- **include_thoughts** (BOOLEAN, optional): Include reasoning content. -- **reasoning_effort** (COMBO, optional): `low`, `high` +- **prompt** (STRING, required): Main text prompt. Used when `messages_json` is empty. +- **role** (COMBO, required): Message role for the prompt. Options: `developer`, `system`, `user`, `assistant`, `tool`. +- **images** (IMAGE, optional): One or more images (batch) to include as media content. +- **video** (VIDEO, optional): A single video input to include as media content. +- **messages_json** (STRING, optional): Full JSON array of message objects. If provided, it overrides `prompt`, `role`, and media inputs. +- **stream** (BOOLEAN, optional): Use streaming responses (SSE). Output is returned after completion. +- **include_thoughts** (BOOLEAN, optional): When enabled, reasoning content is returned in the reasoning output. +- **reasoning_effort** (COMBO, optional): Controls reasoning depth: `low` or `high`. - **tools_json** (STRING, optional): JSON array of tool definitions. Mutually exclusive with `response_format_json`. - **response_format_json** (STRING, optional): JSON schema output format. Mutually exclusive with `tools_json`. -- **log** (BOOLEAN, optional): Enable console logging. +- **log** (BOOLEAN, optional): Enable console logging for uploads and response status. + +## Node UI Reference (What each widget expects) +- **Prompt (text box)**: A natural language instruction or question. Leave empty only if you provide `messages_json` or media. +- **Role (dropdown)**: Sets the `role` field in the message object (default: `user`). +- **Images (input socket)**: Connect a ComfyUI IMAGE or batch. All images are uploaded and appended to message content. +- **Video (input socket)**: Connect a ComfyUI VIDEO. The video is uploaded and appended to message content. +- **Messages JSON (text box)**: Raw JSON array of messages (advanced). Overrides prompt/role/media entirely. +- **Stream (toggle)**: Enables SSE streaming under the hood; output still returns when complete. +- **Include Thoughts (toggle)**: When on, reasoning text is captured and returned in the second output. +- **Reasoning Effort (dropdown)**: `low` is faster, `high` is more thorough. +- **Tools JSON (text box)**: Raw JSON array of tool definitions (advanced). +- **Response Format JSON (text box)**: Raw JSON schema definition (advanced). +- **Log (toggle)**: Emits upload and completion messages to the console. ## Outputs - **STRING**: Assistant response text. @@ -20,4 +36,5 @@ Generate text using Gemini 3 Pro. This node is experimental. ## Helper behavior - Validates `reasoning_effort`. - Enforces mutual exclusivity for tools vs response_format. -- If `messages_json` is provided, it is used as the message array. +- If `messages_json` is provided, it is used as the message array and media inputs are rejected. +- Media inputs are uploaded and added to the message content using the unified `image_url` format.