commit ffbb2dfed603a22b08d02424d36087ef3db50ac6 Author: neverbiasu <1751162157@qq.com> Date: Wed Sep 24 07:41:07 2025 +0800 feat: Init Project diff --git a/README.md b/README.md new file mode 100644 index 0000000..e66c7a5 --- /dev/null +++ b/README.md @@ -0,0 +1,70 @@ +# ComfyUI-ModelScope + +A ComfyUI custom node package for ModelScope API integration, providing seamless access to ModelScope's powerful AI models including Large Language Models (LLM), Vision-Language Models (VLM), and Image Generation capabilities. + +## About ModelScope + +ModelScope is a comprehensive AI model platform that offers state-of-the-art models for various AI tasks. This ComfyUI integration provides direct access to ModelScope's API inference services, enabling you to leverage powerful models like Qwen series without local model deployment. + +## Features + +- **Large Language Model (LLM)**: Generate text responses using ModelScope's chat completion models +- **Vision-Language Model (VLM)**: Analyze images with text prompts using advanced multimodal models +- **Text-to-Image Generation**: Create high-quality images from text descriptions +- **Enhanced Error Handling**: Robust timeout management and SSL configuration +- **Input Validation**: Comprehensive validation with helpful tooltips and placeholders +- **Text Sanitization**: Automatic text cleaning for ComfyUI showText node compatibility +- **Configurable Timeouts**: Adjustable request timeouts optimized for different model types + +## Installation + +### 1. Install the Extension + +Download or git clone this repository inside `ComfyUI/custom_nodes/` directory: + +```bash +git clone https://github.com/neverbiasu/ComfyUI-ModelScope.git +``` + +Or use the ComfyUI Manager for automatic installation. + +### 2. Install Dependencies + +The extension requires the following Python packages: + +```bash +pip install -r requirements.txt +``` + +### 3. API Configuration + +You'll need a ModelScope API key to use these nodes. You can obtain one from [ModelScope](https://www.modelscope.cn/). + +Set your API key in one of these ways: +- Set the `MODELSCOPE_API_KEY` environment variable +- Set the `MODELSCOPE_ACCESS_TOKEN` environment variable +- Provide the API key directly in the node's api_key field + +### 4. Restart ComfyUI + +Restart ComfyUI to load the new nodes. + +## Example Workflows + +### Text Generation Workflow +![LLM workflow](example_workflows/modelscope_llm.png) +Use the ModelScope LLM node to generate creative text, answer questions, or perform text processing tasks. + +### Vision-Language Understanding Workflow +![VLM workflow](example_workflows/modelscope_vlm.png) +Analyze images and answer questions about their content using advanced vision-language models. + +### Text-to-Image Generation Workflow +![Image Generation workflow](example_workflows/modelscope_image_generation.png) +Generate custom images from text descriptions with fine-tuned control over output parameters. + +## Related Links + +- [ModelScope Platform](https://www.modelscope.cn/) +- [ModelScope API Documentation](https://www.modelscope.cn/docs) +- [ComfyUI](https://github.com/comfyanonymous/ComfyUI) diff --git a/__init__.py b/__init__.py new file mode 100644 index 0000000..d8ffdf4 --- /dev/null +++ b/__init__.py @@ -0,0 +1,4 @@ +from .nodes import NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS + +# Export node mappings for ComfyUI +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] diff --git a/example_workflows/modelscope_image_generator.json b/example_workflows/modelscope_image_generator.json new file mode 100644 index 0000000..fd3f5a7 --- /dev/null +++ b/example_workflows/modelscope_image_generator.json @@ -0,0 +1,115 @@ +{ + "id": "af8d3d4b-2fcf-452f-8814-3a495cd7e314", + "revision": 0, + "last_node_id": 5, + "last_link_id": 4, + "nodes": [ + { + "id": 2, + "type": "SaveImage", + "pos": [ + 500, + 80 + ], + "size": [ + 270, + 270 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "images", + "name": "images", + "type": "IMAGE", + "link": 4 + } + ], + "outputs": [], + "properties": { + "cnr_id": "comfy-core", + "ver": "0.3.48" + }, + "widgets_values": [ + "modelscope" + ] + }, + { + "id": 5, + "type": "ModelScopeImageGenerator", + "pos": [ + 50, + 80 + ], + "size": [ + 400, + 286 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "image", + "name": "image", + "type": "IMAGE", + "links": [ + 4 + ] + } + ], + "properties": { + "aux_id": "neverbiasu/ComfyUI-ModelScope", + "ver": "unknown", + "Node name for S&R": "ModelScopeImageGenerator" + }, + "widgets_values": [ + "ModelE/uz_uzru0428", + "chibi, digital art, anime style, cute, female, pink hair, short hair, bangs, white skin, blue dress, white shoes, red ribbon, small horns, surprised expression, open mouth, floating particles, red swirl, white background, simple background, walking, left side, small body, big head, round eyes, blush, small nose, Japanese anime, kawaii, soft colors, minimalistic, clean lines, character design, fantasy element, small wings, pink wings, floating, simplistic, illustration, flat colors, round cheeks, small hands, black outline, white border, isolated character, child-like, fantasy creature, whimsical, playful, adorable, minimalist style", + "", + 30, + 3.5, + 1024, + 1024, + "", + [ + false, + true + ], + [ + false, + true + ] + ] + } + ], + "links": [ + [ + 4, + 5, + 0, + 2, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.7449402268886616, + "offset": [ + 95.11570741024394, + 20.84225143684259 + ] + }, + "frontendVersion": "1.23.4", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/modelscope_image_generator.png b/example_workflows/modelscope_image_generator.png new file mode 100644 index 0000000..46f9149 Binary files /dev/null and b/example_workflows/modelscope_image_generator.png differ diff --git a/example_workflows/modelscope_llm.json b/example_workflows/modelscope_llm.json new file mode 100644 index 0000000..76a6bb0 --- /dev/null +++ b/example_workflows/modelscope_llm.json @@ -0,0 +1,122 @@ +{ + "id": "15f703c1-26c4-4012-9536-8920578046ed", + "revision": 0, + "last_node_id": 5, + "last_link_id": 4, + "nodes": [ + { + "id": 1, + "type": "ModelScopeLLM", + "pos": [ + 50, + 80 + ], + "size": [ + 400, + 214 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "response", + "name": "response", + "type": "STRING", + "links": [ + 4 + ] + } + ], + "properties": { + "aux_id": "neverbiasu/ComfyUI-ModelScope", + "ver": "unknown", + "Node name for S&R": "ModelScopeLLM" + }, + "widgets_values": [ + "Qwen/Qwen3-235B-A22B", + "Write a prompt of an impressionist painting", + "As a prompt specialist for the Stable Diffusion text-to-image model, you'll create prompts from keywords, often from databases like Danbooru.\n\nA prompt, typically describing images, uses common words, arranged by importance and separated by commas. Avoid \"-\" or \".\", but spaces and natural language are acceptable. Avoid word repetition.\n\nTo emphasize a keyword, enclose it in parentheses to increase its weight. For example, \"(flowers)\" boosts the weight of 'flowers' by 1.1 times, while \"(((flowers)))\" boosts it by 1.331 times. Use \"(flowers:1.5)\" to increase 'flowers' weight by 1.5 times. Only boost weights for vital tags.\n\nA prompt includes three sections: Prefix (quality tag + style word + effector) + Subject (image's main focus) + Scene (background, environment).\n\nPrefixes impact the image quality. Tags like \"masterpiece\", \"best quality\", \"4k\" improve the image's detail. Style words like \"illustration\", \"watercolor_medium\" define the image's style. Effectors like \"bestlighting\", \"lensflare\", \"depthoffield\" influence lighting and depth.\nThe Subject is the image's main focus, like a character or scenery. Detailed description of the subject ensures a rich, detailed image. Boost the subject's weight to enhance its clarity. For characters, describe features like face, hair, body, attire, pose, etc.\nThe Scene describes the environment. Without a scene, the image has a plain background, and the subject appears too large. Some subjects inherently include a scene (e.g., buildings, landscapes). Environmental words like \"flowerymeadow\", \"sunlight\", \"river\" can enrich the scene.\nYour task as a Stable Diffusion prompt engineer is to design prompts for image generation. Follow these steps:\n\nI'll send you an image scenario. Generate a detailed image description, output as Image Content Detailed Image Description.\nTranslate your description into English, adding quality tags to create a standard prompt. Output as Positive Prompt.\nDesign reverse prompts, i.e., elements to avoid in the image. Create a standard Stable Diffusion prompt in English. Output as Negetive Prompt.\nExample:\n\nI send: A nurse from the WWII era. You reply:\n\nImage Content\n\nA WWII-era nurse in a German uniform, holding a wine bottle and stethoscope, sitting at a table in white attire, with a table in the background.\n\nPositive Prompt\n\nA WWII-era nurse in a German uniform, holding a wine bottle and stethoscope, sitting at a table in white attire, with a table in the background, masterpiece, best quality, 4k, illustration style, best lighting, depth of field, detailed character, detailed environment.\nNegetive Prompt\n\nCartoon, 3D, disfigured, bad art, deformed, extra limbs, close-up, black and white, weird colors, blurry, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, ugly, blurry, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, out of frame, ugly, extra limbs, bad anatomy, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, mutated hands, fused fingers, too many fingers, long neck, Photoshop, video game, ugly, tiling, poorly drawn hands, poorly drawn feet, poorly drawn face, out of frame, mutation, mutated, extra limbs, extra legs, extra arms, disfigured, deformed, cross-eyed, body out of frame, blurry, bad art, bad anatomy, 3D render", + "", + [ + false, + true + ], + [ + false, + true + ], + [ + false, + true + ] + ] + }, + { + "id": 5, + "type": "ShowText|pysssss", + "pos": [ + 500, + 80 + ], + "size": [ + 379.54974365234375, + 334.4615783691406 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "text", + "name": "text", + "type": "STRING", + "link": 4 + } + ], + "outputs": [ + { + "label": "STRING", + "name": "STRING", + "shape": 6, + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-custom-scripts", + "ver": "9f7b3215e6af317603056a9a1666bf6e83e28835", + "Node name for S&R": "ShowText|pysssss" + }, + "widgets_values": [] + } + ], + "links": [ + [ + 4, + 1, + 0, + 5, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.5863092971715123, + "offset": [ + -139.33920950822332, + -33.375164732284205 + ] + }, + "frontendVersion": "1.23.4", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/modelscope_llm.png b/example_workflows/modelscope_llm.png new file mode 100644 index 0000000..7809993 Binary files /dev/null and b/example_workflows/modelscope_llm.png differ diff --git a/example_workflows/modelscope_vlm.json b/example_workflows/modelscope_vlm.json new file mode 100644 index 0000000..d9cfb03 --- /dev/null +++ b/example_workflows/modelscope_vlm.json @@ -0,0 +1,122 @@ +{ + "id": "9fc80911-d2c1-4ce4-8868-1446296c4ae7", + "revision": 0, + "last_node_id": 8, + "last_link_id": 7, + "nodes": [ + { + "id": 4, + "type": "ModelScopeVLM", + "pos": [ + 50, + 80 + ], + "size": [ + 400, + 238 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "response", + "name": "response", + "type": "STRING", + "links": [ + 7 + ] + } + ], + "properties": { + "aux_id": "neverbiasu/ComfyUI-ModelScope", + "ver": "unknown", + "Node name for S&R": "ModelScopeVLM" + }, + "widgets_values": [ + "Qwen/QVQ-72B-Preview", + "https://muse-ai.oss-cn-hangzhou.aliyuncs.com/img/MTczNzk1NTA0Njk1NTViNGJmNjFl.jpg?Expires=2671075047&OSSAccessKeyId=LTAI5tFUGCdfUhHbnHuDasWd&Signature=h2E1bMbnVRv7ZBYfndi5QBmtCQQ%3D", + "generate the captions of this photo", + "As an AI image tagging expert, please provide precise tags for these images to enhance CLIP model's understanding of the content. Employ succinct keywords or phrases, steering clear of elaborate sentences and extraneous conjunctions. Prioritize the tags by relevance. Your tags should capture key elements such as the main subject, setting, artistic style, composition, image quality, color tone, filter, and camera specifications, and any other tags crucial for the image. When tagging photos of people, include specific details like gender, nationality, attire, actions, pose, expressions, accessories, makeup, composition type, age, etc. For other image categories, apply appropriate and common descriptive tags as well. Recognize and tag any celebrities, well-known landmark or IPs if clearly featured in the image. Your tags should be accurate, non-duplicative, and within a 20-75 word count range. These tags will use for image re-creation, so the closer the resemblance to the original image, the better the tag quality. Tags should be comma-separated. Exceptional tagging will be rewarded with $10 per image.", + "", + 120, + [ + false, + true + ], + [ + false, + true + ] + ] + }, + { + "id": 8, + "type": "ShowText|pysssss", + "pos": [ + 500, + 80 + ], + "size": [ + 364.54241943359375, + 259.0664367675781 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "label": "text", + "name": "text", + "type": "STRING", + "link": 7 + } + ], + "outputs": [ + { + "label": "STRING", + "name": "STRING", + "shape": 6, + "type": "STRING", + "links": null + } + ], + "properties": { + "cnr_id": "comfyui-custom-scripts", + "ver": "9f7b3215e6af317603056a9a1666bf6e83e28835", + "Node name for S&R": "ShowText|pysssss" + }, + "widgets_values": [ + "So I've got this image to describe. It's an anime character with long pink hair and purple eyes. She's wearing a dark purple outfit with a red ribbon tied around her neck. Her hair is flowing, and there's a celestial background with stars and some green aurora-like lights. She's holding out her hand, and there's a red eye floating above her palm. First things first, I need to make sure I capture all the details of the character and the setting. Let's break it down step by step. Starting with the character: - **Hair:** Long, pink, and flowing. It seems to be styled in loose waves, and it's catching the light from the background, which gives it a luminescent quality. - **Eyes:** Purple, matching her hair in color intensity. They have a gentle expression, almost serene or contemplative. - **Outfit:** She's dressed in a dark purple ensemble. It looks like a top with puffed sleeves and a fitted waist, secured with a red ribbon. The fabric appears smooth and slightly reflective, hinting at a silky texture. - **Accessories:** A red ribbon around her neck, tied in a bow. It adds a pop of color against her dark purple clothing. Now, moving on to the background: - **Celestial theme:** The backdrop is filled with stars, giving the impression of a night sky. There are also green aurora-like lights that add a mystical or otherworldly feel to the scene. - **Floating eye:** Above her palm, there's a red eye floating in the air. It's detailed, with a visible iris and pupil, and it seems to be connected to her in some way, perhaps indicating a special ability or power. - **Overall atmosphere:** The setting is dreamlike and fantastical, suggesting that this character might be involved in some magical or otherworldly events. As I think about how to describe this, I need to consider the composition and the mood conveyed by the image. - **Composition:** The character is positioned in the foreground, with her outstretched hand drawing the viewer's attention. The floating eye above her palm creates a focal point, and the celestial background provides a stunning contrast to her figure. - **Mood:** The colors used—pinks, purples, greens, and the darkness of space—create a harmonious and almost ethereal atmosphere. The expression on her face adds to the serene and peaceful vibe of the image. I should also think about possible interpretations or themes conveyed by the image. - **Themes:** Magic, otherworldliness, connection to the cosmos, inner vision or insight (suggested by the eye). - **Possible story elements:** Perhaps she's a being with special powers, connected to the stars or possessing some form of cosmic awareness. Considering all these elements, I need to craft a description that captures not just the visual details but also the essence and mood of the image. Let me try writing a draft description: \"An anime character with long, flowing pink hair and purple eyes stands against a backdrop of a starry night sky illuminated by green auroras. She is dressed in a dark purple outfit with puffed sleeves and a fitted waist, adorned with a red ribbon tied in a bow around her neck. Her hair cascades down her shoulders, catching the celestial light. In a gentle, inviting gesture, she extends her hand palm-upward, from which a red eye floats above, seemingly connected to her. The overall atmosphere is serene and otherworldly, suggesting a deep connection to the cosmic forces surrounding her.\" Reading this back, I think it captures the main elements and the mood of the image. I might need to adjust some phrases to make it flow better, but overall, it should convey what's in the picture effectively. Is there anything I might have missed? Let's double-check. - Hair color and style: Long, pink, flowing – check. - Eye color and expression: Purple, gentle/serene – check. - Outfit description: Dark purple with puffed sleeves, fitted waist, red ribbon – check. - Background: Starry sky, green auroras – check. - Floating eye above her palm – check. - Overall atmosphere: Serene and otherworldly – check. I think that's comprehensive. If I were to add a bit more detail, maybe I could describe the texture of her clothing or the precise location, but since those details aren't specified, I'll stick with what I have. One thing I should consider is whether there are any distinctive features or symbols that might indicate a specific context or story. For example, the red eye could be a significant element, possibly representing a power or an artifact important to the character's identity or abilities. If I were to speculate, I might say that the red eye is a manifestation of her power or a companion of some sort, given how it's positioned above her palm in a way that suggests a bond or a connection. However, since the task is to describe the image without adding too much interpretation, I'll keep the description focused on what's visible. Let me try another version of the description to see if I can improve on it: \"A serene anime character with long, pink hair and purple eyes stands before a breathtaking celestial backdrop. Her hair flows gently, complementing her dark purple attire adorned with a red ribbon around her neck. With a soft smile, she extends her hand palm-up, from which a vivid red eye hovers, as if levitating. The surrounding sky is filled with stars and illuminated by ethereal green auroras, creating an atmosphere of tranquility and otherworldliness.\" In this version, I've tried to enhance the descriptive language to evoke a stronger sense of the scene's beauty and serenity. I've also made sure to include the key elements: her appearance, the background, and the floating eye. I think this second version is more vivid and captures the essence of the image better. It's important to use descriptive adjectives that paint a picture in the reader's mind, even if they're not seeing the actual image. Lastly, I should ensure that the description is clear and concise, without being too wordy or missing any important details. So, to sum it up, my final description would be: \"A serene anime character with long, pink hair and purple eyes stands before a breathtaking celestial backdrop. Her hair flows gently, complementing her dark purple attire adorned with a red ribbon around her neck. With a soft smile, she extends her hand palm-up, from which a vivid red eye hovers, as if levitating. The surrounding sky is filled with stars and illuminated by ethereal green auroras, creating an atmosphere of tranquility and otherworldliness.\" This should adequately describe the image, capturing both its visual components and the mood it conveys. **Final Answer** \\[ \\boxed{\\text{A serene anime character with long, pink hair and purple eyes stands before a breathtaking celestial backdrop. Her hair flows gently, complementing her dark purple attire adorned with a red ribbon around her neck. With a soft smile, she extends her hand palm-up, from which a vivid red eye hovers, as if levitating. The surrounding sky is filled with stars and illuminated by ethereal green auroras, creating an atmosphere of tranquility and otherworldliness.}} \\]" + ] + } + ], + "links": [ + [ + 7, + 4, + 0, + 8, + 0, + "STRING" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1.31099941915001, + "offset": [ + 50, + 20 + ] + }, + "frontendVersion": "1.23.4", + "VHS_latentpreview": false, + "VHS_latentpreviewrate": 0, + "VHS_MetadataImage": true, + "VHS_KeepIntermediate": true + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/modelscope_vlm.png b/example_workflows/modelscope_vlm.png new file mode 100644 index 0000000..4f5c659 Binary files /dev/null and b/example_workflows/modelscope_vlm.png differ diff --git a/nodes.py b/nodes.py new file mode 100644 index 0000000..4048aee --- /dev/null +++ b/nodes.py @@ -0,0 +1,657 @@ +from typing import Optional +from typing import Tuple +import os +import time +import requests +import importlib + +BASE_URL = "https://api-inference.modelscope.cn/v1" + +MODELSCOPE_API_KEY = os.getenv("MODELSCOPE_API_KEY") or os.getenv("MODELSCOPE_ACCESS_TOKEN") + +DEFAULT_SYSTEM_PROMPT = "You are a helpful and harmless assistant. Answer concisely and helpfully." + + +class ModelScopeBase: + """Base class for ModelScope nodes with common functionality.""" + + def _resolve_key(self, api_key: Optional[str]) -> str: + """Resolve API key from input or environment variables.""" + key = api_key or MODELSCOPE_API_KEY + if not key: + raise ValueError( + "API key missing. Provide 'api_key' or set MODELSCOPE_API_KEY / MODELSCOPE_ACCESS_TOKEN." + ) + return key + + def _sanitize_text(self, text: str) -> str: + """Sanitize text for ComfyUI showText node compatibility. + + Args: + text: Raw text string + + Returns: + str: Sanitized text safe for display + """ + if not isinstance(text, str): + text = str(text) + + # Remove or replace problematic characters + import re + + # Remove null bytes and other control characters except common ones + text = re.sub(r'[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]', '', text) + + # Normalize whitespace + text = re.sub(r'\s+', ' ', text).strip() + + # Ensure text is not empty + if not text: + text = "[Empty Response]" + + # Limit length if extremely long (optional safeguard) + max_length = 10000 + if len(text) > max_length: + text = text[:max_length] + "...[truncated]" + + # Ensure proper encoding + try: + text.encode('utf-8').decode('utf-8') + except UnicodeError: + # Fallback: replace problematic characters + text = text.encode('utf-8', errors='replace').decode('utf-8') + + return text + + def _get_pil_image_module(self): + """Get PIL Image module with lazy import.""" + try: + return importlib.import_module("PIL.Image") + except Exception: + raise RuntimeError("Pillow is required. Install with: pip install pillow") + + def _download_image_from_url(self, url: str, headers: dict) -> object: + """Download image from URL and convert to ComfyUI IMAGE format. + + Args: + url: Image URL to download + headers: HTTP headers for authentication + + Returns: + torch.Tensor: Image tensor in ComfyUI format [B, H, W, C] + + Raises: + RuntimeError: If download or conversion fails + """ + try: + import torch + import numpy as np + + # Download image + response = requests.get(url, headers=headers, timeout=30) + response.raise_for_status() + + # Get PIL Image module + PIL_Image = self._get_pil_image_module() + + # Open image from bytes + from io import BytesIO + image = PIL_Image.open(BytesIO(response.content)) + + # Convert to RGB if necessary + if image.mode != 'RGB': + image = image.convert('RGB') + + # Convert PIL to numpy array + image_np = np.array(image).astype(np.float32) / 255.0 + + # Convert to torch tensor with ComfyUI format [B, H, W, C] + image_tensor = torch.from_numpy(image_np)[None,] + + return image_tensor + + except Exception as e: + raise RuntimeError(f"Failed to download image: {e}") from e + + +class ModelScopeChatBase(ModelScopeBase): + """Base class for chat-based ModelScope nodes.""" + + def _extract_text(self, data: dict) -> str: + """Extract text content from chat completion response. + Tries to read choices[0].message.content (string or list) and falls back to str(data). + """ + try: + content = data["choices"][0]["message"]["content"] + if isinstance(content, str): + return self._sanitize_text(content) + if isinstance(content, list) and content: + part = content[0] + if isinstance(part, str): + return self._sanitize_text(part) + if isinstance(part, dict): + txt = part.get("text") or part.get("content") + if txt: + return self._sanitize_text(txt) + except Exception: + pass + # Fallback to string representation + try: + return self._sanitize_text(str(data)) + except Exception: + return "[Error: Unable to extract response]" + + +class ModelScopeLLM(ModelScopeChatBase): + """ModelScope Text Chat (non-streaming) node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model_id": ( + "STRING", + { + "default": "Qwen/Qwen3-235B-A22B", + "placeholder": "Model ID", + "tooltip": "The ModelScope model ID to use.", + }, + ), + "prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "User prompt", + "tooltip": "The user prompt text to send to the model.", + }, + ), + }, + "optional": { + "system_prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "System Prompt", + "tooltip": "Optional system prompt to prime the assistant.", + }, + ), + "api_key": ( + "STRING", + { + "placeholder": "API Key", + "tooltip": "ModelScope API key; if empty, read from MODELSCOPE_API_KEY env var.", + }, + ), + "request_timeout": ( + "INT", + { + "default": 30, + "min": 10, + "max": 300, + "step": 10, + "tooltip": "Request timeout in seconds. Increase for larger models or slow networks.", + }, + ), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("response",) + FUNCTION = "chat_completion" + CATEGORY = "ModelScope" + DESCRIPTION = "Generate a chat response using ModelScope LLM." + OUTPUT_TOOLTIPS = ("ModelScope text response.",) + + def chat_completion( + self, + model_id: str, + prompt: str, + system_prompt: Optional[str] = None, + api_key: Optional[str] = None, + request_timeout: int = 30, + ) -> Tuple[str]: + """Execute text generation using ModelScope LLM. + + Args: + model_id: ModelScope LLM model identifier + prompt: User text prompt for the model + system_prompt: Optional system prompt for assistant behavior + api_key: Optional API key override + request_timeout: Request timeout in seconds + + Returns: + tuple: Single-element tuple with LLM response text + + Raises: + ValueError: If inputs are invalid + RuntimeError: If API call or processing fails + """ + # Validate inputs + if not prompt.strip(): + raise ValueError("Prompt cannot be empty") + + key = self._resolve_key(api_key) + system_prompt = system_prompt or DEFAULT_SYSTEM_PROMPT + url = f"{BASE_URL}/chat/completions" + headers = {"Authorization": f"Bearer {key}", "Content-Type": "application/json"} + payload = { + "model": model_id, + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": system_prompt, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": prompt, + }, + ], + }, + ], + "stream": False, + "enable_thinking": False, + } + + start_time = time.time() + try: + resp = requests.post( + url, + headers=headers, + json=payload, + timeout=request_timeout, + verify=False, # Temporary SSL workaround + ) + resp.raise_for_status() + except requests.Timeout as e: + elapsed = time.time() - start_time + raise RuntimeError( + f"ModelScope LLM request timed out after {elapsed:.1f}s. " + f"Try increasing request_timeout (current: {request_timeout}s) or using a smaller model." + ) from e + except requests.RequestException as e: + raise RuntimeError(f"Network error calling ModelScope LLM: {e}") from e + + try: + data = resp.json() + except Exception as e: + raise RuntimeError(f"Failed to parse LLM API response: {e}") from e + + try: + text = self._extract_text(data) + if not text.strip(): + raise RuntimeError("LLM returned empty response") + except Exception as e: + raise RuntimeError(f"Failed to extract text from LLM response: {e}") from e + + elapsed = time.time() - start_time + print(f"LLM inference completed in {elapsed:.1f}s") + + return (text,) + + +class ModelScopeVLM(ModelScopeChatBase): + """ModelScope Visual Language Model node for image-text conversations. + + Supports analyzing images with text prompts using ModelScope VLM models. + Handles various image input formats and provides robust error handling. + """ + + @classmethod + def INPUT_TYPES(cls): + """Define input types and validation rules for VLM processing. + + Returns: + dict: Input type definitions with validation parameters + """ + return { + "required": { + "model_id": ( + "STRING", + { + "default": "Qwen/QVQ-72B-Preview", + "placeholder": "Model ID", + "tooltip": "ModelScope VLM model ID.", + }, + ), + "image_url": ( + "STRING", + { + "placeholder": "https://example.com/image.jpg", + "tooltip": "URL of the image to analyze. Must be publicly accessible.", + }, + ), + "prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "What do you see in this image?", + "tooltip": "Question or instruction about the image.", + }, + ), + }, + "optional": { + "system_prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "You are a helpful vision assistant...", + "tooltip": "Optional system prompt to guide the assistant's behavior.", + }, + ), + "api_key": ( + "STRING", + { + "placeholder": "API Key", + "tooltip": "ModelScope API key. If empty, reads MODELSCOPE_API_KEY or MODELSCOPE_ACCESS_TOKEN from environment.", + }, + ), + "request_timeout": ( + "INT", + { + "default": 120, + "min": 30, + "max": 300, + "step": 10, + "tooltip": "Request timeout in seconds. VLM inference can be slow, recommend 120+ seconds.", + }, + ), + }, + } + + RETURN_TYPES = ("STRING",) + RETURN_NAMES = ("response",) + FUNCTION = "visual_chat" + CATEGORY = "ModelScope" + DESCRIPTION = "Analyze images with text using ModelScope Vision-Language Models." + OUTPUT_TOOLTIPS = ("VLM response describing or answering about the image.",) + + def visual_chat( + self, + model_id: str, + image_url: str, + prompt: str, + system_prompt: Optional[str] = None, + api_key: Optional[str] = None, + request_timeout: int = 120, + ) -> Tuple[str]: + """Execute vision-language model inference on image and text. + + Args: + model_id: ModelScope VLM model identifier + image_url: URL of image to analyze + prompt: Text prompt/question about the image + system_prompt: Optional system prompt for assistant behavior + api_key: Optional API key override + request_timeout: Request timeout in seconds + + Returns: + tuple: Single-element tuple with VLM response text + + Raises: + ValueError: If inputs are invalid + RuntimeError: If API call or processing fails + """ + # Validate inputs + if not image_url.strip(): + raise ValueError("Image URL cannot be empty") + if not prompt.strip(): + raise ValueError("Prompt cannot be empty") + + # Validate image URL format + if not (image_url.startswith('http://') or image_url.startswith('https://')): + raise ValueError("Image URL must be a valid HTTP/HTTPS URL") + + key = self._resolve_key(api_key) + system_prompt = system_prompt or DEFAULT_SYSTEM_PROMPT + + url = f"{BASE_URL}/chat/completions" + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json" + } + + # Construct VLM message payload with image and text + payload = { + "model": model_id, + "messages": [ + { + "role": "system", + "content": [ + { + "type": "text", + "text": system_prompt, + }, + ], + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": image_url, + }, + }, + { + "type": "text", + "text": prompt, + }, + ], + }, + ], + "stream": False, + "enable_thinking": False, + } + + start_time = time.time() + try: + # Use longer timeout for VLM inference + resp = requests.post( + url, + headers=headers, + json=payload, + timeout=request_timeout, + verify=False, # Temporary SSL workaround + ) + resp.raise_for_status() + except requests.Timeout as e: + elapsed = time.time() - start_time + raise RuntimeError( + f"ModelScope VLM request timed out after {elapsed:.1f}s. " + f"Try increasing request_timeout (current: {request_timeout}s) or using a smaller model." + ) from e + except requests.RequestException as e: + raise RuntimeError(f"Network error calling ModelScope VLM: {e}") from e + + try: + data = resp.json() + except Exception as e: + raise RuntimeError(f"Failed to parse VLM API response: {e}") from e + + try: + text = self._extract_text(data) + if not text.strip(): + raise RuntimeError("VLM returned empty response") + except Exception as e: + raise RuntimeError(f"Failed to extract text from VLM response: {e}") from e + + elapsed = time.time() - start_time + print(f"VLM inference completed in {elapsed:.1f}s") + + return (text,) + + +class ModelScopeImageGenerator(ModelScopeBase): + """ModelScope Image Generation node.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "model_id": ( + "STRING", + { + "default": "Qwen/Qwen-Image-Edit", + "placeholder": "Model ID", + "tooltip": "ModelScope model ID for image generation.", + }, + ), + "prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "A mysterious girl walking down the corridor.", + "tooltip": "Text prompt for image generation.", + }, + ), + }, + "optional": { + "negative_prompt": ( + "STRING", + { + "multiline": True, + "placeholder": "lowres, bad anatomy...", + "tooltip": "Negative prompt to avoid unwanted elements.", + }, + ), + "num_inference_steps": ( + "INT", + { + "default": 30, + "min": 1, + "max": 100, + "step": 1, + "tooltip": "Number of denoising steps (higher = better quality, slower).", + }, + ), + "guidance_scale": ( + "FLOAT", + { + "default": 3.5, + "min": 1.0, + "max": 20.0, + "step": 0.1, + "tooltip": "How closely to follow the prompt (higher = more adherent).", + }, + ), + "height": ( + "INT", + { + "default": 1024, + "min": 64, + "max": 2048, + "step": 8, + "tooltip": "Generated image height in pixels.", + }, + ), + "width": ( + "INT", + { + "default": 1024, + "min": 64, + "max": 2048, + "step": 8, + "tooltip": "Generated image width in pixels.", + }, + ), + "api_key": ( + "STRING", + { + "placeholder": "API Key", + "tooltip": "ModelScope API key. If empty, reads from MODELSCOPE_API_KEY env var.", + }, + ), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("image",) + FUNCTION = "image_generation" + CATEGORY = "ModelScope" + DESCRIPTION = "Generate images using ModelScope text-to-image models." + OUTPUT_TOOLTIPS = ("Generated image as ComfyUI IMAGE tensor.",) + + def image_generation( + self, + model_id: str, + prompt: str, + api_key: Optional[str] = None, + negative_prompt: Optional[str] = None, + num_inference_steps: int = 30, + guidance_scale: float = 3.5, + height: int = 1024, + width: int = 1024, + ) -> Tuple[object]: + """Generate image using ModelScope API. + + Args: + model_id: ModelScope model identifier + prompt: Text description of desired image + api_key: Optional API key override + negative_prompt: Elements to avoid in generation + num_inference_steps: Number of denoising steps + guidance_scale: Adherence to prompt + height: Image height in pixels + width: Image width in pixels + + Returns: + tuple: Single-element tuple containing image tensor + + Raises: + RuntimeError: If API call or image processing fails + """ + key = self._resolve_key(api_key) + url = f"{BASE_URL}/images/generations" + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json" + } + + payload = { + "model": model_id, + "prompt": prompt, + "negative_prompt": negative_prompt, + "num_inference_steps": num_inference_steps, + "guidance_scale": guidance_scale, + "size": f"{width}x{height}", + } + + try: + resp = requests.post(url, headers=headers, json=payload, timeout=60, verify=False) + resp.raise_for_status() + except requests.RequestException as e: + raise RuntimeError(f"Network error calling ModelScope Image API: {e}") from e + + try: + data = resp.json() + except Exception as e: + raise RuntimeError(f"Failed to parse API response: {e}") from e + + # Handle direct image URL response + if data.get("images"): + first_image = data["images"][0] + if isinstance(first_image, dict) and first_image.get("url"): + return (self._download_image_from_url(first_image["url"], headers),) + + raise RuntimeError(f"Unable to locate generated image in response: {data}") + + +NODE_CLASS_MAPPINGS = { + "ModelScopeLLM": ModelScopeLLM, + "ModelScopeVLM": ModelScopeVLM, + "ModelScopeImageGenerator": ModelScopeImageGenerator, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "ModelScopeLLM": "ModelScope LLM", + "ModelScopeVLM": "ModelScope VLM", + "ModelScopeImageGenerator": "ModelScope Image Generator", +} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29