Merge pull request #5 from 2frames/feat/send-image-to-api-node-v3
Feat/send image to api node v3
This commit is contained in:
2frames
+3
-1
@@ -2,7 +2,7 @@ from .nodes.pulid.AQ_MasksAndImagesAsList import AQ_MasksAndImagesAsList
|
||||
from .nodes.transformers.AQ_Qwen import AQ_Qwen
|
||||
from .nodes.transformers.AQ_Qwen import AQ_QwenLoader
|
||||
from .nodes.pulid.AQ_multiface_ApplyPulidFlux import AQ_multiface_ApplyPulidFlux
|
||||
from .nodes.api.AQ_Gemini import AQ_Gemini
|
||||
from .nodes.api.AQ_Gemini import AQ_Gemini, AQ_Gemini_acstep15
|
||||
from .nodes.api.AQ_SendImageToAPI import AQ_SendImageToAPI
|
||||
|
||||
from .nodes.clip.AQ_CLIPSetLastLayer import AQ_CLIPSetLastLayer
|
||||
@@ -20,6 +20,7 @@ NODE_CLASS_MAPPINGS = {
|
||||
"AQ_Qwen": AQ_Qwen,
|
||||
"AQ_QwenLoader": AQ_QwenLoader,
|
||||
"AQ_Gemini": AQ_Gemini,
|
||||
"AQ_Gemini_acstep15": AQ_Gemini_acstep15,
|
||||
"AQ_SendImageToAPI": AQ_SendImageToAPI,
|
||||
"AQ_CLIPSetLastLayer": AQ_CLIPSetLastLayer,
|
||||
"AQ_Increment": AQ_Increment,
|
||||
@@ -42,6 +43,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {
|
||||
"AQ_Qwen": "AQ_Qwen",
|
||||
"AQ_QwenLoader": "AQ_QwenLoader",
|
||||
"AQ_Gemini": "AQ_Gemini",
|
||||
"AQ_Gemini_acstep15": "AQ_Gemini_acstep15",
|
||||
"AQ_SendImageToAPI": "AQ_SendImageToAPI",
|
||||
"AQ_CLIPSetLastLayer": "AQ_CLIPSetLastLayer",
|
||||
"AQ_Increment": "AQ_Increment",
|
||||
|
||||
+252
-1
@@ -273,4 +273,255 @@ Note: If any error occurs, empty strings will be returned instead of raising an
|
||||
return False, text
|
||||
except Exception as e:
|
||||
print(f"Unexpected error while parsing JSON: {str(e)}")
|
||||
return False, text
|
||||
return False, text
|
||||
|
||||
|
||||
class AQ_Gemini_acstep15:
|
||||
@classmethod
|
||||
def INPUT_TYPES(s):
|
||||
return {
|
||||
"required": {
|
||||
"gemini_api_key": ("STRING", {"default": ""}),
|
||||
"model_selection": ([
|
||||
"gemma-3-12b-it", # default
|
||||
"gemma-3-1b-it",
|
||||
"gemma-3-4b-it",
|
||||
"gemma-3-27b-it",
|
||||
"gemini-flash-latest",
|
||||
"gemini-flash-lite-latest",
|
||||
"gemini-2.5-flash",
|
||||
"gemini-2.5-flash-lite",
|
||||
"gemini-2.5-pro",
|
||||
"gemini-3-flash-preview",
|
||||
"gemini-3-pro-preview",
|
||||
"custom" # Option for custom model
|
||||
], {"default": "gemma-3-12b-it"}),
|
||||
"custom_model": ("STRING", {"default": "", "multiline": False}),
|
||||
"prompt": ("STRING", {"default": "", "multiline": True}),
|
||||
"system_message": ("STRING", {"default": "You are a helpful assistant.", "multiline": True}),
|
||||
"temperature": ("FLOAT", {"default": 1, "min": 0.0, "max": 2.0, "step": 0.05}),
|
||||
"top_k": ("INT", {"default": 64, "min": 0, "max": 100}),
|
||||
"top_p": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
},
|
||||
"optional": {
|
||||
"image": ("IMAGE",),
|
||||
}
|
||||
}
|
||||
|
||||
RETURN_TYPES = (
|
||||
"STRING",
|
||||
"STRING",
|
||||
"STRING",
|
||||
"INT",
|
||||
"FLOAT",
|
||||
"STRING",
|
||||
"STRING",
|
||||
"STRING",
|
||||
)
|
||||
RETURN_NAMES = (
|
||||
"response_json",
|
||||
"tags",
|
||||
"lyrics",
|
||||
"bpm",
|
||||
"duration",
|
||||
"timesignature",
|
||||
"language",
|
||||
"keyscale",
|
||||
)
|
||||
FUNCTION = "generate"
|
||||
CATEGORY = "Aquasite/LLM"
|
||||
|
||||
def generate(self, gemini_api_key, model_selection, custom_model, prompt, system_message, temperature=0.8,
|
||||
top_k=64, top_p=0.95, image=None):
|
||||
if not gemini_api_key:
|
||||
return ("", "", "", 120, 120.0, "4", "en", "C major")
|
||||
|
||||
json_schema = {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"description",
|
||||
"tags",
|
||||
"lyrics",
|
||||
"bpm",
|
||||
"keyscale",
|
||||
"durationSeconds",
|
||||
"timesignature",
|
||||
"language"
|
||||
],
|
||||
"properties": {
|
||||
"description": {"type": "string"},
|
||||
"tags": {"type": "string"},
|
||||
"lyrics": {"type": "string"},
|
||||
"bpm": {"type": "integer"},
|
||||
"keyscale": {"type": "string"},
|
||||
"durationSeconds": {"type": "number"},
|
||||
"timesignature": {"type": "string"},
|
||||
"language": {"type": "string"}
|
||||
}
|
||||
}
|
||||
|
||||
json_instruction = (
|
||||
"Return ONLY valid JSON matching this schema (no markdown, no comments): "
|
||||
f"{json.dumps(json_schema)}"
|
||||
)
|
||||
|
||||
try:
|
||||
client = genai.Client(api_key=gemini_api_key)
|
||||
model_name = custom_model if model_selection == "custom" and custom_model else model_selection
|
||||
|
||||
contents = []
|
||||
if system_message:
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[types.Part.from_text(text=system_message)]
|
||||
)
|
||||
)
|
||||
|
||||
combined_prompt = f"{prompt}\n\n{json_instruction}".strip()
|
||||
|
||||
if image is not None:
|
||||
if isinstance(image, torch.Tensor):
|
||||
image = image.cpu().numpy()
|
||||
|
||||
if len(image.shape) == 4:
|
||||
image = image[0]
|
||||
|
||||
if image.dtype != np.uint8:
|
||||
if image.max() <= 1:
|
||||
image = (image * 255).astype(np.uint8)
|
||||
else:
|
||||
image = image.astype(np.uint8)
|
||||
|
||||
img = Image.fromarray(image)
|
||||
img_byte_arr = io.BytesIO()
|
||||
img.save(img_byte_arr, format="PNG")
|
||||
img_byte_arr.seek(0)
|
||||
img_base64 = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
|
||||
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[{
|
||||
"inline_data": {
|
||||
"mime_type": "image/png",
|
||||
"data": img_base64
|
||||
}
|
||||
},
|
||||
{
|
||||
"text": combined_prompt
|
||||
}]
|
||||
)
|
||||
)
|
||||
else:
|
||||
contents.append(
|
||||
types.Content(
|
||||
role="user",
|
||||
parts=[types.Part.from_text(text=combined_prompt)]
|
||||
)
|
||||
)
|
||||
|
||||
generate_config = types.GenerateContentConfig(
|
||||
temperature=temperature,
|
||||
top_k=top_k,
|
||||
top_p=top_p
|
||||
)
|
||||
|
||||
# Use schema-based JSON when supported; otherwise rely on the instruction.
|
||||
is_gemma_model = model_name.startswith("gemma-3")
|
||||
if not is_gemma_model:
|
||||
generate_config.response_mime_type = "application/json"
|
||||
generate_config.response_schema = self._convert_schema_dict_to_genai(json_schema)
|
||||
|
||||
response = ""
|
||||
for chunk in client.models.generate_content_stream(
|
||||
model=model_name,
|
||||
contents=contents,
|
||||
config=generate_config,
|
||||
):
|
||||
if chunk.text:
|
||||
response += chunk.text
|
||||
|
||||
success, json_response = self.try_parse_json(response)
|
||||
if not success or not isinstance(json_response, dict):
|
||||
json_response = {}
|
||||
|
||||
tags = json_response.get("tags") or json_response.get("description") or ""
|
||||
lyrics = json_response.get("lyrics", "")
|
||||
bpm = self._coerce_int(json_response.get("bpm"), default=120)
|
||||
duration = self._coerce_float(json_response.get("durationSeconds"), default=120.0)
|
||||
timesignature = str(json_response.get("timesignature", "4"))
|
||||
language = str(json_response.get("language", "en"))
|
||||
keyscale = str(json_response.get("keyscale", "C major"))
|
||||
|
||||
response_json = json.dumps({
|
||||
"description": json_response.get("description", ""),
|
||||
"tags": tags,
|
||||
"lyrics": lyrics,
|
||||
"bpm": bpm,
|
||||
"keyscale": keyscale,
|
||||
"durationSeconds": duration,
|
||||
"timesignature": timesignature,
|
||||
"language": language
|
||||
})
|
||||
|
||||
return (response_json, tags, lyrics, bpm, duration, timesignature, language, keyscale)
|
||||
except Exception as e:
|
||||
print(f"Error in Gemini API: {str(e)}")
|
||||
return ("", "", "", 0, 120, 120.0, "4", "en", "C major")
|
||||
|
||||
def _convert_schema_dict_to_genai(self, schema_dict):
|
||||
schema_type = schema_dict.get("type", "object")
|
||||
|
||||
if schema_type == "object":
|
||||
properties = {}
|
||||
for prop_name, prop_schema in schema_dict.get("properties", {}).items():
|
||||
properties[prop_name] = self._convert_schema_dict_to_genai(prop_schema)
|
||||
|
||||
return genai.types.Schema(
|
||||
type=genai.types.Type.OBJECT,
|
||||
required=schema_dict.get("required", []),
|
||||
properties=properties
|
||||
)
|
||||
if schema_type == "array":
|
||||
items_schema = self._convert_schema_dict_to_genai(schema_dict.get("items", {}))
|
||||
return genai.types.Schema(
|
||||
type=genai.types.Type.ARRAY,
|
||||
items=items_schema
|
||||
)
|
||||
if schema_type == "string":
|
||||
return genai.types.Schema(type=genai.types.Type.STRING)
|
||||
if schema_type == "number":
|
||||
return genai.types.Schema(type=genai.types.Type.NUMBER)
|
||||
if schema_type == "integer":
|
||||
return genai.types.Schema(type=genai.types.Type.INTEGER)
|
||||
if schema_type == "boolean":
|
||||
return genai.types.Schema(type=genai.types.Type.BOOLEAN)
|
||||
return genai.types.Schema(type=genai.types.Type.STRING)
|
||||
|
||||
def prepare_json(self, text):
|
||||
text = text.replace("```json", "").replace("```", "").strip()
|
||||
return text
|
||||
|
||||
def try_parse_json(self, text):
|
||||
try:
|
||||
cleaned_text = self.prepare_json(text)
|
||||
return True, json.loads(cleaned_text)
|
||||
except json.JSONDecodeError as e:
|
||||
print(f"JSON parsing error: {str(e)}")
|
||||
return False, text
|
||||
except Exception as e:
|
||||
print(f"Unexpected error while parsing JSON: {str(e)}")
|
||||
return False, text
|
||||
|
||||
def _coerce_int(self, value, default=0):
|
||||
try:
|
||||
return int(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def _coerce_float(self, value, default=0.0):
|
||||
try:
|
||||
return float(value)
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
@@ -60,6 +60,7 @@ class AQ_TextEncodeQwenImageEditPlus:
|
||||
"use_image1": ("BOOLEAN", {"default": True}),
|
||||
"use_image2": ("BOOLEAN", {"default": True}),
|
||||
"use_image3": ("BOOLEAN", {"default": True}),
|
||||
"prompt_template": (["original", "detailed", "transformation_focused", "preserving", "creative", "technical", "artistic_generation", "style_transfer", "scene_reimagining", "playful", "descriptive", "precise_surgical", "multi_image_blend", "view_angle_control"], {"default": "original"}),
|
||||
},
|
||||
"optional": {
|
||||
"vae": ("VAE",),
|
||||
@@ -74,12 +75,31 @@ class AQ_TextEncodeQwenImageEditPlus:
|
||||
|
||||
CATEGORY = "advanced/conditioning"
|
||||
|
||||
def encode(self, clip, prompt, use_image1=True, use_image2=True, use_image3=True, vae=None, image1=None, image2=None, image3=None):
|
||||
def encode(self, clip, prompt, use_image1=True, use_image2=True, use_image3=True, prompt_template="original", vae=None, image1=None, image2=None, image3=None):
|
||||
ref_latents = []
|
||||
images = [image1, image2, image3]
|
||||
use_images = [use_image1, use_image2, use_image3]
|
||||
images_vl = []
|
||||
llama_template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
|
||||
|
||||
# Define available prompt templates
|
||||
templates = {
|
||||
"original": "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"detailed": "<|im_start|>system\nAnalyze the input image systematically: identify dominant colors and color palette, describe shapes and compositional elements, note textures and material properties, catalog all visible objects and their spatial relationships, and characterize the background and lighting. Then, interpret the user's modification request and generate a new image that implements these changes while preserving the original aesthetic and maintaining visual coherence.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"transformation_focused": "<|im_start|>system\nExamine the input image and understand its current state. Carefully analyze the user's instruction to determine what specific changes are requested. Apply the transformation precisely as described, modifying only the elements mentioned while keeping all other aspects unchanged. Ensure the modified image maintains photorealistic quality and natural appearance.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"preserving": "<|im_start|>system\nStudy the input image and identify its core visual elements, style, composition, and atmosphere. Understand the user's modification request and implement it in a way that preserves the fundamental character and identity of the original image. Maintain consistency in lighting, perspective, and artistic style while carefully integrating the requested changes.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"creative": "<|im_start|>system\nObserve the input image and capture its visual essence, mood, and artistic qualities. Interpret the user's creative direction and generate a new image that fulfills their vision while enhancing the overall aesthetic. Feel free to refine and improve the image artistically while staying true to the user's intent and maintaining recognizable elements from the original.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"technical": "<|im_start|>system\nProcess the input image data: extract color values (RGB/HSV), identify geometric shapes and dimensions, analyze texture patterns and surface properties, detect and classify objects with bounding regions, and evaluate background composition and depth. Parse the user's modification parameters and generate output image that satisfies the specified constraints while maintaining structural integrity and visual fidelity of unmodified regions.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"artistic_generation": "<|im_start|>system\nLook at the reference image and identify the main subject, character, or object shown. Use this as inspiration to create a new artistic image based on the user's description. The subject from the reference image should be recognizable in the new generation, but feel free to interpret it artistically, change the style, setting, and context according to the user's creative vision. Focus on producing beautiful, imaginative artwork.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"style_transfer": "<|im_start|>system\nIdentify the main subject or character in the reference image. Read the user's description to understand what artistic style, medium, or visual treatment they want applied. Generate a new image that depicts the same subject but rendered in the requested style - whether it's a painting technique (watercolor, oil, pastel), art movement (impressionist, cartoon, anime), or other visual style. Keep the subject recognizable while fully embracing the new artistic style.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"scene_reimagining": "<|im_start|>system\nExamine the reference image to understand the main subject, character, or object. The user will describe a new scene, setting, environment, or situation for this subject. Generate a new image that places the subject from the reference into the described context. The subject should maintain its essential characteristics and identity, but be integrated naturally into the new scene with appropriate lighting, perspective, and atmosphere.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"playful": "<|im_start|>system\nLook at the picture and see what's in it - maybe it's a toy, a pet, a drawing, or something else fun! Now read what the user wants to create. Make a new, fun picture that brings their idea to life! Be creative and imaginative. If they mention colors, styles, places, or activities, include all of those. Make something cheerful and exciting that captures their imagination. The new image should be vibrant, engaging, and full of personality.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"descriptive": "<|im_start|>system\nObserve the reference image to understand what subject is being shown. Read the user's text description carefully - it will tell you exactly what to generate. Follow their description literally and precisely: if they mention specific colors, use those colors; if they describe a style, apply that style; if they mention objects, settings, or actions, include all of them. Generate an image that matches their written description as closely as possible while keeping the main subject from the reference recognizable.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"precise_surgical": "<|im_start|>system\nYou are performing precise, surgical edits. The user will tell you exactly what to ADD, REPLACE, or REMOVE using clear, direct language. Make ONLY the specific change requested - nothing more. Keep everything else completely unchanged: preserve all colors, textures, lighting, shadows, reflections, proportions, perspective, and composition. Do not add distortion, do not warp text, do not create duplicate elements, do not alter the background. Match original font, size, and alignment if editing text. Maintain natural appearance with no artifacts. The edit should be seamless and invisible except for the exact element being modified.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"multi_image_blend": "<|im_start|>system\nYou are blending and combining multiple reference images into one cohesive result. Analyze each input image to identify the key subjects, elements, or features. The user's description will explain how to merge these images together. Integrate all specified elements naturally - match lighting across sources, blend shadows and highlights realistically, ensure consistent perspective and scale, harmonize color palettes. Create seamless transitions between elements from different sources. The final image should look unified and natural, not like a collage or photo-bash. Preserve the identity and key features from each reference while creating visual coherence.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n",
|
||||
"view_angle_control": "<|im_start|>system\nYou are re-rendering the scene from a different camera angle or perspective. Study the reference image to understand the 3D space, object positions, lighting setup, and scene composition. The user will specify a new camera angle, viewpoint, or perspective. Re-project the entire scene from this new angle while maintaining spatial consistency: keep all objects in their correct 3D positions, preserve relative sizes and distances, adjust lighting and shadows to match the new angle, maintain the same materials and textures. The result should look like the same scene photographed from a different position. Do not change object identities, add new elements, or alter the fundamental scene composition - only change the viewing angle.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n"
|
||||
}
|
||||
|
||||
llama_template = templates.get(prompt_template, templates["original"])
|
||||
image_prompt = ""
|
||||
|
||||
for i, (image, use_image) in enumerate(zip(images, use_images)):
|
||||
|
||||
Reference in New Issue
Block a user