diff --git a/node.py b/node.py index 550fc26..faef57d 100644 --- a/node.py +++ b/node.py @@ -8,6 +8,8 @@ from torchvision import transforms import torch import base64 import time +import torchaudio +import soundfile as sf from replicate.client import Client from .schema_to_node import ( schema_to_comfyui_input_types, @@ -32,7 +34,11 @@ def create_comfyui_node(schema): def INPUT_TYPES(cls): return schema_to_comfyui_input_types(schema) - RETURN_TYPES = (return_type,) + RETURN_TYPES = ( + tuple(return_type.values()) + if isinstance(return_type, dict) + else (return_type,) + ) FUNCTION = "run_replicate_model" CATEGORY = "Replicate" @@ -45,6 +51,8 @@ def create_comfyui_node(schema): ) if input_type == "IMAGE": kwargs[key] = self.image_to_base64(value) + elif input_type == "AUDIO": + kwargs[key] = self.audio_to_base64(value) def image_to_base64(self, image): if isinstance(image, torch.Tensor): @@ -60,6 +68,31 @@ def create_comfyui_node(schema): img_str = base64.b64encode(buffer.getvalue()).decode() return f"data:image/png;base64,{img_str}" + def audio_to_base64(self, audio): + if ( + isinstance(audio, dict) + and "waveform" in audio + and "sample_rate" in audio + ): + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + else: + waveform, sample_rate = audio + + # Ensure waveform is 2D + if waveform.dim() == 1: + waveform = waveform.unsqueeze(0) + elif waveform.dim() > 2: + waveform = waveform.squeeze() + if waveform.dim() > 2: + raise ValueError("Waveform must be 1D or 2D") + + buffer = io.BytesIO() + sf.write(buffer, waveform.numpy().T, sample_rate, format="wav") + buffer.seek(0) + audio_str = base64.b64encode(buffer.getvalue()).decode() + return f"data:audio/wav;base64,{audio_str}" + def handle_array_inputs(self, kwargs): array_inputs = inputs_that_need_arrays(schema) for input_name in array_inputs: @@ -75,13 +108,18 @@ def create_comfyui_node(schema): def log_input(self, kwargs): truncated_kwargs = { k: v[:20] + "..." - if isinstance(v, str) and v.startswith("data:image") + if isinstance(v, str) + and (v.startswith("data:image") or v.startswith("data:audio")) else v for k, v in kwargs.items() } print(f"Running {replicate_model} with {truncated_kwargs}") def handle_image_output(self, output): + if output is None: + print("No image output received") + return None + # Handle both string and list outputs output_list = [output] if isinstance(output, str) else list(output) if output_list: @@ -112,19 +150,93 @@ def create_comfyui_node(schema): print("No output received from the model") return None + def handle_audio_output(self, output): + if output is None: + print("No audio output received from the model") + return None + + if isinstance(output, str): + output_list = [output] + elif isinstance(output, (list, tuple)): + output_list = list(output) + else: + print(f"Unexpected output type: {type(output)}") + return None + + if output_list: + audio_data = [] + for audio_url in output_list: + if audio_url: + response = requests.get(audio_url) + if response.status_code == 200: + audio_content = BytesIO(response.content) + waveform, sample_rate = torchaudio.load(audio_content) + audio_data.append( + { + "waveform": waveform.unsqueeze(0), + "sample_rate": sample_rate, + } + ) + else: + print( + f"Failed to download audio. Status code: {response.status_code}" + ) + else: + print("Empty audio URL received") + + # If there's only one audio file, return it directly + if len(audio_data) == 1: + return audio_data[0] + # If there are multiple audio files, return them as a list + return audio_data + else: + print("No valid audio URLs in the output") + return None + + def remove_falsey_optional_inputs(self, kwargs): + optional_inputs = self.INPUT_TYPES().get("optional", {}) + for key in list(kwargs.keys()): + if key in optional_inputs: + if isinstance(kwargs[key], torch.Tensor): + continue + elif not kwargs[key]: + del kwargs[key] + def run_replicate_model(self, **kwargs): self.handle_array_inputs(kwargs) + self.remove_falsey_optional_inputs(kwargs) self.convert_input_images_to_base64(kwargs) self.log_input(kwargs) - output = replicate.run(replicate_model, input=kwargs) + kwargs_without_force_rerun = { + k: v for k, v in kwargs.items() if k != "force_rerun" + } + output = replicate.run(replicate_model, input=kwargs_without_force_rerun) print(f"Output: {output}") - if return_type == "IMAGE": - output = self.handle_image_output(output) + processed_outputs = [] + if isinstance(return_type, dict): + for prop_name, prop_type in return_type.items(): + if prop_type == "IMAGE": + processed_outputs.append( + self.handle_image_output(output.get(prop_name)) + ) + elif prop_type == "AUDIO": + processed_outputs.append( + self.handle_audio_output(output.get(prop_name)) + ) + elif prop_type == "STRING": + processed_outputs.append( + "".join(list(output.get(prop_name, ""))).strip() + ) else: - output = "".join(list(output)).strip() + if return_type == "IMAGE": + processed_outputs.append(self.handle_image_output(output)) + elif return_type == "AUDIO": + processed_outputs.append(self.handle_audio_output(output)) + else: + processed_outputs.append("".join(list(output)).strip()) - return (output,) + return tuple(processed_outputs) return node_name, ReplicateToComfyUI @@ -135,7 +247,9 @@ def create_comfyui_nodes_from_schemas(schemas_dir): schemas_dir_path = os.path.join(current_path, schemas_dir) for schema_file in os.listdir(schemas_dir_path): if schema_file.endswith(".json"): - with open(os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8") as f: + with open( + os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8" + ) as f: schema = json.load(f) node_name, node_class = create_comfyui_node(schema) nodes[node_name] = node_class diff --git a/pyproject.toml b/pyproject.toml index c3fa74a..db3a6fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-replicate" description = "Run Replicate models in ComfyUI" -version = "1.0.4" +version = "1.0.5" license = "LICENSE" [project.urls] diff --git a/schema_to_node.py b/schema_to_node.py index 76df66a..8e91a65 100644 --- a/schema_to_node.py +++ b/schema_to_node.py @@ -1,17 +1,41 @@ DEFAULT_STEP = 0.01 DEFAULT_ROUND = 0.001 +IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp") +VIDEO_EXTENSIONS = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") +AUDIO_EXTENSIONS = (".mp3", ".wav", ".flac", ".mpga", ".m4a") -def convert_to_comfyui_input_type(openapi_type, openapi_format=None): - type_mapping = { - "string": "STRING", - "integer": "INT", - "number": "FLOAT", - "boolean": "BOOLEAN", - } +TYPE_MAPPING = { + "string": "STRING", + "integer": "INT", + "number": "FLOAT", + "boolean": "BOOLEAN", +} + + +def convert_to_comfyui_input_type( + input_name, openapi_type, openapi_format=None, default_example_input=None +): if openapi_type == "string" and openapi_format == "uri": - return "IMAGE" - return type_mapping.get(openapi_type, "STRING") + if ( + default_example_input + and isinstance(default_example_input, dict) + and input_name in default_example_input + ): + if is_type(default_example_input[input_name], IMAGE_EXTENSIONS): + return "IMAGE" + elif is_type(default_example_input[input_name], VIDEO_EXTENSIONS): + return "VIDEO" + elif is_type(default_example_input[input_name], AUDIO_EXTENSIONS): + return "AUDIO" + elif "image" in input_name.lower(): + return "IMAGE" + elif "audio" in input_name.lower(): + return "AUDIO" + else: + return "STRING" + + return TYPE_MAPPING.get(openapi_type, "STRING") def name_and_version(schema): @@ -23,11 +47,13 @@ def name_and_version(schema): return replicate_model, node_name -def resolve_schema(prop_data, openapi_schemas): +def resolve_schema(prop_data, openapi_schema): if "$ref" in prop_data: ref_path = prop_data["$ref"].split("/") - current = openapi_schemas + current = openapi_schema for path in ref_path[1:]: # Skip the first '#' element + if path not in current: + return prop_data # Return original if path is invalid current = current[path] return current return prop_data @@ -37,6 +63,7 @@ def schema_to_comfyui_input_types(schema): openapi_schema = schema["latest_version"]["openapi_schema"] input_schema = openapi_schema["components"]["schemas"]["Input"] input_types = {"required": {}, "optional": {}} + default_example_input = get_default_example_input(schema) required_props = input_schema.get("required", []) @@ -51,7 +78,10 @@ def schema_to_comfyui_input_types(schema): input_type = prop_data["enum"] elif "type" in prop_data: input_type = convert_to_comfyui_input_type( - prop_data["type"], prop_data.get("format") + prop_name, + prop_data["type"], + prop_data.get("format"), + default_example_input, ) else: input_type = "STRING" @@ -134,22 +164,65 @@ def is_type(default_example_output, extensions): return False -def get_return_type(schema): - image_extensions = (".png", ".jpg", ".jpeg", ".gif", ".webp") - video_extensions = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") - audio_extensions = (".mp3", ".wav") - - openapi_schema = schema["latest_version"]["openapi_schema"] - output_schema = openapi_schema["components"]["schemas"].get("Output") +def get_default_example(schema): default_example = schema.get("default_example") - default_example_output = default_example.get("output") if default_example else None + return default_example if default_example else None - if is_type(default_example_output, image_extensions): + +def get_default_example_input(schema): + default_example = get_default_example(schema) + return default_example.get("input") if default_example else None + + +def get_default_example_output(schema): + default_example = get_default_example(schema) + return default_example.get("output") if default_example else None + + +def get_return_type(schema): + openapi_schema = schema["latest_version"]["openapi_schema"] + output_schema = ( + openapi_schema.get("components", {}).get("schemas", {}).get("Output") + ) + default_example_output = get_default_example_output(schema) + + if output_schema and "$ref" in output_schema: + output_schema = resolve_schema(output_schema, openapi_schema) + + if isinstance(output_schema, dict) and output_schema.get("properties"): + return_types = {} + for prop_name, prop_data in output_schema["properties"].items(): + if isinstance(default_example_output, dict): + prop_value = default_example_output.get(prop_name) + + if is_type(prop_value, IMAGE_EXTENSIONS): + return_types[prop_name] = "IMAGE" + elif is_type(prop_value, AUDIO_EXTENSIONS): + return_types[prop_name] = "AUDIO" + elif is_type(prop_value, VIDEO_EXTENSIONS): + return_types[prop_name] = "VIDEO_URI" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("format") == "uri": + if "audio" in prop_name.lower(): + return_types[prop_name] = "AUDIO" + elif "image" in prop_name.lower(): + return_types[prop_name] = "IMAGE" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("type") == "string": + return_types[prop_name] = "STRING" + else: + return_types[prop_name] = "STRING" + + return return_types + + if is_type(default_example_output, IMAGE_EXTENSIONS): return "IMAGE" - elif is_type(default_example_output, video_extensions): + elif is_type(default_example_output, VIDEO_EXTENSIONS): return "VIDEO_URI" - elif is_type(default_example_output, audio_extensions): - return "AUDIO_URI" + elif is_type(default_example_output, AUDIO_EXTENSIONS): + return "AUDIO" if output_schema: if ( @@ -160,8 +233,8 @@ def get_return_type(schema): return "IMAGE" elif ( output_schema.get("type") == "array" - and output_schema["items"].get("type") == "string" - and output_schema["items"].get("format") == "uri" + and output_schema.get("items", {}).get("type") == "string" + and output_schema.get("items", {}).get("format") == "uri" ): # Handle multiple image output return "IMAGE" diff --git a/schemas/ai-forever_kandinsky-2.json b/schemas/ai-forever_kandinsky-2.json new file mode 100644 index 0000000..7705b7d --- /dev/null +++ b/schemas/ai-forever_kandinsky-2.json @@ -0,0 +1,601 @@ +{ + "url": "https://replicate.com/ai-forever/kandinsky-2", + "owner": "ai-forever", + "name": "kandinsky-2", + "description": "text2img model trained on LAION HighRes and fine-tuned on internal datasets", + "visibility": "public", + "github_url": "https://github.com/chenxwh/Kandinsky-2", + "paper_url": null, + "license_url": "https://github.com/ai-forever/Kandinsky-2/blob/main/license", + "run_count": 0, + "cover_image_url": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "default_example": { + "id": "wixq75uo65glnjcdgrqtngdz4q", + "model": "ai-forever/kandinsky-2", + "version": "9c0bf7d5cf2ed934c5921faf61882657c03c4def9d9cb88330c15bd795edb098", + "status": "succeeded", + "input": { + "prompt": "red cat, 4k photo", + "scheduler": "p_sampler", + "prior_steps": "5", + "guidance_scale": 4, + "prior_cf_scale": 4, + "num_inference_steps": 100 + }, + "output": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "logs": "0%| | 0/100 [00:00 #0:0 (pcm_s16le (native) -> pcm_s16le (native))\nPress [q] to stop, [?] for help\nOutput #0, wav, to '/tmp/speaker.wav':\nMetadata:\nISFT : Lavf58.76.100\nStream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 44100 Hz, mono, s16, 705 kb/s\nMetadata:\nencoder : Lavc58.134.100 pcm_s16le\nsize= 4kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 744kB time=00:00:08.63 bitrate= 705.9kbits/s speed=1.75e+03x\nvideo:0kB audio:744kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.010236%\n> Text splitted to sentences.\n[\"Hi there, I'm your new voice clone.\", 'Try your best to upload quality audio']\n> Processing time: 3.3411097526550293\n> Real-time factor: 0.613438166517148", + "error": null, + "metrics": { + "predict_time": 4.303438, + "total_time": 4.319136 + }, + "created_at": "2023-11-16T16:42:06.735772Z", + "started_at": "2023-11-16T16:42:06.751470Z", + "completed_at": "2023-11-16T16:42:11.054908Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu", + "cancel": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu/cancel" + } + }, + "latest_version": { + "id": "684bc3855b37866c0c65add2ff39c78f3dea3f4ff103a436465326e0f438d55e", + "created_at": "2023-11-28T15:39:16.308105+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "required": [ + "speaker" + ], + "properties": { + "text": { + "type": "string", + "title": "Text", + "default": "Hi there, I'm your new voice clone. Try your best to upload quality audio", + "x-order": 0, + "description": "Text to synthesize" + }, + "speaker": { + "type": "string", + "title": "Speaker", + "format": "uri", + "x-order": 1, + "description": "Original speaker audio (wav, mp3, m4a, ogg, or flv)" + }, + "language": { + "allOf": [ + { + "$ref": "#/components/schemas/language" + } + ], + "default": "en", + "x-order": 2, + "description": "Output language for the synthesised speech" + }, + "cleanup_voice": { + "type": "boolean", + "title": "Cleanup Voice", + "default": false, + "x-order": 3, + "description": "Whether to apply denoising to the speaker audio (microphone recordings)" + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "language": { + "enum": [ + "en", + "es", + "fr", + "de", + "it", + "pt", + "pl", + "tr", + "ru", + "nl", + "cs", + "ar", + "zh", + "hu", + "ko", + "hi" + ], + "type": "string", + "title": "language", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/luosiallen_latent-consistency-model.json b/schemas/luosiallen_latent-consistency-model.json new file mode 100644 index 0000000..f1613e9 --- /dev/null +++ b/schemas/luosiallen_latent-consistency-model.json @@ -0,0 +1,496 @@ +{ + "url": "https://replicate.com/luosiallen/latent-consistency-model", + "owner": "luosiallen", + "name": "latent-consistency-model", + "description": "Synthesizing High-Resolution Images with Few-Step Inference", + "visibility": "public", + "github_url": "https://github.com/luosiallen/latent-consistency-model", + "paper_url": "https://arxiv.org/abs/2310.04378", + "license_url": "https://github.com/chenxwh/latent-consistency-model/blob/main/LICENSE", + "run_count": 0, + "cover_image_url": "https://pbxt.replicate.delivery/lE1UZOlbLiY7DpYlLvdUmyyt0Z0pu7tTTpgFn8a0ug3Jb4bE/out-0.png", + "default_example": { + "id": "5lwbdqdbyrilvvsm62mphlbghu", + "model": "luosiallen/latent-consistency-model", + "version": "553803fd018b3cf875a8bc774c99da9b33f36647badfd88a6eec90d61c5f62fc", + "status": "succeeded", + "input": { + "width": 768, + "height": 768, + "prompt": "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k", + "num_images": 1, + "guidance_scale": 8, + "num_inference_steps": 8 + }, + "output": [ + "https://pbxt.replicate.delivery/lE1UZOlbLiY7DpYlLvdUmyyt0Z0pu7tTTpgFn8a0ug3Jb4bE/out-0.png" + ], + "logs": "Using seed: 49296\n 0%| | 0/8 [00:00 #0:0 (pcm_s16le (native) -> mp3 (libmp3lame))\nPress [q] to stop, [?] for help\nOutput #0, mp3, to 'out.mp3':\nMetadata:\nTSSE : Lavf58.76.100\nStream #0:0: Audio: mp3, 32000 Hz, stereo, s16p\nMetadata:\nencoder : Lavc58.134.100 libmp3lame\nsize= 0kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 95kB time=00:00:07.99 bitrate= 97.1kbits/s speed=45.6x\nvideo:0kB audio:94kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.269717%", + "error": null, + "metrics": { + "predict_time": 66.371701, + "total_time": 66.463286 + }, + "created_at": "2024-03-28T12:01:08.688755Z", + "started_at": "2024-03-28T12:01:08.780340Z", + "completed_at": "2024-03-28T12:02:15.152041Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku", + "cancel": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku/cancel" + } + }, + "latest_version": { + "id": "671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb", + "created_at": "2024-03-28T11:07:13.003999+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "seed": { + "type": "integer", + "title": "Seed", + "x-order": 14, + "description": "Seed for random number generator. If None or -1, a random seed will be used." + }, + "top_k": { + "type": "integer", + "title": "Top K", + "default": 250, + "x-order": 9, + "description": "Reduces sampling to the k most likely tokens." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0, + "x-order": 10, + "description": "Reduces sampling to tokens with cumulative probability of p. When set to `0` (default), top_k sampling is used." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "x-order": 1, + "description": "A description of the music you want to generate." + }, + "duration": { + "type": "integer", + "title": "Duration", + "default": 8, + "x-order": 3, + "description": "Duration of the generated audio in seconds." + }, + "input_audio": { + "type": "string", + "title": "Input Audio", + "format": "uri", + "x-order": 2, + "description": "An audio file that will influence the generated music. If `continuation` is `True`, the generated music will be a continuation of the audio file. Otherwise, the generated music will mimic the audio file's melody." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 1, + "x-order": 11, + "description": "Controls the 'conservativeness' of the sampling process. Higher temperature means more diversity." + }, + "continuation": { + "type": "boolean", + "title": "Continuation", + "default": false, + "x-order": 4, + "description": "If `True`, generated music will continue from `input_audio`. Otherwise, generated music will mimic `input_audio`'s melody." + }, + "model_version": { + "allOf": [ + { + "$ref": "#/components/schemas/model_version" + } + ], + "default": "stereo-melody-large", + "x-order": 0, + "description": "Model to use for generation" + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "wav", + "x-order": 13, + "description": "Output format for generated audio." + }, + "continuation_end": { + "type": "integer", + "title": "Continuation End", + "minimum": 0, + "x-order": 6, + "description": "End time of the audio file to use for continuation. If -1 or None, will default to the end of the audio clip." + }, + "continuation_start": { + "type": "integer", + "title": "Continuation Start", + "default": 0, + "minimum": 0, + "x-order": 5, + "description": "Start time of the audio file to use for continuation." + }, + "multi_band_diffusion": { + "type": "boolean", + "title": "Multi Band Diffusion", + "default": false, + "x-order": 7, + "description": "If `True`, the EnCodec tokens will be decoded with MultiBand Diffusion. Only works with non-stereo models." + }, + "normalization_strategy": { + "allOf": [ + { + "$ref": "#/components/schemas/normalization_strategy" + } + ], + "default": "loudness", + "x-order": 8, + "description": "Strategy for normalizing audio." + }, + "classifier_free_guidance": { + "type": "integer", + "title": "Classifier Free Guidance", + "default": 3, + "x-order": 12, + "description": "Increases the influence of inputs on the output. Higher values produce lower-varience outputs that adhere more closely to inputs." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "model_version": { + "enum": [ + "stereo-melody-large", + "stereo-large", + "melody-large", + "large" + ], + "type": "string", + "title": "model_version", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "wav", + "mp3" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "normalization_strategy": { + "enum": [ + "loudness", + "clip", + "peak", + "rms" + ], + "type": "string", + "title": "normalization_strategy", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/nateraw_audio-super-resolution.json b/schemas/nateraw_audio-super-resolution.json new file mode 100644 index 0000000..1c5fac3 --- /dev/null +++ b/schemas/nateraw_audio-super-resolution.json @@ -0,0 +1,469 @@ +{ + "url": "https://replicate.com/nateraw/audio-super-resolution", + "owner": "nateraw", + "name": "audio-super-resolution", + "description": "AudioSR: Versatile Audio Super-resolution at Scale", + "visibility": "public", + "github_url": "https://github.com/haoheliu/versatile_audio_super_resolution", + "paper_url": "https://arxiv.org/abs/2309.07314", + "license_url": "https://huggingface.co/haoheliu/wellsolve_audio_super_resolution_48k/blob/main/README.md", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/6bc4e480-6695-451b-940d-48a3b83a1356/replicate-prediction-jvi5xvlbg4v4.png", + "default_example": { + "id": "qqynzilbxm2t5h6eaywx2p7g54", + "model": "nateraw/audio-super-resolution", + "version": "9c3d3e39fb0cb6aea677264881d8073f835336137b39fdea4e94093319379535", + "status": "succeeded", + "input": { + "seed": 42, + "ddim_steps": 50, + "input_file": "https://replicate.delivery/pbxt/JYv70XQsiZBbSmknfMhGoEb4QYbuyJ9hJkfgjyzCvh4TzPmT/music.wav", + "guidance_scale": 3.5 + }, + "output": "https://pbxt.replicate.delivery/73xEQwFed8UXSCQpBJuLYgg8ym0vjp2nSidaqWKwcxhUvHzIA/out.wav", + "logs": "\u001b[93m Warning: audio is longer than 10.24 seconds, may degrade the model performance. It's recommand to truncate your audio to 5.12 seconds before input to AudioSR to get the best performance.\u001b[00m\n/src/audiosr/utils.py:109: FutureWarning: Pass sr=48000, n_fft=2048, n_mels=256, fmin=20, fmax=24000 as keyword args. From version 0.10 passing these as positional arguments will result in an error\nmel = librosa_mel_fn(sampling_rate, filter_length, n_mel, mel_fmin, mel_fmax)\nRunning DDIM Sampling with 50 timesteps\nDDIM Sampler: 0%| | 0/50 [00:00=3: apply median filtering median filtering to the harvested pitch results." + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "mp3", + "x-order": 18, + "description": "wav for best quality and large file size, mp3 for decent quality and small file size." + }, + "reverb_damping": { + "type": "number", + "title": "Reverb Damping", + "default": 0.7, + "maximum": 1, + "minimum": 0, + "x-order": 17, + "description": "Absorption of high frequencies in the reverb." + }, + "reverb_dryness": { + "type": "number", + "title": "Reverb Dryness", + "default": 0.8, + "maximum": 1, + "minimum": 0, + "x-order": 16, + "description": "Level of AI vocals without reverb." + }, + "reverb_wetness": { + "type": "number", + "title": "Reverb Wetness", + "default": 0.2, + "maximum": 1, + "minimum": 0, + "x-order": 15, + "description": "Level of AI vocals with reverb." + }, + "crepe_hop_length": { + "type": "integer", + "title": "Crepe Hop Length", + "default": 128, + "x-order": 8, + "description": "When `pitch_detection_algo` is set to `mangio-crepe`, this controls how often it checks for pitch changes in milliseconds. Lower values lead to longer conversions and higher risk of voice cracks, but better pitch accuracy." + }, + "pitch_change_all": { + "type": "number", + "title": "Pitch Change All", + "default": 0, + "x-order": 13, + "description": "Change pitch/key of background music, backup vocals and AI vocals in semitones. Reduces sound quality slightly." + }, + "main_vocals_volume_change": { + "type": "number", + "title": "Main Vocals Volume Change", + "default": 0, + "x-order": 10, + "description": "Control volume of main AI vocals. Use -3 to decrease the volume by 3 decibels, or 3 to increase the volume by 3 decibels." + }, + "pitch_detection_algorithm": { + "allOf": [ + { + "$ref": "#/components/schemas/pitch_detection_algorithm" + } + ], + "default": "rmvpe", + "x-order": 7, + "description": "Best option is rmvpe (clarity in vocals), then mangio-crepe (smoother vocals)." + }, + "instrumental_volume_change": { + "type": "number", + "title": "Instrumental Volume Change", + "default": 0, + "x-order": 12, + "description": "Control volume of the background music/instrumentals." + }, + "backup_vocals_volume_change": { + "type": "number", + "title": "Backup Vocals Volume Change", + "default": 0, + "x-order": 11, + "description": "Control volume of backup AI vocals." + }, + "custom_rvc_model_download_url": { + "type": "string", + "title": "Custom Rvc Model Download Url", + "x-order": 2, + "description": "URL to download a custom RVC model. If provided, the model will be downloaded (if it doesn't already exist) and used for prediction, regardless of the 'rvc_model' value." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "rvc_model": { + "enum": [ + "Squidward", + "MrKrabs", + "Plankton", + "Drake", + "Vader", + "Trump", + "Biden", + "Obama", + "Guitar", + "Voilin", + "CUSTOM" + ], + "type": "string", + "title": "rvc_model", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "pitch_change": { + "enum": [ + "no-change", + "male-to-female", + "female-to-male" + ], + "type": "string", + "title": "pitch_change", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "mp3", + "wav" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "pitch_detection_algorithm": { + "enum": [ + "rmvpe", + "mangio-crepe" + ], + "type": "string", + "title": "pitch_detection_algorithm", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/supported_models.json b/supported_models.json index 3138c99..5e5e6ca 100644 --- a/supported_models.json +++ b/supported_models.json @@ -1,5 +1,6 @@ { "models": [ + "ai-forever/kandinsky-2", "ai-forever/kandinsky-2.2", "andreasjansson/blip-2", "batouresearch/high-resolution-controlnet-tile", @@ -8,33 +9,47 @@ "bytedance/sdxl-lightning-4step", "cjwbw/hyper-sdxl-1step-t2i", "cjwbw/pixart-sigma", + "cjwbw/seamless_communication", "cjwbw/supir", "cuuupid/glm-4v-9b", "cuuupid/idm-vton", + "declare-lab/tango", "fofr/consistent-character", "fofr/controlnet-preprocessors", "fofr/face-to-many", + "fofr/latent-consistency-model", "fofr/sd3-with-chaos", "fofr/style-transfer", "lucataco/hunyuandit-v1.1", "lucataco/llama-3-vision-alpha", + "lucataco/magnet", "lucataco/moondream2", "lucataco/paligemma-3b-pt-224", "lucataco/pasd-magnify", "lucataco/qwen-vl-chat", "lucataco/sdxl-clip-interrogator", + "lucataco/xtts-v2", + "luosiallen/latent-consistency-model", "meta/llama-2-70b-chat", "meta/llama-2-7b-chat", + "meta/musicgen", "mistralai/mixtral-8x7b-instruct-v0.1", + "nateraw/audio-super-resolution", + "nateraw/musicgen-songstarter-v0.2", "omniedgeio/face-swap", "philz1337x/clarity-upscaler", + "replicate/deepfloyd-if", "salesforce/blip", "smoretalk/rembg-enhance", "snowflake/snowflake-arctic-instruct", "stability-ai/sdxl", + "stability-ai/stable-diffusion", "stability-ai/stable-diffusion-3", + "suno-ai/bark", + "tstramer/material-diffusion", "yorickvp/llava-13b", "yorickvp/llava-v1.6-34b", - "yorickvp/llava-v1.6-mistral-7b" + "yorickvp/llava-v1.6-mistral-7b", + "zsxkib/realistic-voice-cloning" ] }