diff --git a/example_workflows/bark_and_musicgen.json b/example_workflows/bark_and_musicgen.json new file mode 100644 index 0000000..f5294c8 --- /dev/null +++ b/example_workflows/bark_and_musicgen.json @@ -0,0 +1,162 @@ +{ + "last_node_id": 7, + "last_link_id": 2, + "nodes": [ + { + "id": 5, + "type": "Replicate suno-ai/bark", + "pos": [ + 540, + 264 + ], + "size": { + "0": 400, + "1": 266 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 1 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "STRING", + "type": "STRING", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "Replicate suno-ai/bark" + }, + "widgets_values": [ + "Hello, my name is Suno. And, uh — and I like pizza. [laughs] But I also have other interests such as playing tic tac toe.", + "announcer", + "", + 0.7, + 0.7, + false, + false + ] + }, + { + "id": 7, + "type": "PreviewAudio", + "pos": [ + 1457, + 272 + ], + "size": { + "0": 315, + "1": 76 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "audio", + "type": "AUDIO", + "link": 2 + } + ], + "properties": { + "Node name for S&R": "PreviewAudio" + }, + "widgets_values": [ + null + ] + }, + { + "id": 6, + "type": "Replicate meta/musicgen", + "pos": [ + 1001, + 269 + ], + "size": { + "0": 400, + "1": 436 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "input_audio", + "type": "AUDIO", + "link": 1 + } + ], + "outputs": [ + { + "name": "AUDIO", + "type": "AUDIO", + "links": [ + 2 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Replicate meta/musicgen" + }, + "widgets_values": [ + "stereo-melody-large", + "rap, rock", + 20, + true, + 0, + 6, + false, + "loudness", + 250, + 0, + 1, + 3, + "wav", + 1607, + "randomize", + false + ] + } + ], + "links": [ + [ + 1, + 5, + 0, + 6, + 0, + "AUDIO" + ], + [ + 2, + 6, + 0, + 7, + 0, + "AUDIO" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [ + -123.2666015625, + -48.2666015625 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/flux.json b/example_workflows/flux.json new file mode 100644 index 0000000..02703f9 --- /dev/null +++ b/example_workflows/flux.json @@ -0,0 +1,337 @@ +{ + "last_node_id": 21, + "last_link_id": 27, + "nodes": [ + { + "id": 17, + "type": "DF_Text_Box", + "pos": [ + -55, + -228 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 22, + 23, + 24 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "DF_Text_Box" + }, + "widgets_values": [ + "a neon light saying \"flux now in comfyui-replicate\"" + ] + }, + { + "id": 13, + "type": "SaveImage", + "pos": [ + 1023, + -699 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 25 + } + ], + "properties": {}, + "widgets_values": [ + "flux-schnell" + ] + }, + { + "id": 21, + "type": "Replicate black-forest-labs/flux-schnell", + "pos": [ + 580, + -697 + ], + "size": [ + 400, + 268 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "name": "prompt", + "type": "STRING", + "link": 22, + "widget": { + "name": "prompt" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 25 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Replicate black-forest-labs/flux-schnell" + }, + "widgets_values": [ + "", + "3:2", + 1, + 1331, + "randomize", + "webp", + 80, + false, + false + ] + }, + { + "id": 14, + "type": "SaveImage", + "pos": [ + 1027, + -375 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 26 + } + ], + "properties": {}, + "widgets_values": [ + "flux-dev" + ] + }, + { + "id": 20, + "type": "Replicate black-forest-labs/flux-dev", + "pos": [ + 579, + -377 + ], + "size": [ + 400, + 316 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "name": "image", + "type": "IMAGE", + "link": null + }, + { + "name": "prompt", + "type": "STRING", + "link": 23, + "widget": { + "name": "prompt" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 26 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Replicate black-forest-labs/flux-dev" + }, + "widgets_values": [ + "", + "3:2", + 0.8, + 1, + 3.5, + 1454, + "randomize", + "webp", + 80, + false, + false + ] + }, + { + "id": 15, + "type": "SaveImage", + "pos": [ + 1030, + -7 + ], + "size": { + "0": 315, + "1": 270 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 27 + } + ], + "properties": {}, + "widgets_values": [ + "flux-pro" + ] + }, + { + "id": 19, + "type": "Replicate black-forest-labs/flux-pro", + "pos": [ + 579, + -3 + ], + "size": [ + 400, + 268 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "prompt", + "type": "STRING", + "link": 24, + "widget": { + "name": "prompt" + } + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 27 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Replicate black-forest-labs/flux-pro" + }, + "widgets_values": [ + "", + "3:2", + 25, + 3, + 2, + 2, + 1262, + "randomize", + false + ] + } + ], + "links": [ + [ + 22, + 17, + 0, + 21, + 0, + "STRING" + ], + [ + 23, + 17, + 0, + 20, + 1, + "STRING" + ], + [ + 24, + 17, + 0, + 19, + 0, + "STRING" + ], + [ + 25, + 21, + 0, + 13, + 0, + "IMAGE" + ], + [ + 26, + 20, + 0, + 14, + 0, + "IMAGE" + ], + [ + 27, + 19, + 0, + 15, + 0, + "IMAGE" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 1, + "offset": [ + 189.2626953125, + 773.8525390625 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/llama3-405b.json b/example_workflows/llama3-405b.json new file mode 100644 index 0000000..0b448af --- /dev/null +++ b/example_workflows/llama3-405b.json @@ -0,0 +1,566 @@ +{ + "last_node_id": 10, + "last_link_id": 13, + "nodes": [ + { + "id": 5, + "type": "CLIPTextEncode", + "pos": [ + 1199, + 389 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 4 + }, + { + "name": "text", + "type": "STRING", + "link": 3, + "widget": { + "name": "text" + } + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 5 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 4, + "type": "CLIPTextEncode", + "pos": [ + 1195, + 649 + ], + "size": { + "0": 400, + "1": 200 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "clip", + "type": "CLIP", + "link": 2 + } + ], + "outputs": [ + { + "name": "CONDITIONING", + "type": "CONDITIONING", + "links": [ + 6 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CLIPTextEncode" + }, + "widgets_values": [ + "" + ] + }, + { + "id": 6, + "type": "KSampler", + "pos": [ + 1664, + 404 + ], + "size": { + "0": 315, + "1": 262 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "model", + "type": "MODEL", + "link": 11 + }, + { + "name": "positive", + "type": "CONDITIONING", + "link": 5 + }, + { + "name": "negative", + "type": "CONDITIONING", + "link": 6 + }, + { + "name": "latent_image", + "type": "LATENT", + "link": 7 + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 8 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "KSampler" + }, + "widgets_values": [ + 690393467025334, + "randomize", + 25, + 7, + "dpmpp_2m_sde_gpu", + "karras", + 1 + ] + }, + { + "id": 8, + "type": "VAEDecode", + "pos": [ + 2061, + 406 + ], + "size": { + "0": 210, + "1": 46 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "samples", + "type": "LATENT", + "link": 8 + }, + { + "name": "vae", + "type": "VAE", + "link": 9 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 10 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VAEDecode" + } + }, + { + "id": 3, + "type": "CheckpointLoaderSimple", + "pos": [ + 735, + 527 + ], + "size": { + "0": 315, + "1": 98 + }, + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "MODEL", + "type": "MODEL", + "links": [ + 11 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "CLIP", + "type": "CLIP", + "links": [ + 2, + 4 + ], + "shape": 3, + "slot_index": 1 + }, + { + "name": "VAE", + "type": "VAE", + "links": [ + 9 + ], + "shape": 3, + "slot_index": 2 + } + ], + "properties": { + "Node name for S&R": "CheckpointLoaderSimple" + }, + "widgets_values": [ + "proteusV0.5.safetensors" + ] + }, + { + "id": 7, + "type": "EmptyLatentImage", + "pos": [ + 1270, + 219 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "width", + "type": "INT", + "link": 12, + "widget": { + "name": "width" + } + }, + { + "name": "height", + "type": "INT", + "link": 13, + "widget": { + "name": "height" + } + } + ], + "outputs": [ + { + "name": "LATENT", + "type": "LATENT", + "links": [ + 7 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "EmptyLatentImage" + }, + "widgets_values": [ + 1024, + 1024, + 1 + ] + }, + { + "id": 10, + "type": "Width and height from aspect ratio ðŸŠī", + "pos": [ + 838, + 183 + ], + "size": { + "0": 315, + "1": 126 + }, + "flags": {}, + "order": 1, + "mode": 0, + "outputs": [ + { + "name": "width", + "type": "INT", + "links": [ + 12 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "height", + "type": "INT", + "links": [ + 13 + ], + "shape": 3, + "slot_index": 1 + } + ], + "properties": { + "Node name for S&R": "Width and height from aspect ratio ðŸŠī" + }, + "widgets_values": [ + "16:9", + 1024, + 8 + ] + }, + { + "id": 9, + "type": "SaveImage", + "pos": [ + 157, + 474 + ], + "size": [ + 500.09607697684555, + 307.000781640146 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 10 + } + ], + "properties": {}, + "widgets_values": [ + "ComfyUI" + ] + }, + { + "id": 2, + "type": "ShowText|pysssss", + "pos": [ + 153, + 234 + ], + "size": [ + 450.830078125, + 192.2900390625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "text", + "type": "STRING", + "link": 1, + "widget": { + "name": "text" + } + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": null, + "shape": 6 + } + ], + "properties": { + "Node name for S&R": "ShowText|pysssss" + }, + "widgets_values": [ + "", + "A film still of a sci-fi anime movie, two space explorers in a tense, mystical confrontation on a bizarre, abandoned space station, weird, dreamlike, muted metallic colors, soft, ethereal lighting, unusual, organic architecture, anime style, intricate details, dramatic posing." + ] + }, + { + "id": 1, + "type": "Replicate meta/meta-llama-3.1-405b-instruct", + "pos": [ + -274, + 235 + ], + "size": [ + 398.7880859375, + 447.314453125 + ], + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "links": [ + 1, + 3 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Replicate meta/meta-llama-3.1-405b-instruct" + }, + "widgets_values": [ + "an awesome scifi anime scene, with some dynamic interaction between two people, don't use neon stuff though, be weird", + "You are a helpful text to image prompt assistant. You enhance prompts using short sentences and comma separated keywords, subject first then style. Return just the prompt. Make sure to keep all details from given prompt (like if it's a photo, etc)\n\nGood prompt example:\nA film still of an anime movie, two space explorers in a tense standoff on a desolate alien planet, surreal, muted color palette, eerie atmospheric lighting, unusual architecture, anime style, detailed textures, dramatic posing.", + 0, + 512, + 0.6, + 0.9, + 50, + 0, + 0, + "", + false + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "STRING" + ], + [ + 2, + 3, + 1, + 4, + 0, + "CLIP" + ], + [ + 3, + 1, + 0, + 5, + 1, + "STRING" + ], + [ + 4, + 3, + 1, + 5, + 0, + "CLIP" + ], + [ + 5, + 5, + 0, + 6, + 1, + "CONDITIONING" + ], + [ + 6, + 4, + 0, + 6, + 2, + "CONDITIONING" + ], + [ + 7, + 7, + 0, + 6, + 3, + "LATENT" + ], + [ + 8, + 6, + 0, + 8, + 0, + "LATENT" + ], + [ + 9, + 3, + 2, + 8, + 1, + "VAE" + ], + [ + 10, + 8, + 0, + 9, + 0, + "IMAGE" + ], + [ + 11, + 3, + 0, + 6, + 0, + "MODEL" + ], + [ + 12, + 10, + 0, + 7, + 0, + "INT" + ], + [ + 13, + 10, + 1, + 7, + 1, + "INT" + ] + ], + "groups": [], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591553, + "offset": [ + 357.0363126973379, + 28.753132988760044 + ] + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/import_schemas.py b/import_schemas.py index bc562da..0dda323 100755 --- a/import_schemas.py +++ b/import_schemas.py @@ -2,6 +2,7 @@ import replicate import json import os +import argparse def format_json_file(file_path): @@ -25,13 +26,29 @@ def format_json_files_in_directory(directory): format_json_file(file_path) -with open("supported_models.json", "r", encoding="utf-8") as f: - supported_models = json.load(f) +def update_schemas(update=False): + with open("supported_models.json", "r", encoding="utf-8") as f: + supported_models = json.load(f) -for model in supported_models["models"]: - m = replicate.models.get(model) - with open(f"schemas/{model.replace('/', '_')}.json", "w", encoding="utf-8") as f: - f.write(m.json()) + schemas_directory = "schemas" + existing_schemas = set(os.listdir(schemas_directory)) -schemas_directory = "schemas" -format_json_files_in_directory(schemas_directory) + for model in supported_models["models"]: + schema_filename = f"{model.replace('/', '_')}.json" + schema_path = os.path.join(schemas_directory, schema_filename) + + if update or schema_filename not in existing_schemas: + m = replicate.models.get(model) + with open(schema_path, "w", encoding="utf-8") as f: + f.write(m.json()) + print(f"{'Updated' if update else 'Added'} schema for {model}") + + format_json_files_in_directory(schemas_directory) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Update model schemas") + parser.add_argument("--update", action="store_true", help="Update all schemas, not just new ones") + args = parser.parse_args() + + update_schemas(update=args.update) diff --git a/node.py b/node.py index 550fc26..faef57d 100644 --- a/node.py +++ b/node.py @@ -8,6 +8,8 @@ from torchvision import transforms import torch import base64 import time +import torchaudio +import soundfile as sf from replicate.client import Client from .schema_to_node import ( schema_to_comfyui_input_types, @@ -32,7 +34,11 @@ def create_comfyui_node(schema): def INPUT_TYPES(cls): return schema_to_comfyui_input_types(schema) - RETURN_TYPES = (return_type,) + RETURN_TYPES = ( + tuple(return_type.values()) + if isinstance(return_type, dict) + else (return_type,) + ) FUNCTION = "run_replicate_model" CATEGORY = "Replicate" @@ -45,6 +51,8 @@ def create_comfyui_node(schema): ) if input_type == "IMAGE": kwargs[key] = self.image_to_base64(value) + elif input_type == "AUDIO": + kwargs[key] = self.audio_to_base64(value) def image_to_base64(self, image): if isinstance(image, torch.Tensor): @@ -60,6 +68,31 @@ def create_comfyui_node(schema): img_str = base64.b64encode(buffer.getvalue()).decode() return f"data:image/png;base64,{img_str}" + def audio_to_base64(self, audio): + if ( + isinstance(audio, dict) + and "waveform" in audio + and "sample_rate" in audio + ): + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + else: + waveform, sample_rate = audio + + # Ensure waveform is 2D + if waveform.dim() == 1: + waveform = waveform.unsqueeze(0) + elif waveform.dim() > 2: + waveform = waveform.squeeze() + if waveform.dim() > 2: + raise ValueError("Waveform must be 1D or 2D") + + buffer = io.BytesIO() + sf.write(buffer, waveform.numpy().T, sample_rate, format="wav") + buffer.seek(0) + audio_str = base64.b64encode(buffer.getvalue()).decode() + return f"data:audio/wav;base64,{audio_str}" + def handle_array_inputs(self, kwargs): array_inputs = inputs_that_need_arrays(schema) for input_name in array_inputs: @@ -75,13 +108,18 @@ def create_comfyui_node(schema): def log_input(self, kwargs): truncated_kwargs = { k: v[:20] + "..." - if isinstance(v, str) and v.startswith("data:image") + if isinstance(v, str) + and (v.startswith("data:image") or v.startswith("data:audio")) else v for k, v in kwargs.items() } print(f"Running {replicate_model} with {truncated_kwargs}") def handle_image_output(self, output): + if output is None: + print("No image output received") + return None + # Handle both string and list outputs output_list = [output] if isinstance(output, str) else list(output) if output_list: @@ -112,19 +150,93 @@ def create_comfyui_node(schema): print("No output received from the model") return None + def handle_audio_output(self, output): + if output is None: + print("No audio output received from the model") + return None + + if isinstance(output, str): + output_list = [output] + elif isinstance(output, (list, tuple)): + output_list = list(output) + else: + print(f"Unexpected output type: {type(output)}") + return None + + if output_list: + audio_data = [] + for audio_url in output_list: + if audio_url: + response = requests.get(audio_url) + if response.status_code == 200: + audio_content = BytesIO(response.content) + waveform, sample_rate = torchaudio.load(audio_content) + audio_data.append( + { + "waveform": waveform.unsqueeze(0), + "sample_rate": sample_rate, + } + ) + else: + print( + f"Failed to download audio. Status code: {response.status_code}" + ) + else: + print("Empty audio URL received") + + # If there's only one audio file, return it directly + if len(audio_data) == 1: + return audio_data[0] + # If there are multiple audio files, return them as a list + return audio_data + else: + print("No valid audio URLs in the output") + return None + + def remove_falsey_optional_inputs(self, kwargs): + optional_inputs = self.INPUT_TYPES().get("optional", {}) + for key in list(kwargs.keys()): + if key in optional_inputs: + if isinstance(kwargs[key], torch.Tensor): + continue + elif not kwargs[key]: + del kwargs[key] + def run_replicate_model(self, **kwargs): self.handle_array_inputs(kwargs) + self.remove_falsey_optional_inputs(kwargs) self.convert_input_images_to_base64(kwargs) self.log_input(kwargs) - output = replicate.run(replicate_model, input=kwargs) + kwargs_without_force_rerun = { + k: v for k, v in kwargs.items() if k != "force_rerun" + } + output = replicate.run(replicate_model, input=kwargs_without_force_rerun) print(f"Output: {output}") - if return_type == "IMAGE": - output = self.handle_image_output(output) + processed_outputs = [] + if isinstance(return_type, dict): + for prop_name, prop_type in return_type.items(): + if prop_type == "IMAGE": + processed_outputs.append( + self.handle_image_output(output.get(prop_name)) + ) + elif prop_type == "AUDIO": + processed_outputs.append( + self.handle_audio_output(output.get(prop_name)) + ) + elif prop_type == "STRING": + processed_outputs.append( + "".join(list(output.get(prop_name, ""))).strip() + ) else: - output = "".join(list(output)).strip() + if return_type == "IMAGE": + processed_outputs.append(self.handle_image_output(output)) + elif return_type == "AUDIO": + processed_outputs.append(self.handle_audio_output(output)) + else: + processed_outputs.append("".join(list(output)).strip()) - return (output,) + return tuple(processed_outputs) return node_name, ReplicateToComfyUI @@ -135,7 +247,9 @@ def create_comfyui_nodes_from_schemas(schemas_dir): schemas_dir_path = os.path.join(current_path, schemas_dir) for schema_file in os.listdir(schemas_dir_path): if schema_file.endswith(".json"): - with open(os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8") as f: + with open( + os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8" + ) as f: schema = json.load(f) node_name, node_class = create_comfyui_node(schema) nodes[node_name] = node_class diff --git a/pyproject.toml b/pyproject.toml index a4f12f3..e007bd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-replicate" description = "Run Replicate models in ComfyUI" -version = "1.0.4" +version = "1.0.7" license = { file = "LICENSE" } [project.urls] diff --git a/schema_to_node.py b/schema_to_node.py index 76df66a..8e91a65 100644 --- a/schema_to_node.py +++ b/schema_to_node.py @@ -1,17 +1,41 @@ DEFAULT_STEP = 0.01 DEFAULT_ROUND = 0.001 +IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp") +VIDEO_EXTENSIONS = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") +AUDIO_EXTENSIONS = (".mp3", ".wav", ".flac", ".mpga", ".m4a") -def convert_to_comfyui_input_type(openapi_type, openapi_format=None): - type_mapping = { - "string": "STRING", - "integer": "INT", - "number": "FLOAT", - "boolean": "BOOLEAN", - } +TYPE_MAPPING = { + "string": "STRING", + "integer": "INT", + "number": "FLOAT", + "boolean": "BOOLEAN", +} + + +def convert_to_comfyui_input_type( + input_name, openapi_type, openapi_format=None, default_example_input=None +): if openapi_type == "string" and openapi_format == "uri": - return "IMAGE" - return type_mapping.get(openapi_type, "STRING") + if ( + default_example_input + and isinstance(default_example_input, dict) + and input_name in default_example_input + ): + if is_type(default_example_input[input_name], IMAGE_EXTENSIONS): + return "IMAGE" + elif is_type(default_example_input[input_name], VIDEO_EXTENSIONS): + return "VIDEO" + elif is_type(default_example_input[input_name], AUDIO_EXTENSIONS): + return "AUDIO" + elif "image" in input_name.lower(): + return "IMAGE" + elif "audio" in input_name.lower(): + return "AUDIO" + else: + return "STRING" + + return TYPE_MAPPING.get(openapi_type, "STRING") def name_and_version(schema): @@ -23,11 +47,13 @@ def name_and_version(schema): return replicate_model, node_name -def resolve_schema(prop_data, openapi_schemas): +def resolve_schema(prop_data, openapi_schema): if "$ref" in prop_data: ref_path = prop_data["$ref"].split("/") - current = openapi_schemas + current = openapi_schema for path in ref_path[1:]: # Skip the first '#' element + if path not in current: + return prop_data # Return original if path is invalid current = current[path] return current return prop_data @@ -37,6 +63,7 @@ def schema_to_comfyui_input_types(schema): openapi_schema = schema["latest_version"]["openapi_schema"] input_schema = openapi_schema["components"]["schemas"]["Input"] input_types = {"required": {}, "optional": {}} + default_example_input = get_default_example_input(schema) required_props = input_schema.get("required", []) @@ -51,7 +78,10 @@ def schema_to_comfyui_input_types(schema): input_type = prop_data["enum"] elif "type" in prop_data: input_type = convert_to_comfyui_input_type( - prop_data["type"], prop_data.get("format") + prop_name, + prop_data["type"], + prop_data.get("format"), + default_example_input, ) else: input_type = "STRING" @@ -134,22 +164,65 @@ def is_type(default_example_output, extensions): return False -def get_return_type(schema): - image_extensions = (".png", ".jpg", ".jpeg", ".gif", ".webp") - video_extensions = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") - audio_extensions = (".mp3", ".wav") - - openapi_schema = schema["latest_version"]["openapi_schema"] - output_schema = openapi_schema["components"]["schemas"].get("Output") +def get_default_example(schema): default_example = schema.get("default_example") - default_example_output = default_example.get("output") if default_example else None + return default_example if default_example else None - if is_type(default_example_output, image_extensions): + +def get_default_example_input(schema): + default_example = get_default_example(schema) + return default_example.get("input") if default_example else None + + +def get_default_example_output(schema): + default_example = get_default_example(schema) + return default_example.get("output") if default_example else None + + +def get_return_type(schema): + openapi_schema = schema["latest_version"]["openapi_schema"] + output_schema = ( + openapi_schema.get("components", {}).get("schemas", {}).get("Output") + ) + default_example_output = get_default_example_output(schema) + + if output_schema and "$ref" in output_schema: + output_schema = resolve_schema(output_schema, openapi_schema) + + if isinstance(output_schema, dict) and output_schema.get("properties"): + return_types = {} + for prop_name, prop_data in output_schema["properties"].items(): + if isinstance(default_example_output, dict): + prop_value = default_example_output.get(prop_name) + + if is_type(prop_value, IMAGE_EXTENSIONS): + return_types[prop_name] = "IMAGE" + elif is_type(prop_value, AUDIO_EXTENSIONS): + return_types[prop_name] = "AUDIO" + elif is_type(prop_value, VIDEO_EXTENSIONS): + return_types[prop_name] = "VIDEO_URI" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("format") == "uri": + if "audio" in prop_name.lower(): + return_types[prop_name] = "AUDIO" + elif "image" in prop_name.lower(): + return_types[prop_name] = "IMAGE" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("type") == "string": + return_types[prop_name] = "STRING" + else: + return_types[prop_name] = "STRING" + + return return_types + + if is_type(default_example_output, IMAGE_EXTENSIONS): return "IMAGE" - elif is_type(default_example_output, video_extensions): + elif is_type(default_example_output, VIDEO_EXTENSIONS): return "VIDEO_URI" - elif is_type(default_example_output, audio_extensions): - return "AUDIO_URI" + elif is_type(default_example_output, AUDIO_EXTENSIONS): + return "AUDIO" if output_schema: if ( @@ -160,8 +233,8 @@ def get_return_type(schema): return "IMAGE" elif ( output_schema.get("type") == "array" - and output_schema["items"].get("type") == "string" - and output_schema["items"].get("format") == "uri" + and output_schema.get("items", {}).get("type") == "string" + and output_schema.get("items", {}).get("format") == "uri" ): # Handle multiple image output return "IMAGE" diff --git a/schemas/ai-forever_kandinsky-2.json b/schemas/ai-forever_kandinsky-2.json new file mode 100644 index 0000000..7705b7d --- /dev/null +++ b/schemas/ai-forever_kandinsky-2.json @@ -0,0 +1,601 @@ +{ + "url": "https://replicate.com/ai-forever/kandinsky-2", + "owner": "ai-forever", + "name": "kandinsky-2", + "description": "text2img model trained on LAION HighRes and fine-tuned on internal datasets", + "visibility": "public", + "github_url": "https://github.com/chenxwh/Kandinsky-2", + "paper_url": null, + "license_url": "https://github.com/ai-forever/Kandinsky-2/blob/main/license", + "run_count": 0, + "cover_image_url": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "default_example": { + "id": "wixq75uo65glnjcdgrqtngdz4q", + "model": "ai-forever/kandinsky-2", + "version": "9c0bf7d5cf2ed934c5921faf61882657c03c4def9d9cb88330c15bd795edb098", + "status": "succeeded", + "input": { + "prompt": "red cat, 4k photo", + "scheduler": "p_sampler", + "prior_steps": "5", + "guidance_scale": 4, + "prior_cf_scale": 4, + "num_inference_steps": 100 + }, + "output": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "logs": "0%| | 0/100 [00:00 #0:0 (pcm_s16le (native) -> pcm_s16le (native))\nPress [q] to stop, [?] for help\nOutput #0, wav, to '/tmp/speaker.wav':\nMetadata:\nISFT : Lavf58.76.100\nStream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 44100 Hz, mono, s16, 705 kb/s\nMetadata:\nencoder : Lavc58.134.100 pcm_s16le\nsize= 4kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 744kB time=00:00:08.63 bitrate= 705.9kbits/s speed=1.75e+03x\nvideo:0kB audio:744kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.010236%\n> Text splitted to sentences.\n[\"Hi there, I'm your new voice clone.\", 'Try your best to upload quality audio']\n> Processing time: 3.3411097526550293\n> Real-time factor: 0.613438166517148", + "error": null, + "metrics": { + "predict_time": 4.303438, + "total_time": 4.319136 + }, + "created_at": "2023-11-16T16:42:06.735772Z", + "started_at": "2023-11-16T16:42:06.751470Z", + "completed_at": "2023-11-16T16:42:11.054908Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu", + "cancel": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu/cancel" + } + }, + "latest_version": { + "id": "684bc3855b37866c0c65add2ff39c78f3dea3f4ff103a436465326e0f438d55e", + "created_at": "2023-11-28T15:39:16.308105+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "required": [ + "speaker" + ], + "properties": { + "text": { + "type": "string", + "title": "Text", + "default": "Hi there, I'm your new voice clone. Try your best to upload quality audio", + "x-order": 0, + "description": "Text to synthesize" + }, + "speaker": { + "type": "string", + "title": "Speaker", + "format": "uri", + "x-order": 1, + "description": "Original speaker audio (wav, mp3, m4a, ogg, or flv)" + }, + "language": { + "allOf": [ + { + "$ref": "#/components/schemas/language" + } + ], + "default": "en", + "x-order": 2, + "description": "Output language for the synthesised speech" + }, + "cleanup_voice": { + "type": "boolean", + "title": "Cleanup Voice", + "default": false, + "x-order": 3, + "description": "Whether to apply denoising to the speaker audio (microphone recordings)" + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "language": { + "enum": [ + "en", + "es", + "fr", + "de", + "it", + "pt", + "pl", + "tr", + "ru", + "nl", + "cs", + "ar", + "zh", + "hu", + "ko", + "hi" + ], + "type": "string", + "title": "language", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/luosiallen_latent-consistency-model.json b/schemas/luosiallen_latent-consistency-model.json new file mode 100644 index 0000000..f1613e9 --- /dev/null +++ b/schemas/luosiallen_latent-consistency-model.json @@ -0,0 +1,496 @@ +{ + "url": "https://replicate.com/luosiallen/latent-consistency-model", + "owner": "luosiallen", + "name": "latent-consistency-model", + "description": "Synthesizing High-Resolution Images with Few-Step Inference", + "visibility": "public", + "github_url": "https://github.com/luosiallen/latent-consistency-model", + "paper_url": "https://arxiv.org/abs/2310.04378", + "license_url": "https://github.com/chenxwh/latent-consistency-model/blob/main/LICENSE", + "run_count": 0, + "cover_image_url": "https://pbxt.replicate.delivery/lE1UZOlbLiY7DpYlLvdUmyyt0Z0pu7tTTpgFn8a0ug3Jb4bE/out-0.png", + "default_example": { + "id": "5lwbdqdbyrilvvsm62mphlbghu", + "model": "luosiallen/latent-consistency-model", + "version": "553803fd018b3cf875a8bc774c99da9b33f36647badfd88a6eec90d61c5f62fc", + "status": "succeeded", + "input": { + "width": 768, + "height": 768, + "prompt": "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k", + "num_images": 1, + "guidance_scale": 8, + "num_inference_steps": 8 + }, + "output": [ + "https://pbxt.replicate.delivery/lE1UZOlbLiY7DpYlLvdUmyyt0Z0pu7tTTpgFn8a0ug3Jb4bE/out-0.png" + ], + "logs": "Using seed: 49296\n 0%| | 0/8 [00:00<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nTina has one brother and one sister. How many sisters do Tina's siblings have?<|eot_id|><|start_header_id|>assistant<|end_header_id|>", + "error": null, + "metrics": { + "batch_size": 1.000229398266064, + "total_time": 2.86309, + "input_token_count": 26, + "tokens_per_second": 29.74134035953616, + "output_token_count": 84, + "predict_time_share": 2.8200430870056152, + "predict_time": 2.826425468, + "time_to_first_token": 0.112068943 + }, + "created_at": "2024-07-23T15:04:21.227000Z", + "started_at": "2024-07-23T15:04:21.263665Z", + "completed_at": "2024-07-23T15:04:24.090090Z", + "urls": { + "stream": "https://streaming-api.svc.sea.v.replicate.net/v1/streams/6zoqtc6qgpxby7xzauee36qv6bktfaxugelxfqs6ifgg6q4s77sa", + "get": "https://api.replicate.com/v1/predictions/zjq6szssxdrm20cgw0fsntgvbg", + "cancel": "https://api.replicate.com/v1/predictions/zjq6szssxdrm20cgw0fsntgvbg/cancel" + } + }, + "latest_version": { + "id": "e6cb7fc3ed90eae2c879c48deda8f49152391ad66349fe7694be24089c29f71c", + "created_at": "2024-07-23T15:24:35.230971+00:00", + "cog_version": "0.10.0-alpha17", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/ready": { + "get": { + "summary": "Ready", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Ready Ready Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "ready_ready_get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/trainings": { + "post": { + "summary": "Train", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TrainingResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "operationId": "train_trainings_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/TrainingRequest" + } + } + } + } + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/trainings/{training_id}": { + "put": { + "summary": "Train Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "training_id", + "schema": { + "type": "string", + "title": "Training ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "operationId": "train_idempotent_trainings__training_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/TrainingRequest" + } + ], + "title": "Training Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/trainings/{training_id}/cancel": { + "post": { + "summary": "Cancel Training", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Training Trainings Training Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "training_id", + "schema": { + "type": "string", + "title": "Training ID" + }, + "required": true + } + ], + "operationId": "cancel_training_trainings__training_id__cancel_post" + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "top_k": { + "type": "integer", + "title": "Top K", + "default": 50, + "x-order": 6, + "description": "The number of highest probability tokens to consider for generating the output. If > 0, only keep the top k tokens with highest probability (top-k filtering)." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0.9, + "x-order": 5, + "description": "A probability threshold for generating the output. If < 1.0, only keep the top tokens with cumulative probability >= top_p (nucleus filtering). Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "default": "", + "x-order": 0, + "description": "Prompt" + }, + "max_tokens": { + "type": "integer", + "title": "Max Tokens", + "default": 512, + "x-order": 3, + "description": "The maximum number of tokens the model should generate as output." + }, + "min_tokens": { + "type": "integer", + "title": "Min Tokens", + "default": 0, + "x-order": 2, + "description": "The minimum number of tokens the model should generate as output." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 0.6, + "x-order": 4, + "description": "The value used to modulate the next token probabilities." + }, + "system_prompt": { + "type": "string", + "title": "System Prompt", + "default": "You are a helpful assistant.", + "x-order": 1, + "description": "System prompt to send to the model. This is prepended to the prompt and helps guide system behavior. Ignored for non-chat models." + }, + "stop_sequences": { + "type": "string", + "title": "Stop Sequences", + "x-order": 9, + "description": "A comma-separated list of sequences to stop generation at. For example, ',' will stop generation at the first instance of 'end' or ''." + }, + "presence_penalty": { + "type": "number", + "title": "Presence Penalty", + "default": 0, + "x-order": 7, + "description": "Presence penalty" + }, + "frequency_penalty": { + "type": "number", + "title": "Frequency Penalty", + "default": 0, + "x-order": 8, + "description": "Frequency penalty" + } + } + }, + "Output": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Output", + "x-cog-array-type": "iterator", + "x-cog-array-display": "concatenate" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "TrainingInput": { + "type": "object", + "title": "TrainingInput", + "required": [ + "hf_model_id" + ], + "properties": { + "hf_token": { + "type": "string", + "title": "Hf Token", + "format": "password", + "x-order": 2, + "description": "\n Hugging Face API token. \n Get your token at https://huggingface.co/settings/tokens\n ", + "x-cog-secret": true + }, + "hf_model_id": { + "type": "string", + "title": "Hf Model Id", + "x-order": 0, + "description": "\n Hugging Face model identifier \n (e.g. NousResearch/Hermes-2-Theta-Llama-3-8B).\n " + }, + "hf_model_sha": { + "type": "string", + "title": "Hf Model Sha", + "x-order": 1, + "description": "\n The version of the model.\n If unspecified, the latest version is used.\n " + }, + "allow_patterns": { + "type": "string", + "title": "Allow Patterns", + "x-order": 3, + "description": "\n Patterns constituting the allowlist. \n If provided, item paths must match at least one pattern from the allowlist. \n (e.g. \"*.safetensors\").\n " + }, + "ignore_patterns": { + "type": "string", + "title": "Ignore Patterns", + "default": "*.gguf", + "x-order": 4, + "description": "\n Patterns constituting the denylist. \n If provided, item paths must not match any patterns from the denylist. \n (e.g. \"*.gguf\").\n " + }, + "prompt_template": { + "type": "string", + "title": "Prompt Template", + "x-order": 5, + "description": "Prompt template. This is a Jinja2 template that overrides the HuggingFace tokenizer configuration. If this is set to None and nothing is configured on HuggingFace, no formatting is applied. To override HuggingFace configuration, set it to the string `{{messages[0]['content']}}`." + } + } + }, + "TrainingOutput": { + "type": "object", + "title": "TrainingOutput", + "required": [ + "weights" + ], + "properties": { + "weights": { + "type": "string", + "title": "Weights", + "format": "uri" + } + } + }, + "TrainingRequest": { + "type": "object", + "title": "TrainingRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/TrainingInput" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "TrainingResponse": { + "type": "object", + "title": "TrainingResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/TrainingInput" + }, + "output": { + "$ref": "#/components/schemas/TrainingOutput" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/meta_musicgen.json b/schemas/meta_musicgen.json new file mode 100644 index 0000000..cf4d704 --- /dev/null +++ b/schemas/meta_musicgen.json @@ -0,0 +1,587 @@ +{ + "url": "https://replicate.com/meta/musicgen", + "owner": "meta", + "name": "musicgen", + "description": "Generate music from a prompt or melody", + "visibility": "public", + "github_url": "https://github.com/replicate/cog-musicgen", + "paper_url": "https://arxiv.org/abs/2306.05284", + "license_url": "https://github.com/facebookresearch/audiocraft/blob/main/LICENSE_weights", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/e3d6b391-571b-4e24-8531-936a0fee9fba/musicgen.jpeg", + "default_example": { + "id": "soub5pjbe5g5u62q536t73ltku", + "model": "meta/musicgen", + "version": "671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb", + "status": "succeeded", + "input": { + "top_k": 250, + "top_p": 0, + "prompt": "Edo25 major g melodies that sound triumphant and cinematic. Leading up to a crescendo that resolves in a 9th harmonic", + "duration": 8, + "temperature": 1, + "continuation": false, + "model_version": "stereo-large", + "output_format": "mp3", + "continuation_start": 0, + "multi_band_diffusion": false, + "normalization_strategy": "peak", + "classifier_free_guidance": 3 + }, + "output": "https://replicate.delivery/pbxt/OeLYIQiltdzMaCex1shlEFy6wuvnvExi0kdjF0gYKEzG5wkSA/out.mp3", + "logs": "Loading model stereo-large...\nDownloading models--facebook--musicgen-stereo-large to models\nDownloaded models--facebook--musicgen-stereo-large in 9.88s\nModel stereo-large loaded successfully.\nUsing seed 187926229\n1 / 400\n2 / 400\n3 / 400\n4 / 400\n5 / 400\n6 / 400\n7 / 400\n8 / 400\n9 / 400\n10 / 400\n11 / 400\n12 / 400\n13 / 400\n14 / 400\n15 / 400\n16 / 400\n17 / 400\n18 / 400\n19 / 400\n20 / 400\n21 / 400\n22 / 400\n23 / 400\n24 / 400\n25 / 400\n26 / 400\n27 / 400\n28 / 400\n29 / 400\n30 / 400\n31 / 400\n32 / 400\n33 / 400\n34 / 400\n35 / 400\n36 / 400\n37 / 400\n38 / 400\n39 / 400\n40 / 400\n41 / 400\n42 / 400\n43 / 400\n44 / 400\n45 / 400\n46 / 400\n47 / 400\n48 / 400\n49 / 400\n50 / 400\n51 / 400\n52 / 400\n53 / 400\n54 / 400\n55 / 400\n56 / 400\n57 / 400\n58 / 400\n59 / 400\n60 / 400\n61 / 400\n62 / 400\n63 / 400\n64 / 400\n65 / 400\n66 / 400\n67 / 400\n68 / 400\n69 / 400\n70 / 400\n71 / 400\n72 / 400\n73 / 400\n74 / 400\n75 / 400\n76 / 400\n77 / 400\n78 / 400\n79 / 400\n80 / 400\n81 / 400\n82 / 400\n83 / 400\n 84 / 400\n85 / 400\n86 / 400\n87 / 400\n88 / 400\n89 / 400\n90 / 400\n91 / 400\n92 / 400\n93 / 400\n94 / 400\n95 / 400\n96 / 400\n97 / 400\n98 / 400\n99 / 400\n100 / 400\n101 / 400\n102 / 400\n103 / 400\n104 / 400\n105 / 400\n106 / 400\n107 / 400\n108 / 400\n109 / 400\n110 / 400\n111 / 400\n112 / 400\n113 / 400\n114 / 400\n115 / 400\n116 / 400\n117 / 400\n118 / 400\n119 / 400\n120 / 400\n121 / 400\n122 / 400\n123 / 400\n124 / 400\n125 / 400\n126 / 400\n127 / 400\n128 / 400\n129 / 400\n130 / 400\n131 / 400\n132 / 400\n133 / 400\n134 / 400\n135 / 400\n136 / 400\n137 / 400\n138 / 400\n139 / 400\n140 / 400\n141 / 400\n142 / 400\n143 / 400\n144 / 400\n145 / 400\n146 / 400\n147 / 400\n148 / 400\n149 / 400\n150 / 400\n151 / 400\n152 / 400\n153 / 400\n154 / 400\n155 / 400\n156 / 400\n157 / 400\n158 / 400\n159 / 400\n160 / 400\n161 / 400\n162 / 400\n163 / 400\n164 / 400\n165 / 400\n166 / 400\n167 / 400\n168 / 400\n169 / 400\n170 / 400\n171 / 400\n172 / 400\n173 / 400\n174 / 400\n175 / 400\n176 / 400\n177 / 400\n178 / 400\n179 / 400\n180 / 400\n181 / 400\n182 / 400\n183 / 400\n184 / 400\n185 / 400\n186 / 400\n187 / 400\n188 / 400\n189 / 400\n190 / 400\n191 / 400\n192 / 400\n193 / 400\n194 / 400\n195 / 400\n196 / 400\n197 / 400\n198 / 400\n199 / 400\n200 / 400\n201 / 400\n202 / 400\n203 / 400\n204 / 400\n205 / 400\n206 / 400\n207 / 400\n208 / 400\n209 / 400\n210 / 400\n211 / 400\n212 / 400\n213 / 400\n214 / 400\n215 / 400\n216 / 400\n217 / 400\n218 / 400\n219 / 400\n220 / 400\n221 / 400\n222 / 400\n223 / 400\n224 / 400\n225 / 400\n226 / 400\n227 / 400\n228 / 400\n229 / 400\n230 / 400\n231 / 400\n232 / 400\n233 / 400\n234 / 400\n235 / 400\n236 / 400\n237 / 400\n238 / 400\n239 / 400\n240 / 400\n241 / 400\n242 / 400\n243 / 400\n244 / 400\n245 / 400\n246 / 400\n247 / 400\n248 / 400\n249 / 400\n250 / 400\n251 / 400\n252 / 400\n253 / 400\n254 / 400\n255 / 400\n256 / 400\n257 / 400\n258 / 400\n259 / 400\n260 / 400\n261 / 400\n262 / 400\n263 / 400\n264 / 400\n265 / 400\n266 / 400\n267 / 400\n268 / 400\n269 / 400\n270 / 400\n271 / 400\n272 / 400\n273 / 400\n274 / 400\n275 / 400\n276 / 400\n277 / 400\n278 / 400\n279 / 400\n280 / 400\n281 / 400\n282 / 400\n283 / 400\n284 / 400\n285 / 400\n286 / 400\n287 / 400\n288 / 400\n289 / 400\n290 / 400\n291 / 400\n292 / 400\n293 / 400\n294 / 400\n295 / 400\n296 / 400\n297 / 400\n298 / 400\n299 / 400\n300 / 400\n301 / 400\n302 / 400\n303 / 400\n304 / 400\n305 / 400\n306 / 400\n307 / 400\n308 / 400\n309 / 400\n310 / 400\n311 / 400\n312 / 400\n313 / 400\n314 / 400\n315 / 400\n316 / 400\n317 / 400\n318 / 400\n319 / 400\n320 / 400\n321 / 400\n322 / 400\n323 / 400\n324 / 400\n325 / 400\n326 / 400\n327 / 400\n328 / 400\n329 / 400\n330 / 400\n331 / 400\n332 / 400\n333 / 400\n334 / 400\n335 / 400\n336 / 400\n337 / 400\n338 / 400\n339 / 400\n340 / 400\n341 / 400\n342 / 400\n343 / 400\n344 / 400\n345 / 400\n346 / 400\n347 / 400\n348 / 400\n349 / 400\n350 / 400\n351 / 400\n352 / 400\n353 / 400\n354 / 400\n355 / 400\n356 / 400\n357 / 400\n358 / 400\n359 / 400\n360 / 400\n361 / 400\n362 / 400\n363 / 400\n364 / 400\n365 / 400\n366 / 400\n367 / 400\n368 / 400\n369 / 400\n370 / 400\n371 / 400\n372 / 400\n373 / 400\n374 / 400\n375 / 400\n376 / 400\n377 / 400\n378 / 400\n379 / 400\n380 / 400\n381 / 400\n382 / 400\n383 / 400\n384 / 400\n385 / 400\n386 / 400\n387 / 400\n388 / 400\n389 / 400\n390 / 400\n391 / 400\n392 / 400\n393 / 400\n394 / 400\n395 / 400\n396 / 400\n397 / 400\n398 / 400\n399 / 400\n400 / 400\n401 / 400\n402 / 400\n403 / 400\nffmpeg version 4.4.2-0ubuntu0.22.04.1 Copyright (c) 2000-2021 the FFmpeg developers\nbuilt with gcc 11 (Ubuntu 11.2.0-19ubuntu1)\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.22.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libdav1d --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librabbitmq --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-pocketsphinx --enable-librsvg --enable-libmfx --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\nlibavutil 56. 70.100 / 56. 70.100\nlibavcodec 58.134.100 / 58.134.100\nlibavformat 58. 76.100 / 58. 76.100\nlibavdevice 58. 13.100 / 58. 13.100\nlibavfilter 7.110.100 / 7.110.100\nlibswscale 5. 9.100 / 5. 9.100\nlibswresample 3. 9.100 / 3. 9.100\nlibpostproc 55. 9.100 / 55. 9.100\nGuessed Channel Layout for Input Stream #0.0 : stereo\nInput #0, wav, from 'out.wav':\nMetadata:\nencoder : Lavf58.76.100\nDuration: 00:00:08.00, bitrate: 1024 kb/s\nStream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 32000 Hz, stereo, s16, 1024 kb/s\nStream mapping:\nStream #0:0 -> #0:0 (pcm_s16le (native) -> mp3 (libmp3lame))\nPress [q] to stop, [?] for help\nOutput #0, mp3, to 'out.mp3':\nMetadata:\nTSSE : Lavf58.76.100\nStream #0:0: Audio: mp3, 32000 Hz, stereo, s16p\nMetadata:\nencoder : Lavc58.134.100 libmp3lame\nsize= 0kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 95kB time=00:00:07.99 bitrate= 97.1kbits/s speed=45.6x\nvideo:0kB audio:94kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.269717%", + "error": null, + "metrics": { + "predict_time": 66.371701, + "total_time": 66.463286 + }, + "created_at": "2024-03-28T12:01:08.688755Z", + "started_at": "2024-03-28T12:01:08.780340Z", + "completed_at": "2024-03-28T12:02:15.152041Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku", + "cancel": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku/cancel" + } + }, + "latest_version": { + "id": "671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb", + "created_at": "2024-03-28T11:07:13.003999+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "seed": { + "type": "integer", + "title": "Seed", + "x-order": 14, + "description": "Seed for random number generator. If None or -1, a random seed will be used." + }, + "top_k": { + "type": "integer", + "title": "Top K", + "default": 250, + "x-order": 9, + "description": "Reduces sampling to the k most likely tokens." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0, + "x-order": 10, + "description": "Reduces sampling to tokens with cumulative probability of p. When set to `0` (default), top_k sampling is used." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "x-order": 1, + "description": "A description of the music you want to generate." + }, + "duration": { + "type": "integer", + "title": "Duration", + "default": 8, + "x-order": 3, + "description": "Duration of the generated audio in seconds." + }, + "input_audio": { + "type": "string", + "title": "Input Audio", + "format": "uri", + "x-order": 2, + "description": "An audio file that will influence the generated music. If `continuation` is `True`, the generated music will be a continuation of the audio file. Otherwise, the generated music will mimic the audio file's melody." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 1, + "x-order": 11, + "description": "Controls the 'conservativeness' of the sampling process. Higher temperature means more diversity." + }, + "continuation": { + "type": "boolean", + "title": "Continuation", + "default": false, + "x-order": 4, + "description": "If `True`, generated music will continue from `input_audio`. Otherwise, generated music will mimic `input_audio`'s melody." + }, + "model_version": { + "allOf": [ + { + "$ref": "#/components/schemas/model_version" + } + ], + "default": "stereo-melody-large", + "x-order": 0, + "description": "Model to use for generation" + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "wav", + "x-order": 13, + "description": "Output format for generated audio." + }, + "continuation_end": { + "type": "integer", + "title": "Continuation End", + "minimum": 0, + "x-order": 6, + "description": "End time of the audio file to use for continuation. If -1 or None, will default to the end of the audio clip." + }, + "continuation_start": { + "type": "integer", + "title": "Continuation Start", + "default": 0, + "minimum": 0, + "x-order": 5, + "description": "Start time of the audio file to use for continuation." + }, + "multi_band_diffusion": { + "type": "boolean", + "title": "Multi Band Diffusion", + "default": false, + "x-order": 7, + "description": "If `True`, the EnCodec tokens will be decoded with MultiBand Diffusion. Only works with non-stereo models." + }, + "normalization_strategy": { + "allOf": [ + { + "$ref": "#/components/schemas/normalization_strategy" + } + ], + "default": "loudness", + "x-order": 8, + "description": "Strategy for normalizing audio." + }, + "classifier_free_guidance": { + "type": "integer", + "title": "Classifier Free Guidance", + "default": 3, + "x-order": 12, + "description": "Increases the influence of inputs on the output. Higher values produce lower-varience outputs that adhere more closely to inputs." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "model_version": { + "enum": [ + "stereo-melody-large", + "stereo-large", + "melody-large", + "large" + ], + "type": "string", + "title": "model_version", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "wav", + "mp3" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "normalization_strategy": { + "enum": [ + "loudness", + "clip", + "peak", + "rms" + ], + "type": "string", + "title": "normalization_strategy", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/nateraw_audio-super-resolution.json b/schemas/nateraw_audio-super-resolution.json new file mode 100644 index 0000000..1c5fac3 --- /dev/null +++ b/schemas/nateraw_audio-super-resolution.json @@ -0,0 +1,469 @@ +{ + "url": "https://replicate.com/nateraw/audio-super-resolution", + "owner": "nateraw", + "name": "audio-super-resolution", + "description": "AudioSR: Versatile Audio Super-resolution at Scale", + "visibility": "public", + "github_url": "https://github.com/haoheliu/versatile_audio_super_resolution", + "paper_url": "https://arxiv.org/abs/2309.07314", + "license_url": "https://huggingface.co/haoheliu/wellsolve_audio_super_resolution_48k/blob/main/README.md", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/6bc4e480-6695-451b-940d-48a3b83a1356/replicate-prediction-jvi5xvlbg4v4.png", + "default_example": { + "id": "qqynzilbxm2t5h6eaywx2p7g54", + "model": "nateraw/audio-super-resolution", + "version": "9c3d3e39fb0cb6aea677264881d8073f835336137b39fdea4e94093319379535", + "status": "succeeded", + "input": { + "seed": 42, + "ddim_steps": 50, + "input_file": "https://replicate.delivery/pbxt/JYv70XQsiZBbSmknfMhGoEb4QYbuyJ9hJkfgjyzCvh4TzPmT/music.wav", + "guidance_scale": 3.5 + }, + "output": "https://pbxt.replicate.delivery/73xEQwFed8UXSCQpBJuLYgg8ym0vjp2nSidaqWKwcxhUvHzIA/out.wav", + "logs": "\u001b[93m Warning: audio is longer than 10.24 seconds, may degrade the model performance. It's recommand to truncate your audio to 5.12 seconds before input to AudioSR to get the best performance.\u001b[00m\n/src/audiosr/utils.py:109: FutureWarning: Pass sr=48000, n_fft=2048, n_mels=256, fmin=20, fmax=24000 as keyword args. From version 0.10 passing these as positional arguments will result in an error\nmel = librosa_mel_fn(sampling_rate, filter_length, n_mel, mel_fmin, mel_fmax)\nRunning DDIM Sampling with 50 timesteps\nDDIM Sampler: 0%| | 0/50 [00:00=3: apply median filtering median filtering to the harvested pitch results." + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "mp3", + "x-order": 18, + "description": "wav for best quality and large file size, mp3 for decent quality and small file size." + }, + "reverb_damping": { + "type": "number", + "title": "Reverb Damping", + "default": 0.7, + "maximum": 1, + "minimum": 0, + "x-order": 17, + "description": "Absorption of high frequencies in the reverb." + }, + "reverb_dryness": { + "type": "number", + "title": "Reverb Dryness", + "default": 0.8, + "maximum": 1, + "minimum": 0, + "x-order": 16, + "description": "Level of AI vocals without reverb." + }, + "reverb_wetness": { + "type": "number", + "title": "Reverb Wetness", + "default": 0.2, + "maximum": 1, + "minimum": 0, + "x-order": 15, + "description": "Level of AI vocals with reverb." + }, + "crepe_hop_length": { + "type": "integer", + "title": "Crepe Hop Length", + "default": 128, + "x-order": 8, + "description": "When `pitch_detection_algo` is set to `mangio-crepe`, this controls how often it checks for pitch changes in milliseconds. Lower values lead to longer conversions and higher risk of voice cracks, but better pitch accuracy." + }, + "pitch_change_all": { + "type": "number", + "title": "Pitch Change All", + "default": 0, + "x-order": 13, + "description": "Change pitch/key of background music, backup vocals and AI vocals in semitones. Reduces sound quality slightly." + }, + "main_vocals_volume_change": { + "type": "number", + "title": "Main Vocals Volume Change", + "default": 0, + "x-order": 10, + "description": "Control volume of main AI vocals. Use -3 to decrease the volume by 3 decibels, or 3 to increase the volume by 3 decibels." + }, + "pitch_detection_algorithm": { + "allOf": [ + { + "$ref": "#/components/schemas/pitch_detection_algorithm" + } + ], + "default": "rmvpe", + "x-order": 7, + "description": "Best option is rmvpe (clarity in vocals), then mangio-crepe (smoother vocals)." + }, + "instrumental_volume_change": { + "type": "number", + "title": "Instrumental Volume Change", + "default": 0, + "x-order": 12, + "description": "Control volume of the background music/instrumentals." + }, + "backup_vocals_volume_change": { + "type": "number", + "title": "Backup Vocals Volume Change", + "default": 0, + "x-order": 11, + "description": "Control volume of backup AI vocals." + }, + "custom_rvc_model_download_url": { + "type": "string", + "title": "Custom Rvc Model Download Url", + "x-order": 2, + "description": "URL to download a custom RVC model. If provided, the model will be downloaded (if it doesn't already exist) and used for prediction, regardless of the 'rvc_model' value." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "rvc_model": { + "enum": [ + "Squidward", + "MrKrabs", + "Plankton", + "Drake", + "Vader", + "Trump", + "Biden", + "Obama", + "Guitar", + "Voilin", + "CUSTOM" + ], + "type": "string", + "title": "rvc_model", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "pitch_change": { + "enum": [ + "no-change", + "male-to-female", + "female-to-male" + ], + "type": "string", + "title": "pitch_change", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "mp3", + "wav" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "pitch_detection_algorithm": { + "enum": [ + "rmvpe", + "mangio-crepe" + ], + "type": "string", + "title": "pitch_detection_algorithm", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/supported_models.json b/supported_models.json index 3138c99..a23bc21 100644 --- a/supported_models.json +++ b/supported_models.json @@ -1,40 +1,60 @@ { "models": [ + "ai-forever/kandinsky-2", "ai-forever/kandinsky-2.2", "andreasjansson/blip-2", "batouresearch/high-resolution-controlnet-tile", "batouresearch/magic-image-refiner", "batouresearch/magic-style-transfer", "bytedance/sdxl-lightning-4step", + "black-forest-labs/flux-dev", + "black-forest-labs/flux-pro", + "black-forest-labs/flux-schnell", "cjwbw/hyper-sdxl-1step-t2i", "cjwbw/pixart-sigma", + "cjwbw/seamless_communication", "cjwbw/supir", "cuuupid/glm-4v-9b", "cuuupid/idm-vton", + "declare-lab/tango", "fofr/consistent-character", "fofr/controlnet-preprocessors", "fofr/face-to-many", + "fofr/latent-consistency-model", "fofr/sd3-with-chaos", "fofr/style-transfer", + "fofr/ultrapixel", "lucataco/hunyuandit-v1.1", "lucataco/llama-3-vision-alpha", + "lucataco/magnet", "lucataco/moondream2", "lucataco/paligemma-3b-pt-224", "lucataco/pasd-magnify", "lucataco/qwen-vl-chat", "lucataco/sdxl-clip-interrogator", + "lucataco/xtts-v2", + "luosiallen/latent-consistency-model", + "meta/meta-llama-3.1-405b-instruct", "meta/llama-2-70b-chat", "meta/llama-2-7b-chat", + "meta/musicgen", "mistralai/mixtral-8x7b-instruct-v0.1", + "nateraw/audio-super-resolution", + "nateraw/musicgen-songstarter-v0.2", "omniedgeio/face-swap", "philz1337x/clarity-upscaler", + "replicate/deepfloyd-if", "salesforce/blip", "smoretalk/rembg-enhance", "snowflake/snowflake-arctic-instruct", "stability-ai/sdxl", + "stability-ai/stable-diffusion", "stability-ai/stable-diffusion-3", + "suno-ai/bark", + "tstramer/material-diffusion", "yorickvp/llava-13b", "yorickvp/llava-v1.6-34b", - "yorickvp/llava-v1.6-mistral-7b" + "yorickvp/llava-v1.6-mistral-7b", + "zsxkib/realistic-voice-cloning" ] }