From cdb1c377b72cae39ba8b825091243dd617c98b12 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 1 Jul 2024 15:22:36 +0000 Subject: [PATCH 1/6] Add more txt2img models --- schemas/ai-forever_kandinsky-2.json | 601 +++++++++++++++++ schemas/fofr_latent-consistency-model.json | 615 ++++++++++++++++++ .../luosiallen_latent-consistency-model.json | 496 ++++++++++++++ schemas/omniedgeio_face-swap.json | 4 +- schemas/replicate_deepfloyd-if.json | 447 +++++++++++++ schemas/stability-ai_stable-diffusion.json | 578 ++++++++++++++++ schemas/tstramer_material-diffusion.json | 361 ++++++++++ supported_models.json | 6 + 8 files changed, 3106 insertions(+), 2 deletions(-) create mode 100644 schemas/ai-forever_kandinsky-2.json create mode 100644 schemas/fofr_latent-consistency-model.json create mode 100644 schemas/luosiallen_latent-consistency-model.json create mode 100644 schemas/replicate_deepfloyd-if.json create mode 100644 schemas/stability-ai_stable-diffusion.json create mode 100644 schemas/tstramer_material-diffusion.json diff --git a/schemas/ai-forever_kandinsky-2.json b/schemas/ai-forever_kandinsky-2.json new file mode 100644 index 0000000..7705b7d --- /dev/null +++ b/schemas/ai-forever_kandinsky-2.json @@ -0,0 +1,601 @@ +{ + "url": "https://replicate.com/ai-forever/kandinsky-2", + "owner": "ai-forever", + "name": "kandinsky-2", + "description": "text2img model trained on LAION HighRes and fine-tuned on internal datasets", + "visibility": "public", + "github_url": "https://github.com/chenxwh/Kandinsky-2", + "paper_url": null, + "license_url": "https://github.com/ai-forever/Kandinsky-2/blob/main/license", + "run_count": 0, + "cover_image_url": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "default_example": { + "id": "wixq75uo65glnjcdgrqtngdz4q", + "model": "ai-forever/kandinsky-2", + "version": "9c0bf7d5cf2ed934c5921faf61882657c03c4def9d9cb88330c15bd795edb098", + "status": "succeeded", + "input": { + "prompt": "red cat, 4k photo", + "scheduler": "p_sampler", + "prior_steps": "5", + "guidance_scale": 4, + "prior_cf_scale": 4, + "num_inference_steps": 100 + }, + "output": "https://replicate.delivery/pbxt/NsOpfQRos43e40IzSq4SY7NTtxGodmSWo1m74K17SVpoUzuQA/out.png", + "logs": "0%| | 0/100 [00:00 Date: Tue, 2 Jul 2024 17:52:32 +0000 Subject: [PATCH 2/6] Add some models with audio inputs and outputs --- schemas/cjwbw_seamless_communication.json | 778 ++++++++++++++++++ schemas/cuuupid_glm-4v-9b.json | 53 +- schemas/declare-lab_tango.json | 475 +++++++++++ schemas/lucataco_magnet.json | 565 +++++++++++++ schemas/lucataco_xtts-v2.json | 492 +++++++++++ schemas/meta_musicgen.json | 587 +++++++++++++ schemas/nateraw_audio-super-resolution.json | 469 +++++++++++ .../nateraw_musicgen-songstarter-v0.2.json | 566 +++++++++++++ schemas/omniedgeio_face-swap.json | 11 +- schemas/suno-ai_bark.json | 639 ++++++++++++++ schemas/zsxkib_realistic-voice-cloning.json | 655 +++++++++++++++ supported_models.json | 9 + 12 files changed, 5291 insertions(+), 8 deletions(-) create mode 100644 schemas/cjwbw_seamless_communication.json create mode 100644 schemas/declare-lab_tango.json create mode 100644 schemas/lucataco_magnet.json create mode 100644 schemas/lucataco_xtts-v2.json create mode 100644 schemas/meta_musicgen.json create mode 100644 schemas/nateraw_audio-super-resolution.json create mode 100644 schemas/nateraw_musicgen-songstarter-v0.2.json create mode 100644 schemas/suno-ai_bark.json create mode 100644 schemas/zsxkib_realistic-voice-cloning.json diff --git a/schemas/cjwbw_seamless_communication.json b/schemas/cjwbw_seamless_communication.json new file mode 100644 index 0000000..61e9123 --- /dev/null +++ b/schemas/cjwbw_seamless_communication.json @@ -0,0 +1,778 @@ +{ + "url": "https://replicate.com/cjwbw/seamless_communication", + "owner": "cjwbw", + "name": "seamless_communication", + "description": "SeamlessM4T—Massively Multilingual & Multimodal Machine Translation", + "visibility": "public", + "github_url": "https://github.com/chenxwh/seamless_communication", + "paper_url": "https://dl.fbaipublicfiles.com/seamless/seamless_m4t_paper.pdf", + "license_url": "https://github.com/chenxwh/seamless_communication/blob/main/LICENSE", + "run_count": 0, + "cover_image_url": null, + "default_example": { + "id": "habhkrlbto7c6tm7apuk5amq6u", + "model": "cjwbw/seamless_communication", + "version": "668a4fec05a887143e5fe8d45df25ec4c794dd43169b9a11562309b2d45873b0", + "status": "succeeded", + "input": { + "task_name": "S2ST (Speech to Speech translation)", + "input_audio": "https://replicate.delivery/pbxt/JWSAJpKxUszI0scNYatExIXZX2rJ78UBilGXCTq4Ct9BDwTA/sample_input_2.mp3", + "input_text_language": "None", + "max_input_audio_length": 60, + "target_language_text_only": "Norwegian Nynorsk", + "target_language_with_speech": "French" + }, + "output": { + "text_output": "Le modèle M4T sans faille de MetaAI démocratise la communication parlée à travers les barrières linguistiques.", + "audio_output": "https://pbxt.replicate.delivery/HSuWXcQA0PbWIBQ3renLWSxbFBKcsCQojUtAQK6eys5038jRA/out.wav" + }, + "logs": null, + "error": null, + "metrics": { + "predict_time": 5.103559, + "total_time": 84.660864 + }, + "created_at": "2023-09-13T20:29:20.589970Z", + "started_at": "2023-09-13T20:30:40.147275Z", + "completed_at": "2023-09-13T20:30:45.250834Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/habhkrlbto7c6tm7apuk5amq6u", + "cancel": "https://api.replicate.com/v1/predictions/habhkrlbto7c6tm7apuk5amq6u/cancel" + } + }, + "latest_version": { + "id": "668a4fec05a887143e5fe8d45df25ec4c794dd43169b9a11562309b2d45873b0", + "created_at": "2023-09-13T20:28:44.472205+00:00", + "cog_version": "0.8.3", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "task_name": { + "allOf": [ + { + "$ref": "#/components/schemas/task_name" + } + ], + "default": "S2ST (Speech to Speech translation)", + "x-order": 0, + "description": "Choose a task" + }, + "input_text": { + "type": "string", + "title": "Input Text", + "x-order": 2, + "description": "Provide input for tasks with text: T2ST and T2TT." + }, + "input_audio": { + "type": "string", + "title": "Input Audio", + "format": "uri", + "x-order": 1, + "description": "Provide input file for tasks with speech input: S2ST, S2TT and ASR." + }, + "input_text_language": { + "allOf": [ + { + "$ref": "#/components/schemas/input_text_language" + } + ], + "default": "None", + "x-order": 3, + "description": "Specify language of the input_text for T2ST and T2TT" + }, + "max_input_audio_length": { + "type": "number", + "title": "Max Input Audio Length", + "default": 60, + "x-order": 6, + "description": "Set maximum input audio length." + }, + "target_language_text_only": { + "allOf": [ + { + "$ref": "#/components/schemas/target_language_text_only" + } + ], + "default": "Norwegian Nynorsk", + "x-order": 5, + "description": "Set target language for tasks with text output only: S2TT, T2TT and ASR." + }, + "target_language_with_speech": { + "allOf": [ + { + "$ref": "#/components/schemas/target_language_with_speech" + } + ], + "default": "French", + "x-order": 4, + "description": "Set target language for tasks with speech output: S2ST or T2ST. Less languages are available for speech compared to text output." + } + } + }, + "Output": { + "$ref": "#/components/schemas/ModelOutput", + "title": "Output" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "task_name": { + "enum": [ + "S2ST (Speech to Speech translation)", + "S2TT (Speech to Text translation)", + "T2ST (Text to Speech translation)", + "T2TT (Text to Text translation)", + "ASR (Automatic Speech Recognition)" + ], + "type": "string", + "title": "task_name", + "description": "An enumeration." + }, + "ModelOutput": { + "type": "object", + "title": "ModelOutput", + "required": [ + "text_output" + ], + "properties": { + "text_output": { + "type": "string", + "title": "Text Output" + }, + "audio_output": { + "type": "string", + "title": "Audio Output", + "format": "uri" + } + } + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "completed", + "start", + "logs", + "output" + ], + "uniqueItems": true + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "input_text_language": { + "enum": [ + "None", + "Afrikaans", + "Amharic", + "Armenian", + "Assamese", + "Basque", + "Belarusian", + "Bengali", + "Bosnian", + "Bulgarian", + "Burmese", + "Cantonese", + "Catalan", + "Cebuano", + "Central Kurdish", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Egyptian Arabic", + "English", + "Estonian", + "Finnish", + "French", + "Galician", + "Ganda", + "Georgian", + "German", + "Greek", + "Gujarati", + "Halh Mongolian", + "Hebrew", + "Hindi", + "Hungarian", + "Icelandic", + "Igbo", + "Indonesian", + "Irish", + "Italian", + "Japanese", + "Javanese", + "Kannada", + "Kazakh", + "Khmer", + "Korean", + "Kyrgyz", + "Lao", + "Lithuanian", + "Luo", + "Macedonian", + "Maithili", + "Malayalam", + "Maltese", + "Mandarin Chinese", + "Marathi", + "Meitei", + "Modern Standard Arabic", + "Moroccan Arabic", + "Nepali", + "North Azerbaijani", + "Northern Uzbek", + "Norwegian Bokmål", + "Norwegian Nynorsk", + "Nyanja", + "Odia", + "Polish", + "Portuguese", + "Punjabi", + "Romanian", + "Russian", + "Serbian", + "Shona", + "Sindhi", + "Slovak", + "Slovenian", + "Somali", + "Southern Pashto", + "Spanish", + "Standard Latvian", + "Standard Malay", + "Swahili", + "Swedish", + "Tagalog", + "Tajik", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Ukrainian", + "Urdu", + "Vietnamese", + "Welsh", + "West Central Oromo", + "Western Persian", + "Yoruba", + "Zulu" + ], + "type": "string", + "title": "input_text_language", + "description": "An enumeration." + }, + "target_language_text_only": { + "enum": [ + "Afrikaans", + "Amharic", + "Armenian", + "Assamese", + "Basque", + "Belarusian", + "Bengali", + "Bosnian", + "Bulgarian", + "Burmese", + "Cantonese", + "Catalan", + "Cebuano", + "Central Kurdish", + "Croatian", + "Czech", + "Danish", + "Dutch", + "Egyptian Arabic", + "English", + "Estonian", + "Finnish", + "French", + "Galician", + "Ganda", + "Georgian", + "German", + "Greek", + "Gujarati", + "Halh Mongolian", + "Hebrew", + "Hindi", + "Hungarian", + "Icelandic", + "Igbo", + "Indonesian", + "Irish", + "Italian", + "Japanese", + "Javanese", + "Kannada", + "Kazakh", + "Khmer", + "Korean", + "Kyrgyz", + "Lao", + "Lithuanian", + "Luo", + "Macedonian", + "Maithili", + "Malayalam", + "Maltese", + "Mandarin Chinese", + "Marathi", + "Meitei", + "Modern Standard Arabic", + "Moroccan Arabic", + "Nepali", + "North Azerbaijani", + "Northern Uzbek", + "Norwegian Bokmål", + "Norwegian Nynorsk", + "Nyanja", + "Odia", + "Polish", + "Portuguese", + "Punjabi", + "Romanian", + "Russian", + "Serbian", + "Shona", + "Sindhi", + "Slovak", + "Slovenian", + "Somali", + "Southern Pashto", + "Spanish", + "Standard Latvian", + "Standard Malay", + "Swahili", + "Swedish", + "Tagalog", + "Tajik", + "Tamil", + "Telugu", + "Thai", + "Turkish", + "Ukrainian", + "Urdu", + "Vietnamese", + "Welsh", + "West Central Oromo", + "Western Persian", + "Yoruba", + "Zulu" + ], + "type": "string", + "title": "target_language_text_only", + "description": "An enumeration." + }, + "target_language_with_speech": { + "enum": [ + "Bengali", + "Catalan", + "Czech", + "Danish", + "Dutch", + "English", + "Estonian", + "Finnish", + "French", + "German", + "Hindi", + "Indonesian", + "Italian", + "Japanese", + "Korean", + "Maltese", + "Mandarin Chinese", + "Modern Standard Arabic", + "Northern Uzbek", + "Polish", + "Portuguese", + "Romanian", + "Russian", + "Slovak", + "Spanish", + "Swahili", + "Swedish", + "Tagalog", + "Telugu", + "Thai", + "Turkish", + "Ukrainian", + "Urdu", + "Vietnamese", + "Welsh", + "Western Persian" + ], + "type": "string", + "title": "target_language_with_speech", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/cuuupid_glm-4v-9b.json b/schemas/cuuupid_glm-4v-9b.json index 3ab853e..a1dcaf1 100644 --- a/schemas/cuuupid_glm-4v-9b.json +++ b/schemas/cuuupid_glm-4v-9b.json @@ -34,9 +34,9 @@ } }, "latest_version": { - "id": "a75c919339f65bf00afa96511af653fdbd0ec3cb0f5e6f4350809445eee0e14f", - "created_at": "2024-06-11T18:23:49.034977+00:00", - "cog_version": "v0.9.5+dev", + "id": "69196a237cdc310988a4b12ad64f4b36d10189428c19a18526af708546e1856f", + "created_at": "2024-07-02T06:00:03.657728+00:00", + "cog_version": "v0.10.0-alpha13+dev", "openapi_schema": { "info": { "title": "Cog", @@ -61,6 +61,24 @@ "operationId": "root__get" } }, + "/ready": { + "get": { + "summary": "Ready", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Ready Ready Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "ready_ready_get" + } + }, "/shutdown": { "post": { "summary": "Start Shutdown", @@ -269,17 +287,40 @@ "x-order": 0, "description": "Image input" }, + "top_k": { + "type": "integer", + "title": "Top K", + "default": 1, + "maximum": 1000, + "minimum": 1, + "x-order": 3, + "description": "Top-K sampling" + }, "prompt": { "type": "string", "title": "Prompt", "x-order": 1, - "description": "Propmt" + "description": "Prompt" + }, + "max_length": { + "type": "integer", + "title": "Max Length", + "default": 512, + "maximum": 8192, + "minimum": 1, + "x-order": 2, + "description": "Maximum number of tokens to generate." } } }, "Output": { - "type": "string", - "title": "Output" + "type": "array", + "items": { + "type": "string" + }, + "title": "Output", + "x-cog-array-type": "iterator", + "x-cog-array-display": "concatenate" }, "Status": { "enum": [ diff --git a/schemas/declare-lab_tango.json b/schemas/declare-lab_tango.json new file mode 100644 index 0000000..f21dcce --- /dev/null +++ b/schemas/declare-lab_tango.json @@ -0,0 +1,475 @@ +{ + "url": "https://replicate.com/declare-lab/tango", + "owner": "declare-lab", + "name": "tango", + "description": "Tango 2: Use text prompts to make sound effects", + "visibility": "public", + "github_url": "https://github.com/declare-lab/tango", + "paper_url": "https://arxiv.org/pdf/2304.13731.pdf", + "license_url": "https://github.com/declare-lab/tango/blob/master/LICENSE.md", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/3e074eeb-6f95-4c4c-b7d8-38f5b9c3a2fd/tango2.png", + "default_example": { + "id": "4gahgg8fa1rgm0cf6sg80rcgn4", + "model": "declare-lab/tango", + "version": "740e4f5e59bd3b871c9e5b4efbff7ded516d40aa6abf4e95fd5e8dd149b7bc3f", + "status": "succeeded", + "input": { + "model": "tango2", + "steps": 100, + "prompt": "Quiet speech and then and airplane flying away", + "guidance": 3 + }, + "output": "https://replicate.delivery/pbxt/RqTLAYe5ZM0DEaHQlDwvA9PGLud8TeEwrGkG3fAWY5e62eAWC/output.wav", + "logs": null, + "error": null, + "metrics": { + "predict_time": 12.386586, + "total_time": 184.047861 + }, + "created_at": "2024-05-01T22:56:56.144000Z", + "started_at": "2024-05-01T22:59:47.805275Z", + "completed_at": "2024-05-01T23:00:00.191861Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/4gahgg8fa1rgm0cf6sg80rcgn4", + "cancel": "https://api.replicate.com/v1/predictions/4gahgg8fa1rgm0cf6sg80rcgn4/cancel" + } + }, + "latest_version": { + "id": "740e4f5e59bd3b871c9e5b4efbff7ded516d40aa6abf4e95fd5e8dd149b7bc3f", + "created_at": "2024-05-01T22:29:30.165543+00:00", + "cog_version": "v0.9.6+dev", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "model": { + "allOf": [ + { + "$ref": "#/components/schemas/model" + } + ], + "default": "tango2", + "x-order": 1, + "description": "choose a model" + }, + "steps": { + "type": "integer", + "title": "Steps", + "default": 100, + "x-order": 2, + "description": "inference steps" + }, + "prompt": { + "type": "string", + "title": "Prompt", + "default": "Quiet speech and then and airplane flying away", + "x-order": 0, + "description": "Input prompt" + }, + "guidance": { + "type": "number", + "title": "Guidance", + "default": 3, + "x-order": 3, + "description": "guidance scale" + } + } + }, + "model": { + "enum": [ + "tango2", + "tango2-full" + ], + "type": "string", + "title": "model", + "description": "An enumeration." + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/lucataco_magnet.json b/schemas/lucataco_magnet.json new file mode 100644 index 0000000..c12a4cb --- /dev/null +++ b/schemas/lucataco_magnet.json @@ -0,0 +1,565 @@ +{ + "url": "https://replicate.com/lucataco/magnet", + "owner": "lucataco", + "name": "magnet", + "description": "MAGNeT: Masked Audio Generation using a Single Non-Autoregressive Transformer", + "visibility": "public", + "github_url": "https://github.com/lucataco/audiocraft", + "paper_url": "https://arxiv.org/abs/2401.04577", + "license_url": "https://github.com/facebookresearch/audiocraft?tab=MIT-1-ov-file#readme", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/b958c60f-aa4a-44ac-b251-34317cd96245/magnet.jpeg", + "default_example": { + "id": "6usmrb3bog2on4b52tcnpt74oa", + "model": "lucataco/magnet", + "version": "e8e2ecd4a1dabb58924aa8300b668290cafae166dd36baf65dad9875877de50e", + "status": "succeeded", + "input": { + "model": "facebook/magnet-small-10secs", + "top_p": 0.9, + "prompt": "80s electronic track with melodic synthesizers, catchy beat and groovy bass", + "max_cfg": 10, + "min_cfg": 1, + "span_score": "prod-stride1", + "variations": 3, + "temperature": 3, + "decoding_steps_stage_1": 20, + "decoding_steps_stage_2": 10, + "decoding_steps_stage_3": 10, + "decoding_steps_stage_4": 10 + }, + "output": [ + "https://replicate.delivery/pbxt/A09884lLfO3qWidseZLbU90KjWL3eteHOcSafheMaucmacXjE/0.wav", + "https://replicate.delivery/pbxt/dG0LoAeZINxcf0xueWgwgNHyF2tQv3Lvo7S7HfEew2JbNurRC/1.wav", + "https://replicate.delivery/pbxt/K55hoe139wzvBqXV4pkfDMfqbqRMaYVnJ6O8M1efLbteacXjE/2.wav" + ], + "logs": "/root/.pyenv/versions/3.10.13/lib/python3.10/site-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\nwarnings.warn(\"torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\")\nspiece.model: 0%| | 0.00/792k [00:00 #0:0 (pcm_s16le (native) -> pcm_s16le (native))\nPress [q] to stop, [?] for help\nOutput #0, wav, to '/tmp/speaker.wav':\nMetadata:\nISFT : Lavf58.76.100\nStream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 44100 Hz, mono, s16, 705 kb/s\nMetadata:\nencoder : Lavc58.134.100 pcm_s16le\nsize= 4kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 744kB time=00:00:08.63 bitrate= 705.9kbits/s speed=1.75e+03x\nvideo:0kB audio:744kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.010236%\n> Text splitted to sentences.\n[\"Hi there, I'm your new voice clone.\", 'Try your best to upload quality audio']\n> Processing time: 3.3411097526550293\n> Real-time factor: 0.613438166517148", + "error": null, + "metrics": { + "predict_time": 4.303438, + "total_time": 4.319136 + }, + "created_at": "2023-11-16T16:42:06.735772Z", + "started_at": "2023-11-16T16:42:06.751470Z", + "completed_at": "2023-11-16T16:42:11.054908Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu", + "cancel": "https://api.replicate.com/v1/predictions/adinmbzbslyshwze5s6qmpzmzu/cancel" + } + }, + "latest_version": { + "id": "684bc3855b37866c0c65add2ff39c78f3dea3f4ff103a436465326e0f438d55e", + "created_at": "2023-11-28T15:39:16.308105+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "required": [ + "speaker" + ], + "properties": { + "text": { + "type": "string", + "title": "Text", + "default": "Hi there, I'm your new voice clone. Try your best to upload quality audio", + "x-order": 0, + "description": "Text to synthesize" + }, + "speaker": { + "type": "string", + "title": "Speaker", + "format": "uri", + "x-order": 1, + "description": "Original speaker audio (wav, mp3, m4a, ogg, or flv)" + }, + "language": { + "allOf": [ + { + "$ref": "#/components/schemas/language" + } + ], + "default": "en", + "x-order": 2, + "description": "Output language for the synthesised speech" + }, + "cleanup_voice": { + "type": "boolean", + "title": "Cleanup Voice", + "default": false, + "x-order": 3, + "description": "Whether to apply denoising to the speaker audio (microphone recordings)" + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "language": { + "enum": [ + "en", + "es", + "fr", + "de", + "it", + "pt", + "pl", + "tr", + "ru", + "nl", + "cs", + "ar", + "zh", + "hu", + "ko", + "hi" + ], + "type": "string", + "title": "language", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/meta_musicgen.json b/schemas/meta_musicgen.json new file mode 100644 index 0000000..cf4d704 --- /dev/null +++ b/schemas/meta_musicgen.json @@ -0,0 +1,587 @@ +{ + "url": "https://replicate.com/meta/musicgen", + "owner": "meta", + "name": "musicgen", + "description": "Generate music from a prompt or melody", + "visibility": "public", + "github_url": "https://github.com/replicate/cog-musicgen", + "paper_url": "https://arxiv.org/abs/2306.05284", + "license_url": "https://github.com/facebookresearch/audiocraft/blob/main/LICENSE_weights", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/e3d6b391-571b-4e24-8531-936a0fee9fba/musicgen.jpeg", + "default_example": { + "id": "soub5pjbe5g5u62q536t73ltku", + "model": "meta/musicgen", + "version": "671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb", + "status": "succeeded", + "input": { + "top_k": 250, + "top_p": 0, + "prompt": "Edo25 major g melodies that sound triumphant and cinematic. Leading up to a crescendo that resolves in a 9th harmonic", + "duration": 8, + "temperature": 1, + "continuation": false, + "model_version": "stereo-large", + "output_format": "mp3", + "continuation_start": 0, + "multi_band_diffusion": false, + "normalization_strategy": "peak", + "classifier_free_guidance": 3 + }, + "output": "https://replicate.delivery/pbxt/OeLYIQiltdzMaCex1shlEFy6wuvnvExi0kdjF0gYKEzG5wkSA/out.mp3", + "logs": "Loading model stereo-large...\nDownloading models--facebook--musicgen-stereo-large to models\nDownloaded models--facebook--musicgen-stereo-large in 9.88s\nModel stereo-large loaded successfully.\nUsing seed 187926229\n1 / 400\n2 / 400\n3 / 400\n4 / 400\n5 / 400\n6 / 400\n7 / 400\n8 / 400\n9 / 400\n10 / 400\n11 / 400\n12 / 400\n13 / 400\n14 / 400\n15 / 400\n16 / 400\n17 / 400\n18 / 400\n19 / 400\n20 / 400\n21 / 400\n22 / 400\n23 / 400\n24 / 400\n25 / 400\n26 / 400\n27 / 400\n28 / 400\n29 / 400\n30 / 400\n31 / 400\n32 / 400\n33 / 400\n34 / 400\n35 / 400\n36 / 400\n37 / 400\n38 / 400\n39 / 400\n40 / 400\n41 / 400\n42 / 400\n43 / 400\n44 / 400\n45 / 400\n46 / 400\n47 / 400\n48 / 400\n49 / 400\n50 / 400\n51 / 400\n52 / 400\n53 / 400\n54 / 400\n55 / 400\n56 / 400\n57 / 400\n58 / 400\n59 / 400\n60 / 400\n61 / 400\n62 / 400\n63 / 400\n64 / 400\n65 / 400\n66 / 400\n67 / 400\n68 / 400\n69 / 400\n70 / 400\n71 / 400\n72 / 400\n73 / 400\n74 / 400\n75 / 400\n76 / 400\n77 / 400\n78 / 400\n79 / 400\n80 / 400\n81 / 400\n82 / 400\n83 / 400\n 84 / 400\n85 / 400\n86 / 400\n87 / 400\n88 / 400\n89 / 400\n90 / 400\n91 / 400\n92 / 400\n93 / 400\n94 / 400\n95 / 400\n96 / 400\n97 / 400\n98 / 400\n99 / 400\n100 / 400\n101 / 400\n102 / 400\n103 / 400\n104 / 400\n105 / 400\n106 / 400\n107 / 400\n108 / 400\n109 / 400\n110 / 400\n111 / 400\n112 / 400\n113 / 400\n114 / 400\n115 / 400\n116 / 400\n117 / 400\n118 / 400\n119 / 400\n120 / 400\n121 / 400\n122 / 400\n123 / 400\n124 / 400\n125 / 400\n126 / 400\n127 / 400\n128 / 400\n129 / 400\n130 / 400\n131 / 400\n132 / 400\n133 / 400\n134 / 400\n135 / 400\n136 / 400\n137 / 400\n138 / 400\n139 / 400\n140 / 400\n141 / 400\n142 / 400\n143 / 400\n144 / 400\n145 / 400\n146 / 400\n147 / 400\n148 / 400\n149 / 400\n150 / 400\n151 / 400\n152 / 400\n153 / 400\n154 / 400\n155 / 400\n156 / 400\n157 / 400\n158 / 400\n159 / 400\n160 / 400\n161 / 400\n162 / 400\n163 / 400\n164 / 400\n165 / 400\n166 / 400\n167 / 400\n168 / 400\n169 / 400\n170 / 400\n171 / 400\n172 / 400\n173 / 400\n174 / 400\n175 / 400\n176 / 400\n177 / 400\n178 / 400\n179 / 400\n180 / 400\n181 / 400\n182 / 400\n183 / 400\n184 / 400\n185 / 400\n186 / 400\n187 / 400\n188 / 400\n189 / 400\n190 / 400\n191 / 400\n192 / 400\n193 / 400\n194 / 400\n195 / 400\n196 / 400\n197 / 400\n198 / 400\n199 / 400\n200 / 400\n201 / 400\n202 / 400\n203 / 400\n204 / 400\n205 / 400\n206 / 400\n207 / 400\n208 / 400\n209 / 400\n210 / 400\n211 / 400\n212 / 400\n213 / 400\n214 / 400\n215 / 400\n216 / 400\n217 / 400\n218 / 400\n219 / 400\n220 / 400\n221 / 400\n222 / 400\n223 / 400\n224 / 400\n225 / 400\n226 / 400\n227 / 400\n228 / 400\n229 / 400\n230 / 400\n231 / 400\n232 / 400\n233 / 400\n234 / 400\n235 / 400\n236 / 400\n237 / 400\n238 / 400\n239 / 400\n240 / 400\n241 / 400\n242 / 400\n243 / 400\n244 / 400\n245 / 400\n246 / 400\n247 / 400\n248 / 400\n249 / 400\n250 / 400\n251 / 400\n252 / 400\n253 / 400\n254 / 400\n255 / 400\n256 / 400\n257 / 400\n258 / 400\n259 / 400\n260 / 400\n261 / 400\n262 / 400\n263 / 400\n264 / 400\n265 / 400\n266 / 400\n267 / 400\n268 / 400\n269 / 400\n270 / 400\n271 / 400\n272 / 400\n273 / 400\n274 / 400\n275 / 400\n276 / 400\n277 / 400\n278 / 400\n279 / 400\n280 / 400\n281 / 400\n282 / 400\n283 / 400\n284 / 400\n285 / 400\n286 / 400\n287 / 400\n288 / 400\n289 / 400\n290 / 400\n291 / 400\n292 / 400\n293 / 400\n294 / 400\n295 / 400\n296 / 400\n297 / 400\n298 / 400\n299 / 400\n300 / 400\n301 / 400\n302 / 400\n303 / 400\n304 / 400\n305 / 400\n306 / 400\n307 / 400\n308 / 400\n309 / 400\n310 / 400\n311 / 400\n312 / 400\n313 / 400\n314 / 400\n315 / 400\n316 / 400\n317 / 400\n318 / 400\n319 / 400\n320 / 400\n321 / 400\n322 / 400\n323 / 400\n324 / 400\n325 / 400\n326 / 400\n327 / 400\n328 / 400\n329 / 400\n330 / 400\n331 / 400\n332 / 400\n333 / 400\n334 / 400\n335 / 400\n336 / 400\n337 / 400\n338 / 400\n339 / 400\n340 / 400\n341 / 400\n342 / 400\n343 / 400\n344 / 400\n345 / 400\n346 / 400\n347 / 400\n348 / 400\n349 / 400\n350 / 400\n351 / 400\n352 / 400\n353 / 400\n354 / 400\n355 / 400\n356 / 400\n357 / 400\n358 / 400\n359 / 400\n360 / 400\n361 / 400\n362 / 400\n363 / 400\n364 / 400\n365 / 400\n366 / 400\n367 / 400\n368 / 400\n369 / 400\n370 / 400\n371 / 400\n372 / 400\n373 / 400\n374 / 400\n375 / 400\n376 / 400\n377 / 400\n378 / 400\n379 / 400\n380 / 400\n381 / 400\n382 / 400\n383 / 400\n384 / 400\n385 / 400\n386 / 400\n387 / 400\n388 / 400\n389 / 400\n390 / 400\n391 / 400\n392 / 400\n393 / 400\n394 / 400\n395 / 400\n396 / 400\n397 / 400\n398 / 400\n399 / 400\n400 / 400\n401 / 400\n402 / 400\n403 / 400\nffmpeg version 4.4.2-0ubuntu0.22.04.1 Copyright (c) 2000-2021 the FFmpeg developers\nbuilt with gcc 11 (Ubuntu 11.2.0-19ubuntu1)\nconfiguration: --prefix=/usr --extra-version=0ubuntu0.22.04.1 --toolchain=hardened --libdir=/usr/lib/x86_64-linux-gnu --incdir=/usr/include/x86_64-linux-gnu --arch=amd64 --enable-gpl --disable-stripping --enable-gnutls --enable-ladspa --enable-libaom --enable-libass --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 --enable-libdav1d --enable-libflite --enable-libfontconfig --enable-libfreetype --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus --enable-libpulse --enable-librabbitmq --enable-librubberband --enable-libshine --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis --enable-libvpx --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid --enable-libzimg --enable-libzmq --enable-libzvbi --enable-lv2 --enable-omx --enable-openal --enable-opencl --enable-opengl --enable-sdl2 --enable-pocketsphinx --enable-librsvg --enable-libmfx --enable-libdc1394 --enable-libdrm --enable-libiec61883 --enable-chromaprint --enable-frei0r --enable-libx264 --enable-shared\nlibavutil 56. 70.100 / 56. 70.100\nlibavcodec 58.134.100 / 58.134.100\nlibavformat 58. 76.100 / 58. 76.100\nlibavdevice 58. 13.100 / 58. 13.100\nlibavfilter 7.110.100 / 7.110.100\nlibswscale 5. 9.100 / 5. 9.100\nlibswresample 3. 9.100 / 3. 9.100\nlibpostproc 55. 9.100 / 55. 9.100\nGuessed Channel Layout for Input Stream #0.0 : stereo\nInput #0, wav, from 'out.wav':\nMetadata:\nencoder : Lavf58.76.100\nDuration: 00:00:08.00, bitrate: 1024 kb/s\nStream #0:0: Audio: pcm_s16le ([1][0][0][0] / 0x0001), 32000 Hz, stereo, s16, 1024 kb/s\nStream mapping:\nStream #0:0 -> #0:0 (pcm_s16le (native) -> mp3 (libmp3lame))\nPress [q] to stop, [?] for help\nOutput #0, mp3, to 'out.mp3':\nMetadata:\nTSSE : Lavf58.76.100\nStream #0:0: Audio: mp3, 32000 Hz, stereo, s16p\nMetadata:\nencoder : Lavc58.134.100 libmp3lame\nsize= 0kB time=00:00:00.00 bitrate=N/A speed=N/A\nsize= 95kB time=00:00:07.99 bitrate= 97.1kbits/s speed=45.6x\nvideo:0kB audio:94kB subtitle:0kB other streams:0kB global headers:0kB muxing overhead: 0.269717%", + "error": null, + "metrics": { + "predict_time": 66.371701, + "total_time": 66.463286 + }, + "created_at": "2024-03-28T12:01:08.688755Z", + "started_at": "2024-03-28T12:01:08.780340Z", + "completed_at": "2024-03-28T12:02:15.152041Z", + "urls": { + "get": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku", + "cancel": "https://api.replicate.com/v1/predictions/soub5pjbe5g5u62q536t73ltku/cancel" + } + }, + "latest_version": { + "id": "671ac645ce5e552cc63a54a2bbff63fcf798043055d2dac5fc9e36a837eedcfb", + "created_at": "2024-03-28T11:07:13.003999+00:00", + "cog_version": "0.8.6", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "seed": { + "type": "integer", + "title": "Seed", + "x-order": 14, + "description": "Seed for random number generator. If None or -1, a random seed will be used." + }, + "top_k": { + "type": "integer", + "title": "Top K", + "default": 250, + "x-order": 9, + "description": "Reduces sampling to the k most likely tokens." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0, + "x-order": 10, + "description": "Reduces sampling to tokens with cumulative probability of p. When set to `0` (default), top_k sampling is used." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "x-order": 1, + "description": "A description of the music you want to generate." + }, + "duration": { + "type": "integer", + "title": "Duration", + "default": 8, + "x-order": 3, + "description": "Duration of the generated audio in seconds." + }, + "input_audio": { + "type": "string", + "title": "Input Audio", + "format": "uri", + "x-order": 2, + "description": "An audio file that will influence the generated music. If `continuation` is `True`, the generated music will be a continuation of the audio file. Otherwise, the generated music will mimic the audio file's melody." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 1, + "x-order": 11, + "description": "Controls the 'conservativeness' of the sampling process. Higher temperature means more diversity." + }, + "continuation": { + "type": "boolean", + "title": "Continuation", + "default": false, + "x-order": 4, + "description": "If `True`, generated music will continue from `input_audio`. Otherwise, generated music will mimic `input_audio`'s melody." + }, + "model_version": { + "allOf": [ + { + "$ref": "#/components/schemas/model_version" + } + ], + "default": "stereo-melody-large", + "x-order": 0, + "description": "Model to use for generation" + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "wav", + "x-order": 13, + "description": "Output format for generated audio." + }, + "continuation_end": { + "type": "integer", + "title": "Continuation End", + "minimum": 0, + "x-order": 6, + "description": "End time of the audio file to use for continuation. If -1 or None, will default to the end of the audio clip." + }, + "continuation_start": { + "type": "integer", + "title": "Continuation Start", + "default": 0, + "minimum": 0, + "x-order": 5, + "description": "Start time of the audio file to use for continuation." + }, + "multi_band_diffusion": { + "type": "boolean", + "title": "Multi Band Diffusion", + "default": false, + "x-order": 7, + "description": "If `True`, the EnCodec tokens will be decoded with MultiBand Diffusion. Only works with non-stereo models." + }, + "normalization_strategy": { + "allOf": [ + { + "$ref": "#/components/schemas/normalization_strategy" + } + ], + "default": "loudness", + "x-order": 8, + "description": "Strategy for normalizing audio." + }, + "classifier_free_guidance": { + "type": "integer", + "title": "Classifier Free Guidance", + "default": 3, + "x-order": 12, + "description": "Increases the influence of inputs on the output. Higher values produce lower-varience outputs that adhere more closely to inputs." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "model_version": { + "enum": [ + "stereo-melody-large", + "stereo-large", + "melody-large", + "large" + ], + "type": "string", + "title": "model_version", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "wav", + "mp3" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "normalization_strategy": { + "enum": [ + "loudness", + "clip", + "peak", + "rms" + ], + "type": "string", + "title": "normalization_strategy", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/nateraw_audio-super-resolution.json b/schemas/nateraw_audio-super-resolution.json new file mode 100644 index 0000000..1c5fac3 --- /dev/null +++ b/schemas/nateraw_audio-super-resolution.json @@ -0,0 +1,469 @@ +{ + "url": "https://replicate.com/nateraw/audio-super-resolution", + "owner": "nateraw", + "name": "audio-super-resolution", + "description": "AudioSR: Versatile Audio Super-resolution at Scale", + "visibility": "public", + "github_url": "https://github.com/haoheliu/versatile_audio_super_resolution", + "paper_url": "https://arxiv.org/abs/2309.07314", + "license_url": "https://huggingface.co/haoheliu/wellsolve_audio_super_resolution_48k/blob/main/README.md", + "run_count": 0, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/6bc4e480-6695-451b-940d-48a3b83a1356/replicate-prediction-jvi5xvlbg4v4.png", + "default_example": { + "id": "qqynzilbxm2t5h6eaywx2p7g54", + "model": "nateraw/audio-super-resolution", + "version": "9c3d3e39fb0cb6aea677264881d8073f835336137b39fdea4e94093319379535", + "status": "succeeded", + "input": { + "seed": 42, + "ddim_steps": 50, + "input_file": "https://replicate.delivery/pbxt/JYv70XQsiZBbSmknfMhGoEb4QYbuyJ9hJkfgjyzCvh4TzPmT/music.wav", + "guidance_scale": 3.5 + }, + "output": "https://pbxt.replicate.delivery/73xEQwFed8UXSCQpBJuLYgg8ym0vjp2nSidaqWKwcxhUvHzIA/out.wav", + "logs": "\u001b[93m Warning: audio is longer than 10.24 seconds, may degrade the model performance. It's recommand to truncate your audio to 5.12 seconds before input to AudioSR to get the best performance.\u001b[00m\n/src/audiosr/utils.py:109: FutureWarning: Pass sr=48000, n_fft=2048, n_mels=256, fmin=20, fmax=24000 as keyword args. From version 0.10 passing these as positional arguments will result in an error\nmel = librosa_mel_fn(sampling_rate, filter_length, n_mel, mel_fmin, mel_fmax)\nRunning DDIM Sampling with 50 timesteps\nDDIM Sampler: 0%| | 0/50 [00:00=3: apply median filtering median filtering to the harvested pitch results." + }, + "output_format": { + "allOf": [ + { + "$ref": "#/components/schemas/output_format" + } + ], + "default": "mp3", + "x-order": 18, + "description": "wav for best quality and large file size, mp3 for decent quality and small file size." + }, + "reverb_damping": { + "type": "number", + "title": "Reverb Damping", + "default": 0.7, + "maximum": 1, + "minimum": 0, + "x-order": 17, + "description": "Absorption of high frequencies in the reverb." + }, + "reverb_dryness": { + "type": "number", + "title": "Reverb Dryness", + "default": 0.8, + "maximum": 1, + "minimum": 0, + "x-order": 16, + "description": "Level of AI vocals without reverb." + }, + "reverb_wetness": { + "type": "number", + "title": "Reverb Wetness", + "default": 0.2, + "maximum": 1, + "minimum": 0, + "x-order": 15, + "description": "Level of AI vocals with reverb." + }, + "crepe_hop_length": { + "type": "integer", + "title": "Crepe Hop Length", + "default": 128, + "x-order": 8, + "description": "When `pitch_detection_algo` is set to `mangio-crepe`, this controls how often it checks for pitch changes in milliseconds. Lower values lead to longer conversions and higher risk of voice cracks, but better pitch accuracy." + }, + "pitch_change_all": { + "type": "number", + "title": "Pitch Change All", + "default": 0, + "x-order": 13, + "description": "Change pitch/key of background music, backup vocals and AI vocals in semitones. Reduces sound quality slightly." + }, + "main_vocals_volume_change": { + "type": "number", + "title": "Main Vocals Volume Change", + "default": 0, + "x-order": 10, + "description": "Control volume of main AI vocals. Use -3 to decrease the volume by 3 decibels, or 3 to increase the volume by 3 decibels." + }, + "pitch_detection_algorithm": { + "allOf": [ + { + "$ref": "#/components/schemas/pitch_detection_algorithm" + } + ], + "default": "rmvpe", + "x-order": 7, + "description": "Best option is rmvpe (clarity in vocals), then mangio-crepe (smoother vocals)." + }, + "instrumental_volume_change": { + "type": "number", + "title": "Instrumental Volume Change", + "default": 0, + "x-order": 12, + "description": "Control volume of the background music/instrumentals." + }, + "backup_vocals_volume_change": { + "type": "number", + "title": "Backup Vocals Volume Change", + "default": 0, + "x-order": 11, + "description": "Control volume of backup AI vocals." + }, + "custom_rvc_model_download_url": { + "type": "string", + "title": "Custom Rvc Model Download Url", + "x-order": 2, + "description": "URL to download a custom RVC model. If provided, the model will be downloaded (if it doesn't already exist) and used for prediction, regardless of the 'rvc_model' value." + } + } + }, + "Output": { + "type": "string", + "title": "Output", + "format": "uri" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "rvc_model": { + "enum": [ + "Squidward", + "MrKrabs", + "Plankton", + "Drake", + "Vader", + "Trump", + "Biden", + "Obama", + "Guitar", + "Voilin", + "CUSTOM" + ], + "type": "string", + "title": "rvc_model", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "pitch_change": { + "enum": [ + "no-change", + "male-to-female", + "female-to-male" + ], + "type": "string", + "title": "pitch_change", + "description": "An enumeration." + }, + "output_format": { + "enum": [ + "mp3", + "wav" + ], + "type": "string", + "title": "output_format", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + }, + "pitch_detection_algorithm": { + "enum": [ + "rmvpe", + "mangio-crepe" + ], + "type": "string", + "title": "pitch_detection_algorithm", + "description": "An enumeration." + } + } + } + } + } +} \ No newline at end of file diff --git a/supported_models.json b/supported_models.json index 78f2a27..f5b7307 100644 --- a/supported_models.json +++ b/supported_models.json @@ -1,5 +1,14 @@ { "models": [ + "suno-ai/bark", + "meta/musicgen", + "lucataco/magnet", + "nateraw/musicgen-songstarter-v0.2", + "nateraw/audio-super-resolution", + "zsxkib/realistic-voice-cloning", + "lucataco/xtts-v2", + "declare-lab/tango", + "cjwbw/seamless_communication", "ai-forever/kandinsky-2.2", "ai-forever/kandinsky-2", "andreasjansson/blip-2", From f3b74e2d3e8094a1c65258a6c75486e3d8915883 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 2 Jul 2024 17:58:21 +0000 Subject: [PATCH 3/6] Initial audio input and output handling --- node.py | 127 +++++++++++++++++++++++++++++++++++++++++++--- schema_to_node.py | 125 +++++++++++++++++++++++++++++++++++---------- 2 files changed, 218 insertions(+), 34 deletions(-) diff --git a/node.py b/node.py index 550fc26..3588199 100644 --- a/node.py +++ b/node.py @@ -8,6 +8,8 @@ from torchvision import transforms import torch import base64 import time +import torchaudio +import soundfile as sf from replicate.client import Client from .schema_to_node import ( schema_to_comfyui_input_types, @@ -32,7 +34,11 @@ def create_comfyui_node(schema): def INPUT_TYPES(cls): return schema_to_comfyui_input_types(schema) - RETURN_TYPES = (return_type,) + RETURN_TYPES = ( + tuple(return_type.values()) + if isinstance(return_type, dict) + else (return_type,) + ) FUNCTION = "run_replicate_model" CATEGORY = "Replicate" @@ -45,6 +51,8 @@ def create_comfyui_node(schema): ) if input_type == "IMAGE": kwargs[key] = self.image_to_base64(value) + elif input_type == "AUDIO": + kwargs[key] = self.audio_to_base64(value) def image_to_base64(self, image): if isinstance(image, torch.Tensor): @@ -60,6 +68,31 @@ def create_comfyui_node(schema): img_str = base64.b64encode(buffer.getvalue()).decode() return f"data:image/png;base64,{img_str}" + def audio_to_base64(self, audio): + if ( + isinstance(audio, dict) + and "waveform" in audio + and "sample_rate" in audio + ): + waveform = audio["waveform"] + sample_rate = audio["sample_rate"] + else: + waveform, sample_rate = audio + + # Ensure waveform is 2D + if waveform.dim() == 1: + waveform = waveform.unsqueeze(0) + elif waveform.dim() > 2: + waveform = waveform.squeeze() + if waveform.dim() > 2: + raise ValueError("Waveform must be 1D or 2D") + + buffer = io.BytesIO() + sf.write(buffer, waveform.numpy().T, sample_rate, format="wav") + buffer.seek(0) + audio_str = base64.b64encode(buffer.getvalue()).decode() + return f"data:audio/wav;base64,{audio_str}" + def handle_array_inputs(self, kwargs): array_inputs = inputs_that_need_arrays(schema) for input_name in array_inputs: @@ -75,13 +108,18 @@ def create_comfyui_node(schema): def log_input(self, kwargs): truncated_kwargs = { k: v[:20] + "..." - if isinstance(v, str) and v.startswith("data:image") + if isinstance(v, str) + and (v.startswith("data:image") or v.startswith("data:audio")) else v for k, v in kwargs.items() } print(f"Running {replicate_model} with {truncated_kwargs}") def handle_image_output(self, output): + if output is None: + print("No image output received") + return None + # Handle both string and list outputs output_list = [output] if isinstance(output, str) else list(output) if output_list: @@ -112,19 +150,90 @@ def create_comfyui_node(schema): print("No output received from the model") return None + def handle_audio_output(self, output): + if output is None: + print("No audio output received from the model") + return None + + if isinstance(output, str): + output_list = [output] + elif isinstance(output, (list, tuple)): + output_list = list(output) + else: + print(f"Unexpected output type: {type(output)}") + return None + + if output_list: + audio_data = [] + for audio_url in output_list: + if audio_url: + response = requests.get(audio_url) + if response.status_code == 200: + audio_content = BytesIO(response.content) + waveform, sample_rate = torchaudio.load(audio_content) + audio_data.append( + { + "waveform": waveform.unsqueeze(0), + "sample_rate": sample_rate, + } + ) + else: + print( + f"Failed to download audio. Status code: {response.status_code}" + ) + else: + print("Empty audio URL received") + + # If there's only one audio file, return it directly + if len(audio_data) == 1: + return audio_data[0] + # If there are multiple audio files, return them as a list + return audio_data + else: + print("No valid audio URLs in the output") + return None + + def remove_falsey_optional_inputs(self, kwargs): + optional_inputs = self.INPUT_TYPES().get("optional", {}) + for key in list(kwargs.keys()): + if key in optional_inputs and not kwargs[key]: + del kwargs[key] + def run_replicate_model(self, **kwargs): self.handle_array_inputs(kwargs) + self.remove_falsey_optional_inputs(kwargs) self.convert_input_images_to_base64(kwargs) self.log_input(kwargs) - output = replicate.run(replicate_model, input=kwargs) + kwargs_without_force_rerun = { + k: v for k, v in kwargs.items() if k != "force_rerun" + } + output = replicate.run(replicate_model, input=kwargs_without_force_rerun) print(f"Output: {output}") - if return_type == "IMAGE": - output = self.handle_image_output(output) + processed_outputs = [] + if isinstance(return_type, dict): + for prop_name, prop_type in return_type.items(): + if prop_type == "IMAGE": + processed_outputs.append( + self.handle_image_output(output.get(prop_name)) + ) + elif prop_type == "AUDIO": + processed_outputs.append( + self.handle_audio_output(output.get(prop_name)) + ) + elif prop_type == "STRING": + processed_outputs.append( + "".join(list(output.get(prop_name, ""))).strip() + ) else: - output = "".join(list(output)).strip() + if return_type == "IMAGE": + processed_outputs.append(self.handle_image_output(output)) + elif return_type == "AUDIO": + processed_outputs.append(self.handle_audio_output(output)) + else: + processed_outputs.append("".join(list(output)).strip()) - return (output,) + return tuple(processed_outputs) return node_name, ReplicateToComfyUI @@ -135,7 +244,9 @@ def create_comfyui_nodes_from_schemas(schemas_dir): schemas_dir_path = os.path.join(current_path, schemas_dir) for schema_file in os.listdir(schemas_dir_path): if schema_file.endswith(".json"): - with open(os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8") as f: + with open( + os.path.join(schemas_dir_path, schema_file), "r", encoding="utf-8" + ) as f: schema = json.load(f) node_name, node_class = create_comfyui_node(schema) nodes[node_name] = node_class diff --git a/schema_to_node.py b/schema_to_node.py index 76df66a..8e91a65 100644 --- a/schema_to_node.py +++ b/schema_to_node.py @@ -1,17 +1,41 @@ DEFAULT_STEP = 0.01 DEFAULT_ROUND = 0.001 +IMAGE_EXTENSIONS = (".png", ".jpg", ".jpeg", ".gif", ".webp") +VIDEO_EXTENSIONS = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") +AUDIO_EXTENSIONS = (".mp3", ".wav", ".flac", ".mpga", ".m4a") -def convert_to_comfyui_input_type(openapi_type, openapi_format=None): - type_mapping = { - "string": "STRING", - "integer": "INT", - "number": "FLOAT", - "boolean": "BOOLEAN", - } +TYPE_MAPPING = { + "string": "STRING", + "integer": "INT", + "number": "FLOAT", + "boolean": "BOOLEAN", +} + + +def convert_to_comfyui_input_type( + input_name, openapi_type, openapi_format=None, default_example_input=None +): if openapi_type == "string" and openapi_format == "uri": - return "IMAGE" - return type_mapping.get(openapi_type, "STRING") + if ( + default_example_input + and isinstance(default_example_input, dict) + and input_name in default_example_input + ): + if is_type(default_example_input[input_name], IMAGE_EXTENSIONS): + return "IMAGE" + elif is_type(default_example_input[input_name], VIDEO_EXTENSIONS): + return "VIDEO" + elif is_type(default_example_input[input_name], AUDIO_EXTENSIONS): + return "AUDIO" + elif "image" in input_name.lower(): + return "IMAGE" + elif "audio" in input_name.lower(): + return "AUDIO" + else: + return "STRING" + + return TYPE_MAPPING.get(openapi_type, "STRING") def name_and_version(schema): @@ -23,11 +47,13 @@ def name_and_version(schema): return replicate_model, node_name -def resolve_schema(prop_data, openapi_schemas): +def resolve_schema(prop_data, openapi_schema): if "$ref" in prop_data: ref_path = prop_data["$ref"].split("/") - current = openapi_schemas + current = openapi_schema for path in ref_path[1:]: # Skip the first '#' element + if path not in current: + return prop_data # Return original if path is invalid current = current[path] return current return prop_data @@ -37,6 +63,7 @@ def schema_to_comfyui_input_types(schema): openapi_schema = schema["latest_version"]["openapi_schema"] input_schema = openapi_schema["components"]["schemas"]["Input"] input_types = {"required": {}, "optional": {}} + default_example_input = get_default_example_input(schema) required_props = input_schema.get("required", []) @@ -51,7 +78,10 @@ def schema_to_comfyui_input_types(schema): input_type = prop_data["enum"] elif "type" in prop_data: input_type = convert_to_comfyui_input_type( - prop_data["type"], prop_data.get("format") + prop_name, + prop_data["type"], + prop_data.get("format"), + default_example_input, ) else: input_type = "STRING" @@ -134,22 +164,65 @@ def is_type(default_example_output, extensions): return False -def get_return_type(schema): - image_extensions = (".png", ".jpg", ".jpeg", ".gif", ".webp") - video_extensions = (".mp4", ".mkv", ".webm", ".mov", ".mpg", ".mpeg") - audio_extensions = (".mp3", ".wav") - - openapi_schema = schema["latest_version"]["openapi_schema"] - output_schema = openapi_schema["components"]["schemas"].get("Output") +def get_default_example(schema): default_example = schema.get("default_example") - default_example_output = default_example.get("output") if default_example else None + return default_example if default_example else None - if is_type(default_example_output, image_extensions): + +def get_default_example_input(schema): + default_example = get_default_example(schema) + return default_example.get("input") if default_example else None + + +def get_default_example_output(schema): + default_example = get_default_example(schema) + return default_example.get("output") if default_example else None + + +def get_return_type(schema): + openapi_schema = schema["latest_version"]["openapi_schema"] + output_schema = ( + openapi_schema.get("components", {}).get("schemas", {}).get("Output") + ) + default_example_output = get_default_example_output(schema) + + if output_schema and "$ref" in output_schema: + output_schema = resolve_schema(output_schema, openapi_schema) + + if isinstance(output_schema, dict) and output_schema.get("properties"): + return_types = {} + for prop_name, prop_data in output_schema["properties"].items(): + if isinstance(default_example_output, dict): + prop_value = default_example_output.get(prop_name) + + if is_type(prop_value, IMAGE_EXTENSIONS): + return_types[prop_name] = "IMAGE" + elif is_type(prop_value, AUDIO_EXTENSIONS): + return_types[prop_name] = "AUDIO" + elif is_type(prop_value, VIDEO_EXTENSIONS): + return_types[prop_name] = "VIDEO_URI" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("format") == "uri": + if "audio" in prop_name.lower(): + return_types[prop_name] = "AUDIO" + elif "image" in prop_name.lower(): + return_types[prop_name] = "IMAGE" + else: + return_types[prop_name] = "STRING" + elif prop_data.get("type") == "string": + return_types[prop_name] = "STRING" + else: + return_types[prop_name] = "STRING" + + return return_types + + if is_type(default_example_output, IMAGE_EXTENSIONS): return "IMAGE" - elif is_type(default_example_output, video_extensions): + elif is_type(default_example_output, VIDEO_EXTENSIONS): return "VIDEO_URI" - elif is_type(default_example_output, audio_extensions): - return "AUDIO_URI" + elif is_type(default_example_output, AUDIO_EXTENSIONS): + return "AUDIO" if output_schema: if ( @@ -160,8 +233,8 @@ def get_return_type(schema): return "IMAGE" elif ( output_schema.get("type") == "array" - and output_schema["items"].get("type") == "string" - and output_schema["items"].get("format") == "uri" + and output_schema.get("items", {}).get("type") == "string" + and output_schema.get("items", {}).get("format") == "uri" ): # Handle multiple image output return "IMAGE" From 6a61a5b9bb1457b61e969c62ffd9edc1c19fe04d Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 22 Jul 2024 10:54:47 +0000 Subject: [PATCH 4/6] Ignore tensors in optional arg checks, they are not falsey --- node.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/node.py b/node.py index 3588199..faef57d 100644 --- a/node.py +++ b/node.py @@ -196,8 +196,11 @@ def create_comfyui_node(schema): def remove_falsey_optional_inputs(self, kwargs): optional_inputs = self.INPUT_TYPES().get("optional", {}) for key in list(kwargs.keys()): - if key in optional_inputs and not kwargs[key]: - del kwargs[key] + if key in optional_inputs: + if isinstance(kwargs[key], torch.Tensor): + continue + elif not kwargs[key]: + del kwargs[key] def run_replicate_model(self, **kwargs): self.handle_array_inputs(kwargs) From 834240896cf020a2def521fb2ee0adf996974912 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 22 Jul 2024 10:56:30 +0000 Subject: [PATCH 5/6] Sort models --- supported_models.json | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/supported_models.json b/supported_models.json index f5b7307..5e5e6ca 100644 --- a/supported_models.json +++ b/supported_models.json @@ -1,16 +1,7 @@ { "models": [ - "suno-ai/bark", - "meta/musicgen", - "lucataco/magnet", - "nateraw/musicgen-songstarter-v0.2", - "nateraw/audio-super-resolution", - "zsxkib/realistic-voice-cloning", - "lucataco/xtts-v2", - "declare-lab/tango", - "cjwbw/seamless_communication", - "ai-forever/kandinsky-2.2", "ai-forever/kandinsky-2", + "ai-forever/kandinsky-2.2", "andreasjansson/blip-2", "batouresearch/high-resolution-controlnet-tile", "batouresearch/magic-image-refiner", @@ -18,26 +9,33 @@ "bytedance/sdxl-lightning-4step", "cjwbw/hyper-sdxl-1step-t2i", "cjwbw/pixart-sigma", + "cjwbw/seamless_communication", "cjwbw/supir", "cuuupid/glm-4v-9b", "cuuupid/idm-vton", + "declare-lab/tango", "fofr/consistent-character", "fofr/controlnet-preprocessors", "fofr/face-to-many", + "fofr/latent-consistency-model", "fofr/sd3-with-chaos", "fofr/style-transfer", - "fofr/latent-consistency-model", "lucataco/hunyuandit-v1.1", "lucataco/llama-3-vision-alpha", + "lucataco/magnet", "lucataco/moondream2", "lucataco/paligemma-3b-pt-224", "lucataco/pasd-magnify", "lucataco/qwen-vl-chat", "lucataco/sdxl-clip-interrogator", + "lucataco/xtts-v2", "luosiallen/latent-consistency-model", "meta/llama-2-70b-chat", "meta/llama-2-7b-chat", + "meta/musicgen", "mistralai/mixtral-8x7b-instruct-v0.1", + "nateraw/audio-super-resolution", + "nateraw/musicgen-songstarter-v0.2", "omniedgeio/face-swap", "philz1337x/clarity-upscaler", "replicate/deepfloyd-if", @@ -47,9 +45,11 @@ "stability-ai/sdxl", "stability-ai/stable-diffusion", "stability-ai/stable-diffusion-3", + "suno-ai/bark", "tstramer/material-diffusion", "yorickvp/llava-13b", "yorickvp/llava-v1.6-34b", - "yorickvp/llava-v1.6-mistral-7b" + "yorickvp/llava-v1.6-mistral-7b", + "zsxkib/realistic-voice-cloning" ] } From 7b3cc9bcbfde837dbf314adb460af75930643ac3 Mon Sep 17 00:00:00 2001 From: Paul Date: Mon, 22 Jul 2024 11:11:53 +0000 Subject: [PATCH 6/6] Bump version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c3fa74a..db3a6fc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "comfyui-replicate" description = "Run Replicate models in ComfyUI" -version = "1.0.4" +version = "1.0.5" license = "LICENSE" [project.urls]