diff --git a/import_schemas.py b/import_schemas.py new file mode 100644 index 0000000..babcad9 --- /dev/null +++ b/import_schemas.py @@ -0,0 +1,38 @@ +import replicate +import json +import os + +models_to_import = [ + "fofr/face-to-many", + "meta/meta-llama-3-70b-instruct", + "meta/meta-llama-3-8b-instruct", +] + + +def format_json_file(file_path): + try: + with open(file_path, "r") as f: + data = json.load(f) + + with open(file_path, "w") as f: + json.dump(data, f, indent=4, ensure_ascii=False) + except json.JSONDecodeError: + print(f"Error: {file_path} contains invalid JSON") + except IOError: + print(f"Error: Could not read or write to {file_path}") + + +def format_json_files_in_directory(directory): + for filename in os.listdir(directory): + if filename.endswith(".json"): + file_path = os.path.join(directory, filename) + format_json_file(file_path) + + +for model in models_to_import: + m = replicate.models.get(model) + with open(f"schemas/{model.replace('/', '_')}.json", "w") as f: + f.write(m.json()) + +schemas_directory = "schemas" +format_json_files_in_directory(schemas_directory) diff --git a/node.py b/node.py index 8c09709..c706536 100644 --- a/node.py +++ b/node.py @@ -1,90 +1,157 @@ +import os +import json import replicate -class Llama3Replicate: - @classmethod - def INPUT_TYPES(cls): - return { - "required": { - "prompt": ( - "STRING", - {"default": "", "multiline": True, "dynamicPrompts": True}, - ), - "system_prompt": ( - "STRING", - { - "default": "You are a helpful assistant", - "multiline": True, - "dynamicPrompts": True, - }, - ), - "top_p": ( - "FLOAT", - {"default": 0.95, "max": 1.0, "min": -1.0}, - ), - "top_k": ( - "INT", - {"default": 0, "min": -1}, - ), - "max_tokens": ( - "INT", - {"default": 512, "min": 1}, - ), - "min_tokens": ( - "INT", - {"default": 0, "min": 0}, - ), - "temperature": ( - "FLOAT", - {"default": 0.7, "max": 5.0, "min": 0.0}, - ), - "length_penalty": ( - "FLOAT", - {"default": 1.0, "max": 5.0, "min": 0.0}, - ), - "presence_penalty": ( - "FLOAT", - {"default": 0}, - ), - "seed": ("INT", {"default": 0, "min": 0, "max": 0xFFFFFFFFFFFFFFFF}), - } - } - - RETURN_TYPES = ("STRING",) - FUNCTION = "run_llama3_replicate" - CATEGORY = "Replicate" - - def run_llama3_replicate( - self, - top_p, - top_k, - prompt, - system_prompt, - max_tokens, - min_tokens, - temperature, - length_penalty, - presence_penalty, - seed, - ): - input = { - "system_prompt": system_prompt, - "prompt": prompt, - "top_p": top_p, - "top_k": top_k, - "max_tokens": max_tokens, - "min_tokens": min_tokens, - "temperature": temperature, - "length_penalty": length_penalty, - "presence_penalty": presence_penalty, - "seed": seed, - } - - output = replicate.run("meta/meta-llama-3-70b-instruct", input=input) - output = "".join(output).strip() - return (output,) +def convert_type(openapi_type): + type_mapping = { + "string": "STRING", + "integer": "INT", + "number": "FLOAT", + "boolean": "BOOL", + } + return type_mapping.get(openapi_type, "STRING") -NODE_CLASS_MAPPINGS = { - "Llama 3 Replicate": Llama3Replicate, -} +def resolve_schema(prop_data, schemas): + if "$ref" in prop_data: + ref_path = prop_data["$ref"].split("/") + current = schemas + for path in ref_path[1:]: # Skip the first '#' element + current = current[path] + return current + return prop_data + + +def convert_schema_to_comfyui(schema, schemas): + input_types = {"required": {}} + + for prop_name, prop_data in schema["properties"].items(): + prop_data = resolve_schema(prop_data, schemas) + + if "allOf" in prop_data: + prop_data = resolve_schema(prop_data["allOf"][0], schemas) + + if "enum" in prop_data: + input_type = prop_data["enum"] + elif "type" in prop_data: + input_type = convert_type(prop_data["type"]) + else: + input_type = "STRING" + + default_value = prop_data.get("default", "") + + input_config = {"default": default_value} + + if "minimum" in prop_data: + input_config["min"] = prop_data["minimum"] + if "maximum" in prop_data: + input_config["max"] = prop_data["maximum"] + + if prop_data.get("type") == "string" and prop_data.get("format") == "uri": + input_config["multiline"] = True + + if "prompt" in prop_name and prop_data.get("type") == "string": + input_config["multiline"] = True + + if "template" not in prop_name: + input_config["dynamicPrompts"] = True + + input_types["required"][prop_name] = (input_type, input_config) + + # Reorder input_types to put prompt and negative_prompt first + ordered_input_types = {"required": {}} + for key in ["prompt", "negative_prompt"]: + if key in input_types["required"]: + ordered_input_types["required"][key] = input_types["required"][key] + for key in list(input_types["required"].keys()): + if "prompt" in key: + ordered_input_types["required"][key] = input_types["required"][key] + for key in list(input_types["required"].keys()): + if key not in ordered_input_types["required"] and key != "seed": + ordered_input_types["required"][key] = input_types["required"][key] + if "seed" in input_types["required"]: + ordered_input_types["required"]["seed"] = input_types["required"]["seed"] + + return ordered_input_types + + +def create_comfyui_node(schemas, model_info): + author = model_info["owner"] + name = model_info["name"] + version = model_info["latest_version"]["id"] + + replicate_model = f"{author}/{name}:{version}" + node_name = f"Replicate {author}/{name}" + input_schema = schemas["components"]["schemas"]["Input"] + + class ReplicateToComfyUI: + @classmethod + def INPUT_TYPES(cls): + return convert_schema_to_comfyui(input_schema, schemas) + + RETURN_TYPES = ("STRING",) + FUNCTION = "run_openapi_to_comfyui" + CATEGORY = "Replicate" + + def run_openapi_to_comfyui(self, **kwargs): + print(f"Running {replicate_model} with {kwargs}") + output = replicate.run(replicate_model, input=kwargs) + output = "".join(output).strip() + # print(f"Output: {output}") + return (output,) + + return node_name, ReplicateToComfyUI + + +def create_comfyui_nodes_from_schemas(schemas_dir): + nodes = {} + current_path = os.path.dirname(os.path.abspath(__file__)) + schemas_dir_path = os.path.join(current_path, schemas_dir) + for schema_file in os.listdir(schemas_dir_path): + if schema_file.endswith(".json"): + with open(os.path.join(schemas_dir_path, schema_file), "r") as f: + schema = json.load(f) + openapi_schema = schema["latest_version"]["openapi_schema"] + model_info = schema + node_name, node_class = create_comfyui_node(openapi_schema, model_info) + nodes[node_name] = node_class + return nodes + + +# Create ComfyUI nodes for all schema files in the "schemas" directory +comfyui_nodes = create_comfyui_nodes_from_schemas("schemas") + +# Print the resulting node classes +for schema_file, node_class in comfyui_nodes.items(): + print(f"Node class for {schema_file}:") + print(node_class.INPUT_TYPES()) + + +NODE_CLASS_MAPPINGS = comfyui_nodes +print(NODE_CLASS_MAPPINGS) + +# # Load the schema +# with open("schema.json", "r") as f: +# schema = json.load(f) +# openapi_schema = schema["latest_version"]["openapi_schema"] + +# # Create the ComfyUI node +# ComfyUINode = create_comfyui_node(openapi_schema, schema) + +# # Print the resulting node class +# print(ComfyUINode.INPUT_TYPES()) + +# # Create an instance of the node and pass in defaults +# node_instance = ComfyUINode() +# defaults = { +# "seed": 0, +# "image": "https://example.com/default_image.png", +# "style": "3D", +# "prompt": "a person", +# "lora_scale": 1.0 +# } + +# # Run the node with the defaults +# result = node_instance.run_openapi_to_comfyui(**defaults) +# print(result) diff --git a/schemas/fofr_face-to-many.json b/schemas/fofr_face-to-many.json new file mode 100644 index 0000000..1005dfb --- /dev/null +++ b/schemas/fofr_face-to-many.json @@ -0,0 +1,543 @@ +{ + "url": "https://replicate.com/fofr/face-to-many", + "owner": "fofr", + "name": "face-to-many", + "description": "Turn a face into 3D, emoji, pixel art, video game, claymation or toy", + "visibility": "public", + "github_url": "https://github.com/fofr/cog-face-to-many", + "paper_url": null, + "license_url": "https://github.com/fofr/cog-face-to-many/blob/main/weights_licenses.md", + "run_count": 10959380, + "cover_image_url": "https://replicate.delivery/pbxt/R1ayGe5efoQbaoRzgDEJdLsIZ20lWRiprvoW1F4uKAZIha6kA/ComfyUI_00001_.png", + "default_example": { + "id": "pft2dadbcdnh25z5ktctsj3sze", + "model": "fofr/face-to-many", + "version": "edc6439ac55af138defbca7c472b38bcdd62c61797e8e0c2fae88696cd8afb25", + "status": "succeeded", + "input": { + "image": "https://replicate.delivery/pbxt/KW7Getr2zD5ECxySdBZtLmPa322lNkXrpkMdKcmxeaDmq2b1/MTk4MTczMTkzNzI1Mjg5NjYy.webp", + "style": "Clay", + "prompt": "a person in a post apocalyptic war game", + "negative_prompt": "", + "prompt_strength": 4.5, + "denoising_strength": 0.65, + "instant_id_strength": 0.8 + }, + "output": [ + "https://replicate.delivery/pbxt/R1ayGe5efoQbaoRzgDEJdLsIZ20lWRiprvoW1F4uKAZIha6kA/ComfyUI_00001_.png" + ], + "logs": "Random seed set to: 3672888193\nChecking inputs\n✅ /tmp/inputs/input.webp\n====================================\nRunning workflow\ngot prompt\nExecuting node 3, title: LoRA Stacker, class type: LoRA Stacker\nExecuting node 2, title: Efficient Loader, class type: Efficient Loader\nRequested to load SDXLClipModel\nLoading 1 new model\n----------------------------------------\n\u001b[36mEfficient Loader Models Cache:\u001b[0m\nCkpt:\n[1] albedobaseXL_v13\nLora:\n[1] base_ckpt: albedobaseXL_v13\nlora(mod,clip): ClayAnimationRedm(1,1)\nExecuting node 41, title: Apply InstantID, class type: ApplyInstantID\nExecuting node 51, title: VAE Encode, class type: VAEEncode\nExecuting node 4, title: KSampler (Efficient), class type: KSampler (Efficient)\nRequested to load SDXL\nRequested to load ControlNet\nRequested to load ControlNet\nLoading 3 new models\n 0%| | 0/20 [00:00<|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "presence_penalty": 1.15, + "frequency_penalty": 0.2 + }, + "output": [ + "Let", + "'s", + " break", + " this", + " problem", + " down", + " step", + " by", + " step", + ".\n\n", + "Step", + " ", + "1", + ":", + " Sarah", + " already", + " has", + " ", + "7", + " ll", + "amas", + ".\n\n", + "Step", + " ", + "2", + ":", + " Her", + " friend", + " gives", + " her", + " ", + "3", + " trucks", + " of", + " ll", + "amas", + ".", + " We", + " need", + " to", + " find", + " out", + " how", + " many", + " ll", + "amas", + " are", + " in", + " these", + " ", + "3", + " trucks", + ".\n\n", + "Step", + " ", + "3", + ":", + " Each", + " truck", + " has", + " ", + "5", + " ll", + "amas", + ",", + " so", + " we", + " multiply", + " the", + " number", + " of", + " trucks", + " (", + "3", + ")", + " by", + " the", + " number", + " of", + " ll", + "amas", + " per", + " truck", + " (", + "5", + "):\n\n", + "3", + " trucks", + " x", + " ", + "5", + " ll", + "amas", + "/tr", + "uck", + " =", + " ", + "3", + " x", + " ", + "5", + " =", + " ", + "15", + " ll", + "amas", + "\n\n", + "Step", + " ", + "4", + ":", + " Sarah", + " already", + " had", + " ", + "7", + " ll", + "amas", + ",", + " and", + " now", + " she", + " gets", + " ", + "15", + " more", + " ll", + "amas", + " from", + " her", + " friend", + ".", + " To", + " find", + " the", + " total", + " number", + " of", + " ll", + "amas", + " Sarah", + " has", + ",", + " we", + " add", + " the", + " two", + " numbers", + " together", + ":\n\n", + "7", + " ll", + "amas", + " (", + "already", + " had", + ")", + " +", + " ", + "15", + " ll", + "amas", + " (", + "from", + " her", + " friend", + ")", + " =", + " ", + "22", + " ll", + "amas", + "\n\n", + "Therefore", + ",", + " Sarah", + " has", + " a", + " total", + " of", + " ", + "22", + " ll", + "amas", + ".", + "" + ], + "logs": "", + "error": "", + "metrics": { + "total_time": 3.47, + "input_token_count": 54, + "tokens_per_second": 41.72998441835564, + "output_token_count": 166, + "predict_time": 4.045511 + }, + "created_at": "2024-04-18T16:31:19.530000Z", + "started_at": "2024-04-18T16:31:19Z", + "completed_at": "2024-04-18T16:31:23Z", + "urls": { + "stream": "https://streaming-api.svc.us.c.replicate.net/v1/streams/cvg64spwkdzlpdltjkmv6jotmwn2cq5btgxirmsbejvsxx7xp2la", + "get": "https://api.replicate.com/v1/predictions/7zr9g2asx9rgj0cey8698vm4d4", + "cancel": "https://api.replicate.com/v1/predictions/7zr9g2asx9rgj0cey8698vm4d4/cancel" + } + }, + "latest_version": { + "id": "fbfb20b472b2f3bdd101412a9f70a0ed4fc0ced78a77ff00970ee7a2383c575d", + "created_at": "2024-04-17T21:58:54.806446+00:00", + "cog_version": "0.9.4", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/ready": { + "get": { + "summary": "Ready", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Ready Ready Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "ready_ready_get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "top_k": { + "type": "integer", + "title": "Top K", + "default": 50, + "x-order": 5, + "description": "The number of highest probability tokens to consider for generating the output. If > 0, only keep the top k tokens with highest probability (top-k filtering)." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0.9, + "x-order": 4, + "description": "A probability threshold for generating the output. If < 1.0, only keep the top tokens with cumulative probability >= top_p (nucleus filtering). Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "default": "", + "x-order": 0, + "description": "Prompt" + }, + "max_tokens": { + "type": "integer", + "title": "Max Tokens", + "default": 512, + "x-order": 2, + "description": "The maximum number of tokens the model should generate as output." + }, + "min_tokens": { + "type": "integer", + "title": "Min Tokens", + "default": 0, + "x-order": 1, + "description": "The minimum number of tokens the model should generate as output." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 0.6, + "x-order": 3, + "description": "The value used to modulate the next token probabilities." + }, + "prompt_template": { + "type": "string", + "title": "Prompt Template", + "default": "{prompt}", + "x-order": 8, + "description": "Prompt template. The string `{prompt}` will be substituted for the input prompt. If you want to generate dialog output, use this template as a starting point and construct the prompt string manually, leaving `prompt_template={prompt}`." + }, + "presence_penalty": { + "type": "number", + "title": "Presence Penalty", + "default": 1.15, + "x-order": 6, + "description": "Presence penalty" + }, + "frequency_penalty": { + "type": "number", + "title": "Frequency Penalty", + "default": 0.2, + "x-order": 7, + "description": "Frequency penalty" + } + } + }, + "Output": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Output", + "x-cog-array-type": "iterator", + "x-cog-array-display": "concatenate" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/schemas/meta_meta-llama-3-8b-instruct.json b/schemas/meta_meta-llama-3-8b-instruct.json new file mode 100644 index 0000000..2ccec83 --- /dev/null +++ b/schemas/meta_meta-llama-3-8b-instruct.json @@ -0,0 +1,683 @@ +{ + "url": "https://replicate.com/meta/meta-llama-3-8b-instruct", + "owner": "meta", + "name": "meta-llama-3-8b-instruct", + "description": "An 8 billion parameter language model from Meta, fine tuned for chat completions", + "visibility": "public", + "github_url": "https://github.com/meta-llama/llama3", + "paper_url": null, + "license_url": "https://github.com/meta-llama/llama3/blob/main/LICENSE", + "run_count": 21377091, + "cover_image_url": "https://tjzk.replicate.delivery/models_models_cover_image/927d3fce-75e3-4af9-92da-f537bc34072a/meta-logo.png", + "default_example": { + "id": "855g9wxd7hrgp0cf7tsv1ewzgc", + "model": "replicate-internal/llama-3-8b-instruct-int8-triton", + "version": "b63acc3f54e3c08cb3b081f049ebc881420035dfc6db48f554530e9c4bc02ba3", + "status": "succeeded", + "input": { + "top_p": 0.95, + "prompt": "Johnny has 8 billion parameters. His friend Tommy has 70 billion parameters. What does this mean when it comes to speed?", + "temperature": 0.7, + "system_prompt": "You are a helpful assistant", + "length_penalty": 1, + "max_new_tokens": 512, + "stop_sequences": "<|end_of_text|>,<|eot_id|>", + "prompt_template": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{system_prompt}<|eot_id|><|start_header_id|>user<|end_header_id|>\n\n{prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "presence_penalty": 0 + }, + "output": [ + "The", + " number", + " of", + " parameters", + " in", + " a", + " neural", + " network", + " can", + " impact", + " its", + " speed", + ",", + " but", + " it", + "'s", + " not", + " the", + " only", + " factor", + ".\n\n", + "In", + " general", + ",", + " a", + " larger", + " number", + " of", + " parameters", + " can", + " lead", + " to", + ":\n\n", + "1", + ".", + " Increased", + " computational", + " complexity", + ":", + " More", + " parameters", + " mean", + " more", + " calculations", + " are", + " required", + " to", + " process", + " the", + " data", + ".\n", + "2", + ".", + " Increased", + " memory", + " requirements", + ":", + " Larger", + " models", + " require", + " more", + " memory", + " to", + " store", + " their", + " parameters", + ",", + " which", + " can", + " impact", + " system", + " performance", + ".\n\n", + "However", + ",", + " it", + "'s", + " worth", + " noting", + " that", + " the", + " relationship", + " between", + " the", + " number", + " of", + " parameters", + " and", + " speed", + " is", + " not", + " always", + " linear", + ".", + " Other", + " factors", + ",", + " such", + " as", + ":\n\n", + "*", + " Model", + " architecture", + "\n", + "*", + " Optim", + "izer", + " choice", + "\n", + "*", + " Hyper", + "parameter", + " tuning", + "\n\n", + "can", + " also", + " impact", + " the", + " speed", + " of", + " a", + " neural", + " network", + ".\n\n", + "In", + " the", + " case", + " of", + " Johnny", + " and", + " Tommy", + ",", + " it", + "'s", + " difficult", + " to", + " say", + " which", + " one", + "'s", + " model", + " will", + " be", + " faster", + " without", + " more", + " information", + " about", + " the", + " models", + " themselves", + "." + ], + "logs": "Random seed used: `57440`\nNote: Random seed will not impact output if greedy decoding is used.\nFormatted prompt: `<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nJohnny has 8 billion parameters. His friend Tommy has 70 billion parameters. What does this mean when it comes to speed?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`Random seed used: `57440`\nNote: Random seed will not impact output if greedy decoding is used.\nFormatted prompt: `<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nYou are a helpful assistant<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nJohnny has 8 billion parameters. His friend Tommy has 70 billion parameters. What does this mean when it comes to speed?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n`", + "error": null, + "metrics": { + "total_time": 1.657073, + "input_token_count": 39, + "tokens_per_second": 92.80206135476371, + "output_token_count": 149, + "predict_time": 1.652461, + "time_to_first_token": 0.060728942999999994 + }, + "created_at": "2024-05-03T13:45:13.788000Z", + "started_at": "2024-05-03T13:45:13.792612Z", + "completed_at": "2024-05-03T13:45:15.445073Z", + "urls": { + "stream": "https://streaming-api.svc.us.c.replicate.net/v1/streams/hscsfwedhigbbnorfpq7c4i3lbac5srhaqgvb4b5m2iuof3rotwq", + "get": "https://api.replicate.com/v1/predictions/855g9wxd7hrgp0cf7tsv1ewzgc", + "cancel": "https://api.replicate.com/v1/predictions/855g9wxd7hrgp0cf7tsv1ewzgc/cancel" + } + }, + "latest_version": { + "id": "5a6809ca6288247d06daf6365557e5e429063f32a21146b2a807c682652136b8", + "created_at": "2024-04-17T21:58:45.726276+00:00", + "cog_version": "0.9.4", + "openapi_schema": { + "info": { + "title": "Cog", + "version": "0.1.0" + }, + "paths": { + "/": { + "get": { + "summary": "Root", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Root Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "root__get" + } + }, + "/ready": { + "get": { + "summary": "Ready", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Ready Ready Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "ready_ready_get" + } + }, + "/shutdown": { + "post": { + "summary": "Start Shutdown", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Start Shutdown Shutdown Post" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "start_shutdown_shutdown_post" + } + }, + "/predictions": { + "post": { + "summary": "Predict", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model", + "operationId": "predict_predictions_post", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionRequest" + } + } + } + } + } + }, + "/health-check": { + "get": { + "summary": "Healthcheck", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Healthcheck Health Check Get" + } + } + }, + "description": "Successful Response" + } + }, + "operationId": "healthcheck_health_check_get" + } + }, + "/predictions/{prediction_id}": { + "put": { + "summary": "Predict Idempotent", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/PredictionResponse" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + }, + { + "in": "header", + "name": "prefer", + "schema": { + "type": "string", + "title": "Prefer" + }, + "required": false + } + ], + "description": "Run a single prediction on the model (idempotent creation).", + "operationId": "predict_idempotent_predictions__prediction_id__put", + "requestBody": { + "content": { + "application/json": { + "schema": { + "allOf": [ + { + "$ref": "#/components/schemas/PredictionRequest" + } + ], + "title": "Prediction Request" + } + } + }, + "required": true + } + } + }, + "/predictions/{prediction_id}/cancel": { + "post": { + "summary": "Cancel", + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "title": "Response Cancel Predictions Prediction Id Cancel Post" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "parameters": [ + { + "in": "path", + "name": "prediction_id", + "schema": { + "type": "string", + "title": "Prediction ID" + }, + "required": true + } + ], + "description": "Cancel a running prediction", + "operationId": "cancel_predictions__prediction_id__cancel_post" + } + } + }, + "openapi": "3.0.2", + "components": { + "schemas": { + "Input": { + "type": "object", + "title": "Input", + "properties": { + "top_k": { + "type": "integer", + "title": "Top K", + "default": 50, + "x-order": 5, + "description": "The number of highest probability tokens to consider for generating the output. If > 0, only keep the top k tokens with highest probability (top-k filtering)." + }, + "top_p": { + "type": "number", + "title": "Top P", + "default": 0.9, + "x-order": 4, + "description": "A probability threshold for generating the output. If < 1.0, only keep the top tokens with cumulative probability >= top_p (nucleus filtering). Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751)." + }, + "prompt": { + "type": "string", + "title": "Prompt", + "default": "", + "x-order": 0, + "description": "Prompt" + }, + "max_tokens": { + "type": "integer", + "title": "Max Tokens", + "default": 512, + "x-order": 2, + "description": "The maximum number of tokens the model should generate as output." + }, + "min_tokens": { + "type": "integer", + "title": "Min Tokens", + "default": 0, + "x-order": 1, + "description": "The minimum number of tokens the model should generate as output." + }, + "temperature": { + "type": "number", + "title": "Temperature", + "default": 0.6, + "x-order": 3, + "description": "The value used to modulate the next token probabilities." + }, + "prompt_template": { + "type": "string", + "title": "Prompt Template", + "default": "{prompt}", + "x-order": 8, + "description": "Prompt template. The string `{prompt}` will be substituted for the input prompt. If you want to generate dialog output, use this template as a starting point and construct the prompt string manually, leaving `prompt_template={prompt}`." + }, + "presence_penalty": { + "type": "number", + "title": "Presence Penalty", + "default": 1.15, + "x-order": 6, + "description": "Presence penalty" + }, + "frequency_penalty": { + "type": "number", + "title": "Frequency Penalty", + "default": 0.2, + "x-order": 7, + "description": "Frequency penalty" + } + } + }, + "Output": { + "type": "array", + "items": { + "type": "string" + }, + "title": "Output", + "x-cog-array-type": "iterator", + "x-cog-array-display": "concatenate" + }, + "Status": { + "enum": [ + "starting", + "processing", + "succeeded", + "canceled", + "failed" + ], + "type": "string", + "title": "Status", + "description": "An enumeration." + }, + "WebhookEvent": { + "enum": [ + "start", + "output", + "logs", + "completed" + ], + "type": "string", + "title": "WebhookEvent", + "description": "An enumeration." + }, + "ValidationError": { + "type": "object", + "title": "ValidationError", + "required": [ + "loc", + "msg", + "type" + ], + "properties": { + "loc": { + "type": "array", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Location" + }, + "msg": { + "type": "string", + "title": "Message" + }, + "type": { + "type": "string", + "title": "Error Type" + } + } + }, + "PredictionRequest": { + "type": "object", + "title": "PredictionRequest", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "webhook": { + "type": "string", + "title": "Webhook", + "format": "uri", + "maxLength": 65536, + "minLength": 1 + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "output_file_prefix": { + "type": "string", + "title": "Output File Prefix" + }, + "webhook_events_filter": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + }, + "default": [ + "start", + "output", + "logs", + "completed" + ] + } + } + }, + "PredictionResponse": { + "type": "object", + "title": "PredictionResponse", + "properties": { + "id": { + "type": "string", + "title": "Id" + }, + "logs": { + "type": "string", + "title": "Logs", + "default": "" + }, + "error": { + "type": "string", + "title": "Error" + }, + "input": { + "$ref": "#/components/schemas/Input" + }, + "output": { + "$ref": "#/components/schemas/Output" + }, + "status": { + "$ref": "#/components/schemas/Status" + }, + "metrics": { + "type": "object", + "title": "Metrics" + }, + "version": { + "type": "string", + "title": "Version" + }, + "created_at": { + "type": "string", + "title": "Created At", + "format": "date-time" + }, + "started_at": { + "type": "string", + "title": "Started At", + "format": "date-time" + }, + "completed_at": { + "type": "string", + "title": "Completed At", + "format": "date-time" + } + } + }, + "HTTPValidationError": { + "type": "object", + "title": "HTTPValidationError", + "properties": { + "detail": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ValidationError" + }, + "title": "Detail" + } + } + } + } + } + } + } +} \ No newline at end of file