From 73d366b2a137f850ebd0506e85758bf0c644e3ce Mon Sep 17 00:00:00 2001 From: CentralFloridaAttorney <71934273+centralfloridaattorney@users.noreply.github.com> Date: Fri, 29 May 2026 13:25:45 -0400 Subject: [PATCH] work on chunked documents --- .gitignore | 1 + ...XED_CHUNK_NAME_SYSTEM_WORKFLOW_CANVAS.json | 1689 +++++++++++++++ .../zmongo_gemini_chunk_loop_workflow.json | 1802 +++++++++++++++++ nodes/__init__.py | 7 +- nodes/document_api_nodes.py | 1619 ++++++++++++++- nodes/gemini_api_nodes.py | 1 - nodes/generic_helpers.py | 66 +- nodes/local_file_store.py | 65 + nodes/zmongo_helper_nodes.py | 679 ++++++- nodes/zmongo_preset_nodes.py | 129 +- nodes/zmongo_text_agent_nodes.py | 1517 ++++++++++++++ requirements.txt | 4 +- 12 files changed, 7475 insertions(+), 104 deletions(-) create mode 100644 example_workflows/zmongo_gemini_chunk_loop_FIXED_CHUNK_NAME_SYSTEM_WORKFLOW_CANVAS.json create mode 100644 example_workflows/zmongo_gemini_chunk_loop_workflow.json create mode 100644 nodes/zmongo_text_agent_nodes.py diff --git a/.gitignore b/.gitignore index 5ae760c..febf3fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ **/__pycache__/ *.pyc +/nodes/local_store/ diff --git a/example_workflows/zmongo_gemini_chunk_loop_FIXED_CHUNK_NAME_SYSTEM_WORKFLOW_CANVAS.json b/example_workflows/zmongo_gemini_chunk_loop_FIXED_CHUNK_NAME_SYSTEM_WORKFLOW_CANVAS.json new file mode 100644 index 0000000..e9446c9 --- /dev/null +++ b/example_workflows/zmongo_gemini_chunk_loop_FIXED_CHUNK_NAME_SYSTEM_WORKFLOW_CANVAS.json @@ -0,0 +1,1689 @@ +{ + "id": "f13fe622-dcda-452d-bc99-87d6eb58092b", + "revision": 0, + "last_node_id": 13, + "last_link_id": 17, + "nodes": [ + { + "id": 1, + "type": "ZMongoApiKeySessionNode", + "pos": [ + -1280, + -160 + ], + "size": [ + 360, + 290 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "base_url", + "name": "base_url", + "type": "STRING", + "widget": { + "name": "base_url" + }, + "link": null + }, + { + "localized_name": "zai_api_key", + "name": "zai_api_key", + "type": "STRING", + "widget": { + "name": "zai_api_key" + }, + "link": null + }, + { + "localized_name": "username", + "name": "username", + "type": "STRING", + "widget": { + "name": "username" + }, + "link": null + }, + { + "localized_name": "comfy_zmongo_prefix", + "name": "comfy_zmongo_prefix", + "type": "STRING", + "widget": { + "name": "comfy_zmongo_prefix" + }, + "link": null + }, + { + "localized_name": "fleet_prefix", + "name": "fleet_prefix", + "type": "STRING", + "widget": { + "name": "fleet_prefix" + }, + "link": null + }, + { + "localized_name": "comfy_zmongo_fleet_prefix", + "name": "comfy_zmongo_fleet_prefix", + "type": "STRING", + "widget": { + "name": "comfy_zmongo_fleet_prefix" + }, + "link": null + }, + { + "localized_name": "timeout_seconds", + "name": "timeout_seconds", + "type": "INT", + "widget": { + "name": "timeout_seconds" + }, + "link": null + }, + { + "localized_name": "verify_tls", + "name": "verify_tls", + "type": "BOOLEAN", + "widget": { + "name": "verify_tls" + }, + "link": null + }, + { + "localized_name": "test_whoami", + "name": "test_whoami", + "type": "BOOLEAN", + "widget": { + "name": "test_whoami" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "links": [ + 1, + 3, + 6, + 7 + ] + }, + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": null + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": null + } + ], + "title": "00 API Key Session", + "properties": { + "Node name for S&R": "ZMongoApiKeySessionNode" + }, + "widgets_values": [ + "https://businessprocessapplications.com", + "PASTE_API_KEY_HERE", + "PASTE_USERNAME_HERE", + "/comfy-zmongo", + "/fleet", + "/comfy-zmongo-fleet", + 30, + true, + true + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 2, + "type": "ZMongoDocumentFilePathBrowser", + "pos": [ + -1280, + 240 + ], + "size": [ + 360, + 620 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "root_mode", + "name": "root_mode", + "type": "COMBO", + "widget": { + "name": "root_mode" + }, + "link": null + }, + { + "localized_name": "selected_file", + "name": "selected_file", + "type": "COMBO", + "widget": { + "name": "selected_file" + }, + "link": null + }, + { + "localized_name": "manual_path", + "name": "manual_path", + "type": "STRING", + "widget": { + "name": "manual_path" + }, + "link": null + }, + { + "localized_name": "custom_root", + "name": "custom_root", + "type": "STRING", + "widget": { + "name": "custom_root" + }, + "link": null + }, + { + "localized_name": "recursive", + "name": "recursive", + "type": "BOOLEAN", + "widget": { + "name": "recursive" + }, + "link": null + }, + { + "localized_name": "extensions", + "name": "extensions", + "type": "STRING", + "widget": { + "name": "extensions" + }, + "link": null + }, + { + "localized_name": "search_text", + "name": "search_text", + "type": "STRING", + "widget": { + "name": "search_text" + }, + "link": null + }, + { + "localized_name": "max_results", + "name": "max_results", + "type": "INT", + "widget": { + "name": "max_results" + }, + "link": null + }, + { + "localized_name": "max_depth", + "name": "max_depth", + "type": "INT", + "widget": { + "name": "max_depth" + }, + "link": null + }, + { + "localized_name": "include_hidden", + "name": "include_hidden", + "type": "BOOLEAN", + "widget": { + "name": "include_hidden" + }, + "link": null + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "shape": 7, + "widget": { + "name": "refresh_token" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "file_path", + "name": "file_path", + "type": "STRING", + "links": null + }, + { + "localized_name": "file_path_link", + "name": "file_path_link", + "type": "ZMONGO_FILE_PATH", + "links": [ + 2 + ] + }, + { + "localized_name": "filename", + "name": "filename", + "type": "STRING", + "links": null + }, + { + "localized_name": "filename_link", + "name": "filename_link", + "type": "ZMONGO_FILENAME", + "links": null + }, + { + "localized_name": "content_type", + "name": "content_type", + "type": "STRING", + "links": null + }, + { + "localized_name": "size_bytes", + "name": "size_bytes", + "type": "INT", + "links": null + }, + { + "localized_name": "exists", + "name": "exists", + "type": "BOOLEAN", + "links": null + }, + { + "localized_name": "browser_root", + "name": "browser_root", + "type": "STRING", + "links": null + }, + { + "localized_name": "refresh", + "name": "refresh", + "type": "STRING", + "links": null + }, + { + "localized_name": "file_paths", + "name": "file_paths", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "filenames", + "name": "filenames", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "indexed_file_paths", + "name": "indexed_file_paths", + "type": "STRING", + "links": null + }, + { + "localized_name": "discovered_count", + "name": "discovered_count", + "type": "INT", + "links": null + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": null + }, + { + "localized_name": "selectable_items", + "name": "selectable_items", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "indexed_items", + "name": "indexed_items", + "type": "STRING", + "links": null + }, + { + "localized_name": "item_count", + "name": "item_count", + "type": "INT", + "links": null + } + ], + "title": "01 Choose Local Document File", + "properties": { + "Node name for S&R": "ZMongoDocumentFilePathBrowser" + }, + "widgets_values": [ + "Documents", + "", + "", + "", + true, + ".pdf,.docx,.txt,.md,.csv,.json,.log,.rtf", + "", + 500, + 6, + false, + "" + ], + "color": "#322", + "bgcolor": "#533" + }, + { + "id": 3, + "type": "ZMongoDocumentIngestTextFile", + "pos": [ + -780, + 160 + ], + "size": [ + 430, + 640 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 1 + }, + { + "localized_name": "file_path_link", + "name": "file_path_link", + "type": "*", + "shape": 7, + "link": 2 + }, + { + "localized_name": "file_path", + "name": "file_path", + "type": "STRING", + "widget": { + "name": "file_path" + }, + "link": null + }, + { + "localized_name": "document_name", + "name": "document_name", + "type": "STRING", + "widget": { + "name": "document_name" + }, + "link": null + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "widget": { + "name": "collection_name" + }, + "link": null + }, + { + "localized_name": "field_root", + "name": "field_root", + "type": "STRING", + "widget": { + "name": "field_root" + }, + "link": null + }, + { + "localized_name": "chunk_text", + "name": "chunk_text", + "type": "BOOLEAN", + "widget": { + "name": "chunk_text" + }, + "link": null + }, + { + "localized_name": "chunk_size", + "name": "chunk_size", + "type": "INT", + "widget": { + "name": "chunk_size" + }, + "link": null + }, + { + "localized_name": "chunk_overlap", + "name": "chunk_overlap", + "type": "INT", + "widget": { + "name": "chunk_overlap" + }, + "link": null + }, + { + "localized_name": "overwrite", + "name": "overwrite", + "type": "BOOLEAN", + "widget": { + "name": "overwrite" + }, + "link": null + }, + { + "localized_name": "metadata_json", + "name": "metadata_json", + "type": "STRING", + "widget": { + "name": "metadata_json" + }, + "link": null + }, + { + "localized_name": "tags_json", + "name": "tags_json", + "type": "STRING", + "widget": { + "name": "tags_json" + }, + "link": null + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "shape": 7, + "widget": { + "name": "refresh_token" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 11 + ] + }, + { + "localized_name": "document_id", + "name": "document_id", + "type": "STRING", + "links": null + }, + { + "localized_name": "document_id_link", + "name": "document_id_link", + "type": "ZMONGO_DOCUMENT_ID", + "links": [ + 4 + ] + }, + { + "localized_name": "document_name", + "name": "document_name", + "type": "STRING", + "links": null + }, + { + "localized_name": "filename_link", + "name": "filename_link", + "type": "ZMONGO_FILENAME", + "links": null + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "links": null + }, + { + "localized_name": "text_length", + "name": "text_length", + "type": "INT", + "links": null + }, + { + "localized_name": "chunk_count", + "name": "chunk_count", + "type": "INT", + "links": null + }, + { + "localized_name": "success", + "name": "success", + "type": "BOOLEAN", + "links": null + }, + { + "localized_name": "refresh", + "name": "refresh", + "type": "STRING", + "links": null + }, + { + "localized_name": "document_ids", + "name": "document_ids", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "text_items", + "name": "text_items", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "selectable_items", + "name": "selectable_items", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "indexed_items", + "name": "indexed_items", + "type": "STRING", + "links": null + }, + { + "localized_name": "item_count", + "name": "item_count", + "type": "INT", + "links": null + } + ], + "title": "02 Ingest/Reuse Document With Chunks FIRST", + "properties": { + "Node name for S&R": "ZMongoDocumentIngestTextFile" + }, + "widgets_values": [ + "", + "", + "text_documents", + "document_text", + true, + 3000, + 300, + false, + "{}", + "[]", + "" + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 4, + "type": "ZMongoDocumentChunkLoopManagerNode", + "pos": [ + -220, + 120 + ], + "size": [ + 500, + 620 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 3 + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "widget": { + "name": "collection_name" + }, + "link": null + }, + { + "localized_name": "document_id", + "name": "document_id", + "type": "STRING", + "widget": { + "name": "document_id" + }, + "link": null + }, + { + "localized_name": "chunk_field_root", + "name": "chunk_field_root", + "type": "STRING", + "widget": { + "name": "chunk_field_root" + }, + "link": null + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "widget": { + "name": "index" + }, + "link": null + }, + { + "localized_name": "fallback_text", + "name": "fallback_text", + "type": "STRING", + "widget": { + "name": "fallback_text" + }, + "link": null + }, + { + "localized_name": "user_instruction", + "name": "user_instruction", + "type": "STRING", + "widget": { + "name": "user_instruction" + }, + "link": null + }, + { + "localized_name": "result_field_root", + "name": "result_field_root", + "type": "STRING", + "widget": { + "name": "result_field_root" + }, + "link": null + }, + { + "localized_name": "wrap", + "name": "wrap", + "type": "BOOLEAN", + "widget": { + "name": "wrap" + }, + "link": null + }, + { + "localized_name": "advance_on_execute", + "name": "advance_on_execute", + "type": "BOOLEAN", + "widget": { + "name": "advance_on_execute" + }, + "link": null + }, + { + "localized_name": "state_name", + "name": "state_name", + "type": "STRING", + "widget": { + "name": "state_name" + }, + "link": null + }, + { + "localized_name": "reset_state", + "name": "reset_state", + "type": "BOOLEAN", + "widget": { + "name": "reset_state" + }, + "link": null + }, + { + "localized_name": "fallback_to_raw_text", + "name": "fallback_to_raw_text", + "type": "BOOLEAN", + "widget": { + "name": "fallback_to_raw_text" + }, + "link": null + }, + { + "localized_name": "raw_text_field_path", + "name": "raw_text_field_path", + "type": "STRING", + "widget": { + "name": "raw_text_field_path" + }, + "link": null + }, + { + "localized_name": "dynamic_chunk_size", + "name": "dynamic_chunk_size", + "type": "INT", + "widget": { + "name": "dynamic_chunk_size" + }, + "link": null + }, + { + "localized_name": "dynamic_chunk_overlap", + "name": "dynamic_chunk_overlap", + "type": "INT", + "widget": { + "name": "dynamic_chunk_overlap" + }, + "link": null + }, + { + "localized_name": "max_chunk_chars", + "name": "max_chunk_chars", + "type": "INT", + "widget": { + "name": "max_chunk_chars" + }, + "link": null + }, + { + "localized_name": "document_id_link", + "name": "document_id_link", + "type": "*", + "shape": 7, + "link": 4 + }, + { + "localized_name": "collection_name_link", + "name": "collection_name_link", + "type": "*", + "shape": 7, + "link": null + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "shape": 7, + "widget": { + "name": "refresh_token" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "links": [ + 5 + ] + }, + { + "localized_name": "chunk_text", + "name": "chunk_text", + "type": "STRING", + "links": [ + 14 + ] + }, + { + "localized_name": "document_id_link", + "name": "document_id_link", + "type": "STRING", + "links": [ + 8 + ] + }, + { + "localized_name": "result_field_path", + "name": "result_field_path", + "type": "STRING", + "links": [ + 9 + ] + }, + { + "localized_name": "chunk_index", + "name": "chunk_index", + "type": "INT", + "links": null + }, + { + "localized_name": "chunk_count", + "name": "chunk_count", + "type": "INT", + "links": null + }, + { + "localized_name": "done", + "name": "done", + "type": "BOOLEAN", + "links": null + }, + { + "localized_name": "next_index", + "name": "next_index", + "type": "INT", + "links": null + }, + { + "localized_name": "visible_index", + "name": "visible_index", + "type": "STRING", + "links": null + }, + { + "localized_name": "visible_next_index", + "name": "visible_next_index", + "type": "STRING", + "links": null + }, + { + "localized_name": "state_json", + "name": "state_json", + "type": "STRING", + "links": [ + 13 + ] + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": [ + 17 + ] + }, + { + "localized_name": "progress_text", + "name": "progress_text", + "type": "STRING", + "links": [ + 12 + ] + }, + { + "localized_name": "timestamp", + "name": "timestamp", + "type": "STRING", + "links": null + }, + { + "localized_name": "source_path", + "name": "source_path", + "type": "STRING", + "links": null + } + ], + "title": "03 Chunk Loop Manager - document_text.chunks..text", + "properties": { + "Node name for S&R": "ZMongoDocumentChunkLoopManagerNode" + }, + "widgets_values": [ + "text_documents", + "", + "document_text.chunks", + 0, + "", + "Analyze this document chunk. Identify legally important facts, issues, rules, reasoning, citations, names, dates, and procedural posture. Do not invent facts. If the chunk is incomplete, state what appears partial.", + "analysis.gemini.chunk_results", + false, + true, + "gemini_chunk_loop_uploaded_document_chunks_list", + false, + true, + "gemini_chunk_loop_flattened_chunks_list", + true, + true, + 12000, + "local_iso" + ], + "color": "#352", + "bgcolor": "#463" + }, + { + "id": 5, + "type": "GeminiChatNode", + "pos": [ + 390, + 210 + ], + "size": [ + 430, + 340 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 6 + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "link": 5 + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "widget": { + "name": "model" + }, + "link": null + }, + { + "localized_name": "max_output_tokens", + "name": "max_output_tokens", + "type": "INT", + "widget": { + "name": "max_output_tokens" + }, + "link": null + }, + { + "localized_name": "temperature", + "name": "temperature", + "type": "FLOAT", + "widget": { + "name": "temperature" + }, + "link": null + }, + { + "localized_name": "system_instruction", + "name": "system_instruction", + "type": "STRING", + "shape": 7, + "widget": { + "name": "system_instruction" + }, + "link": null + }, + { + "localized_name": "gemini_prefix", + "name": "gemini_prefix", + "type": "STRING", + "shape": 7, + "widget": { + "name": "gemini_prefix" + }, + "link": null + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "shape": 7, + "widget": { + "name": "refresh_token" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 15 + ] + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "links": [ + 10 + ] + }, + { + "localized_name": "success", + "name": "success", + "type": "BOOLEAN", + "links": null + } + ], + "title": "04 Gemini Analyze Current Chunk", + "properties": { + "Node name for S&R": "GeminiChatNode" + }, + "widgets_values": [ + "gemini-2.5-flash", + 8000, + 0.2, + "You are a careful legal/document analysis assistant. Analyze only the supplied chunk. Preserve citations and quote short key phrases when useful. Do not infer beyond the supplied chunk.", + "/gemini", + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 6, + "type": "ZMongoApiSaveValueNode", + "pos": [ + 950, + 190 + ], + "size": [ + 470, + 450 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 7 + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "widget": { + "name": "collection_name" + }, + "link": null + }, + { + "localized_name": "query_json", + "name": "query_json", + "type": "STRING", + "widget": { + "name": "query_json" + }, + "link": null + }, + { + "localized_name": "document_id", + "name": "document_id", + "type": "STRING", + "link": 8 + }, + { + "localized_name": "field_path", + "name": "field_path", + "type": "STRING", + "link": 9 + }, + { + "localized_name": "value_json", + "name": "value_json", + "type": "STRING", + "link": 10 + }, + { + "localized_name": "parse_value_json", + "name": "parse_value_json", + "type": "BOOLEAN", + "widget": { + "name": "parse_value_json" + }, + "link": null + }, + { + "localized_name": "upsert_if_missing", + "name": "upsert_if_missing", + "type": "BOOLEAN", + "widget": { + "name": "upsert_if_missing" + }, + "link": null + }, + { + "localized_name": "parse_json_strings", + "name": "parse_json_strings", + "type": "BOOLEAN", + "widget": { + "name": "parse_json_strings" + }, + "link": null + }, + { + "localized_name": "normalize_for_storage", + "name": "normalize_for_storage", + "type": "BOOLEAN", + "widget": { + "name": "normalize_for_storage" + }, + "link": null + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 16 + ] + }, + { + "localized_name": "refresh", + "name": "refresh", + "type": "STRING", + "links": null + }, + { + "localized_name": "success", + "name": "success", + "type": "BOOLEAN", + "links": null + } + ], + "title": "05 Save Gemini Chunk Result", + "properties": { + "Node name for S&R": "ZMongoApiSaveValueNode" + }, + "widgets_values": [ + "text_documents", + "{}", + "", + "", + "", + false, + true, + false, + false + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 7, + "type": "PreviewAny", + "pos": [ + -320, + 810 + ], + "size": [ + 450, + 180 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 11 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Ingest JSON Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 8, + "type": "PreviewAny", + "pos": [ + 330, + 650 + ], + "size": [ + 430, + 180 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 12 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Chunk Progress Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 9, + "type": "PreviewAny", + "pos": [ + 330, + 850 + ], + "size": [ + 430, + 180 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 13 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Chunk State JSON Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 10, + "type": "PreviewAny", + "pos": [ + 330, + -20 + ], + "size": [ + 430, + 190 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 14 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Current Chunk Text Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 11, + "type": "PreviewAny", + "pos": [ + 880, + -60 + ], + "size": [ + 430, + 190 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 15 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Gemini JSON Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 12, + "type": "PreviewAny", + "pos": [ + 1510, + 220 + ], + "size": [ + 430, + 190 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 16 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Save Result Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 13, + "type": "PreviewAny", + "pos": [ + 880, + 690 + ], + "size": [ + 430, + 190 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Chunk Status Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 3, + 0, + "ZMONGO_API_SESSION" + ], + [ + 2, + 2, + 1, + 3, + 1, + "ZMONGO_FILE_PATH" + ], + [ + 3, + 1, + 0, + 4, + 0, + "ZMONGO_API_SESSION" + ], + [ + 4, + 3, + 2, + 4, + 17, + "*" + ], + [ + 5, + 4, + 0, + 5, + 1, + "STRING" + ], + [ + 6, + 1, + 0, + 5, + 0, + "ZMONGO_API_SESSION" + ], + [ + 7, + 1, + 0, + 6, + 0, + "ZMONGO_API_SESSION" + ], + [ + 8, + 4, + 2, + 6, + 3, + "STRING" + ], + [ + 9, + 4, + 3, + 6, + 4, + "STRING" + ], + [ + 10, + 5, + 1, + 6, + 5, + "STRING" + ], + [ + 11, + 3, + 0, + 7, + 0, + "STRING" + ], + [ + 12, + 4, + 12, + 8, + 0, + "STRING" + ], + [ + 13, + 4, + 10, + 9, + 0, + "STRING" + ], + [ + 14, + 4, + 1, + 10, + 0, + "STRING" + ], + [ + 15, + 5, + 0, + 11, + 0, + "STRING" + ], + [ + 16, + 6, + 0, + 12, + 0, + "STRING" + ], + [ + 17, + 4, + 11, + 13, + 0, + "STRING" + ] + ], + "groups": [ + { + "id": 1, + "title": "1. Select file and ingest document into chunks", + "bounding": [ + -1320, + 80, + 1010, + 820 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "2. Emit one stored chunk per run", + "bounding": [ + -260, + 60, + 570, + 980 + ], + "color": "#8A8", + "flags": {} + }, + { + "id": 3, + "title": "3. Gemini analyzes the current chunk and saves result", + "bounding": [ + 360, + 140, + 1600, + 540 + ], + "color": "#3f789e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.82, + "offset": [ + 640, + 260 + ] + }, + "workflow_note": "Uses fixed chunk-name system: reads document_text.chunks..text and saves Gemini response to analysis.gemini.chunk_results..text. Reset true on first import; set false after first run." + }, + "version": 0.4 +} \ No newline at end of file diff --git a/example_workflows/zmongo_gemini_chunk_loop_workflow.json b/example_workflows/zmongo_gemini_chunk_loop_workflow.json new file mode 100644 index 0000000..125a415 --- /dev/null +++ b/example_workflows/zmongo_gemini_chunk_loop_workflow.json @@ -0,0 +1,1802 @@ +{ + "id": "3af15d4c-2fc7-4853-9612-5042638e1556", + "revision": 0, + "last_node_id": 17, + "last_link_id": 25, + "nodes": [ + { + "id": 1, + "type": "ZMongoApiKeySessionNode", + "pos": [ + -1180, + -300 + ], + "size": [ + 340, + 300 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [ + { + "localized_name": "base_url", + "name": "base_url", + "type": "STRING", + "link": null, + "widget": { + "name": "base_url" + } + }, + { + "localized_name": "zai_api_key", + "name": "zai_api_key", + "type": "STRING", + "link": null, + "widget": { + "name": "zai_api_key" + } + }, + { + "localized_name": "username", + "name": "username", + "type": "STRING", + "link": null, + "widget": { + "name": "username" + } + }, + { + "localized_name": "comfy_zmongo_prefix", + "name": "comfy_zmongo_prefix", + "type": "STRING", + "link": null, + "widget": { + "name": "comfy_zmongo_prefix" + } + }, + { + "localized_name": "fleet_prefix", + "name": "fleet_prefix", + "type": "STRING", + "link": null, + "widget": { + "name": "fleet_prefix" + } + }, + { + "localized_name": "comfy_zmongo_fleet_prefix", + "name": "comfy_zmongo_fleet_prefix", + "type": "STRING", + "link": null, + "widget": { + "name": "comfy_zmongo_fleet_prefix" + } + }, + { + "localized_name": "timeout_seconds", + "name": "timeout_seconds", + "type": "INT", + "link": null, + "widget": { + "name": "timeout_seconds" + } + }, + { + "localized_name": "verify_tls", + "name": "verify_tls", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "verify_tls" + } + }, + { + "localized_name": "test_whoami", + "name": "test_whoami", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "test_whoami" + } + } + ], + "outputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "links": [ + 1, + 2, + 7, + 11, + 25 + ] + }, + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 15 + ] + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "ZMongoApiKeySessionNode" + }, + "title": "00 API Key Session", + "widgets_values": [ + "https://businessprocessapplications.com", + "PASTE_API_KEY_HERE", + "PASTE_USERNAME_HERE", + "/comfy-zmongo", + "/fleet", + "/comfy-zmongo-fleet", + 30, + true, + true + ], + "color": "#233", + "bgcolor": "#355" + }, + { + "id": 2, + "type": "ZMongoApiListCollectionsNode", + "pos": [ + -800, + -320 + ], + "size": [ + 360, + 150 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 1 + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "link": null, + "widget": { + "name": "refresh_token" + }, + "shape": 7 + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": null + }, + { + "localized_name": "collections", + "name": "collections", + "type": "*", + "links": [ + 3 + ], + "shape": 6 + }, + { + "localized_name": "indexed", + "name": "indexed", + "type": "STRING", + "links": [ + 16 + ] + } + ], + "properties": { + "Node name for S&R": "ZMongoApiListCollectionsNode" + }, + "title": "01 List Collections", + "widgets_values": [ + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 3, + "type": "ZMongoApiSelectNthItemNode", + "pos": [ + -800, + -90 + ], + "size": [ + 410, + 278 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [ + { + "localized_name": "items_list", + "name": "items_list", + "type": "*", + "link": 3 + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "link": null, + "widget": { + "name": "index" + } + }, + { + "localized_name": "fallback", + "name": "fallback", + "type": "STRING", + "link": null, + "widget": { + "name": "fallback" + } + }, + { + "localized_name": "selection", + "name": "selection", + "type": "STRING", + "link": null, + "widget": { + "name": "selection" + } + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "link": null, + "widget": { + "name": "mode" + } + }, + { + "localized_name": "include_end", + "name": "include_end", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "include_end" + } + }, + { + "localized_name": "dedupe", + "name": "dedupe", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "dedupe" + } + } + ], + "outputs": [ + { + "localized_name": "item", + "name": "item", + "type": "STRING", + "links": [ + 4, + 8, + 12 + ] + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": null + }, + { + "localized_name": "selected_items", + "name": "selected_items", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "selected_json", + "name": "selected_json", + "type": "STRING", + "links": null + }, + { + "localized_name": "indexed_selected_items", + "name": "indexed_selected_items", + "type": "STRING", + "links": null + }, + { + "localized_name": "count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ZMongoApiSelectNthItemNode" + }, + "title": "02 Select Collection", + "widgets_values": [ + 0, + "", + "", + "auto", + true, + true + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 4, + "type": "ZMongoApiListDocsNode", + "pos": [ + -320, + -170 + ], + "size": [ + 420, + 240 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 2 + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "link": 4, + "widget": { + "name": "collection_name" + } + }, + { + "localized_name": "query_json", + "name": "query_json", + "type": "STRING", + "link": null, + "widget": { + "name": "query_json" + } + }, + { + "localized_name": "limit", + "name": "limit", + "type": "INT", + "link": null, + "widget": { + "name": "limit" + } + }, + { + "localized_name": "skip", + "name": "skip", + "type": "INT", + "link": null, + "widget": { + "name": "skip" + } + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "link": null, + "widget": { + "name": "refresh_token" + }, + "shape": 7 + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": null + }, + { + "localized_name": "ids", + "name": "ids", + "type": "*", + "links": [ + 5 + ], + "shape": 6 + }, + { + "localized_name": "indexed", + "name": "indexed", + "type": "STRING", + "links": [ + 17 + ] + } + ], + "properties": { + "Node name for S&R": "ZMongoApiListDocsNode" + }, + "title": "03 List Document IDs", + "widgets_values": [ + "", + "{}", + 50, + 0, + "" + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 5, + "type": "ZMongoApiSelectNthItemNode", + "pos": [ + -320, + 160 + ], + "size": [ + 410, + 278 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "localized_name": "items_list", + "name": "items_list", + "type": "*", + "link": 5 + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "link": null, + "widget": { + "name": "index" + } + }, + { + "localized_name": "fallback", + "name": "fallback", + "type": "STRING", + "link": null, + "widget": { + "name": "fallback" + } + }, + { + "localized_name": "selection", + "name": "selection", + "type": "STRING", + "link": null, + "widget": { + "name": "selection" + } + }, + { + "localized_name": "mode", + "name": "mode", + "type": "COMBO", + "link": null, + "widget": { + "name": "mode" + } + }, + { + "localized_name": "include_end", + "name": "include_end", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "include_end" + } + }, + { + "localized_name": "dedupe", + "name": "dedupe", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "dedupe" + } + } + ], + "outputs": [ + { + "localized_name": "item", + "name": "item", + "type": "STRING", + "links": [ + 6, + 9 + ] + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": [ + 14 + ] + }, + { + "localized_name": "selected_items", + "name": "selected_items", + "type": "*", + "links": null, + "shape": 6 + }, + { + "localized_name": "selected_json", + "name": "selected_json", + "type": "STRING", + "links": null + }, + { + "localized_name": "indexed_selected_items", + "name": "indexed_selected_items", + "type": "STRING", + "links": null + }, + { + "localized_name": "count", + "name": "count", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "ZMongoApiSelectNthItemNode" + }, + "title": "04 Select Document", + "widgets_values": [ + 0, + "", + "", + "auto", + true, + true + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 6, + "type": "ZMongoDocumentChunkLoopManagerNode", + "pos": [ + 160, + -230 + ], + "size": [ + 520, + 640 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 7 + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "link": null, + "widget": { + "name": "collection_name" + } + }, + { + "localized_name": "document_id", + "name": "document_id", + "type": "STRING", + "link": null, + "widget": { + "name": "document_id" + } + }, + { + "localized_name": "chunk_field_root", + "name": "chunk_field_root", + "type": "STRING", + "link": null, + "widget": { + "name": "chunk_field_root" + } + }, + { + "localized_name": "index", + "name": "index", + "type": "INT", + "link": null, + "widget": { + "name": "index" + } + }, + { + "localized_name": "fallback_text", + "name": "fallback_text", + "type": "STRING", + "link": null, + "widget": { + "name": "fallback_text" + } + }, + { + "localized_name": "user_instruction", + "name": "user_instruction", + "type": "STRING", + "link": null, + "widget": { + "name": "user_instruction" + } + }, + { + "localized_name": "result_field_root", + "name": "result_field_root", + "type": "STRING", + "link": null, + "widget": { + "name": "result_field_root" + } + }, + { + "localized_name": "wrap", + "name": "wrap", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "wrap" + } + }, + { + "localized_name": "advance_on_execute", + "name": "advance_on_execute", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "advance_on_execute" + } + }, + { + "localized_name": "state_name", + "name": "state_name", + "type": "STRING", + "link": null, + "widget": { + "name": "state_name" + } + }, + { + "localized_name": "reset_state", + "name": "reset_state", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "reset_state" + } + }, + { + "localized_name": "fallback_to_raw_text", + "name": "fallback_to_raw_text", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "fallback_to_raw_text" + } + }, + { + "localized_name": "raw_text_field_path", + "name": "raw_text_field_path", + "type": "STRING", + "link": null, + "widget": { + "name": "raw_text_field_path" + } + }, + { + "localized_name": "dynamic_chunk_size", + "name": "dynamic_chunk_size", + "type": "INT", + "link": null, + "widget": { + "name": "dynamic_chunk_size" + } + }, + { + "localized_name": "dynamic_chunk_overlap", + "name": "dynamic_chunk_overlap", + "type": "INT", + "link": null, + "widget": { + "name": "dynamic_chunk_overlap" + } + }, + { + "localized_name": "max_chunk_chars", + "name": "max_chunk_chars", + "type": "INT", + "link": null, + "widget": { + "name": "max_chunk_chars" + } + }, + { + "localized_name": "timestamp_format", + "name": "timestamp_format", + "type": "COMBO", + "link": null, + "widget": { + "name": "timestamp_format" + } + }, + { + "localized_name": "document_id_link", + "name": "document_id_link", + "type": "*", + "link": 6, + "shape": 7 + }, + { + "localized_name": "collection_name_link", + "name": "collection_name_link", + "type": "*", + "link": 8, + "shape": 7 + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "link": null, + "widget": { + "name": "refresh_token" + }, + "shape": 7 + } + ], + "outputs": [ + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "links": [ + 10 + ] + }, + { + "localized_name": "chunk_text", + "name": "chunk_text", + "type": "STRING", + "links": [ + 13 + ] + }, + { + "localized_name": "document_id_link", + "name": "document_id_link", + "type": "STRING", + "links": [ + 9 + ] + }, + { + "localized_name": "result_field_path", + "name": "result_field_path", + "type": "STRING", + "links": [ + 12 + ] + }, + { + "localized_name": "chunk_index", + "name": "chunk_index", + "type": "INT", + "links": null + }, + { + "localized_name": "chunk_count", + "name": "chunk_count", + "type": "INT", + "links": null + }, + { + "localized_name": "done", + "name": "done", + "type": "BOOLEAN", + "links": null + }, + { + "localized_name": "next_index", + "name": "next_index", + "type": "INT", + "links": null + }, + { + "localized_name": "visible_index", + "name": "visible_index", + "type": "STRING", + "links": null + }, + { + "localized_name": "visible_next_index", + "name": "visible_next_index", + "type": "STRING", + "links": null + }, + { + "localized_name": "state_json", + "name": "state_json", + "type": "STRING", + "links": [ + 18 + ] + }, + { + "localized_name": "status", + "name": "status", + "type": "STRING", + "links": [ + 24 + ] + }, + { + "localized_name": "progress_text", + "name": "progress_text", + "type": "STRING", + "links": [ + 20 + ] + }, + { + "localized_name": "timestamp", + "name": "timestamp", + "type": "STRING", + "links": null + }, + { + "localized_name": "source_path", + "name": "source_path", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "ZMongoDocumentChunkLoopManagerNode" + }, + "title": "05 Chunk Loop Manager - one chunk per run", + "widgets_values": [ + "text_documents", + "", + "document_text.chunks_by_index", + 0, + "", + "Analyze this document chunk. Identify legally important facts, issues, rules, reasoning, citations, names, dates, and procedural posture. Do not invent facts. If the chunk is incomplete, state what appears partial.", + "analysis.gemini.chunk_results", + false, + true, + "gemini_chunk_loop", + false, + true, + "document_text.raw_text", + 6000, + 300, + 12000, + "iso_utc", + "" + ], + "color": "#352", + "bgcolor": "#463" + }, + { + "id": 7, + "type": "GeminiChatNode", + "pos": [ + 820, + -230 + ], + "size": [ + 430, + 320 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 11 + }, + { + "localized_name": "prompt", + "name": "prompt", + "type": "STRING", + "link": 10, + "widget": { + "name": "prompt" + } + }, + { + "localized_name": "model", + "name": "model", + "type": "STRING", + "link": null, + "widget": { + "name": "model" + } + }, + { + "localized_name": "max_output_tokens", + "name": "max_output_tokens", + "type": "INT", + "link": null, + "widget": { + "name": "max_output_tokens" + } + }, + { + "localized_name": "temperature", + "name": "temperature", + "type": "FLOAT", + "link": null, + "widget": { + "name": "temperature" + } + }, + { + "localized_name": "system_instruction", + "name": "system_instruction", + "type": "STRING", + "link": null, + "widget": { + "name": "system_instruction" + }, + "shape": 7 + }, + { + "localized_name": "gemini_prefix", + "name": "gemini_prefix", + "type": "STRING", + "link": null, + "widget": { + "name": "gemini_prefix" + }, + "shape": 7 + }, + { + "localized_name": "refresh_token", + "name": "refresh_token", + "type": "STRING", + "link": null, + "widget": { + "name": "refresh_token" + }, + "shape": 7 + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 21 + ] + }, + { + "localized_name": "text", + "name": "text", + "type": "STRING", + "links": [ + 22, + 13 + ] + }, + { + "localized_name": "success", + "name": "success", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "GeminiChatNode" + }, + "title": "06 Gemini Analyze Current Chunk", + "widgets_values": [ + "", + "gemini-3.5-flash", + 8000, + 0.2, + "You are a careful legal/document analysis assistant. Analyze only the supplied chunk. Preserve citations and quote short key phrases when useful. Do not infer beyond the text.", + "/gemini", + "" + ], + "color": "#223", + "bgcolor": "#335" + }, + { + "id": 8, + "type": "ZMongoApiSaveValueNode", + "pos": [ + 1320, + -210 + ], + "size": [ + 460, + 460 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "localized_name": "session", + "name": "session", + "type": "ZMONGO_API_SESSION", + "link": 25 + }, + { + "localized_name": "collection_name", + "name": "collection_name", + "type": "STRING", + "link": 12, + "widget": { + "name": "collection_name" + } + }, + { + "localized_name": "query_json", + "name": "query_json", + "type": "STRING", + "link": null, + "widget": { + "name": "query_json" + } + }, + { + "localized_name": "document_id", + "name": "document_id", + "type": "STRING", + "link": 9, + "widget": { + "name": "document_id" + } + }, + { + "localized_name": "field_path", + "name": "field_path", + "type": "STRING", + "link": 13, + "widget": { + "name": "field_path" + } + }, + { + "localized_name": "value_json", + "name": "value_json", + "type": "STRING", + "link": 22, + "widget": { + "name": "value_json" + } + }, + { + "localized_name": "parse_value_json", + "name": "parse_value_json", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "parse_value_json" + } + }, + { + "localized_name": "upsert_if_missing", + "name": "upsert_if_missing", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "upsert_if_missing" + } + }, + { + "localized_name": "parse_json_strings", + "name": "parse_json_strings", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "parse_json_strings" + } + }, + { + "localized_name": "normalize_for_storage", + "name": "normalize_for_storage", + "type": "BOOLEAN", + "link": null, + "widget": { + "name": "normalize_for_storage" + } + } + ], + "outputs": [ + { + "localized_name": "json", + "name": "json", + "type": "STRING", + "links": [ + 23 + ] + }, + { + "localized_name": "refresh", + "name": "refresh", + "type": "STRING", + "links": null + }, + { + "localized_name": "success", + "name": "success", + "type": "BOOLEAN", + "links": null + } + ], + "properties": { + "Node name for S&R": "ZMongoApiSaveValueNode" + }, + "title": "07 Save Gemini Chunk Result", + "widgets_values": [ + "", + "{}", + "", + "analysis.gemini.chunk_results.c0000.text", + "", + false, + true, + false, + false + ], + "color": "#232", + "bgcolor": "#353" + }, + { + "id": 9, + "type": "PreviewAny", + "pos": [ + -1180, + 80 + ], + "size": [ + 340, + 160 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 15 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Session JSON Preview", + "widgets_values": [ + null, + null, + false + ] + }, + { + "id": 10, + "type": "PreviewAny", + "pos": [ + -800, + -520 + ], + "size": [ + 360, + 180 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 16 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Collections Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 11, + "type": "PreviewAny", + "pos": [ + -320, + -430 + ], + "size": [ + 420, + 220 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 17 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Document IDs Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 12, + "type": "PreviewAny", + "pos": [ + 160, + 440 + ], + "size": [ + 520, + 170 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 20 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Chunk Progress Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 13, + "type": "PreviewAny", + "pos": [ + 160, + 650 + ], + "size": [ + 520, + 220 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 18 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Chunk State JSON Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 14, + "type": "PreviewAny", + "pos": [ + 820, + 130 + ], + "size": [ + 430, + 190 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 13 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Current Chunk Text Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 15, + "type": "PreviewAny", + "pos": [ + 820, + 360 + ], + "size": [ + 430, + 200 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 21 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Gemini JSON Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 16, + "type": "PreviewAny", + "pos": [ + 1320, + 300 + ], + "size": [ + 460, + 220 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 23 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "PreviewAny" + }, + "title": "Save Result Preview", + "widgets_values": [ + null, + null, + null + ] + }, + { + "id": 17, + "type": "PreviewAny", + "pos": [ + 160, + 910 + ], + "size": [ + 520, + 170 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "localized_name": "source", + "name": "source", + "type": "*", + "link": 24 + } + ], + "outputs": [ + { + "localized_name": "STRING", + "name": "STRING", + "type": "STRING", + "links": null + } + ], + "title": "Chunk Status Preview", + "properties": { + "Node name for S&R": "PreviewAny" + }, + "widgets_values": [ + null, + null, + null + ] + } + ], + "links": [ + [ + 1, + 1, + 0, + 2, + 0, + "ZMONGO_API_SESSION" + ], + [ + 2, + 1, + 0, + 4, + 0, + "ZMONGO_API_SESSION" + ], + [ + 3, + 2, + 1, + 3, + 0, + "*" + ], + [ + 4, + 3, + 0, + 4, + 1, + "STRING" + ], + [ + 5, + 4, + 1, + 5, + 0, + "*" + ], + [ + 6, + 5, + 0, + 6, + 18, + "*" + ], + [ + 7, + 1, + 0, + 6, + 0, + "ZMONGO_API_SESSION" + ], + [ + 8, + 3, + 0, + 6, + 19, + "*" + ], + [ + 9, + 6, + 2, + 8, + 3, + "STRING" + ], + [ + 10, + 6, + 0, + 7, + 1, + "STRING" + ], + [ + 11, + 1, + 0, + 7, + 0, + "ZMONGO_API_SESSION" + ], + [ + 12, + 3, + 0, + 8, + 1, + "STRING" + ], + [ + 13, + 6, + 3, + 8, + 4, + "STRING" + ], + [ + 14, + 5, + 1, + 14, + 0, + "STRING" + ], + [ + 15, + 1, + 1, + 9, + 0, + "STRING" + ], + [ + 16, + 2, + 2, + 10, + 0, + "STRING" + ], + [ + 17, + 4, + 2, + 11, + 0, + "STRING" + ], + [ + 18, + 6, + 10, + 13, + 0, + "STRING" + ], + [ + 20, + 6, + 12, + 12, + 0, + "STRING" + ], + [ + 21, + 7, + 0, + 15, + 0, + "STRING" + ], + [ + 22, + 7, + 1, + 8, + 5, + "STRING" + ], + [ + 23, + 8, + 0, + 16, + 0, + "STRING" + ], + [ + 24, + 6, + 11, + 17, + 0, + "STRING" + ], + [ + 25, + 1, + 0, + 8, + 0, + "ZMONGO_API_SESSION" + ] + ], + "groups": [ + { + "id": 1, + "title": "1. Connect and choose document", + "bounding": [ + -1220, + -420, + 1120, + 820 + ], + "color": "#3f789e", + "flags": {} + }, + { + "id": 2, + "title": "2. Emit one chunk per queue run", + "bounding": [ + -40, + -380, + 760, + 800 + ], + "color": "#8A8", + "flags": {} + }, + { + "id": 3, + "title": "3. Gemini analyzes chunk and save result", + "bounding": [ + 780, + -380, + 1240, + 860 + ], + "color": "#3f789e", + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.78, + "offset": [ + 760, + 410 + ] + }, + "workflow_note": "Requires fixed zmongo_helper_nodes.py with ZMongoDocumentChunkLoopManagerNode. This workflow processes ONE chunk per queue run and saves Gemini output to analysis.gemini.chunk_results.cXXXX.text. Reset chunk loop by setting reset_state=true for one run, then false." + }, + "version": 0.4 +} \ No newline at end of file diff --git a/nodes/__init__.py b/nodes/__init__.py index d042b20..1c806e7 100644 --- a/nodes/__init__.py +++ b/nodes/__init__.py @@ -8,7 +8,7 @@ from . import zmongo_image_nodes from . import zmongo_images_nodes from . import document_api_nodes from . import zmongo_preset_nodes -from . import zmongo_discovery_nodes +from . import zmongo_text_agent_nodes from . import zmongo_helper_nodes @@ -68,6 +68,11 @@ NODE_CLASS_MAPPINGS, NODE_DISPLAY_NAME_MAPPINGS = _merge_node_mappings( zmongo_preset_nodes.NODE_CLASS_MAPPINGS, zmongo_preset_nodes.NODE_DISPLAY_NAME_MAPPINGS, ), + ( + "zmongo_text_agent_nodes", + zmongo_text_agent_nodes.NODE_CLASS_MAPPINGS, + zmongo_text_agent_nodes.NODE_DISPLAY_NAME_MAPPINGS, + ), ( "zmongo_helper_nodes", zmongo_helper_nodes.NODE_CLASS_MAPPINGS, diff --git a/nodes/document_api_nodes.py b/nodes/document_api_nodes.py index c5ec5df..e2ad022 100644 --- a/nodes/document_api_nodes.py +++ b/nodes/document_api_nodes.py @@ -1,7 +1,10 @@ from __future__ import annotations +import json import mimetypes import os +import re +import time import urllib.parse from pathlib import Path from typing import Any, Optional @@ -21,6 +24,203 @@ DEFAULT_DOCUMENT_PREFIX = os.getenv("ZMONGO_DOCUMENT_API_PREFIX", "/documents"). DEFAULT_DOCUMENT_COLLECTION = os.getenv("ZMONGO_DOCUMENT_COLLECTION", "documents").strip() or "documents" DEFAULT_MAX_UPLOAD_BYTES = int(os.getenv("ZMONGO_DOCUMENT_NODE_MAX_UPLOAD_BYTES", str(64 * 1024 * 1024))) +DEFAULT_DOCUMENT_DISCOVERY_EXTENSIONS = os.getenv( + "ZMONGO_DOCUMENT_BROWSER_EXTENSIONS", + ".pdf,.docx,.txt,.md,.csv,.json,.log,.rtf", +).strip() or ".pdf,.docx,.txt,.md,.csv,.json,.log,.rtf" +DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS = int(os.getenv("ZMONGO_DOCUMENT_BROWSER_MAX_RESULTS", "500")) +DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH = int(os.getenv("ZMONGO_DOCUMENT_BROWSER_MAX_DEPTH", "6")) + +DOCUMENT_BROWSER_ROOT_MODES = [ + "Documents", + "Comfy Input", + "Home", + "Current Working Directory", + "Filesystem Root", + "Custom Root", +] + + +def _split_extensions(extensions: str) -> set[str]: + cleaned: set[str] = set() + for item in re.split(r"[,;\s]+", str(extensions or "")): + item = item.strip().lower() + if not item: + continue + if item in {"*", ".*", "all"}: + return set() + if not item.startswith("."): + item = "." + item + cleaned.add(item) + return cleaned + + +def _safe_resolve_path(value: Any) -> Path: + raw = _clean_scalar(value) + if not raw: + return Path.home().resolve() + return Path(os.path.expandvars(os.path.expanduser(raw))).resolve() + + +def _document_browser_root_for_mode(root_mode: str, custom_root: str = "") -> Path: + mode = str(root_mode or "Documents").strip() + + if mode == "Home": + return Path.home().resolve() + + if mode == "Current Working Directory": + return Path.cwd().resolve() + + if mode == "Filesystem Root": + return Path(Path.home().anchor or "/").resolve() + + if mode == "Custom Root": + return _safe_resolve_path(custom_root) + + try: + return Path(_get_document_browser_root(root_mode=mode, custom_root=custom_root)).resolve() + except Exception: + if mode == "Comfy Input" and folder_paths is not None: + try: + return Path(folder_paths.get_input_directory()).resolve() + except Exception: + pass + return (Path.home() / "Documents").resolve() + + +def _is_probably_hidden(path: Path) -> bool: + return any(part.startswith(".") for part in path.parts if part not in {path.anchor, "/"}) + + +def _should_skip_discovery_dir(path: Path, include_hidden: bool) -> bool: + name = path.name + if not include_hidden and name.startswith("."): + return True + return name in { + "__pycache__", + ".git", + ".hg", + ".svn", + "node_modules", + ".venv", + "venv", + "env", + "site-packages", + "dist-packages", + "proc", + "sys", + "dev", + "run", + "tmp", + "var", + } + + +def _discover_document_files( + *, + root: Path, + extensions: str = DEFAULT_DOCUMENT_DISCOVERY_EXTENSIONS, + recursive: bool = True, + max_results: int = DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS, + max_depth: int = DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH, + search_text: str = "", + include_hidden: bool = False, +) -> list[Path]: + root = Path(root).resolve() + allowed_exts = _split_extensions(extensions) + limit = max(1, min(int(max_results or DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS), 10000)) + depth_limit = max(0, min(int(max_depth or DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH), 25)) + needle = str(search_text or "").strip().lower() + + if not root.exists(): + return [] + + if root.is_file(): + if allowed_exts and root.suffix.lower() not in allowed_exts: + return [] + if needle and needle not in root.name.lower() and needle not in str(root).lower(): + return [] + return [root] + + found: list[Path] = [] + stack: list[tuple[Path, int]] = [(root, 0)] + seen_dirs: set[str] = set() + + while stack and len(found) < limit: + current, depth = stack.pop() + try: + current_key = str(current.resolve()) + except Exception: + current_key = str(current) + if current_key in seen_dirs: + continue + seen_dirs.add(current_key) + + try: + entries = sorted(current.iterdir(), key=lambda p: (not p.is_dir(), str(p).lower())) + except (PermissionError, OSError): + continue + + for entry in entries: + if len(found) >= limit: + break + + try: + if entry.is_dir(): + if recursive and depth < depth_limit and not _should_skip_discovery_dir(entry, include_hidden): + stack.append((entry, depth + 1)) + continue + + if not entry.is_file(): + continue + + if not include_hidden and _is_probably_hidden(entry): + continue + + if allowed_exts and entry.suffix.lower() not in allowed_exts: + continue + + entry_text = f"{entry.name}\n{entry}".lower() + if needle and needle not in entry_text: + continue + + found.append(entry.resolve()) + except (PermissionError, OSError): + continue + + found.sort(key=lambda p: str(p).lower()) + return found + + +def _discovered_files_for_dropdown() -> list[str]: + roots: list[Path] = [] + for mode in ("Documents", "Comfy Input", "Home"): + try: + root = _document_browser_root_for_mode(mode) + if root.exists() and root not in roots: + roots.append(root) + except Exception: + continue + + values: list[str] = [""] + seen: set[str] = set() + per_root_limit = max(25, min(DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS, 250)) + for root in roots: + for file_path in _discover_document_files( + root=root, + recursive=True, + max_results=per_root_limit, + max_depth=DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH, + ): + value = str(file_path) + if value in seen: + continue + seen.add(value) + values.append(value) + + return values or [""] + + def _is_route_not_found(payload: Any) -> bool: payload = _ensure_payload_dict(payload) @@ -112,6 +312,280 @@ def _generic_document_create_fallback( json_body={"collection": collection_name, "document": document}, ) + +# ----------------------------------------------------------------------------- +# Local text-document ingestion helpers +# ----------------------------------------------------------------------------- + +TEXT_DOCUMENT_SCHEMA_KIND = "zmongo_text_document" +TEXT_DOCUMENT_SCHEMA_VERSION = "1.0.0" +DEFAULT_TEXT_DOCUMENT_FIELD_ROOT = "document_text" +DEFAULT_TEXT_DOCUMENT_COLLECTION = os.getenv("ZMONGO_TEXT_DOCUMENT_COLLECTION", "text_documents").strip() or "text_documents" +DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES = int(os.getenv("ZMONGO_LOCAL_TEXT_DOCUMENT_MAX_BYTES", str(256 * 1024))) + + +def _is_local_storage_session(session: Any) -> bool: + return str(getattr(session, "storage_backend", "") or "").strip() == "local_file_store" + + +def _utf8_size(value: Any) -> int: + return len(str(value or "").encode("utf-8")) + + +def _local_storage_limit_payload(*, path: Path, filename: str, text_size_bytes: int, refresh: str) -> dict[str, Any]: + return _ensure_payload_dict({ + "success": False, + "message": ( + f"Local File Store is a small CRUD store and will not save extracted text larger than " + f"{DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES} bytes. Use BusinessProcessApplications.com / hosted ZMongo " + "for large files, where ZEmbedder.py handles document chunking and embeddings." + ), + "data": { + "effective_file_path": str(path), + "filename": filename, + "text_size_bytes": int(text_size_bytes), + "local_max_text_document_bytes": int(DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES), + "storage_backend": "local_file_store", + "recommended_backend": "https://businessprocessapplications.com", + "recommended_pipeline": "Upload the original file to hosted ZMongo; let backend ZEmbedder.py chunk/embed it.", + "refresh": refresh, + }, + "error": {"type": "LocalStorageItemTooLarge", "msg": "local_text_document_limit_exceeded"}, + "status_code": 413, + }) + + +def _clean_dot_path_segment(value: Any, fallback: str = "field") -> str: + cleaned = re.sub(r"[^A-Za-z0-9_]+", "_", str(value or "").strip()).strip("_") + return cleaned or fallback + + +def _safe_read_text_file(path: Path) -> str: + raw = path.read_bytes() + for encoding in ("utf-8", "utf-8-sig", "cp1252", "latin-1"): + try: + return raw.decode(encoding) + except Exception: + continue + return raw.decode("utf-8", errors="replace") + + +def _extract_text_from_pdf_file(path: Path) -> str: + try: + import fitz # PyMuPDF + except Exception as exc: + raise RuntimeError( + "PDF text extraction requires PyMuPDF. Install it in the ComfyUI Python environment with: pip install pymupdf" + ) from exc + + pieces: list[str] = [] + with fitz.open(str(path)) as doc: + for page_index, page in enumerate(doc): + page_text = page.get_text("text") or "" + if page_text.strip(): + pieces.append(f"\n\n--- Page {page_index + 1} ---\n{page_text.strip()}") + return "".join(pieces).strip() + + +def _extract_text_from_docx_file(path: Path) -> str: + try: + from docx import Document + except Exception as exc: + raise RuntimeError( + "DOCX text extraction requires python-docx. Install it in the ComfyUI Python environment with: pip install python-docx" + ) from exc + + document = Document(str(path)) + pieces: list[str] = [] + + for paragraph in document.paragraphs: + text = paragraph.text.strip() + if text: + pieces.append(text) + + for table_index, table in enumerate(document.tables): + rows: list[str] = [] + for row in table.rows: + cells = [cell.text.strip() for cell in row.cells] + row_text = "\t".join(cells).strip() + if row_text: + rows.append(row_text) + if rows: + pieces.append(f"\n[Table {table_index + 1}]\n" + "\n".join(rows)) + + return "\n\n".join(pieces).strip() + + +def _extract_text_from_supported_document(path: Path) -> tuple[str, str]: + suffix = path.suffix.lower() + if suffix == ".pdf": + return _extract_text_from_pdf_file(path), "pymupdf" + if suffix == ".docx": + return _extract_text_from_docx_file(path), "python-docx" + if suffix in {".txt", ".md", ".markdown", ".csv", ".json", ".log"}: + return _safe_read_text_file(path), "plain-text" + raise ValueError(f"Unsupported text-ingestion file type: {suffix}. Supported: .pdf, .docx, .txt, .md, .csv, .json, .log") + + + +def _text_quality_score(value: Any) -> int: + """ + Score likely extracted document text. Higher is better. + + This intentionally penalizes short strings containing control/binary-looking + characters because local document APIs can sometimes expose file bytes or a + stale/corrupt top-level ``text`` value. Long readable text from + ``document_text.raw_text`` should win over a short binary-looking ``text``. + """ + if not isinstance(value, str): + return -1 + + text = value.strip() + if not text: + return 0 + + length = len(text) + printable = sum(1 for ch in text if ch.isprintable() or ch in "\r\n\t") + controls = sum(1 for ch in text if (ord(ch) < 32 and ch not in "\r\n\t")) + replacement = text.count("\ufffd") + whitespace = sum(1 for ch in text if ch.isspace()) + letters = sum(1 for ch in text if ch.isalpha()) + + printable_ratio = printable / max(1, length) + letter_ratio = letters / max(1, length) + whitespace_ratio = whitespace / max(1, length) + + score = min(length, 100000) + score += int(printable_ratio * 1000) + score += int(letter_ratio * 500) + score += int(whitespace_ratio * 250) + score -= controls * 1000 + score -= replacement * 500 + + if length < 32 and (controls or printable_ratio < 0.85 or letter_ratio < 0.25): + score -= 10000 + + return score + + +def _looks_like_bad_text_value(value: Any) -> bool: + if not isinstance(value, str): + return False + text = value.strip() + if not text: + return False + length = len(text) + printable = sum(1 for ch in text if ch.isprintable() or ch in "\r\n\t") + controls = sum(1 for ch in text if (ord(ch) < 32 and ch not in "\r\n\t")) + letters = sum(1 for ch in text if ch.isalpha()) + return ( + (length < 64 and controls > 0) + or (length > 0 and printable / max(1, length) < 0.85) + or (length < 64 and letters / max(1, length) < 0.20 and controls > 0) + ) + + +def _best_text_from_document(document: Any) -> tuple[str, str, int]: + """Return the best text-like field from a document plus its source path.""" + if not isinstance(document, dict): + return "", "", 0 + + candidates: list[tuple[str, Any]] = [] + for candidate_path in ( + "document_text.raw_text", + "document_text.text", + "content.text", + "extracted_text", + "raw_text", + "text", + ): + marker = object() + value = _safe_get_by_path(document, candidate_path, marker) + if value is not marker: + candidates.append((candidate_path, value)) + + best_text = "" + best_path = "" + best_score = -1 + for source_path, value in candidates: + if not isinstance(value, str): + continue + score = _text_quality_score(value) + if score > best_score: + best_text = value + best_path = source_path + best_score = score + + return best_text, best_path, best_score + + +def _query_existing_text_document( + session: Any, + *, + collection_name: str, + document_name: str, +) -> tuple[dict[str, Any], str, dict[str, Any]]: + query = {"schema_kind": TEXT_DOCUMENT_SCHEMA_KIND, "document_name": document_name} + payload = _generic_document_query_fallback( + session, + query=query, + sort=[["updated_at_unix", -1]], + limit=1, + skip=0, + collection_name=collection_name, + ) + documents = _extract_documents(payload) + document = documents[0] if documents else {} + document_id = _coerce_document_id(document) if document else "" + return document, document_id, payload + + +def _save_text_document_by_dot_paths( + session: Any, + *, + collection_name: str, + document_id: str, + document: dict[str, Any], + field_root: str, +) -> dict[str, Any]: + failures: list[str] = [] + saved_paths: list[str] = [] + + root = _clean_dot_path_segment(field_root, DEFAULT_TEXT_DOCUMENT_FIELD_ROOT) + save_items = { + "schema_kind": document.get("schema_kind"), + "schema_version": document.get("schema_version"), + "document_name": document.get("document_name"), + "collection_name": document.get("collection_name"), + "source_file": document.get("source_file") or {}, + "metadata": document.get("metadata") or {}, + root: document.get(root) or {}, + "updated_at_unix": document.get("updated_at_unix"), + } + + for field_path, value in save_items.items(): + payload = _session_save_value( + session, + collection_name=collection_name, + document_id=document_id, + query=None, + field_path=field_path, + value=value, + upsert=False, + ) + if payload.get("success"): + saved_paths.append(field_path) + else: + failures.append(f"{field_path}: {payload.get('message') or payload.get('error')}") + + return _ensure_payload_dict({ + "success": not failures, + "message": "Updated existing ingested text document." if not failures else "Some text document field writes failed.", + "data": {"document_id": document_id, "saved_paths": saved_paths, "failures": failures}, + "error": {"msg": "; ".join(failures[:5])} if failures else None, + "status_code": 200 if not failures else 500, + }) + try: import folder_paths # ComfyUI runtime helper for input/output/temp directories except Exception: # Allows py_compile outside ComfyUI @@ -162,33 +636,239 @@ class ZMongoDocumentWhoAmI(AlwaysDirtyMixin): class ZMongoDocumentFilePathBrowser(AlwaysDirtyMixin): @classmethod def INPUT_TYPES(cls): - return {"required": {"root_mode": (["Documents", "Comfy Input"], {"default": "Documents"}), "selected_file": (_list_document_files(),), "manual_path": ("STRING", {"default": ""}), "custom_root": ("STRING", {"default": ""})}, "optional": {"refresh_token": ("STRING", {"default": ""})}} + return { + "required": { + "root_mode": (DOCUMENT_BROWSER_ROOT_MODES, {"default": "Documents"}), + "selected_file": (_discovered_files_for_dropdown(),), + "manual_path": ("STRING", {"default": ""}), + "custom_root": ("STRING", {"default": ""}), + "recursive": ("BOOLEAN", {"default": True}), + "extensions": ( + "STRING", + { + "default": DEFAULT_DOCUMENT_DISCOVERY_EXTENSIONS, + "tooltip": "Comma/space-separated extensions. Use * for all files.", + }, + ), + "search_text": ( + "STRING", + { + "default": "", + "tooltip": "Optional case-insensitive filename/path filter.", + }, + ), + "max_results": ( + "INT", + {"default": DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS, "min": 1, "max": 10000}, + ), + "max_depth": ( + "INT", + {"default": DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH, "min": 0, "max": 25}, + ), + "include_hidden": ("BOOLEAN", {"default": False}), + }, + "optional": {"refresh_token": ("STRING", {"default": ""})}, + } - RETURN_TYPES = ("STRING", ZMONGO_FILE_PATH, "STRING", ZMONGO_FILENAME, "STRING", "INT", "BOOLEAN", "STRING", "STRING", "*", "*") + SELECTABLE_RETURN_TYPES - RETURN_NAMES = ("file_path", "file_path_link", "filename", "filename_link", "content_type", "size_bytes", "exists", "browser_root", "refresh", "file_paths", "filenames") + SELECTABLE_RETURN_NAMES - OUTPUT_IS_LIST = (False, False, False, False, False, False, False, False, False, True, True) + SELECTABLE_OUTPUT_IS_LIST + RETURN_TYPES = ( + "STRING", + ZMONGO_FILE_PATH, + "STRING", + ZMONGO_FILENAME, + "STRING", + "INT", + "BOOLEAN", + "STRING", + "STRING", + "*", + "*", + "STRING", + "INT", + "STRING", + ) + SELECTABLE_RETURN_TYPES + RETURN_NAMES = ( + "file_path", + "file_path_link", + "filename", + "filename_link", + "content_type", + "size_bytes", + "exists", + "browser_root", + "refresh", + "file_paths", + "filenames", + "indexed_file_paths", + "discovered_count", + "status", + ) + SELECTABLE_RETURN_NAMES + OUTPUT_IS_LIST = (False, False, False, False, False, False, False, False, False, True, True, False, False, False) + SELECTABLE_OUTPUT_IS_LIST FUNCTION = "resolve_file_path" CATEGORY = "ZMongo/06 Documents" OUTPUT_NODE = True - def resolve_file_path(self, root_mode: str = "Documents", selected_file: str = "", manual_path: str = "", custom_root: str = "", refresh_token: str = ""): + @staticmethod + def _resolve_selected_file(selected_file: str, manual_path: str, browser_root: Path) -> Path: + manual = _clean_scalar(manual_path) + selected = _clean_scalar(selected_file) + + if manual: + return _safe_resolve_path(manual) + + if selected: + selected_path = Path(os.path.expandvars(os.path.expanduser(selected))) + if selected_path.is_absolute(): + return selected_path.resolve() + return (browser_root / selected_path).resolve() + + return browser_root + + def resolve_file_path( + self, + root_mode: str = "Documents", + selected_file: str = "", + manual_path: str = "", + custom_root: str = "", + recursive: bool = True, + extensions: str = DEFAULT_DOCUMENT_DISCOVERY_EXTENSIONS, + search_text: str = "", + max_results: int = DEFAULT_DOCUMENT_DISCOVERY_MAX_RESULTS, + max_depth: int = DEFAULT_DOCUMENT_DISCOVERY_MAX_DEPTH, + include_hidden: bool = False, + refresh_token: str = "", + ): + refresh = _dirty_token( + "document_file_browser", + refresh_token, + root_mode, + selected_file, + manual_path, + custom_root, + recursive, + extensions, + search_text, + max_results, + max_depth, + include_hidden, + ) + try: - browser_root = _get_document_browser_root(root_mode=root_mode, custom_root=custom_root) - resolved = _resolve_document_file_path(selected_file, manual_path, root_mode=root_mode, custom_root=custom_root) + browser_root = _document_browser_root_for_mode(root_mode=root_mode, custom_root=custom_root) + discovered = _discover_document_files( + root=browser_root, + extensions=extensions, + recursive=bool(recursive), + max_results=int(max_results), + max_depth=int(max_depth), + search_text=search_text, + include_hidden=bool(include_hidden), + ) + + resolved = self._resolve_selected_file(selected_file, manual_path, browser_root) exists = resolved.exists() and resolved.is_file() + + # If the user did not select/type a file, expose the first discovered file + # as the primary scalar while still returning the complete discovered list. + if not exists and not _clean_scalar(manual_path) and not _clean_scalar(selected_file) and discovered: + resolved = discovered[0] + exists = True + size_bytes = resolved.stat().st_size if exists else 0 content_type = mimetypes.guess_type(str(resolved))[0] or "application/octet-stream" - filename = resolved.name if str(resolved) else "" + filename = resolved.name if exists else "" file_path = str(resolved) if exists else "" - file_paths = [file_path] if file_path else [] - filenames = [filename] if filename else [] - return (file_path, file_path, filename, filename, content_type, int(size_bytes), bool(exists), str(browser_root), _dirty_token("document_file_browser", refresh_token, root_mode, selected_file, manual_path, custom_root), _as_comfy_list(file_paths), _as_comfy_list(filenames), *_selectable_tail(file_paths)) - except Exception: + + file_paths = [str(path) for path in discovered] + if file_path and file_path not in file_paths: + file_paths.insert(0, file_path) + + filenames = [Path(path).name for path in file_paths] + indexed = _indexed_list_text(file_paths) + + payload = _ensure_payload_dict( + { + "success": bool(exists or file_paths), + "message": f"Discovered {len(file_paths)} file(s) under {browser_root}.", + "data": { + "root_mode": root_mode, + "browser_root": str(browser_root), + "selected_file": _clean_scalar(selected_file), + "manual_path": _clean_scalar(manual_path), + "custom_root": _clean_scalar(custom_root), + "effective_file_path": file_path, + "exists": bool(exists), + "recursive": bool(recursive), + "extensions": sorted(_split_extensions(extensions)) or ["*"], + "search_text": _clean_scalar(search_text), + "max_results": int(max_results), + "max_depth": int(max_depth), + "include_hidden": bool(include_hidden), + "discovered_count": len(file_paths), + "file_paths": file_paths, + "filenames": filenames, + "refresh": refresh, + }, + "error": None if (exists or file_paths) else {"msg": "No matching files discovered."}, + "status_code": 200 if (exists or file_paths) else 404, + } + ) + + status = payload.get("message") or "" + return ( + file_path, + file_path, + filename, + filename, + content_type, + int(size_bytes), + bool(exists), + str(browser_root), + refresh, + _as_comfy_list(file_paths), + _as_comfy_list(filenames), + indexed, + len(file_paths), + status, + *_selectable_tail(file_paths), + ) + except Exception as exc: try: - browser_root = str(_get_document_browser_root(root_mode=root_mode, custom_root=custom_root)) + browser_root = str(_document_browser_root_for_mode(root_mode=root_mode, custom_root=custom_root)) except Exception: browser_root = "" - return ("", "", "", "", "application/octet-stream", 0, False, browser_root, _dirty_token("document_file_browser_error", refresh_token), [], [], *_selectable_tail([])) + payload = _ensure_payload_dict( + { + "success": False, + "message": f"Document file discovery failed: {exc}", + "data": { + "root_mode": root_mode, + "browser_root": browser_root, + "selected_file": _clean_scalar(selected_file), + "manual_path": _clean_scalar(manual_path), + "custom_root": _clean_scalar(custom_root), + "refresh": refresh, + }, + "error": {"type": exc.__class__.__name__, "msg": str(exc)}, + "status_code": 0, + } + ) + return ( + "", + "", + "", + "", + "application/octet-stream", + 0, + False, + browser_root, + refresh, + _as_comfy_list([]), + _as_comfy_list([]), + _indexed_list_text([]), + 0, + _json_text(payload), + *_selectable_tail([]), + ) class ZMongoDocumentUploadFile(AlwaysDirtyMixin): @@ -346,6 +1026,355 @@ class ZMongoDocumentUploadFile(AlwaysDirtyMixin): }, effective_path.name if effective_path is not None else "") + +class ZMongoDocumentIngestTextFile(AlwaysDirtyMixin): + """ + Pick a local text-based PDF, DOCX, or plain text file, extract text locally, + and save it as one bounded ZMongo text-document record. + + Local File Store mode deliberately does not chunk documents. Large files + should be uploaded to the hosted Business Process Applications / ZMongo + backend, where ZEmbedder.py performs chunking and embedding. + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("ZMONGO_API_SESSION",), + "file_path": ("STRING", {"default": ""}), + "document_name": ("STRING", {"default": ""}), + "collection_name": ("STRING", {"default": DEFAULT_TEXT_DOCUMENT_COLLECTION}), + "field_root": ("STRING", {"default": DEFAULT_TEXT_DOCUMENT_FIELD_ROOT}), + "overwrite": ("BOOLEAN", {"default": True}), + "metadata_json": ("STRING", {"default": "{}", "multiline": True}), + "tags_json": ("STRING", {"default": "[]", "multiline": True}), + }, + "optional": { + "file_path_link": ("*",), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ( + "STRING", + "STRING", + ZMONGO_DOCUMENT_ID, + "STRING", + ZMONGO_FILENAME, + "STRING", + "INT", + "INT", + "BOOLEAN", + "STRING", + "*", + "*", + ) + SELECTABLE_RETURN_TYPES + RETURN_NAMES = ( + "json", + "document_id", + "document_id_link", + "document_name", + "filename_link", + "text", + "text_length", + "local_limit_bytes", + "success", + "refresh", + "document_ids", + "text_items", + ) + SELECTABLE_RETURN_NAMES + OUTPUT_IS_LIST = (False, False, False, False, False, False, False, False, False, False, True, True) + SELECTABLE_OUTPUT_IS_LIST + FUNCTION = "ingest_text_file" + CATEGORY = "ZMongo/06 Documents" + OUTPUT_NODE = True + + def _nonfatal( + self, + *, + payload: dict[str, Any], + document_name: str = "", + filename: str = "", + refresh: str = "", + ): + payload = _ensure_payload_dict(payload) + return ( + _json_text(payload), + "", + "", + document_name or "", + filename or "", + "", + 0, + 0, + False, + refresh, + _as_comfy_list([]), + _as_comfy_list([]), + *_selectable_tail([]), + ) + + def ingest_text_file( + self, + session: Any, + file_path: str, + document_name: str, + collection_name: str, + field_root: str, + overwrite: bool, + metadata_json: str, + tags_json: str, + file_path_link: Any = "", + refresh_token: str = "", + ): + primary_path = _clean_scalar(file_path) + linked_path = _coerce_file_path_link(file_path_link) + effective_file_path = linked_path or primary_path + refresh = _dirty_token("document_ingest_text_file", refresh_token, effective_file_path, document_name, collection_name) + + if not effective_file_path: + return self._nonfatal( + payload={ + "success": False, + "message": "Text document ingestion skipped. No file path was provided.", + "data": { + "file_path": primary_path, + "file_path_link": _clean_scalar(file_path_link), + "checks": [ + "Connect 06 Document File Browser.file_path_link to this node's file_path_link.", + "Or enter a full path to a .pdf, .docx, .txt, .md, .csv, .json, or .log file.", + ], + "refresh": refresh, + }, + "error": {"type": "MissingFilePath", "msg": "No file path supplied."}, + "status_code": 400, + }, + document_name=document_name, + refresh=refresh, + ) + + path = Path(os.path.expanduser(effective_file_path)).resolve() + filename = path.name + + if not path.exists() or not path.is_file(): + return self._nonfatal( + payload={ + "success": False, + "message": f"Text document ingestion skipped. File was not found: {path}", + "data": { + "effective_file_path": str(path), + "exists": path.exists(), + "is_file": path.is_file() if path.exists() else False, + "checks": [ + "The file must exist on the machine running ComfyUI.", + "Use 06 Document File Browser to avoid browser/client path confusion.", + ], + "refresh": refresh, + }, + "error": {"type": "FileNotFound", "msg": "File not found."}, + "status_code": 404, + }, + document_name=document_name, + filename=filename, + refresh=refresh, + ) + + clean_collection = _clean_scalar(collection_name) or DEFAULT_TEXT_DOCUMENT_COLLECTION + clean_root = _clean_dot_path_segment(field_root, DEFAULT_TEXT_DOCUMENT_FIELD_ROOT) + clean_document_name = _clean_scalar(document_name) or path.stem + + try: + extracted_text, extraction_engine = _extract_text_from_supported_document(path) + except Exception as exc: + return self._nonfatal( + payload={ + "success": False, + "message": f"Text extraction failed: {exc}", + "data": { + "effective_file_path": str(path), + "filename": filename, + "extension": path.suffix.lower(), + "refresh": refresh, + }, + "error": {"type": exc.__class__.__name__, "msg": str(exc)}, + "status_code": 0, + }, + document_name=clean_document_name, + filename=filename, + refresh=refresh, + ) + + if not extracted_text.strip(): + return self._nonfatal( + payload={ + "success": False, + "message": "No text was extracted. This may be a scanned PDF; use OCR ingestion instead.", + "data": { + "effective_file_path": str(path), + "filename": filename, + "extension": path.suffix.lower(), + "extraction_engine": extraction_engine, + "refresh": refresh, + }, + "error": {"type": "NoExtractedText", "msg": "No text extracted."}, + "status_code": 422, + }, + document_name=clean_document_name, + filename=filename, + refresh=refresh, + ) + + text_size_bytes = _utf8_size(extracted_text) + if _is_local_storage_session(session) and text_size_bytes > DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES: + return self._nonfatal( + payload=_local_storage_limit_payload( + path=path, + filename=filename, + text_size_bytes=text_size_bytes, + refresh=refresh, + ), + document_name=clean_document_name, + filename=filename, + refresh=refresh, + ) + + now = time.time() + metadata = _parse_json_object(metadata_json, "metadata_json") + tags = _parse_json_list(tags_json, "tags_json") + + text_payload = { + "raw_text": extracted_text, + "text_length": len(extracted_text), + "text_size_bytes": text_size_bytes, + "line_count": len(extracted_text.splitlines()), + "chunked": False, + "chunk_count": 0, + "local_max_text_document_bytes": DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES, + "large_file_strategy": "Use hosted ZMongo on BusinessProcessApplications.com; backend ZEmbedder.py handles chunking and embeddings.", + "extraction_engine": extraction_engine, + } + + document = { + "schema_kind": TEXT_DOCUMENT_SCHEMA_KIND, + "schema_version": TEXT_DOCUMENT_SCHEMA_VERSION, + "document_name": clean_document_name, + "collection_name": clean_collection, + "source_file": { + "path": str(path), + "filename": filename, + "extension": path.suffix.lower(), + "content_type": mimetypes.guess_type(str(path))[0] or "application/octet-stream", + "size_bytes": int(path.stat().st_size), + "ingested_at_unix": now, + }, + clean_root: text_payload, + "text": extracted_text, + "text_length": len(extracted_text), + "metadata": { + "tags": tags, + "user_metadata": metadata, + }, + "created_at_unix": now, + "updated_at_unix": now, + } + + existing_doc, existing_id, existing_query_payload = _query_existing_text_document( + session, + collection_name=clean_collection, + document_name=clean_document_name, + ) + + if existing_id and not bool(overwrite): + payload = _ensure_payload_dict({ + "success": False, + "message": f"Text document {clean_document_name!r} already exists. Enable overwrite to replace/update it.", + "data": { + "collection_name": clean_collection, + "document_name": clean_document_name, + "document_id": existing_id, + "existing_query_payload": existing_query_payload, + "refresh": refresh, + }, + "error": {"type": "DocumentExists", "msg": "overwrite is false."}, + "status_code": 409, + }) + ids = [existing_id] + return ( + _json_text(payload), + existing_id, + existing_id, + clean_document_name, + filename, + extracted_text, + len(extracted_text), + DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES, + False, + refresh, + _as_comfy_list(ids), + _as_comfy_list([extracted_text]), + *_selectable_tail(ids), + ) + + if existing_id: + payload = _save_text_document_by_dot_paths( + session, + collection_name=clean_collection, + document_id=existing_id, + document=document, + field_root=clean_root, + ) + document_id = existing_id + else: + payload = _generic_document_create_fallback( + session, + document=document, + collection_name=clean_collection, + ) + document_id = _extract_doc_id(payload) + + success = bool(payload.get("success")) + ids = [document_id] if document_id else [] + response_payload = _ensure_payload_dict({ + "success": success, + "message": ( + f"Ingested {filename!r} into collection {clean_collection!r}." + if success else + f"Text was extracted from {filename!r}, but database save failed." + ), + "data": { + "document_id": document_id, + "document_name": clean_document_name, + "collection_name": clean_collection, + "field_root": clean_root, + "filename": filename, + "text_length": len(extracted_text), + "chunked": False, + "chunk_count": 0, + "local_max_text_document_bytes": DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES, + "save_payload": payload, + "refresh": refresh, + }, + "error": None if success else payload.get("error"), + "status_code": int(payload.get("status_code") or (200 if success else 0)), + }) + + return ( + _json_text(response_payload), + document_id, + document_id, + clean_document_name, + filename, + extracted_text, + len(extracted_text), + DEFAULT_LOCAL_TEXT_DOCUMENT_MAX_BYTES, + success, + refresh, + _as_comfy_list(ids), + _as_comfy_list([extracted_text] if extracted_text else []), + *_selectable_tail(ids), + ) + + class ZMongoDocumentCreateText(AlwaysDirtyMixin): @classmethod def INPUT_TYPES(cls): @@ -438,7 +1467,20 @@ class ZMongoDocumentGet(AlwaysDirtyMixin): class ZMongoDocumentGetText(AlwaysDirtyMixin): @classmethod def INPUT_TYPES(cls): - return {"required": {"session": ("ZMONGO_API_SESSION",), "document_id": ("STRING", {"default": ""}), "document_prefix": ("STRING", {"default": DEFAULT_DOCUMENT_PREFIX})}, "optional": {"document_id_link": ("*",), "refresh_token": ("STRING", {"default": ""})}} + return { + "required": { + "session": ("ZMONGO_API_SESSION",), + "document_id": ("STRING", {"default": ""}), + "document_prefix": ("STRING", {"default": DEFAULT_DOCUMENT_PREFIX}), + }, + "optional": { + "document_id_link": ("*",), + "collection_name": ("STRING", {"default": DEFAULT_DOCUMENT_COLLECTION}), + "collection_name_link": ("*",), + "fallback_text": ("STRING", {"default": "", "multiline": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } RETURN_TYPES = ("STRING", "STRING", "INT", "BOOLEAN", "STRING", "*") + SELECTABLE_RETURN_TYPES RETURN_NAMES = ("json", "text", "text_length", "success", "refresh", "text_items") + SELECTABLE_RETURN_NAMES @@ -447,16 +1489,158 @@ class ZMongoDocumentGetText(AlwaysDirtyMixin): CATEGORY = "ZMongo/06 Documents" OUTPUT_NODE = True - def get_text(self, session: Any, document_id: str, document_prefix: str, document_id_link: str = "", refresh_token: str = ""): - clean_id = _coerce_document_id(_prefer_link_value(document_id, document_id_link)) - quoted = urllib.parse.quote(clean_id, safe="") - payload = _session_request(session, "GET", document_prefix, f"/api/text/{quoted}") - if not payload.get("success") and _is_route_not_found(payload): - payload = _generic_document_get_fallback(session, clean_id) - _raise_if_failed(payload, "Document text get") - text = _extract_text(payload) + def _nonfatal_get_text_result( + self, + *, + payload: dict[str, Any], + text: str = "", + refresh: str, + ): + text = text or "" items = [text] if text else [] - return (_json_text(payload), text, len(text), bool(payload.get("success")), _dirty_token("document_text", refresh_token), _as_comfy_list(items), *_selectable_tail(items)) + return ( + _json_text(_ensure_payload_dict(payload)), + text, + len(text), + bool(payload.get("success")), + refresh, + _as_comfy_list(items), + *_selectable_tail(items), + ) + + def get_text( + self, + session: Any, + document_id: str, + document_prefix: str, + document_id_link: str = "", + collection_name: str = DEFAULT_DOCUMENT_COLLECTION, + collection_name_link: Any = "", + fallback_text: str = "", + refresh_token: str = "", + ): + clean_id = _coerce_document_id(_prefer_link_value(document_id, document_id_link)) + clean_collection = _clean_scalar(_prefer_link_value(collection_name, collection_name_link)) or DEFAULT_DOCUMENT_COLLECTION + refresh = _dirty_token("document_text", refresh_token, clean_collection, clean_id) + + if not clean_id: + payload = _ensure_payload_dict({ + "success": False, + "message": "Document text get skipped: document_id is empty.", + "data": { + "collection_name": clean_collection, + "document_id": clean_id, + "refresh": refresh, + "checks": [ + "Connect document_id_link from 06 Select Nth Document Item.", + "Confirm the selected value is a document id, not a filename, file path, or field path.", + ], + }, + "error": {"type": "MissingDocumentId", "msg": "document_id is required."}, + "status_code": 400, + }) + return self._nonfatal_get_text_result(payload=payload, refresh=refresh) + + quoted = urllib.parse.quote(clean_id, safe="") + text_payload = _session_request(session, "GET", document_prefix, f"/api/text/{quoted}") + text = _extract_text(text_payload) + + if text_payload.get("success") or text: + if not text_payload.get("success") and text: + text_payload = _ensure_payload_dict({ + "success": True, + "message": "Document text route returned a non-success payload, but text was extracted from the response.", + "data": { + "collection_name": clean_collection, + "document_id": clean_id, + "text": text, + "text_length": len(text), + "source_payload": text_payload, + "refresh": refresh, + }, + "error": None, + "status_code": 200, + }) + return self._nonfatal_get_text_result(payload=text_payload, text=text, refresh=refresh) + + # Route-based document APIs may not exist for local_file_store sessions. + # Fall back to the generic local/session document loader and extract text + # from common fields. This also fixes documents saved outside the default + # "documents" collection by allowing collection_name to be supplied. + fallback_payload = _generic_document_get_fallback( + session, + clean_id, + collection_name=clean_collection, + ) + fallback_document = _extract_document(fallback_payload) + fallback_extracted_text = _extract_text(fallback_payload) + best_document_text, best_document_text_path, best_document_text_score = _best_text_from_document(fallback_document) + + if best_document_text and ( + not fallback_extracted_text + or _looks_like_bad_text_value(fallback_extracted_text) + or _text_quality_score(best_document_text) > _text_quality_score(fallback_extracted_text) + ): + fallback_extracted_text = best_document_text + + if fallback_payload.get("success") or fallback_extracted_text: + if fallback_extracted_text: + combined_payload = _ensure_payload_dict({ + "success": True, + "message": f"Loaded document text using generic/local document fallback from {best_document_text_path or 'payload text'}.", + "data": { + "collection_name": clean_collection, + "document_id": clean_id, + "text": fallback_extracted_text, + "text_length": len(fallback_extracted_text), + "resolved_text_path": best_document_text_path or "payload", + "text_quality_score": best_document_text_score, + "source": "generic_document_get_fallback", + "text_route_payload": text_payload, + "fallback_payload": fallback_payload, + "refresh": refresh, + }, + "error": None, + "status_code": 200, + }) + return self._nonfatal_get_text_result( + payload=combined_payload, + text=fallback_extracted_text, + refresh=refresh, + ) + + # Final behavior is intentionally nonfatal. A stale local id or wrong + # collection should not crash the whole ComfyUI graph. + final_text = fallback_text or "" + final_payload = _ensure_payload_dict({ + "success": bool(final_text), + "message": ( + "Document text was not found; returned fallback_text." + if final_text else + "Document text was not found. The workflow continued without raising." + ), + "data": { + "collection_name": clean_collection, + "document_id": clean_id, + "text": final_text, + "text_length": len(final_text), + "text_route_payload": text_payload, + "fallback_payload": fallback_payload, + "refresh": refresh, + "checks": [ + "The selected local document id may be stale; run 06 List Documents again and select a current id.", + "Confirm collection_name matches the collection used when the document was created or ingested.", + "For text files ingested by 06 Ingest Text PDF / DOCX, try collection_name='documents' unless you changed it.", + "Confirm the local_store JSON path shown in the payload exists on the ComfyUI machine.", + ], + }, + "error": { + "type": "DocumentTextNotFound", + "msg": "Document text could not be loaded from the route or generic fallback.", + }, + "status_code": 404, + }) + return self._nonfatal_get_text_result(payload=final_payload, text=final_text, refresh=refresh) class ZMongoDocumentFieldPaths(AlwaysDirtyMixin): @@ -602,22 +1786,55 @@ class ZMongoDocumentGetValue(AlwaysDirtyMixin): def get_value(self, session: Any, document_id: str, field_path: str, fallback: str, document_prefix: str, document_id_link: str = "", field_path_link: str = "", refresh_token: str = ""): clean_id = _coerce_document_id(_prefer_link_value(document_id, document_id_link)) clean_path = _coerce_field_path(_prefer_link_value(field_path, field_path_link)) + refresh = _dirty_token("document_get_value", refresh_token, clean_id, clean_path) quoted = urllib.parse.quote(clean_id, safe="") + payload = _session_request(session, "GET", document_prefix, f"/api/get/{quoted}") if not payload.get("success") and _is_route_not_found(payload): payload = _generic_document_get_fallback(session, clean_id) + if not payload.get("success"): empty_tail = _selectable_tail([]) return (_json_text(payload), fallback or "", False, "missing_document", _dirty_token("document_get_value_error", refresh_token), _as_comfy_list([]), *empty_tail) + document = _extract_document(payload) marker = object() value = _safe_get_by_path(document, clean_path, marker) exists = value is not marker + + resolved_path = clean_path + smart_replacement_used = False if not exists: value = fallback + elif clean_path in {"text", "raw_text", "document_text.raw_text", "document_text.text"} and _looks_like_bad_text_value(value): + best_text, best_path, best_score = _best_text_from_document(document) + if best_text and best_path and best_path != clean_path and _text_quality_score(best_text) > _text_quality_score(value): + value = best_text + resolved_path = best_path + smart_replacement_used = True + value_text = _json_text(value) if isinstance(value, (dict, list, tuple)) else str(value if value is not None else "") items = _value_items(value) - return (_json_text(payload), value_text, bool(exists), type(value).__name__ if exists else "missing", _dirty_token("document_get_value", refresh_token), _as_comfy_list(items), *_selectable_tail(items)) + + if smart_replacement_used: + payload = _ensure_payload_dict({ + "success": True, + "message": f"Field path {clean_path!r} looked binary/corrupt; returned better text from {resolved_path!r}.", + "data": { + "document_id": clean_id, + "requested_field_path": clean_path, + "resolved_path": resolved_path, + "exists": True, + "value_type": type(value).__name__, + "value": value, + "value_text": value_text, + "refresh": refresh, + }, + "error": None, + "status_code": 200, + }) + + return (_json_text(payload), value_text, bool(exists), type(value).__name__ if exists else "missing", refresh, _as_comfy_list(items), *_selectable_tail(items)) class ZMongoDocumentSaveText(AlwaysDirtyMixin): @@ -1044,12 +2261,362 @@ class ZMongoDocumentSelectNthItem(AlwaysDirtyMixin): ) return (selected, document_id, field_path, file_path, filename, text, status) + + +class ZMongoDocumentIndexSelector(AlwaysDirtyMixin): + """ + Select one or more items from a ComfyUI list by single index, range, + comma-separated series, mixed expression, or all. + + Designed for document workflows: + 06/List Docs.ids -> 06 Document Index Selector -> 06 Get Document Text + + Selection examples: + 3 + 2-7 + 0,2,5 + 0,2-5,9 + * + all + """ + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "items_list": ("*",), + "selection": ( + "STRING", + { + "default": "0", + "tooltip": "Index expression: 0, 2-7, 0,2,5, 0,2-5,9, *, or all.", + }, + ), + "fallback": ( + "STRING", + { + "default": "", + "tooltip": "Returned only when the input list is empty or no selection matches.", + }, + ), + "clamp_indexes": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Clamp out-of-range indexes to the nearest valid item instead of dropping them.", + }, + ), + "dedupe": ( + "BOOLEAN", + { + "default": True, + "tooltip": "Remove duplicate selected items while preserving order.", + }, + ), + "semantic_hint": ( + ["auto", "document_id", "field_path", "file_path", "filename", "text"], + { + "default": "document_id", + "tooltip": "Controls the guarded semantic link outputs.", + }, + ), + }, + "optional": { + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ( + "*", + ZMONGO_DOCUMENT_ID, + "STRING", + ZMONGO_DOCUMENT_ID, + ZMONGO_FIELD_PATH, + ZMONGO_FILE_PATH, + ZMONGO_FILENAME, + ZMONGO_TEXT, + "STRING", + "STRING", + "INT", + "BOOLEAN", + "STRING", + ) + RETURN_NAMES = ( + "selected_items", + "document_id_links", + "first_item", + "first_document_id_link", + "first_field_path_link", + "first_file_path_link", + "first_filename_link", + "first_text_link", + "selected_json", + "indexed_selected_items", + "count", + "success", + "status", + ) + OUTPUT_IS_LIST = ( + True, + True, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + False, + ) + FUNCTION = "select_indexes" + CATEGORY = "ZMongo/06 Documents" + INPUT_IS_LIST = True + OUTPUT_NODE = True + + @staticmethod + def _unwrap_scalar(value: Any, default: Any = None) -> Any: + if isinstance(value, list): + if not value: + return default + return value[0] + return value if value is not None else default + + @staticmethod + def _normalize_items(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, tuple): + value = list(value) + if isinstance(value, list): + if len(value) == 1 and isinstance(value[0], (list, tuple)): + return list(value[0]) + return value + return [value] + + @staticmethod + def _selected_as_string(value: Any) -> str: + if isinstance(value, (dict, list, tuple)): + return _json_text(value) + return _strip_index_prefix(value) + + @staticmethod + def _parse_index_token(token: str, max_index: int, clamp_indexes: bool) -> list[int]: + token = str(token or "").strip() + if not token: + return [] + + # Allow bracket-ish user input copied from previews. + token = token.strip().strip("[](){}") + + def coerce_index(raw: str) -> Optional[int]: + try: + idx = int(str(raw).strip()) + except Exception: + return None + + if idx < 0: + idx = max_index + 1 + idx + + if 0 <= idx <= max_index: + return idx + + if clamp_indexes and max_index >= 0: + return max(0, min(idx, max_index)) + + return None + + # Range syntax: 2-7, 7-2, 2:7, or 2..7. End is inclusive. + for sep in ("..", ":", "-"): + if sep in token: + left, right = token.split(sep, 1) + start = coerce_index(left) + end = coerce_index(right) + if start is None or end is None: + return [] + step = 1 if end >= start else -1 + return list(range(start, end + step, step)) + + idx = coerce_index(token) + return [idx] if idx is not None else [] + + @classmethod + def _parse_selection_expression( + cls, + selection: str, + item_count: int, + clamp_indexes: bool, + ) -> list[int]: + max_index = item_count - 1 + expression = str(selection or "").strip().lower() + + if not expression or expression in {"0", "first"}: + return [0] if item_count else [] + + if expression in {"*", "all", "each", "every"}: + return list(range(item_count)) + + if expression in {"last", "-1"}: + return [max_index] if item_count else [] + + # Normalize semicolons and whitespace-separated lists, but preserve range separators. + expression = expression.replace(";", ",") + raw_tokens: list[str] = [] + for comma_part in expression.split(","): + comma_part = comma_part.strip() + if not comma_part: + continue + if " " in comma_part and not any(sep in comma_part for sep in ("-", ":", "..")): + raw_tokens.extend(part for part in comma_part.split() if part.strip()) + else: + raw_tokens.append(comma_part) + + indexes: list[int] = [] + for token in raw_tokens: + indexes.extend(cls._parse_index_token(token, max_index, clamp_indexes)) + + return indexes + + @staticmethod + def _dedupe_indexes(indexes: list[int]) -> list[int]: + seen: set[int] = set() + out: list[int] = [] + for idx in indexes: + if idx in seen: + continue + seen.add(idx) + out.append(idx) + return out + + def _semantic_outputs(self, selected: str, semantic_hint: str = "document_id") -> tuple[str, str, str, str, str]: + hint = str(semantic_hint or "document_id").strip().lower() + selected = self._selected_as_string(selected) + + document_id = _coerce_document_id_link(selected) + field_path = _coerce_field_path(selected) + file_path = _coerce_file_path_link(selected) + filename = _coerce_filename_link(selected) + text = _coerce_text_link(selected) + + if hint == "document_id": + return (document_id, "", "", "", "") + if hint == "field_path": + return ("", field_path, "", "", "") + if hint == "file_path": + return ("", "", file_path, filename, "") + if hint == "filename": + return ("", "", "", filename, "") + if hint == "text": + return ("", "", "", "", selected) + + if document_id: + return (document_id, "", "", "", "") + if file_path: + return ("", "", file_path, filename, "") + if field_path: + return ("", field_path, "", filename, "") + return ("", "", "", filename, text) + + def select_indexes( + self, + items_list, + selection, + fallback, + clamp_indexes, + dedupe, + semantic_hint, + refresh_token="", + ): + raw_items = self._normalize_items(items_list) + clean_items = [ + self._selected_as_string(item).strip() + for item in raw_items + if self._selected_as_string(item).strip() + ] + + selection_value = str(self._unwrap_scalar(selection, "0") or "0") + fallback_value = str(self._unwrap_scalar(fallback, "") or "") + clamp_value = bool(self._unwrap_scalar(clamp_indexes, True)) + dedupe_value = bool(self._unwrap_scalar(dedupe, True)) + semantic_value = str(self._unwrap_scalar(semantic_hint, "document_id") or "document_id") + refresh = _dirty_token("document_index_selector", refresh_token, selection_value, len(clean_items)) + + if not clean_items: + selected = [fallback_value] if fallback_value else [] + first = selected[0] if selected else "" + doc_id, field_path, file_path, filename, text = self._semantic_outputs(first, semantic_value) + status = "Input list was empty. Returned fallback." if fallback_value else "Input list was empty." + return ( + selected, + [doc_id] if doc_id else [], + first, + doc_id, + field_path, + file_path, + filename, + text, + _json_text(selected), + _indexed_list_text(selected), + len(selected), + bool(selected), + status, + ) + + indexes = self._parse_selection_expression( + selection=selection_value, + item_count=len(clean_items), + clamp_indexes=clamp_value, + ) + if dedupe_value: + indexes = self._dedupe_indexes(indexes) + + selected = [clean_items[idx] for idx in indexes if 0 <= idx < len(clean_items)] + + if not selected and fallback_value: + selected = [fallback_value] + + first = selected[0] if selected else "" + doc_id, field_path, file_path, filename, text = self._semantic_outputs(first, semantic_value) + + document_id_links = [] + for item in selected: + item_doc_id, _, _, _, _ = self._semantic_outputs(item, "document_id") + if item_doc_id: + document_id_links.append(item_doc_id) + + selected_json = _json_text(selected) + indexed_selected = _indexed_list_text(selected) + status = ( + f"Selected {len(selected)} item(s) from {len(clean_items)} using selection={selection_value!r}; " + f"indexes={indexes}; first={first or ''}; refresh={refresh}" + ) + + return ( + selected, + document_id_links, + first, + doc_id, + field_path, + file_path, + filename, + text, + selected_json, + indexed_selected, + len(selected), + bool(selected), + status, + ) + NODE_CLASS_MAPPINGS = { + "ZMongoDocumentIndexSelector": ZMongoDocumentIndexSelector, "ZMongoDocumentSelectNthItem": ZMongoDocumentSelectNthItem, "ZMongoDocumentFilePathBrowser": ZMongoDocumentFilePathBrowser, "ZMongoDocumentHealth": ZMongoDocumentHealth, "ZMongoDocumentWhoAmI": ZMongoDocumentWhoAmI, "ZMongoDocumentUploadFile": ZMongoDocumentUploadFile, + "ZMongoDocumentIngestTextFile": ZMongoDocumentIngestTextFile, "ZMongoDocumentCreateText": ZMongoDocumentCreateText, "ZMongoDocumentList": ZMongoDocumentList, "ZMongoDocumentGet": ZMongoDocumentGet, @@ -1066,11 +2633,13 @@ NODE_CLASS_MAPPINGS = { } NODE_DISPLAY_NAME_MAPPINGS = { + "ZMongoDocumentIndexSelector": "06 Document Index Selector", "ZMongoDocumentSelectNthItem": "06 Select Nth Document Item", "ZMongoDocumentFilePathBrowser": "06 Document File Browser", "ZMongoDocumentHealth": "06 Document API Health", "ZMongoDocumentWhoAmI": "06 Document WhoAmI", "ZMongoDocumentUploadFile": "06 Upload Document File", + "ZMongoDocumentIngestTextFile": "06 Ingest Text PDF / DOCX", "ZMongoDocumentCreateText": "06 Create Text Document", "ZMongoDocumentList": "06 List Documents", "ZMongoDocumentGet": "06 Get Document", diff --git a/nodes/gemini_api_nodes.py b/nodes/gemini_api_nodes.py index 17211b8..9c85cba 100644 --- a/nodes/gemini_api_nodes.py +++ b/nodes/gemini_api_nodes.py @@ -13,7 +13,6 @@ from .generic_helpers import AlwaysDirtyMixin, DEFAULT_GEMINI_PREFIX, _session_a # Gemini route nodes # ----------------------------------------------------------------------------- -# TODO: Make endpoints in the backend to handle image and text for gemini use class GeminiApiKeyStatusNode(AlwaysDirtyMixin): @classmethod diff --git a/nodes/generic_helpers.py b/nodes/generic_helpers.py index db933f3..67176d1 100644 --- a/nodes/generic_helpers.py +++ b/nodes/generic_helpers.py @@ -65,6 +65,8 @@ DEFAULT_COMFY_ZMONGO_FLEET_PREFIX = os.getenv("COMFY_ZMONGO_FLEET_PREFIX", "/com DEFAULT_DOCUMENT_PREFIX = os.getenv("ZMONGO_DOCUMENT_API_PREFIX", DEFAULT_COMFY_ZMONGO_PREFIX).strip().rstrip( "/") or DEFAULT_COMFY_ZMONGO_PREFIX DEFAULT_MAX_UPLOAD_BYTES = int(os.getenv("ZMONGO_MAX_UPLOAD_BYTES", str(50 * 1024 * 1024))) +DEFAULT_LOCAL_MAX_VALUE_BYTES = int(os.getenv("ZMONGO_LOCAL_MAX_VALUE_BYTES", str(256 * 1024))) +DEFAULT_LOCAL_MAX_DOCUMENT_BYTES = int(os.getenv("ZMONGO_LOCAL_MAX_DOCUMENT_BYTES", str(1024 * 1024))) DOCUMENT_FILE_EXTENSIONS = tuple( ext.strip().lower() @@ -1318,6 +1320,29 @@ _local_payload_error = lambda message, data=None, status_code=400, error_type="L message, data=data, status_code=status_code, error_type=error_type) +def _local_json_size_bytes(value: Any) -> int: + return len(json.dumps(value, ensure_ascii=False, default=str).encode("utf-8")) + + +def _local_limit_payload(kind: str, size_bytes: int, max_bytes: int, extra: Optional[dict[str, Any]] = None) -> dict[str, Any]: + data = { + "storage_backend": "local_file_store", + "kind": kind, + "size_bytes": int(size_bytes), + "max_bytes": int(max_bytes), + "recommended_backend": "https://businessprocessapplications.com", + "recommended_pipeline": "Use hosted ZMongo for large documents/files; backend ZEmbedder.py handles chunking and embeddings.", + } + if extra: + data.update(extra) + return _local_payload_error( + f"Local File Store {kind} is too large: {size_bytes} bytes exceeds {max_bytes} bytes.", + data=data, + status_code=413, + error_type="LocalStorageItemTooLarge", + ) + + class LocalZMongoSession: """ Local file-backed ZMongo-compatible session. @@ -1338,6 +1363,16 @@ class LocalZMongoSession: comfy_zmongo_prefix = "/local-file-store" fleet_prefix = "/local-disabled" comfy_zmongo_fleet_prefix = "/local-disabled" + max_value_bytes = DEFAULT_LOCAL_MAX_VALUE_BYTES + max_document_bytes = DEFAULT_LOCAL_MAX_DOCUMENT_BYTES + + def _limit_info(self) -> dict[str, Any]: + return { + "local_max_value_bytes": int(self.max_value_bytes), + "local_max_document_bytes": int(self.max_document_bytes), + "large_document_backend": "https://businessprocessapplications.com", + "large_document_chunker": "ZEmbedder.py", + } def __init__(self, root_dir: Optional[str | Path] = None) -> None: plugin_root = Path(__file__).resolve().parent @@ -1419,7 +1454,8 @@ class LocalZMongoSession: "storage_backend": "local_file_store", "root_dir": str(self.root_dir), "manifest": str(self.manifest_path), - "message": "This stores JSON and images on this machine only. No hosted backend, API key, R2, dashboard, sync, metering, or backup.", + "message": "This stores small JSON items on this machine only. Large document storage, chunking, embeddings, R2, dashboard, sync, metering, and backup require hosted ZMongo.", + **self._limit_info(), }) def whoami(self) -> dict[str, Any]: @@ -1592,7 +1628,11 @@ class LocalZMongoSession: def create_doc(self, *, collection: str, document: dict[str, Any]) -> dict[str, Any]: clean = _local_safe_name(collection, "documents") - saved = self._save_doc(clean, document or {}) + candidate = dict(document or {}) + candidate_size = _local_json_size_bytes(candidate) + if candidate_size > self.max_document_bytes: + return _local_limit_payload("document", candidate_size, self.max_document_bytes, {"collection": clean}) + saved = self._save_doc(clean, candidate) doc_id = str(saved["_id"]) return _local_payload_ok("Created local document.", { "document": saved, @@ -1602,6 +1642,7 @@ class LocalZMongoSession: "collection": clean, "collection_name": clean, "storage_backend": "local_file_store", + **self._limit_info(), }) def update_doc( @@ -1650,6 +1691,12 @@ class LocalZMongoSession: _local_set_by_path(doc, key, item) else: _local_set_by_path(doc, field_path, value) + doc_size = _local_json_size_bytes(doc) + if doc_size > self.max_document_bytes: + return _local_limit_payload("document", doc_size, self.max_document_bytes, { + "collection": clean, + "document_id": clean_id, + }) saved = self._save_doc(clean, doc) doc_id = str(saved["_id"]) return _local_payload_ok("Updated local document.", { @@ -1734,7 +1781,21 @@ class LocalZMongoSession: value = json.loads(value) except Exception: pass + value_size = _local_json_size_bytes(value) + if value_size > self.max_value_bytes: + return _local_limit_payload("value", value_size, self.max_value_bytes, { + "collection": clean, + "document_id": clean_id, + "field_path": clean_field, + }) _local_set_by_path(doc, clean_field, value) + doc_size = _local_json_size_bytes(doc) + if doc_size > self.max_document_bytes: + return _local_limit_payload("document", doc_size, self.max_document_bytes, { + "collection": clean, + "document_id": clean_id, + "field_path": clean_field, + }) saved = self._save_doc(clean, doc) doc_id = str(saved["_id"]) return _local_payload_ok("Saved local value.", { @@ -1749,6 +1810,7 @@ class LocalZMongoSession: "document": saved, "document_path": str(self._doc_path(clean, doc_id).relative_to(self.root_dir)), "storage_backend": "local_file_store", + **self._limit_info(), }) def fetch_image_field( diff --git a/nodes/local_file_store.py b/nodes/local_file_store.py index ad7b0f5..3006d7b 100644 --- a/nodes/local_file_store.py +++ b/nodes/local_file_store.py @@ -32,6 +32,11 @@ from pathlib import Path from typing import Any, Dict, Iterable, List, Optional, Union +DEFAULT_LOCAL_MAX_VALUE_BYTES = int(os.getenv("ZMONGO_LOCAL_MAX_VALUE_BYTES", str(256 * 1024))) +DEFAULT_LOCAL_MAX_DOCUMENT_BYTES = int(os.getenv("ZMONGO_LOCAL_MAX_DOCUMENT_BYTES", str(1024 * 1024))) +DEFAULT_LOCAL_MAX_IMAGE_BYTES = int(os.getenv("ZMONGO_LOCAL_MAX_IMAGE_BYTES", str(10 * 1024 * 1024))) + + class DataProcessor: @staticmethod def to_json_compatible(data: Any, _seen: Optional[set] = None) -> Any: @@ -250,6 +255,43 @@ class SafeResult: class LocalFileStore: DOCUMENT_SUFFIX = ".document.json" STORAGE_BACKEND = "local_file_store" + max_value_bytes = DEFAULT_LOCAL_MAX_VALUE_BYTES + max_document_bytes = DEFAULT_LOCAL_MAX_DOCUMENT_BYTES + max_image_bytes = DEFAULT_LOCAL_MAX_IMAGE_BYTES + + @staticmethod + def _json_size_bytes(value: Any) -> int: + return len(json.dumps(DataProcessor.to_json_compatible(value), ensure_ascii=False, default=str).encode("utf-8")) + + def _limit_data(self, kind: str, size_bytes: int, max_bytes: int, extra: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + data = { + "storage_backend": self.STORAGE_BACKEND, + "kind": kind, + "size_bytes": int(size_bytes), + "max_bytes": int(max_bytes), + "recommended_backend": "https://businessprocessapplications.com", + "recommended_pipeline": "Use hosted ZMongo for large documents/files; backend ZEmbedder.py handles chunking and embeddings.", + } + if extra: + data.update(extra) + return data + + def _limit_result(self, kind: str, size_bytes: int, max_bytes: int, extra: Optional[Dict[str, Any]] = None) -> SafeResult: + return SafeResult.fail( + error={"error_type": "LocalStorageItemTooLarge", **self._limit_data(kind, size_bytes, max_bytes, extra)}, + data=self._limit_data(kind, size_bytes, max_bytes, extra), + status_code=413, + message=f"Local File Store {kind} is too large: {size_bytes} bytes exceeds {max_bytes} bytes.", + ) + + def _limit_info(self) -> Dict[str, Any]: + return { + "local_max_value_bytes": int(self.max_value_bytes), + "local_max_document_bytes": int(self.max_document_bytes), + "local_max_image_bytes": int(self.max_image_bytes), + "large_document_backend": "https://businessprocessapplications.com", + "large_document_chunker": "ZEmbedder.py", + } def __init__(self, root_dir: Optional[Union[str, Path]] = None) -> None: if root_dir is None: @@ -355,6 +397,7 @@ class LocalFileStore: "updated_at": self._now(), "root_dir": str(self.root_dir), "description": "ComfyUI-ZMongo Local File Store", + "limits": self._limit_info(), } self._write_json(self.manifest_path, manifest) @@ -463,6 +506,7 @@ class LocalFileStore: "storage_backend": self.STORAGE_BACKEND, "root_dir": str(self.root_dir), "manifest": str(self.manifest_path), + **self._limit_info(), } ) except Exception as exc: @@ -564,6 +608,9 @@ class LocalFileStore: ) safe_doc = DataProcessor.to_json_compatible(document) + safe_doc_size = self._json_size_bytes(safe_doc) + if safe_doc_size > self.max_document_bytes: + return self._limit_result("document", safe_doc_size, self.max_document_bytes, {"collection": coll}) target_id = document_id or safe_doc.get("_id") existing_path = self._find_document_path(coll, document_id=target_id) if target_id else None @@ -721,6 +768,13 @@ class LocalFileStore: parsed_value = self._json_loads_maybe(value) if parse_json_strings else value parsed_value = DataProcessor.to_json_compatible(parsed_value) + parsed_value_size = self._json_size_bytes(parsed_value) + if parsed_value_size > self.max_value_bytes: + return self._limit_result("value", parsed_value_size, self.max_value_bytes, { + "collection": coll, + "document_id": document_id, + "field_path": field_path, + }) path = self._find_document_path( coll, @@ -773,6 +827,14 @@ class LocalFileStore: doc["collection"] = coll doc["storage_backend"] = self.STORAGE_BACKEND + doc_size = self._json_size_bytes(doc) + if doc_size > self.max_document_bytes: + return self._limit_result("document", doc_size, self.max_document_bytes, { + "collection": coll, + "document_id": doc.get("_id"), + "field_path": field_path, + }) + self._write_json(path, doc) self._touch_manifest() @@ -899,6 +961,8 @@ class LocalFileStore: with self._lock: coll = self._safe_collection_name(collection) image_bytes = self._coerce_image_bytes(image) + if len(image_bytes) > self.max_image_bytes: + return self._limit_result("image", len(image_bytes), self.max_image_bytes, {"collection": coll, "filename": filename}) safe_filename = self._safe_name(filename, "comfy_image.png") path = self._find_document_path( @@ -1143,6 +1207,7 @@ class LocalFileStore: "files": file_count, "total_bytes": total_bytes, "total_mb": round(total_bytes / (1024 * 1024), 4), + **self._limit_info(), } ) except Exception as exc: diff --git a/nodes/zmongo_helper_nodes.py b/nodes/zmongo_helper_nodes.py index 02ee323..be43d67 100644 --- a/nodes/zmongo_helper_nodes.py +++ b/nodes/zmongo_helper_nodes.py @@ -1,78 +1,653 @@ from __future__ import annotations import json -import os +import re import time -import uuid -import urllib.parse +from datetime import datetime, timezone from pathlib import Path from typing import Any, Optional -import requests -from dotenv import load_dotenv -from .generic_helpers import AlwaysDirtyMixin, DEFAULT_BASE_URL, DEFAULT_COMFY_ZMONGO_PREFIX, DEFAULT_FLEET_PREFIX, \ - DEFAULT_COMFY_ZMONGO_FLEET_PREFIX, DEFAULT_TIMEOUT, _normalize_base_url, _clean_prefix, _json_text, _error_payload, \ - _success_payload, _indexed_list_text, _extract_collections, _as_comfy_list, _dirty_token, _parse_json_object, \ - _parse_json_list, _extract_doc_ids, _extract_count, _parse_any_json, _extract_document_from_payload, \ - ZMongoLocalFileStoreSessionNode, safe_get_by_path, _ensure_payload_dict +try: + from .generic_helpers import ( + AlwaysDirtyMixin, + _as_comfy_list, + _dirty_token, + _ensure_payload_dict, + _indexed_list_text, + _json_text, + safe_get_by_path, + ) +except Exception: # pragma: no cover - allows syntax checking outside ComfyUI package + class AlwaysDirtyMixin: + @classmethod + def IS_CHANGED(cls, *args, **kwargs): + return float('nan') + + def _as_comfy_list(value): + return value if isinstance(value, list) else ([] if value is None else [value]) + + def _dirty_token(*parts): + return "::".join(str(p) for p in parts if p is not None) + + def _ensure_payload_dict(value): + return value if isinstance(value, dict) else {"success": False, "message": str(value), "data": value} + + def _indexed_list_text(items): + return json.dumps([f"{i}: {x}" for i, x in enumerate(items)], indent=2) + + def _json_text(value): + try: + return json.dumps(value, indent=2, ensure_ascii=False, default=str) + except Exception: + return str(value) + + def safe_get_by_path(data, path, default=None): + return _get_by_dot_path(data, path, default) + +try: # Newer ComfyUI-ZMongo helper API. + from .generic_helpers import _session_get_doc as _GENERIC_SESSION_GET_DOC # type: ignore +except Exception: # pragma: no cover + _GENERIC_SESSION_GET_DOC = None + +try: + from .generic_helpers import _session_api_request as _GENERIC_SESSION_API_REQUEST # type: ignore +except Exception: # pragma: no cover + _GENERIC_SESSION_API_REQUEST = None + + +# ----------------------------------------------------------------------------- +# Shared helpers +# ----------------------------------------------------------------------------- + +def _unwrap_scalar(value: Any, default: Any = None) -> Any: + if isinstance(value, list): + if not value: + return default + return value[0] + return value if value is not None else default + + +def _normalize_items(value: Any) -> list[Any]: + if value is None: + return [] + if isinstance(value, tuple): + value = list(value) + if isinstance(value, list): + if len(value) == 1 and isinstance(value[0], (list, tuple)): + return list(value[0]) + return value + return [value] + + +def _item_to_string(value: Any) -> str: + if isinstance(value, (dict, list, tuple)): + return _json_text(value) + return str(value if value is not None else "") + + +def _clean_items(raw_items: Any) -> list[str]: + return [ + _item_to_string(item).strip() + for item in _normalize_items(raw_items) + if _item_to_string(item).strip() + ] + + +def _coerce_bool(value: Any, default: bool = False) -> bool: + value = _unwrap_scalar(value, value) + if isinstance(value, bool): + return value + if value is None: + return default + if isinstance(value, (int, float)): + return bool(value) + if isinstance(value, str): + text = value.strip().lower() + if text in {"true", "1", "yes", "y", "on"}: + return True + if text in {"false", "0", "no", "n", "off", ""}: + return default if text == "" else False + return bool(value) + + +def _coerce_int(value: Any, default: int = 0) -> int: + value = _unwrap_scalar(value, value) + if value is None: + return default + if isinstance(value, str) and not value.strip(): + return default + try: + return int(float(value)) + except Exception: + return default + + +def _timestamp(format_name: str = "iso_utc") -> str: + fmt = str(format_name or "iso_utc").strip().lower() + if fmt == "unix": + return str(time.time()) + if fmt == "local_iso": + return datetime.now().isoformat(timespec="seconds") + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _safe_state_name(name: str, fallback: str = "default_loop") -> str: + clean = re.sub(r"[^a-zA-Z0-9_.-]+", "_", str(name or "").strip()).strip("._") + return clean or fallback + + +def _state_dir() -> Path: + root = Path(__file__).resolve().parent + out = root / "local_store" / "loop_state" + out.mkdir(parents=True, exist_ok=True) + return out + + +def _state_path(name: str) -> Path: + return _state_dir() / f"{_safe_state_name(name)}.json" + + +def _load_state(name: str) -> dict[str, Any]: + path = _state_path(name) + if not path.exists(): + return {} + try: + data = json.loads(path.read_text(encoding="utf-8")) + return data if isinstance(data, dict) else {} + except Exception: + return {} + + +def _save_state(name: str, state: dict[str, Any]) -> None: + path = _state_path(name) + path.write_text(_json_text(state), encoding="utf-8") + + +def _parse_json_object(text: Any, default: Optional[dict[str, Any]] = None) -> dict[str, Any]: + if isinstance(text, dict): + return text + if text is None: + return default or {} + raw = str(text).strip() + if not raw: + return default or {} + try: + value = json.loads(raw) + return value if isinstance(value, dict) else (default or {}) + except Exception: + return default or {} + + +def _get_by_dot_path(data: Any, path: str, default: Any = None) -> Any: + if not path: + return data + current = data + for raw_part in str(path).split("."): + part = raw_part.strip() + if not part: + continue + if isinstance(current, dict): + if part in current: + current = current[part] + continue + return default + if isinstance(current, list): + try: + index = int(part) + except Exception: + return default + if 0 <= index < len(current): + current = current[index] + continue + return default + return default + return current + + +def _extract_document_from_response(payload: Any) -> dict[str, Any]: + if not isinstance(payload, dict): + return {} + + data = payload.get("data") + if isinstance(data, dict): + for key in ("document", "doc", "result"): + value = data.get(key) + if isinstance(value, dict): + return value + + docs = data.get("documents") or data.get("docs") or data.get("results") or data.get("items") + if isinstance(docs, list) and docs and isinstance(docs[0], dict): + return docs[0] + + if any(key in data for key in ("_id", "document_id", "id", "document_text", "text")): + return data + + for key in ("document", "doc", "result"): + value = payload.get(key) + if isinstance(value, dict): + return value + + docs = payload.get("documents") or payload.get("docs") or payload.get("results") or payload.get("items") + if isinstance(docs, list) and docs and isinstance(docs[0], dict): + return docs[0] + + if any(key in payload for key in ("_id", "document_id", "id", "document_text", "text")): + return payload + + return {} + + +def _session_get_document(session: Any, collection_name: str, document_id: str) -> tuple[dict[str, Any], dict[str, Any]]: + """ + Load a document through the real ComfyUI-ZMongo session contract. + + This deliberately does NOT call session.get_value(). ZMongoApiSession does + not expose get_value; read the full document through get_doc/API routes and + resolve flattened paths locally. + """ + collection = str(collection_name or "").strip() + clean_id = str(document_id or "").strip() + + if not session: + payload = _ensure_payload_dict({ + "success": False, + "message": "No session was provided.", + "data": {"collection_name": collection, "document_id": clean_id}, + "error": {"msg": "missing_session"}, + "status_code": 400, + }) + return {}, payload + + attempts: list[dict[str, Any]] = [] + + # 1. Preferred generic helper used by existing document nodes. + if callable(_GENERIC_SESSION_GET_DOC): + try: + payload = _ensure_payload_dict(_GENERIC_SESSION_GET_DOC(session, collection, clean_id, cache=False)) + document = _extract_document_from_response(payload) + if document: + return document, payload + attempts.append({"method": "_session_get_doc", "payload": payload}) + except Exception as exc: + attempts.append({"method": "_session_get_doc", "error": str(exc), "type": exc.__class__.__name__}) + + # 2. ZMongoApiSession / local session direct methods, if present. + get_doc = getattr(session, "get_doc", None) + if callable(get_doc): + call_variants = [ + dict(collection=collection, document_id=clean_id, cache=False), + dict(collection_name=collection, document_id=clean_id, cache=False), + dict(coll=collection, document_id=clean_id, cache=False), + dict(collection=collection, id=clean_id, cache=False), + dict(collection=collection, document_id=clean_id), + dict(collection_name=collection, document_id=clean_id), + dict(coll=collection, doc_id=clean_id), + ] + for kwargs in call_variants: + try: + payload = _ensure_payload_dict(get_doc(**kwargs)) + document = _extract_document_from_response(payload) + if document: + return document, payload + attempts.append({"method": "get_doc", "kwargs": kwargs, "payload": payload}) + except TypeError: + continue + except Exception as exc: + attempts.append({"method": "get_doc", "kwargs": kwargs, "error": str(exc), "type": exc.__class__.__name__}) + + # 3. API route fallback. The backend exposes GET /api/doc//. + if callable(_GENERIC_SESSION_API_REQUEST): + try: + import urllib.parse + q_coll = urllib.parse.quote(collection, safe="") + q_id = urllib.parse.quote(clean_id, safe="") + payload = _ensure_payload_dict(_GENERIC_SESSION_API_REQUEST(session, "GET", f"/api/doc/{q_coll}/{q_id}")) + document = _extract_document_from_response(payload) + if document: + return document, payload + attempts.append({"method": "_session_api_request.GET /api/doc", "payload": payload}) + except Exception as exc: + attempts.append({"method": "_session_api_request.GET /api/doc", "error": str(exc), "type": exc.__class__.__name__}) + + # 4. Query fallback. + query_docs = getattr(session, "query_docs", None) + if callable(query_docs): + for query in ({"_id": clean_id}, {"document_id": clean_id}, {"id": clean_id}): + call_variants = [ + dict(collection=collection, query=query, many=False, limit=1, skip=0, cache=False), + dict(collection_name=collection, query=query, many=False, limit=1, skip=0, cache=False), + dict(coll=collection, query=query, many=False, limit=1, skip=0, cache=False), + ] + for kwargs in call_variants: + try: + payload = _ensure_payload_dict(query_docs(**kwargs)) + document = _extract_document_from_response(payload) + if document: + return document, payload + attempts.append({"method": "query_docs", "kwargs": kwargs, "payload": payload}) + except TypeError: + continue + except Exception as exc: + attempts.append({"method": "query_docs", "kwargs": kwargs, "error": str(exc), "type": exc.__class__.__name__}) + + payload = _ensure_payload_dict({ + "success": False, + "message": "Could not load document from session.", + "data": { + "collection_name": collection, + "document_id": clean_id, + "attempts": attempts[-10:], + }, + "error": {"msg": "document_not_found"}, + "status_code": 404, + }) + return {}, payload + + + # ----------------------------------------------------------------------------- # 99 Helper nodes # ----------------------------------------------------------------------------- class ZMongoApiSelectNthItemNode(AlwaysDirtyMixin): + """Generic item selector for any ComfyUI list output.""" + + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "items_list": ("*",), + "index": ("INT", {"default": 0, "min": -1000000, "max": 1000000}), + "fallback": ("STRING", {"default": ""}), + "selection": ( + "STRING", + { + "default": "", + "tooltip": "Examples: 0, 2-7, 7-2, 0,2,5, 0,2-5,9, *, all, first, last, -1. Empty uses index.", + }, + ), + "mode": (["auto", "single", "range", "series", "all"], {"default": "auto"}), + "include_end": ("BOOLEAN", {"default": True}), + "dedupe": ("BOOLEAN", {"default": True}), + } + } + + RETURN_TYPES = ("STRING", "STRING", "*", "STRING", "STRING", "INT") + RETURN_NAMES = ("item", "status", "selected_items", "selected_json", "indexed_selected_items", "count") + OUTPUT_IS_LIST = (False, False, True, False, False, False) + FUNCTION = "select_nth_item" + CATEGORY = "ZMongo/99 Helpers" + INPUT_IS_LIST = True + + @staticmethod + def _normalize_index(index: Any, count: int) -> Optional[int]: + if count <= 0: + return None + raw = _coerce_int(index, 0) + if raw < 0: + raw = count + raw + return max(0, min(raw, count - 1)) + + @classmethod + def _parse_selection_indexes( + cls, + *, + selection: str, + index: Any, + mode: str, + count: int, + include_end: bool, + ) -> tuple[list[int], list[str]]: + warnings: list[str] = [] + if count <= 0: + return [], warnings + + clean_mode = str(mode or "auto").strip().lower() + expression = str(selection or "").strip().lower() + + if clean_mode == "all" or expression in {"*", "all"}: + return list(range(count)), warnings + + if clean_mode in {"single", "range"} and not expression: + selected = cls._normalize_index(index, count) + return ([selected] if selected is not None else []), warnings + + if not expression: + selected = cls._normalize_index(index, count) + return ([selected] if selected is not None else []), warnings + + aliases = {"first": "0", "last": "-1"} + expression = aliases.get(expression, expression) + indexes: list[int] = [] + + for part in [p.strip() for p in expression.split(",") if p.strip()]: + part = aliases.get(part, part) + if part in {"*", "all"}: + indexes.extend(range(count)) + continue + + if "-" in part[1:]: + split_at = part.find("-", 1) + left = part[:split_at].strip() + right = part[split_at + 1 :].strip() + try: + start = cls._normalize_index(int(left), count) + end = cls._normalize_index(int(right), count) + except Exception: + warnings.append(f"Ignored invalid range {part!r}.") + continue + if start is None or end is None: + continue + step = 1 if start <= end else -1 + final = end + step if include_end else end + indexes.extend(range(start, final, step)) + continue + + try: + one = cls._normalize_index(int(part), count) + except Exception: + warnings.append(f"Ignored invalid index {part!r}.") + continue + if one is not None: + indexes.append(one) + + return indexes, warnings + + @staticmethod + def _dedupe_indexes(indexes: list[int]) -> list[int]: + seen: set[int] = set() + output: list[int] = [] + for value in indexes: + if value in seen: + continue + seen.add(value) + output.append(value) + return output + + def select_nth_item( + self, + items_list, + index, + fallback, + selection="", + mode="auto", + include_end=True, + dedupe=True, + ): + cleaned = _clean_items(items_list) + fallback_value = str(_unwrap_scalar(fallback, "") or "") + index_value = _unwrap_scalar(index, 0) + selection_value = str(_unwrap_scalar(selection, "") or "") + mode_value = str(_unwrap_scalar(mode, "auto") or "auto") + include_end_value = _coerce_bool(_unwrap_scalar(include_end, True), True) + dedupe_value = _coerce_bool(_unwrap_scalar(dedupe, True), True) + + if not cleaned: + selected_items = [fallback_value] if fallback_value else [] + return ( + fallback_value, + "Input list was empty; returned fallback." if fallback_value else "Input list was empty.", + _as_comfy_list(selected_items), + _json_text(selected_items), + _indexed_list_text(selected_items), + len(selected_items), + ) + + indexes, warnings = self._parse_selection_indexes( + selection=selection_value, + index=index_value, + mode=mode_value, + count=len(cleaned), + include_end=include_end_value, + ) + if dedupe_value: + indexes = self._dedupe_indexes(indexes) + + selected_items = [cleaned[i] for i in indexes if 0 <= i < len(cleaned)] + if not selected_items and fallback_value: + selected_items = [fallback_value] + + first = selected_items[0] if selected_items else "" + status = f"Selected {len(selected_items)} item(s) from {len(cleaned)} total." + if indexes: + status += f" First index={indexes[0]}." + if warnings: + status += " " + " ".join(warnings) + + return ( + first, + status, + _as_comfy_list(selected_items), + _json_text(selected_items), + _indexed_list_text(selected_items), + len(selected_items), + ) + + +class ZMongoRecordLoopManagerNode(AlwaysDirtyMixin): + """Self-advancing loop manager for document ids or arbitrary records.""" + @classmethod def INPUT_TYPES(cls): return { "required": { "items_list": ("*",), "index": ("INT", {"default": 0, "min": 0, "max": 1000000}), - "fallback": ("STRING", {"default": ""}), - } + "wrap": ("BOOLEAN", {"default": False}), + "advance_on_execute": ("BOOLEAN", {"default": True}), + "state_name": ("STRING", {"default": "default_record_loop"}), + "reset_state": ("BOOLEAN", {"default": False}), + "timestamp_format": (["iso_utc", "unix", "local_iso"], {"default": "iso_utc"}), + }, + "optional": { + "refresh_token": ("STRING", {"default": ""}), + }, } - RETURN_TYPES = ("STRING", "STRING") - RETURN_NAMES = ("item", "status") - FUNCTION = "select_nth_item" + RETURN_TYPES = ("STRING", "STRING", "BOOLEAN", "INT", "INT", "INT", "STRING", "*", "STRING", "STRING", "INT", "STRING", "STRING", "STRING") + RETURN_NAMES = ("item", "document_id_link", "done", "index", "item_count", "remaining_count", "reviewed_timestamp", "selected_items", "state_json", "status", "next_index", "visible_index", "visible_next_index", "progress_text") + OUTPUT_IS_LIST = (False, False, False, False, False, False, False, True, False, False, False, False, False, False) + FUNCTION = "loop_record" CATEGORY = "ZMongo/99 Helpers" INPUT_IS_LIST = True + OUTPUT_NODE = True - @staticmethod - def _unwrap_scalar(value: Any, default: Any = None) -> Any: - if isinstance(value, list): - if not value: - return default - return value[0] - return value if value is not None else default + def loop_record( + self, + items_list, + index, + wrap, + advance_on_execute, + state_name, + reset_state, + timestamp_format, + refresh_token="", + ): + cleaned = _clean_items(items_list) + count = len(cleaned) + widget_index = max(0, _coerce_int(_unwrap_scalar(index, 0), 0)) + wrap_value = _coerce_bool(_unwrap_scalar(wrap, False), False) + advance_value = _coerce_bool(_unwrap_scalar(advance_on_execute, True), True) + reset_value = _coerce_bool(_unwrap_scalar(reset_state, False), False) + state_base = str(_unwrap_scalar(state_name, "default_record_loop") or "default_record_loop") + state_key = _safe_state_name(state_base, "default_record_loop") + ts = _timestamp(str(_unwrap_scalar(timestamp_format, "iso_utc") or "iso_utc")) + refresh_value = str(_unwrap_scalar(refresh_token, "") or "") - @staticmethod - def _normalize_items(value: Any) -> list[Any]: - if value is None: - return [] - if isinstance(value, tuple): - value = list(value) - if isinstance(value, list): - if len(value) == 1 and isinstance(value[0], (list, tuple)): - return list(value[0]) - return value - return [value] + prior = {} if reset_value else _load_state(state_key) + persisted_index = _coerce_int(prior.get("next_index"), widget_index) + current_index = widget_index if reset_value or not prior else max(0, persisted_index) - def select_nth_item(self, items_list, index, fallback): - raw_items = self._normalize_items(items_list) - fallback_value = str(self._unwrap_scalar(fallback, "") or "") - index_value = self._unwrap_scalar(index, 0) + if count <= 0: + state = { + "done": True, + "state_name": state_key, + "index": current_index, + "next_index": current_index, + "item_count": 0, + "timestamp": ts, + "refresh": _dirty_token("record_loop_empty", refresh_value, state_key), + } + if advance_value or reset_value: + _save_state(state_key, state) + return ("", "", True, current_index, 0, 0, ts, _as_comfy_list([]), _json_text(state), "Record loop is empty.", current_index, "0 / 0", "0 / 0", "No items to process.") - try: - safe_index = int(index_value or 0) - except Exception: - safe_index = 0 + done = current_index >= count + effective_index = current_index + if done and wrap_value: + effective_index = current_index % count + done = False - cleaned = [str(item).strip() for item in raw_items if str(item).strip()] - if not cleaned: - return (fallback_value, "Input list was empty.") + if done: + item = "" + next_index = current_index + status = f"Loop complete: index={current_index}, item_count={count}. Reset state to start again." + else: + item = cleaned[effective_index] + next_index = effective_index + 1 + if wrap_value and next_index >= count: + next_index = 0 + status = f"Selected record {effective_index + 1}/{count}; next_index={next_index}." + + remaining = max(0, count - next_index) if not wrap_value else count + state = { + "done": bool(done), + "state_name": state_key, + "index": int(effective_index if not done else current_index), + "current_index": int(current_index), + "next_index": int(next_index), + "item_count": int(count), + "item": item, + "timestamp": ts, + "updated_at_unix": time.time(), + "refresh": _dirty_token("record_loop", refresh_value, state_key, current_index, count), + } + if advance_value or reset_value: + _save_state(state_key, state) + if not done: + status += " Persisted next_index for the next run." + + visible_current = int(effective_index if not done else current_index) + visible_next = int(next_index) + progress = f"item_index={visible_current}; next_index={visible_next}; item={visible_current + 1 if not done else count}/{count}; done={bool(done)}; item_value={item}" + selected_items = [item] if item else [] + return ( + item, + item, + bool(done), + visible_current, + int(count), + int(remaining), + ts, + _as_comfy_list(selected_items), + _json_text(state), + status, + visible_next, + f"{visible_current} / {max(count - 1, 0)}", + f"{visible_next} / {max(count - 1, 0)}", + progress, + ) - selected_index = max(0, min(safe_index, len(cleaned) - 1)) - selected = cleaned[selected_index] - return (selected, f"Selected {selected_index + 1}/{len(cleaned)}: {selected}") class ZMongoApiJsonPickNode(AlwaysDirtyMixin): @@ -116,19 +691,15 @@ class ZMongoApiJsonPickNode(AlwaysDirtyMixin): # ----------------------------------------------------------------------------- NODE_CLASS_MAPPINGS = { - # 99 Helpers "ZMongoApiSelectNthItemNode": ZMongoApiSelectNthItemNode, + "ZMongoRecordLoopManagerNode": ZMongoRecordLoopManagerNode, "ZMongoApiJsonPickNode": ZMongoApiJsonPickNode, } NODE_DISPLAY_NAME_MAPPINGS = { - # 99 Helpers "ZMongoApiSelectNthItemNode": "99 Select Nth Item", + "ZMongoRecordLoopManagerNode": "99 Record Loop Manager", "ZMongoApiJsonPickNode": "99 JSON Pick", } - -__all__ = [ - "NODE_CLASS_MAPPINGS", - "NODE_DISPLAY_NAME_MAPPINGS", -] \ No newline at end of file +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS"] \ No newline at end of file diff --git a/nodes/zmongo_preset_nodes.py b/nodes/zmongo_preset_nodes.py index 4d3e37e..2e48fbf 100644 --- a/nodes/zmongo_preset_nodes.py +++ b/nodes/zmongo_preset_nodes.py @@ -343,6 +343,9 @@ def _reconstruct_preset_payload_from_dot_path_document(document: Any) -> dict[st if isinstance(document.get("unresolved_links"), list): payload["unresolved_links"] = document.get("unresolved_links") + if isinstance(document.get("unresolved_link_details"), list): + payload["unresolved_link_details"] = document.get("unresolved_link_details") + if document.get("warning"): payload["warning"] = document.get("warning") @@ -776,6 +779,9 @@ def _reconstruct_preset_payload_from_api_document(document: Any) -> dict[str, An if isinstance(document.get("unresolved_links"), list): payload["unresolved_links"] = document.get("unresolved_links") + if isinstance(document.get("unresolved_link_details"), list): + payload["unresolved_link_details"] = document.get("unresolved_link_details") + return payload @@ -880,6 +886,19 @@ def _save_preset_api( if not ok: failures.append(message) + if isinstance(payload.get("unresolved_link_details"), list): + ok, message, document_id, raw_response = _api_save_value( + session=session, + collection_name=collection, + query=query, + field_path="unresolved_link_details", + value=payload.get("unresolved_link_details"), + ) + if document_id: + latest_document_id = document_id + if not ok: + failures.append(message) + if payload.get("warning"): ok, message, document_id, raw_response = _api_save_value( session=session, @@ -1406,6 +1425,30 @@ def _extract_dynamic_preset_payload_from_prompt_node( return {} +def _link_source_node_info( + *, + prompt: Any, + link_value: Any, +) -> tuple[str, Optional[int], str, Dict[str, Any]]: + """ + Return source node information for a ComfyUI link value. + + ComfyUI prompt inputs represent links as [source_node_id, output_index]. + This helper keeps link detection centralized so preset saving can distinguish + ordinary graph wiring from ZMongo dynamic preset value links. + """ + source_node_id, output_index = _parse_link_value(link_value) + if not source_node_id or output_index is None: + return "", None, "", {} + + source_node = _prompt_node_by_id(prompt, source_node_id) + if not source_node: + return source_node_id, output_index, "", {} + + source_class = str(source_node.get("class_type") or "") + return source_node_id, output_index, source_class, source_node + + def _resolve_dynamic_loader_field_from_link( *, prompt: Any, @@ -1413,15 +1456,14 @@ def _resolve_dynamic_loader_field_from_link( target_input_name: str, session: Any = None, ) -> Dict[str, Any]: - source_node_id, output_index = _parse_link_value(link_value) - if not source_node_id or output_index is None: + source_node_id, output_index, source_class, source_node = _link_source_node_info( + prompt=prompt, + link_value=link_value, + ) + + if not source_node_id or output_index is None or not source_node: return {} - source_node = _prompt_node_by_id(prompt, source_node_id) - if not source_node: - return {} - - source_class = str(source_node.get("class_type") or "") if source_class != DYNAMIC_PRESET_NODE_CLASS: return {} @@ -1435,23 +1477,29 @@ def _resolve_dynamic_loader_field_from_link( if not isinstance(fields, list): return {} + target_name = str(target_input_name or "").strip() + + # First choice: exact output-index match AND exact field-name match. + # This is the only safe index-based resolution because dynamic outputs are + # positional but target inputs are named. if 0 <= output_index < len(fields) and isinstance(fields[output_index], dict): candidate = fields[output_index] - candidate_name = str(candidate.get("input_name") or "") - - if candidate_name == target_input_name: + candidate_name = str(candidate.get("input_name") or "").strip() + if candidate_name == target_name: return candidate - for field in fields: - if isinstance(field, dict) and str(field.get("input_name") or "") == target_input_name: - return field - - return candidate - + # Second choice: exact field-name match anywhere in the loaded preset. + # This survives field-order changes while still avoiding wrong-value saves. for field in fields: - if isinstance(field, dict) and str(field.get("input_name") or "") == target_input_name: + if not isinstance(field, dict): + continue + field_name = str(field.get("input_name") or "").strip() + if field_name == target_name: return field + # Deliberately do not fall back to returning fields[output_index]. + # Saving an indexed field with a mismatched name silently stores the wrong + # preset value and is harder to diagnose than an unresolved-link warning. return {} @@ -1529,6 +1577,7 @@ def _build_preset_payload( fields: List[Dict[str, Any]] = [] unresolved_links: List[str] = [] + unresolved_link_details: List[Dict[str, Any]] = [] for input_name, current_value in current_inputs.items(): if input_name not in declarations: @@ -1545,6 +1594,12 @@ def _build_preset_payload( is_link = bool(base_field.get("is_link")) if is_link: + source_node_id, output_index, link_source_class, _source_node = _link_source_node_info( + prompt=prompt, + link_value=current_value, + ) + is_dynamic_loader_link = link_source_class == DYNAMIC_PRESET_NODE_CLASS + resolved_field = _resolve_dynamic_loader_field_from_link( prompt=prompt, link_value=current_value, @@ -1559,6 +1614,10 @@ def _build_preset_payload( target_input_name=input_name, link_value=current_value, ) + merged_field["resolved_source_node_id"] = source_node_id + merged_field["resolved_source_output_index"] = output_index + merged_field["resolved_source_node_class"] = link_source_class + merged_field["resolved_source_input_name"] = str(resolved_field.get("input_name") or "") if save_mode in {"widgets_only", "widgets_and_inputs"}: fields.append(merged_field) @@ -1568,6 +1627,27 @@ def _build_preset_payload( continue unresolved_links.append(input_name) + unresolved_link_details.append( + { + "input_name": input_name, + "link_value": current_value, + "source_node_id": source_node_id, + "source_output_index": output_index, + "source_node_class": link_source_class, + "dynamic_loader_link": is_dynamic_loader_link, + "reason": ( + "Dynamic preset link could not be resolved to an underlying field value." + if is_dynamic_loader_link + else "Ordinary ComfyUI runtime link; no serializable widget value available." + ), + } + ) + + # A dynamic preset output is a value-provider link. If resolution + # fails, do not store the ComfyUI graph link as the preset value in + # the normal save modes; that is the bug this branch prevents. + if is_dynamic_loader_link and save_mode in {"widgets_only", "widgets_and_inputs"}: + continue if save_mode == "widgets_only": continue @@ -1597,10 +1677,11 @@ def _build_preset_payload( if unresolved_links: payload["unresolved_links"] = unresolved_links + payload["unresolved_link_details"] = unresolved_link_details payload["warning"] = ( - "Some linked inputs could not be resolved. Links from " - "ZMongoDynamicPresetOutputs are resolvable; ordinary runtime " - "object links are not saved in widgets_only mode." + "Some linked inputs could not be resolved. Dynamic preset output " + "links are not saved as raw ComfyUI link arrays in widgets_only or " + "widgets_and_inputs mode; they must resolve to underlying preset values." ) return payload, "" @@ -2115,6 +2196,13 @@ class ZMongoPresetDebugInfo: "widget_kind": field.get("widget_kind"), "value": field.get("value"), "resolved_from_link": field.get("resolved_from_link", False), + "resolved_link_target": field.get("resolved_link_target"), + "resolved_source_node_id": field.get("resolved_source_node_id"), + "resolved_source_output_index": field.get("resolved_source_output_index"), + "resolved_source_node_class": field.get("resolved_source_node_class"), + "resolved_source_input_name": field.get("resolved_source_input_name"), + "is_link": field.get("is_link", False), + "link_target": field.get("link_target"), } ) @@ -2126,6 +2214,7 @@ class ZMongoPresetDebugInfo: "field_count": len(field_names), "fields": field_names, "unresolved_links": payload.get("unresolved_links", []), + "unresolved_link_details": payload.get("unresolved_link_details", []), "warning": payload.get("warning", ""), } diff --git a/nodes/zmongo_text_agent_nodes.py b/nodes/zmongo_text_agent_nodes.py new file mode 100644 index 0000000..44a2590 --- /dev/null +++ b/nodes/zmongo_text_agent_nodes.py @@ -0,0 +1,1517 @@ + +from __future__ import annotations + +# pylint: disable=too-many-lines,too-many-locals,too-many-branches,too-many-statements,broad-exception-caught,line-too-long,missing-class-docstring,missing-function-docstring + +import json +import re +import time +import uuid +from datetime import datetime, timezone +from typing import Any, Optional + +from .generic_helpers import ( + AlwaysDirtyMixin, + DEFAULT_COMFY_ZMONGO_PREFIX, + DEFAULT_GEMINI_PREFIX, + _dirty_token, + _error_payload, + _extract_text_from_gemini_payload, + _json_text, + _parse_json_object, + _safe_get_by_path, + _session_api_request, + _success_payload, +) + + +TEXT_AGENT_CATEGORY = "ZMongo/06 Text Agents" +DEFAULT_MODEL = "gemini-2.5-flash" +DEFAULT_MEMORY_COLLECTION = "text_agent_memory" +DEFAULT_LEDGER_COLLECTION = "text_agent_ledger" +DEFAULT_CAPSULE_COLLECTION = "text_agent_capsules" +DEFAULT_CONTEXT_COLLECTION = "text_agent_context_packs" +DEFAULT_DOC_COLLECTION = "text_agent_documents" + + +# ----------------------------------------------------------------------------- +# Internal helpers +# ----------------------------------------------------------------------------- + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _clean(value: Any, default: str = "") -> str: + text = str(value or "").strip() + return text or default + + +def _safe_int(value: Any, default: int, minimum: int, maximum: int) -> int: + try: + number = int(value) + except Exception: + number = default + return max(minimum, min(number, maximum)) + + +def _safe_float(value: Any, default: float, minimum: float, maximum: float) -> float: + try: + number = float(value) + except Exception: + number = default + return max(minimum, min(number, maximum)) + + +def _csv_list(value: str) -> list[str]: + return [part.strip() for part in (value or "").split(",") if part.strip()] + + +def _new_run_id(prefix: str = "text_agent") -> str: + return f"{prefix}_{time.strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:10]}" + + +def _json_loads_any(value: Any, default: Any = None) -> Any: + if value is None: + return default + if isinstance(value, (dict, list)): + return value + text = str(value or "").strip() + if not text: + return default + try: + return json.loads(text) + except Exception: + return default + + +def _parse_object(value: Any, default: Optional[dict[str, Any]] = None) -> dict[str, Any]: + if isinstance(value, dict): + return value + parsed = _json_loads_any(value, default={}) + return parsed if isinstance(parsed, dict) else (default or {}) + + +def _parse_schema(value: str) -> Optional[dict[str, Any]]: + parsed = _json_loads_any(value, default=None) + return parsed if isinstance(parsed, dict) and parsed else None + + +def _extract_json_from_text(text: str) -> Any: + """Best-effort extraction of JSON from Gemini text responses.""" + raw = (text or "").strip() + if not raw: + return None + + # Remove common Markdown code fences. + fenced = re.search(r"```(?:json)?\s*(.*?)```", raw, flags=re.IGNORECASE | re.DOTALL) + if fenced: + raw = fenced.group(1).strip() + + try: + return json.loads(raw) + except Exception: + pass + + # Try first JSON object. + start = raw.find("{") + end = raw.rfind("}") + if start >= 0 and end > start: + try: + return json.loads(raw[start : end + 1]) + except Exception: + pass + + # Try first JSON array. + start = raw.find("[") + end = raw.rfind("]") + if start >= 0 and end > start: + try: + return json.loads(raw[start : end + 1]) + except Exception: + pass + + return None + + +def _payload_data(payload: Any) -> dict[str, Any]: + if isinstance(payload, dict) and isinstance(payload.get("data"), dict): + return payload["data"] + return {} + + +def _extract_document_id(payload: Any) -> str: + data = _payload_data(payload) + candidates = ( + data.get("document_id"), + data.get("_id"), + data.get("id"), + data.get("inserted_id"), + data.get("upserted_id"), + data.get("doc_id"), + ) + for candidate in candidates: + if candidate: + return str(candidate) + + if isinstance(data.get("document"), dict): + doc = data["document"] + for key in ("_id", "id", "document_id"): + if doc.get(key): + return str(doc[key]) + + return "" + + +def _extract_documents(payload: Any) -> list[dict[str, Any]]: + data = _payload_data(payload) + + for key in ("documents", "docs", "items", "results", "records"): + value = data.get(key) + if isinstance(value, list): + return [item for item in value if isinstance(item, dict)] + + value = data.get("document") + if isinstance(value, dict): + return [value] + + if isinstance(data, list): + return [item for item in data if isinstance(item, dict)] + + return [] + + +def _limit_text(text: str, max_chars: int) -> str: + clean = str(text or "") + if max_chars <= 0 or len(clean) <= max_chars: + return clean + return clean[: max_chars - 40].rstrip() + "\n...[truncated]" + + +def _get_by_paths(document: dict[str, Any], field_paths: list[str]) -> dict[str, Any]: + result: dict[str, Any] = {} + for path in field_paths: + clean_path = path.strip().strip(".") + if not clean_path: + continue + value = _safe_get_by_path(document, clean_path) + if value is not None: + result[clean_path] = value + return result + + +def _stringify_record(record: dict[str, Any], field_paths: list[str], max_chars_per_record: int) -> str: + doc_id = str(record.get("_id") or record.get("id") or record.get("document_id") or "") + selected = _get_by_paths(record, field_paths) if field_paths else record + if not selected: + selected = record + text = json.dumps( + {"document_id": doc_id, "fields": selected}, + ensure_ascii=False, + indent=2, + default=str, + ) + return _limit_text(text, max_chars_per_record) + + +def _extract_score(parsed: Any) -> float: + if not isinstance(parsed, dict): + return 0.0 + + direct = parsed.get("score") + if isinstance(direct, (int, float)): + return float(direct) + + scores = parsed.get("scores") + if isinstance(scores, dict): + numeric = [float(v) for v in scores.values() if isinstance(v, (int, float))] + if numeric: + return round(sum(numeric) / len(numeric), 2) + + return 0.0 + + +def _extract_best_prompt(parsed: Any, fallback: str = "") -> str: + if isinstance(parsed, dict): + for path in ( + "best.positive_prompt", + "best.prompt", + "selected.positive_prompt", + "selected.prompt", + "revised_prompt", + "positive_prompt", + "prompt", + ): + value = _safe_get_by_path(parsed, path) + if isinstance(value, str) and value.strip(): + return value.strip() + + variants = parsed.get("variants") + if isinstance(variants, list) and variants: + first = variants[0] + if isinstance(first, dict): + for key in ("positive_prompt", "prompt", "text"): + if isinstance(first.get(key), str) and first[key].strip(): + return first[key].strip() + + return fallback + + +def _extract_negative_prompt(parsed: Any) -> str: + if isinstance(parsed, dict): + for path in ( + "best.negative_prompt", + "selected.negative_prompt", + "negative_prompt", + ): + value = _safe_get_by_path(parsed, path) + if isinstance(value, str): + return value.strip() + + variants = parsed.get("variants") + if isinstance(variants, list) and variants: + first = variants[0] + if isinstance(first, dict) and isinstance(first.get("negative_prompt"), str): + return first["negative_prompt"].strip() + return "" + + +def _agent_config( + session: Any, + *, + project_name: str, + memory_collection: str, + ledger_collection: str, + capsule_collection: str, + context_collection: str, + model: str, + max_output_tokens: int, + temperature: float, + system_instruction: str, + gemini_prefix: str, + zmongo_prefix: str, +) -> dict[str, Any]: + return { + "session": session, + "project_name": _clean(project_name, "default"), + "memory_collection": _clean(memory_collection, DEFAULT_MEMORY_COLLECTION), + "ledger_collection": _clean(ledger_collection, DEFAULT_LEDGER_COLLECTION), + "capsule_collection": _clean(capsule_collection, DEFAULT_CAPSULE_COLLECTION), + "context_collection": _clean(context_collection, DEFAULT_CONTEXT_COLLECTION), + "model": _clean(model, DEFAULT_MODEL), + "max_output_tokens": _safe_int(max_output_tokens, 2048, 1, 65536), + "temperature": _safe_float(temperature, 0.4, 0.0, 2.0), + "system_instruction": system_instruction or "", + "gemini_prefix": _clean(gemini_prefix, DEFAULT_GEMINI_PREFIX), + "zmongo_prefix": _clean(zmongo_prefix, DEFAULT_COMFY_ZMONGO_PREFIX), + } + + +def _resolve_config(config: Any, session: Any = None) -> dict[str, Any]: + if isinstance(config, dict): + resolved = dict(config) + else: + resolved = {} + if session is not None: + resolved["session"] = session + resolved.setdefault("project_name", "default") + resolved.setdefault("memory_collection", DEFAULT_MEMORY_COLLECTION) + resolved.setdefault("ledger_collection", DEFAULT_LEDGER_COLLECTION) + resolved.setdefault("capsule_collection", DEFAULT_CAPSULE_COLLECTION) + resolved.setdefault("context_collection", DEFAULT_CONTEXT_COLLECTION) + resolved.setdefault("model", DEFAULT_MODEL) + resolved.setdefault("max_output_tokens", 2048) + resolved.setdefault("temperature", 0.4) + resolved.setdefault("system_instruction", "") + resolved.setdefault("gemini_prefix", DEFAULT_GEMINI_PREFIX) + resolved.setdefault("zmongo_prefix", DEFAULT_COMFY_ZMONGO_PREFIX) + return resolved + + +def _require_session(config: dict[str, Any]) -> Any: + session = config.get("session") + if session is None: + raise ValueError("Missing ZMongo API session. Connect a ZMongo API Key Session node.") + return session + + +def _query_docs( + session: Any, + *, + collection: str, + query: Optional[dict[str, Any]] = None, + limit: int = 20, + skip: int = 0, + projection: Optional[dict[str, Any]] = None, + sort: Optional[list[Any]] = None, + zmongo_prefix: str = DEFAULT_COMFY_ZMONGO_PREFIX, +) -> dict[str, Any]: + body: dict[str, Any] = { + "collection": collection, + "query": query or {}, + "many": True, + "limit": _safe_int(limit, 20, 1, 500), + "skip": max(0, int(skip or 0)), + "cache": False, + } + if projection: + body["projection"] = projection + if sort: + body["sort"] = sort + return _session_api_request( + session, + "POST", + "/api/query", + json_body=body, + gemini_prefix=zmongo_prefix, + ) + + +def _create_doc( + session: Any, + *, + collection: str, + document: dict[str, Any], + zmongo_prefix: str = DEFAULT_COMFY_ZMONGO_PREFIX, +) -> dict[str, Any]: + return _session_api_request( + session, + "POST", + "/api/doc/create", + json_body={"collection": collection, "document": document}, + gemini_prefix=zmongo_prefix, + ) + + +def _save_value( + session: Any, + *, + collection: str, + document_id: str, + field_path: str, + value: Any, + upsert: bool = True, + query: Optional[dict[str, Any]] = None, + zmongo_prefix: str = DEFAULT_COMFY_ZMONGO_PREFIX, +) -> dict[str, Any]: + body: dict[str, Any] = { + "collection": collection, + "query": query or {}, + "field_path": field_path, + "value": value, + "upsert_if_missing": bool(upsert), + "parse_json_strings": False, + "normalize_for_storage": False, + } + if document_id: + body["document_id"] = document_id + return _session_api_request( + session, + "POST", + "/api/save-value", + json_body=body, + gemini_prefix=zmongo_prefix, + ) + + +def _gemini_chat( + session: Any, + *, + prompt: str, + model: str, + max_output_tokens: int, + temperature: float, + system_instruction: str = "", + response_schema: Optional[dict[str, Any]] = None, + gemini_prefix: str = DEFAULT_GEMINI_PREFIX, +) -> dict[str, Any]: + body: dict[str, Any] = { + "prompt": prompt or "", + "model": _clean(model, DEFAULT_MODEL), + "max_output_tokens": _safe_int(max_output_tokens, 2048, 1, 65536), + "temperature": _safe_float(temperature, 0.4, 0.0, 2.0), + } + if (system_instruction or "").strip(): + body["system_instruction"] = system_instruction.strip() + if response_schema: + body["response_schema"] = response_schema + body["response_mime_type"] = "application/json" + + return _session_api_request( + session, + "POST", + "/api/chat", + json_body=body, + gemini_prefix=gemini_prefix, + ) + + +def _save_agent_record( + session: Any, + *, + collection: str, + project_name: str, + record_type: str, + payload: dict[str, Any], + zmongo_prefix: str, +) -> tuple[dict[str, Any], str]: + doc = { + "project_name": project_name, + "record_type": record_type, + "created_at": _utc_now(), + "updated_at": _utc_now(), + "payload": payload, + } + save_payload = _create_doc( + session, + collection=collection, + document=doc, + zmongo_prefix=zmongo_prefix, + ) + return save_payload, _extract_document_id(save_payload) + + +def _context_records_text(records: list[dict[str, Any]], field_paths: list[str], max_chars: int) -> str: + if not records: + return "" + max_per_record = max(500, int(max_chars / max(1, len(records)))) + chunks = [] + for index, record in enumerate(records, start=1): + chunks.append(f"[Memory {index}]\n{_stringify_record(record, field_paths, max_per_record)}") + return _limit_text("\n\n".join(chunks), max_chars) + + +def _json_schema_for_prompt_audit() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "score": {"type": "number"}, + "scores": {"type": "object"}, + "strengths": {"type": "array", "items": {"type": "string"}}, + "weaknesses": {"type": "array", "items": {"type": "string"}}, + "recommendations": {"type": "array", "items": {"type": "string"}}, + "revised_prompt": {"type": "string"}, + "negative_prompt": {"type": "string"}, + "backend_paid_value": {"type": "string"}, + }, + "required": ["score", "scores", "strengths", "weaknesses", "recommendations", "revised_prompt"], + } + + +def _json_schema_for_prompt_evolution() -> dict[str, Any]: + return { + "type": "object", + "properties": { + "best": { + "type": "object", + "properties": { + "index": {"type": "integer"}, + "positive_prompt": {"type": "string"}, + "negative_prompt": {"type": "string"}, + "reason": {"type": "string"}, + "score": {"type": "number"}, + }, + }, + "variants": { + "type": "array", + "items": { + "type": "object", + "properties": { + "index": {"type": "integer"}, + "positive_prompt": {"type": "string"}, + "negative_prompt": {"type": "string"}, + "reason": {"type": "string"}, + "score": {"type": "number"}, + }, + "required": ["index", "positive_prompt", "reason", "score"], + }, + }, + }, + "required": ["best", "variants"], + } + + +# ----------------------------------------------------------------------------- +# Nodes +# ----------------------------------------------------------------------------- + + +class ZMongoTextAgentSessionNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "session": ("ZMONGO_API_SESSION",), + "project_name": ("STRING", {"default": "default", "multiline": False}), + "model": ("STRING", {"default": DEFAULT_MODEL}), + "max_output_tokens": ("INT", {"default": 2048, "min": 1, "max": 65536}), + "temperature": ("FLOAT", {"default": 0.4, "min": 0.0, "max": 2.0, "step": 0.05}), + }, + "optional": { + "memory_collection": ("STRING", {"default": DEFAULT_MEMORY_COLLECTION}), + "ledger_collection": ("STRING", {"default": DEFAULT_LEDGER_COLLECTION}), + "capsule_collection": ("STRING", {"default": DEFAULT_CAPSULE_COLLECTION}), + "context_collection": ("STRING", {"default": DEFAULT_CONTEXT_COLLECTION}), + "system_instruction": ("STRING", {"default": "You are a precise ComfyUI prompt and workflow text agent. Return compact, useful output.", "multiline": True}), + "gemini_prefix": ("STRING", {"default": DEFAULT_GEMINI_PREFIX}), + "zmongo_prefix": ("STRING", {"default": DEFAULT_COMFY_ZMONGO_PREFIX}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("ZMONGO_TEXT_AGENT_CONFIG", "STRING", "STRING") + RETURN_NAMES = ("text_agent_config", "project_name", "json") + FUNCTION = "build" + CATEGORY = TEXT_AGENT_CATEGORY + + def build( + self, + session, + project_name: str, + model: str, + max_output_tokens: int, + temperature: float, + memory_collection: str = DEFAULT_MEMORY_COLLECTION, + ledger_collection: str = DEFAULT_LEDGER_COLLECTION, + capsule_collection: str = DEFAULT_CAPSULE_COLLECTION, + context_collection: str = DEFAULT_CONTEXT_COLLECTION, + system_instruction: str = "", + gemini_prefix: str = DEFAULT_GEMINI_PREFIX, + zmongo_prefix: str = DEFAULT_COMFY_ZMONGO_PREFIX, + refresh_token: str = "", + ): + config = _agent_config( + session, + project_name=project_name, + memory_collection=memory_collection, + ledger_collection=ledger_collection, + capsule_collection=capsule_collection, + context_collection=context_collection, + model=model, + max_output_tokens=max_output_tokens, + temperature=temperature, + system_instruction=system_instruction, + gemini_prefix=gemini_prefix, + zmongo_prefix=zmongo_prefix, + ) + public = {key: value for key, value in config.items() if key != "session"} + public["refresh"] = _dirty_token("text_agent_session", refresh_token) + return (config, config["project_name"], _json_text(_success_payload("Text agent session configured.", public))) + + +class ZMongoSavePromptMemoryNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "memory_name": ("STRING", {"default": "new_memory"}), + "text": ("STRING", {"default": "", "multiline": True}), + "memory_type": ("STRING", {"default": "prompt"}), + }, + "optional": { + "tags_csv": ("STRING", {"default": "prompt,comfyui"}), + "notes": ("STRING", {"default": "", "multiline": True}), + "collection_override": ("STRING", {"default": ""}), + "extra_json": ("STRING", {"default": "{}", "multiline": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "BOOLEAN", "STRING") + RETURN_NAMES = ("json", "document_id", "success", "refresh") + FUNCTION = "save_memory" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Memory" + + def save_memory( + self, + text_agent_config, + memory_name: str, + text: str, + memory_type: str, + tags_csv: str = "", + notes: str = "", + collection_override: str = "", + extra_json: str = "{}", + refresh_token: str = "", + ): + refresh = _dirty_token("save_prompt_memory", refresh_token) + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + collection = _clean(collection_override, config["memory_collection"]) + extra = _parse_object(extra_json) + + document = { + "project_name": config["project_name"], + "memory_name": _clean(memory_name, "new_memory"), + "memory_type": _clean(memory_type, "prompt"), + "tags": _csv_list(tags_csv), + "text": text or "", + "notes": notes or "", + "extra": extra, + "created_at": _utc_now(), + "updated_at": _utc_now(), + "source": "ComfyUI-ZMongo Text Agents", + } + payload = _create_doc(session, collection=collection, document=document, zmongo_prefix=config["zmongo_prefix"]) + document_id = _extract_document_id(payload) + return (_json_text(payload), document_id, bool(payload.get("success")), refresh) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return (_json_text(payload), "", False, refresh) + + +class ZMongoLoadMemoryCapsuleNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "capsule_name": ("STRING", {"default": "default_style"}), + }, + "optional": { + "collection_override": ("STRING", {"default": ""}), + "field_paths_csv": ("STRING", {"default": "capsule_text,text,summary,payload.context_pack"}), + "max_chars": ("INT", {"default": 6000, "min": 500, "max": 64000}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING", "BOOLEAN") + RETURN_NAMES = ("capsule_context", "json", "document_id", "success") + FUNCTION = "load_capsule" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Memory" + + def load_capsule( + self, + text_agent_config, + capsule_name: str, + collection_override: str = "", + field_paths_csv: str = "capsule_text,text,summary,payload.context_pack", + max_chars: int = 6000, + refresh_token: str = "", + ): + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + collection = _clean(collection_override, config["capsule_collection"]) + query = {"project_name": config["project_name"], "capsule_name": _clean(capsule_name)} + payload = _query_docs( + session, + collection=collection, + query=query, + limit=1, + sort=[["updated_at", -1]], + zmongo_prefix=config["zmongo_prefix"], + ) + docs = _extract_documents(payload) + field_paths = _csv_list(field_paths_csv) + context = _context_records_text(docs, field_paths, _safe_int(max_chars, 6000, 500, 64000)) + document_id = str(docs[0].get("_id") or docs[0].get("document_id") or "") if docs else "" + merged = _success_payload( + "Memory capsule loaded." if docs else "No memory capsule matched the requested name.", + {"query_payload": payload, "capsule_name": capsule_name, "document_id": document_id, "refresh": _dirty_token("load_capsule", refresh_token)}, + ) + merged["success"] = bool(docs) and bool(payload.get("success", True)) + return (context, _json_text(merged), document_id, bool(merged["success"])) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return ("", _json_text(payload), "", False) + + +class ZMongoBuildContextPackNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "query_text": ("STRING", {"default": "", "multiline": True}), + "source_collection": ("STRING", {"default": DEFAULT_MEMORY_COLLECTION}), + "limit": ("INT", {"default": 8, "min": 1, "max": 50}), + }, + "optional": { + "field_paths_csv": ("STRING", {"default": "memory_name,memory_type,tags,text,summary,notes,payload"}), + "max_source_chars": ("INT", {"default": 12000, "min": 1000, "max": 64000}), + "save_context_pack": ("BOOLEAN", {"default": True}), + "context_pack_name": ("STRING", {"default": "latest_context_pack"}), + "extra_filter_json": ("STRING", {"default": "{}", "multiline": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING", "BOOLEAN") + RETURN_NAMES = ("context_pack", "json", "context_document_id", "success") + FUNCTION = "build_context" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Memory" + + def build_context( + self, + text_agent_config, + query_text: str, + source_collection: str, + limit: int, + field_paths_csv: str = "memory_name,memory_type,tags,text,summary,notes,payload", + max_source_chars: int = 12000, + save_context_pack: bool = True, + context_pack_name: str = "latest_context_pack", + extra_filter_json: str = "{}", + refresh_token: str = "", + ): + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + extra_filter = _parse_object(extra_filter_json) + query = {"project_name": config["project_name"]} + query.update(extra_filter) + + source_payload = _query_docs( + session, + collection=_clean(source_collection, config["memory_collection"]), + query=query, + limit=limit, + sort=[["updated_at", -1]], + zmongo_prefix=config["zmongo_prefix"], + ) + records = _extract_documents(source_payload) + field_paths = _csv_list(field_paths_csv) + raw_context = _context_records_text(records, field_paths, _safe_int(max_source_chars, 12000, 1000, 64000)) + + prompt = f""" +Build a compact ComfyUI prompt-memory context pack. + +User query: +{query_text or "(no query provided)"} + +Project: +{config["project_name"]} + +Source records: +{raw_context} + +Return useful context only. Include: +- reusable style constraints +- prompt fragments worth reusing +- project-specific warnings +- prior successful patterns +- exact memory/document references when available + +Do not invent stored facts. If records are weak, say what is missing. +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=config["temperature"], + system_instruction=config["system_instruction"], + gemini_prefix=config["gemini_prefix"], + ) + context_pack = _extract_text_from_gemini_payload(gemini_payload) + context_document_id = "" + + save_payload: dict[str, Any] = {} + if save_context_pack and gemini_payload.get("success"): + save_payload, context_document_id = _save_agent_record( + session, + collection=config["context_collection"], + project_name=config["project_name"], + record_type="context_pack", + payload={ + "context_pack_name": _clean(context_pack_name, "latest_context_pack"), + "query_text": query_text or "", + "source_collection": source_collection, + "source_document_ids": [str(doc.get("_id") or doc.get("document_id") or "") for doc in records], + "field_paths": field_paths, + "context_pack": context_pack, + "source_count": len(records), + }, + zmongo_prefix=config["zmongo_prefix"], + ) + + merged = _success_payload( + "Context pack built.", + { + "source_payload": source_payload, + "gemini_payload": gemini_payload, + "save_payload": save_payload, + "source_count": len(records), + "context_document_id": context_document_id, + "refresh": _dirty_token("build_context_pack", refresh_token), + }, + ) + merged["success"] = bool(gemini_payload.get("success")) and (not save_context_pack or bool(save_payload.get("success"))) + return (context_pack, _json_text(merged), context_document_id, bool(merged["success"])) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return ("", _json_text(payload), "", False) + + +class ZMongoPromptCriticNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "prompt": ("STRING", {"default": "", "multiline": True}), + }, + "optional": { + "context_pack": ("STRING", {"default": "", "multiline": True}), + "target_model": ("STRING", {"default": "ComfyUI image/video model"}), + "rubric": ("STRING", {"default": "Score clarity, visual specificity, consistency, contradiction risk, and model fit.", "multiline": True}), + "save_to_ledger": ("BOOLEAN", {"default": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "FLOAT", "BOOLEAN", "STRING") + RETURN_NAMES = ("json", "revised_prompt", "score", "success", "refresh") + FUNCTION = "critique" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Prompt Agents" + + def critique( + self, + text_agent_config, + prompt: str, + context_pack: str = "", + target_model: str = "ComfyUI image/video model", + rubric: str = "", + save_to_ledger: bool = True, + refresh_token: str = "", + ): + refresh = _dirty_token("prompt_critic", refresh_token) + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + schema = _json_schema_for_prompt_audit() + critic_prompt = f""" +Critique this ComfyUI generation prompt as a professional prompt engineer. + +Target model: +{target_model} + +Rubric: +{rubric or "Score clarity, visual specificity, consistency, contradiction risk, and model fit."} + +Project context: +{context_pack or "(none)"} + +Prompt: +{prompt or ""} + +Return JSON only. The revised_prompt should be directly usable as a ComfyUI positive prompt. +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=critic_prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=0.2, + system_instruction=config["system_instruction"], + response_schema=schema, + gemini_prefix=config["gemini_prefix"], + ) + text = _extract_text_from_gemini_payload(gemini_payload) + parsed = _extract_json_from_text(text) + if parsed is None: + parsed = {"raw_text": text, "score": 0, "revised_prompt": prompt or ""} + + score = _extract_score(parsed) + revised_prompt = _extract_best_prompt(parsed, prompt or "") + + save_payload: dict[str, Any] = {} + ledger_id = "" + if save_to_ledger and gemini_payload.get("success"): + save_payload, ledger_id = _save_agent_record( + session, + collection=config["ledger_collection"], + project_name=config["project_name"], + record_type="prompt_critic", + payload={ + "run_id": _new_run_id("critic"), + "original_prompt": prompt or "", + "context_pack": context_pack or "", + "target_model": target_model, + "rubric": rubric, + "audit": parsed, + "score": score, + "revised_prompt": revised_prompt, + }, + zmongo_prefix=config["zmongo_prefix"], + ) + + merged = _success_payload( + "Prompt critique completed.", + { + "audit": parsed, + "score": score, + "revised_prompt": revised_prompt, + "gemini_payload": gemini_payload, + "save_payload": save_payload, + "ledger_id": ledger_id, + "refresh": refresh, + }, + ) + merged["success"] = bool(gemini_payload.get("success")) and (not save_to_ledger or bool(save_payload.get("success"))) + return (_json_text(merged), revised_prompt, float(score), bool(merged["success"]), refresh) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return (_json_text(payload), prompt or "", 0.0, False, refresh) + + +class ZMongoPromptEvolverNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "base_prompt": ("STRING", {"default": "", "multiline": True}), + "variant_count": ("INT", {"default": 4, "min": 1, "max": 20}), + }, + "optional": { + "context_pack": ("STRING", {"default": "", "multiline": True}), + "mutation_mode": ("STRING", {"default": "cinematic, specific, production-ready"}), + "target_model": ("STRING", {"default": "ComfyUI image/video model"}), + "save_to_ledger": ("BOOLEAN", {"default": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING", "BOOLEAN", "STRING") + RETURN_NAMES = ("best_positive_prompt", "best_negative_prompt", "json", "success", "refresh") + FUNCTION = "evolve" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Prompt Agents" + + def evolve( + self, + text_agent_config, + base_prompt: str, + variant_count: int, + context_pack: str = "", + mutation_mode: str = "cinematic, specific, production-ready", + target_model: str = "ComfyUI image/video model", + save_to_ledger: bool = True, + refresh_token: str = "", + ): + refresh = _dirty_token("prompt_evolver", refresh_token) + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + count = _safe_int(variant_count, 4, 1, 20) + + evolve_prompt = f""" +Create {count} improved ComfyUI prompt variants and select the best one. + +Target model: +{target_model} + +Mutation mode: +{mutation_mode} + +Project memory/context: +{context_pack or "(none)"} + +Base prompt: +{base_prompt or ""} + +Return JSON only. Each variant must include: +- index +- positive_prompt +- negative_prompt +- reason +- score + +The best positive_prompt must be directly usable in ComfyUI. +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=evolve_prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=max(0.2, float(config["temperature"])), + system_instruction=config["system_instruction"], + response_schema=_json_schema_for_prompt_evolution(), + gemini_prefix=config["gemini_prefix"], + ) + text = _extract_text_from_gemini_payload(gemini_payload) + parsed = _extract_json_from_text(text) + if parsed is None: + parsed = { + "best": {"index": 1, "positive_prompt": base_prompt or "", "negative_prompt": "", "reason": "Gemini did not return parseable JSON.", "score": 0}, + "variants": [], + "raw_text": text, + } + + best_positive = _extract_best_prompt(parsed, base_prompt or "") + best_negative = _extract_negative_prompt(parsed) + + save_payload: dict[str, Any] = {} + ledger_id = "" + if save_to_ledger and gemini_payload.get("success"): + save_payload, ledger_id = _save_agent_record( + session, + collection=config["ledger_collection"], + project_name=config["project_name"], + record_type="prompt_evolution", + payload={ + "run_id": _new_run_id("evolve"), + "base_prompt": base_prompt or "", + "context_pack": context_pack or "", + "mutation_mode": mutation_mode, + "target_model": target_model, + "variant_count": count, + "evolution": parsed, + "best_positive_prompt": best_positive, + "best_negative_prompt": best_negative, + }, + zmongo_prefix=config["zmongo_prefix"], + ) + + merged = _success_payload( + "Prompt evolution completed.", + { + "evolution": parsed, + "best_positive_prompt": best_positive, + "best_negative_prompt": best_negative, + "gemini_payload": gemini_payload, + "save_payload": save_payload, + "ledger_id": ledger_id, + "refresh": refresh, + }, + ) + merged["success"] = bool(gemini_payload.get("success")) and (not save_to_ledger or bool(save_payload.get("success"))) + return (best_positive, best_negative, _json_text(merged), bool(merged["success"]), refresh) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return (base_prompt or "", "", _json_text(payload), False, refresh) + + +class ZMongoBestVariantSelectorNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "variants_json": ("STRING", {"default": "{}", "multiline": True}), + }, + "optional": { + "fallback_prompt": ("STRING", {"default": "", "multiline": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "FLOAT", "STRING") + RETURN_NAMES = ("positive_prompt", "negative_prompt", "score", "json") + FUNCTION = "select_best" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Prompt Agents" + + def select_best(self, variants_json: str, fallback_prompt: str = "", refresh_token: str = ""): + parsed = _extract_json_from_text(variants_json) or _parse_object(variants_json) + # Accept either raw evolution object or full wrapper from Prompt Evolver. + evolution = _safe_get_by_path(parsed, "data.evolution") if isinstance(parsed, dict) else None + if isinstance(evolution, dict): + parsed = evolution + + positive = _extract_best_prompt(parsed, fallback_prompt or "") + negative = _extract_negative_prompt(parsed) + score = _extract_score(_safe_get_by_path(parsed, "best") if isinstance(parsed, dict) else parsed) + + payload = _success_payload( + "Best variant selected.", + {"positive_prompt": positive, "negative_prompt": negative, "score": score, "refresh": _dirty_token("best_variant", refresh_token)}, + ) + return (positive, negative, float(score), _json_text(payload)) + + +class ZMongoPromptProvenanceLedgerNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "original_prompt": ("STRING", {"default": "", "multiline": True}), + "final_prompt": ("STRING", {"default": "", "multiline": True}), + }, + "optional": { + "negative_prompt": ("STRING", {"default": "", "multiline": True}), + "context_pack": ("STRING", {"default": "", "multiline": True}), + "image_document_id": ("STRING", {"default": ""}), + "workflow_id": ("STRING", {"default": ""}), + "notes": ("STRING", {"default": "", "multiline": True}), + "extra_json": ("STRING", {"default": "{}", "multiline": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "BOOLEAN", "STRING") + RETURN_NAMES = ("json", "ledger_document_id", "success", "refresh") + FUNCTION = "save_ledger" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Provenance" + + def save_ledger( + self, + text_agent_config, + original_prompt: str, + final_prompt: str, + negative_prompt: str = "", + context_pack: str = "", + image_document_id: str = "", + workflow_id: str = "", + notes: str = "", + extra_json: str = "{}", + refresh_token: str = "", + ): + refresh = _dirty_token("prompt_provenance", refresh_token) + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + payload = { + "run_id": _new_run_id("provenance"), + "original_prompt": original_prompt or "", + "final_prompt": final_prompt or "", + "negative_prompt": negative_prompt or "", + "context_pack": context_pack or "", + "image_document_id": image_document_id or "", + "workflow_id": workflow_id or "", + "notes": notes or "", + "extra": _parse_object(extra_json), + } + save_payload, ledger_id = _save_agent_record( + session, + collection=config["ledger_collection"], + project_name=config["project_name"], + record_type="prompt_provenance", + payload=payload, + zmongo_prefix=config["zmongo_prefix"], + ) + return (_json_text(save_payload), ledger_id, bool(save_payload.get("success")), refresh) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return (_json_text(payload), "", False, refresh) + + +class ZMongoFindSimilarSuccessfulPromptsNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "query_prompt": ("STRING", {"default": "", "multiline": True}), + }, + "optional": { + "source_collection": ("STRING", {"default": DEFAULT_LEDGER_COLLECTION}), + "limit": ("INT", {"default": 12, "min": 1, "max": 50}), + "field_paths_csv": ("STRING", {"default": "payload.final_prompt,payload.best_positive_prompt,payload.evolution.best.positive_prompt,payload.score,payload.notes,record_type,created_at"}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "BOOLEAN") + RETURN_NAMES = ("ranked_context", "json", "success") + FUNCTION = "find_similar" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Provenance" + + def find_similar( + self, + text_agent_config, + query_prompt: str, + source_collection: str = DEFAULT_LEDGER_COLLECTION, + limit: int = 12, + field_paths_csv: str = "payload.final_prompt,payload.best_positive_prompt,payload.evolution.best.positive_prompt,payload.score,payload.notes,record_type,created_at", + refresh_token: str = "", + ): + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + source_payload = _query_docs( + session, + collection=_clean(source_collection, config["ledger_collection"]), + query={"project_name": config["project_name"]}, + limit=limit, + sort=[["created_at", -1]], + zmongo_prefix=config["zmongo_prefix"], + ) + records = _extract_documents(source_payload) + raw_context = _context_records_text(records, _csv_list(field_paths_csv), 16000) + + prompt = f""" +Rank the stored prompt records by usefulness for improving this new prompt. + +New prompt: +{query_prompt} + +Stored records: +{raw_context} + +Return a compact ranked list. Include exact document references when present. Focus on successful reusable prompt fragments and warnings. +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=0.2, + system_instruction=config["system_instruction"], + gemini_prefix=config["gemini_prefix"], + ) + ranked_context = _extract_text_from_gemini_payload(gemini_payload) + merged = _success_payload( + "Similar successful prompts ranked.", + { + "source_payload": source_payload, + "gemini_payload": gemini_payload, + "source_count": len(records), + "refresh": _dirty_token("find_similar", refresh_token), + }, + ) + merged["success"] = bool(gemini_payload.get("success")) + return (ranked_context, _json_text(merged), bool(merged["success"])) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return ("", _json_text(payload), False) + + +class ZMongoCaptionPackGeneratorNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "source_prompt": ("STRING", {"default": "", "multiline": True}), + }, + "optional": { + "context_pack": ("STRING", {"default": "", "multiline": True}), + "platform": ("STRING", {"default": "YouTube Shorts"}), + "tone": ("STRING", {"default": "clear, energetic, non-clickbait"}), + "save_to_ledger": ("BOOLEAN", {"default": True}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING", "BOOLEAN") + RETURN_NAMES = ("caption_text", "json", "ledger_document_id", "success") + FUNCTION = "caption_pack" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Publishing" + + def caption_pack( + self, + text_agent_config, + source_prompt: str, + context_pack: str = "", + platform: str = "YouTube Shorts", + tone: str = "clear, energetic, non-clickbait", + save_to_ledger: bool = True, + refresh_token: str = "", + ): + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + + prompt = f""" +Create a publishing caption pack for a ComfyUI output. + +Platform: +{platform} + +Tone: +{tone} + +Project context: +{context_pack or "(none)"} + +Source prompt / image description: +{source_prompt or ""} + +Return: +- title options +- short caption +- description +- hashtags/tags +- alt text +- one-line internal note about what memory should be saved +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=config["temperature"], + system_instruction=config["system_instruction"], + gemini_prefix=config["gemini_prefix"], + ) + caption_text = _extract_text_from_gemini_payload(gemini_payload) + save_payload: dict[str, Any] = {} + ledger_id = "" + + if save_to_ledger and gemini_payload.get("success"): + save_payload, ledger_id = _save_agent_record( + session, + collection=config["ledger_collection"], + project_name=config["project_name"], + record_type="caption_pack", + payload={ + "run_id": _new_run_id("caption"), + "source_prompt": source_prompt or "", + "context_pack": context_pack or "", + "platform": platform, + "tone": tone, + "caption_pack": caption_text, + }, + zmongo_prefix=config["zmongo_prefix"], + ) + + merged = _success_payload( + "Caption pack generated.", + {"gemini_payload": gemini_payload, "save_payload": save_payload, "ledger_id": ledger_id, "refresh": _dirty_token("caption_pack", refresh_token)}, + ) + merged["success"] = bool(gemini_payload.get("success")) and (not save_to_ledger or bool(save_payload.get("success"))) + return (caption_text, _json_text(merged), ledger_id, bool(merged["success"])) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return ("", _json_text(payload), "", False) + + +class ZMongoWorkflowDocumentationAgentNode(AlwaysDirtyMixin): + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "text_agent_config": ("ZMONGO_TEXT_AGENT_CONFIG",), + "workflow_name": ("STRING", {"default": "ComfyUI-ZMongo Workflow"}), + "workflow_summary": ("STRING", {"default": "", "multiline": True}), + }, + "optional": { + "context_pack": ("STRING", {"default": "", "multiline": True}), + "include_sections_csv": ("STRING", {"default": "overview,requirements,node setup,usage steps,troubleshooting,release notes"}), + "save_to_collection": ("BOOLEAN", {"default": True}), + "collection_override": ("STRING", {"default": DEFAULT_DOC_COLLECTION}), + "refresh_token": ("STRING", {"default": ""}), + }, + } + + RETURN_TYPES = ("STRING", "STRING", "STRING", "BOOLEAN") + RETURN_NAMES = ("markdown", "json", "document_id", "success") + FUNCTION = "write_docs" + CATEGORY = f"{TEXT_AGENT_CATEGORY}/Publishing" + + def write_docs( + self, + text_agent_config, + workflow_name: str, + workflow_summary: str, + context_pack: str = "", + include_sections_csv: str = "overview,requirements,node setup,usage steps,troubleshooting,release notes", + save_to_collection: bool = True, + collection_override: str = DEFAULT_DOC_COLLECTION, + refresh_token: str = "", + ): + try: + config = _resolve_config(text_agent_config) + session = _require_session(config) + sections = _csv_list(include_sections_csv) + + prompt = f""" +Write professional Markdown documentation for a ComfyUI workflow. + +Workflow name: +{workflow_name} + +Requested sections: +{sections} + +Workflow summary / raw notes: +{workflow_summary or ""} + +Project context: +{context_pack or "(none)"} + +Requirements: +- Use clear markdown headings. +- Include a setup checklist. +- Explain what the hosted backend does and why an API account is needed. +- Include troubleshooting for auth/session/API-key/context-pack failures. +- Do not include code unless necessary. +""".strip() + + gemini_payload = _gemini_chat( + session, + prompt=prompt, + model=config["model"], + max_output_tokens=config["max_output_tokens"], + temperature=0.3, + system_instruction=config["system_instruction"], + gemini_prefix=config["gemini_prefix"], + ) + markdown = _extract_text_from_gemini_payload(gemini_payload) + save_payload: dict[str, Any] = {} + document_id = "" + + if save_to_collection and gemini_payload.get("success"): + save_payload, document_id = _save_agent_record( + session, + collection=_clean(collection_override, DEFAULT_DOC_COLLECTION), + project_name=config["project_name"], + record_type="workflow_documentation", + payload={ + "workflow_name": workflow_name, + "workflow_summary": workflow_summary, + "context_pack": context_pack, + "sections": sections, + "markdown": markdown, + }, + zmongo_prefix=config["zmongo_prefix"], + ) + + merged = _success_payload( + "Workflow documentation generated.", + {"gemini_payload": gemini_payload, "save_payload": save_payload, "document_id": document_id, "refresh": _dirty_token("workflow_docs", refresh_token)}, + ) + merged["success"] = bool(gemini_payload.get("success")) and (not save_to_collection or bool(save_payload.get("success"))) + return (markdown, _json_text(merged), document_id, bool(merged["success"])) + except Exception as exc: + payload = _error_payload(str(exc), error_type=exc.__class__.__name__) + return ("", _json_text(payload), "", False) + + +# ----------------------------------------------------------------------------- +# ComfyUI mappings +# ----------------------------------------------------------------------------- + + +NODE_CLASS_MAPPINGS = { + "ZMongoTextAgentSessionNode": ZMongoTextAgentSessionNode, + "ZMongoSavePromptMemoryNode": ZMongoSavePromptMemoryNode, + "ZMongoLoadMemoryCapsuleNode": ZMongoLoadMemoryCapsuleNode, + "ZMongoBuildContextPackNode": ZMongoBuildContextPackNode, + "ZMongoPromptCriticNode": ZMongoPromptCriticNode, + "ZMongoPromptEvolverNode": ZMongoPromptEvolverNode, + "ZMongoBestVariantSelectorNode": ZMongoBestVariantSelectorNode, + "ZMongoPromptProvenanceLedgerNode": ZMongoPromptProvenanceLedgerNode, + "ZMongoFindSimilarSuccessfulPromptsNode": ZMongoFindSimilarSuccessfulPromptsNode, + "ZMongoCaptionPackGeneratorNode": ZMongoCaptionPackGeneratorNode, + "ZMongoWorkflowDocumentationAgentNode": ZMongoWorkflowDocumentationAgentNode, +} + +NODE_DISPLAY_NAME_MAPPINGS = { + "ZMongoTextAgentSessionNode": "06 Text Agent Session", + "ZMongoSavePromptMemoryNode": "06 Save Prompt Memory", + "ZMongoLoadMemoryCapsuleNode": "06 Load Memory Capsule", + "ZMongoBuildContextPackNode": "06 Build Context Pack", + "ZMongoPromptCriticNode": "06 Prompt Critic", + "ZMongoPromptEvolverNode": "06 Prompt Evolver", + "ZMongoBestVariantSelectorNode": "06 Best Variant Selector", + "ZMongoPromptProvenanceLedgerNode": "06 Save Prompt Provenance Ledger", + "ZMongoFindSimilarSuccessfulPromptsNode": "06 Find Similar Successful Prompts", + "ZMongoCaptionPackGeneratorNode": "06 Caption Pack Generator", + "ZMongoWorkflowDocumentationAgentNode": "06 Workflow Documentation Agent", +} + +__all__ = [ + "NODE_CLASS_MAPPINGS", + "NODE_DISPLAY_NAME_MAPPINGS", +] diff --git a/requirements.txt b/requirements.txt index 6e4f539..3fb9926 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,4 +20,6 @@ FlagEmbedding pytest quart pillow -python-dotenv \ No newline at end of file +python-dotenv +python-docx +pymupdf \ No newline at end of file