diff --git a/__init__.py b/__init__.py index 3eb45d9..26b3c21 100644 --- a/__init__.py +++ b/__init__.py @@ -2,16 +2,19 @@ from typing import Dict, Type, TypeVar from .modules.matplotlib.mpl_bar import MPLBar from .modules.matplotlib.mpl_line import MPLLine from .modules.matplotlib.mpl_scatter import MPLScatter -from .modules.pandas.pandas_head import PandasHead -from .modules.pandas.pandas_join import PandasJoin -from .modules.pandas.pandas_load_csv import PandasLoadCSV -from .modules.pandas.pandas_save_csv import PandasSaveCSV -from .modules.pandas.pandas_select_columns import PandasSelectColumns -from .modules.pandas.pandas_select_rows import PandasSelectRows -from .modules.pandas.pandas_show_dataframe import PandasShowDataFrame -from .modules.pandas.pandas_sort import PandasSort -from .modules.pandas.pandas_summary import PandasSummary -from .modules.pandas.pandas_to_string import PandasToString +from .modules.pandas_wrapper.pandas_columns import PandasColumns +from .modules.pandas_wrapper.pandas_head import PandasHead +from .modules.pandas_wrapper.pandas_index import PandasIndex +from .modules.pandas_wrapper.pandas_join import PandasJoin +from .modules.pandas_wrapper.pandas_load_csv import PandasLoadCSV +from .modules.pandas_wrapper.pandas_save_csv import PandasSaveCSV +from .modules.pandas_wrapper.pandas_select_columns import PandasSelectColumns +from .modules.pandas_wrapper.pandas_select_rows import PandasSelectRows +from .modules.pandas_wrapper.pandas_show_dataframe import PandasShowDataFrame +from .modules.pandas_wrapper.pandas_show_text import PandasShowText +from .modules.pandas_wrapper.pandas_sort import PandasSort +from .modules.pandas_wrapper.pandas_summary import PandasSummary +from .modules.pandas_wrapper.pandas_to_string import PandasToString T = TypeVar("T") @@ -19,13 +22,16 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "MPLBar": MPLBar, "MPLLine": MPLLine, "MPLScatter": MPLScatter, + "PandasColumns": PandasColumns, "PandasHead": PandasHead, + "PandasIndex": PandasIndex, "PandasJoin": PandasJoin, "PandasLoadCSV": PandasLoadCSV, "PandasSaveCSV": PandasSaveCSV, "PandasSelectColumns": PandasSelectColumns, "PandasSelectRows": PandasSelectRows, "PandasShowDataFrame": PandasShowDataFrame, + "PandasShowText": PandasShowText, "PandasSort": PandasSort, "PandasSummary": PandasSummary, "PandasToString": PandasToString, @@ -33,33 +39,22 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { """ NODE_CLASS_MAPPINGS (Dict[str, Type[T]]): A dictionary mapping node names to their corresponding class implementations. - - - "MPLBar": Generates a bar chart from DataFrame data. - - "MPLLine": Generates a line plot from DataFrame data. - - "MPLScatter": Generates a scatter plot from DataFrame data. - - "PandasHead": Handles retrieving the first few rows of a DataFrame. - - "PandasJoin": Provides functionality to join multiple DataFrames. - - "PandasLoadCSV": Loads CSV data into a DataFrame. - - "PandasSaveCSV": Save a DataFrame to a CSV file. - - "PandasSelectColumns": Selects specific columns from a DataFrame. - - "PandasSelectRows": Filters specific rows from a DataFrame. - - "PandasShowDataFrame": Displays the content of the DataFrame. - - "PandasSort": Sorts a DataFrame based on given criteria. - - "PandasSummary": Provides a summary of a DataFrame. - - "PandasToString": Converts a DataFrame to a string representation. """ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "MPLBar": "MPL Bar Chart", "MPLLine": "MPL Line Plot", "MPLScatter": "MPL Scatter Plot", + "PandasColumns": "Pandas Columns", "PandasHead": "Pandas Head", + "PandasIndex": "Pandas Index", "PandasJoin": "Pandas Join", "PandasLoadCSV": "Pandas Load CSV", "PandasSaveCSV": "Pandas Save CSV", "PandasSelectColumns": "Pandas Select Columns", "PandasSelectRows": "Pandas Select Rows", "PandasShowDataFrame": "Pandas Show DataFrame", + "PandasShowText": "Pandas Show Text", "PandasSort": "Pandas Sort", "PandasSummary": "Pandas Summary", "PandasToString": "Pandas To String", diff --git a/docs/images/columns_index.png b/docs/images/columns_index.png new file mode 100644 index 0000000..756048f Binary files /dev/null and b/docs/images/columns_index.png differ diff --git a/docs/ug.md b/docs/ug.md index a2fcb34..33ba102 100644 --- a/docs/ug.md +++ b/docs/ug.md @@ -3,6 +3,8 @@ ### How DataFrames is passed between nodes DataFrames are transmitted as JSON strings. When a DataFrame enters a node, it is reconstructed by deserializing the JSON string. Upon exiting, it is serialized back into a JSON string. +Pandas's Index object is also processed the same way. For example, Pandas Columns and Pandas Index node return labels in a serialized Index format. + ### How to connect nodes for DataFrame JSON strings To connect the DataFrame JSON string output from one Pandas node to another, move the wire toward the top-left of the text field. You may need to hover around the corner to find the correct connection point. @@ -13,11 +15,14 @@ You can right-click, select **Add Node**, go to **Data Analysis**, and look for |-------------------------|-------------------------------------| | **Pandas Load CSV** | Load a CSV file | | **Pandas Save CSV** | Save a DataFrame to a CSV file | +| **Pandas Columns** | Extract column labels | +| **Pandas Index** | Extract row labels (index) | | **Pandas Select Columns** | Select specific columns from a DataFrame | | **Pandas Select Rows** | Filter rows based on conditions | | **Pandas Join** | Join two DataFrames | | **Pandas Head** | Extract the first few rows | | **Pandas Show DataFrame** | Display DataFrame contents | +| **Pandas Show Text** | Display Text | | **Pandas Summary** | Show DataFrame statistics | | **Pandas Sort** | Sort DataFrame by a column | | **Pandas To String** | Convert DataFrame to a string | @@ -39,6 +44,11 @@ CSV file path is relative to the ComfyUI installation directory unless you speci ## Saving data Use **Pandas Save CSV** node to save the DataFrame to a CSV file. +## Extracting column and row labels +Use **Pandas Columns** and **Pandas Index** nodes. As these nodes outputs JSON-serialized Index object, to display the value, use **Pandas Show Text** node as shown below: + +![Pandas Columns and Pandas Index](images/columns_index.png) + ## Selecting columns and rows Use **Pandas Select Columns** for selecting columns, **Pandas Select Rows** to select rows by a filter condition. @@ -71,6 +81,13 @@ The workflow file examples/workflows/join.json contains all of the above cases. ## Extract the first few rows Use **Pandas Head**. +## Display text ## +Use **Pandas Show Text** node. + +Currently this node is using the code taken from ComfyUI-Custom-Scripts's Show Text custom node ( +https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/py/show_text.py). +This node has been copied to display text without requiring the user to install the above custom nodes package. However, if you already have ComfyUI-Custom-Scripts, you can also use the Show Text node from the package. + ## Displaying DataFrame contents Use **Pandas Show DataFrame** node. When the number of rows is large, Pandas automatically hides middle rows during string conversion, so the display size should be always manageable. diff --git a/modules/pandas/__init__.py b/modules/pandas_wrapper/__init__.py similarity index 100% rename from modules/pandas/__init__.py rename to modules/pandas_wrapper/__init__.py diff --git a/modules/pandas_wrapper/pandas_columns.py b/modules/pandas_wrapper/pandas_columns.py new file mode 100644 index 0000000..a8447be --- /dev/null +++ b/modules/pandas_wrapper/pandas_columns.py @@ -0,0 +1,43 @@ +from typing import Any, Dict +from io import StringIO +import pandas as pd +from .utils import index_to_jsons + +class PandasColumns: + """ + PandasColumns: + A class for retrieving the column labels of a pandas DataFrame. + """ + + @classmethod + def INPUT_TYPES(cls) -> Dict[str, Any]: + """ + Defines the input types for the `columns` function. + + Returns: + Dict[str, Any]: A dictionary specifying required input types. + """ + return { + "required": { + "dataframe_json": ("STRING", {"multiline": True}) + } + } + + RETURN_TYPES: tuple = ("STRING",) + FUNCTION: str = "columns" + CATEGORY: str = "Data Analysis" + + def columns(self, dataframe_json: str) -> tuple: + """ + Returns the column labels of a pandas DataFrame. + + Args: + dataframe_json (str): A JSON string representation of the DataFrame. + + Returns: + tuple: A tuple containing a JSON string for the serialized pandas.Index containing columns. + """ + # Deserialize JSON string to DataFrame + df = pd.read_json(StringIO(dataframe_json)) + columns = df.columns + return (index_to_jsons(columns),) diff --git a/modules/pandas/pandas_head.py b/modules/pandas_wrapper/pandas_head.py similarity index 100% rename from modules/pandas/pandas_head.py rename to modules/pandas_wrapper/pandas_head.py diff --git a/modules/pandas_wrapper/pandas_index.py b/modules/pandas_wrapper/pandas_index.py new file mode 100644 index 0000000..68990f3 --- /dev/null +++ b/modules/pandas_wrapper/pandas_index.py @@ -0,0 +1,43 @@ +from typing import Any, Dict +from io import StringIO +import pandas as pd +from .utils import index_to_jsons + +class PandasIndex: + """ + PandasColumns: + A class for retrieving the row labels (index) of a pandas DataFrame. + """ + + @classmethod + def INPUT_TYPES(cls) -> Dict[str, Any]: + """ + Defines the input types for the `row_index` function. + + Returns: + Dict[str, Any]: A dictionary specifying required input types. + """ + return { + "required": { + "dataframe_json": ("STRING", {"multiline": True}) + } + } + + RETURN_TYPES: tuple = ("STRING",) + FUNCTION: str = "row_index" + CATEGORY: str = "Data Analysis" + + def row_index(self, dataframe_json: str) -> tuple: + """ + Returns the row labels (index) of a pandas DataFrame. + + Args: + dataframe_json (str): A JSON string representation of the DataFrame. + + Returns: + tuple: A tuple containing a JSON string for the serialized pandas.Index containing row labels. + """ + # Deserialize JSON string to DataFrame + df = pd.read_json(StringIO(dataframe_json)) + row_index = df.index + return (index_to_jsons(row_index),) diff --git a/modules/pandas/pandas_join.py b/modules/pandas_wrapper/pandas_join.py similarity index 96% rename from modules/pandas/pandas_join.py rename to modules/pandas_wrapper/pandas_join.py index 423cd0f..2c1da74 100644 --- a/modules/pandas/pandas_join.py +++ b/modules/pandas_wrapper/pandas_join.py @@ -12,7 +12,7 @@ class PandasJoin: @classmethod def INPUT_TYPES(cls) -> Dict[str, Any]: """ - Defines the input types for the `sort_rows` function. + Defines the input types for the `join` function. Returns: Dict[str, Any]: A dictionary specifying required input types. diff --git a/modules/pandas/pandas_load_csv.py b/modules/pandas_wrapper/pandas_load_csv.py similarity index 100% rename from modules/pandas/pandas_load_csv.py rename to modules/pandas_wrapper/pandas_load_csv.py diff --git a/modules/pandas/pandas_save_csv.py b/modules/pandas_wrapper/pandas_save_csv.py similarity index 100% rename from modules/pandas/pandas_save_csv.py rename to modules/pandas_wrapper/pandas_save_csv.py diff --git a/modules/pandas/pandas_select_columns.py b/modules/pandas_wrapper/pandas_select_columns.py similarity index 100% rename from modules/pandas/pandas_select_columns.py rename to modules/pandas_wrapper/pandas_select_columns.py diff --git a/modules/pandas/pandas_select_rows.py b/modules/pandas_wrapper/pandas_select_rows.py similarity index 100% rename from modules/pandas/pandas_select_rows.py rename to modules/pandas_wrapper/pandas_select_rows.py diff --git a/modules/pandas/pandas_show_dataframe.py b/modules/pandas_wrapper/pandas_show_dataframe.py similarity index 100% rename from modules/pandas/pandas_show_dataframe.py rename to modules/pandas_wrapper/pandas_show_dataframe.py diff --git a/modules/pandas_wrapper/pandas_show_text.py b/modules/pandas_wrapper/pandas_show_text.py new file mode 100644 index 0000000..d64dc81 --- /dev/null +++ b/modules/pandas_wrapper/pandas_show_text.py @@ -0,0 +1,48 @@ +""" +Almost all of this code was taken from +https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/py/show_text.py + +See credit/credit.md for the full license. +""" + +class PandasShowText: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "text": ("STRING", {"forceInput": True}), + }, + "hidden": { + "unique_id": "UNIQUE_ID", + "extra_pnginfo": "EXTRA_PNGINFO", + }, + } + + INPUT_IS_LIST = True + RETURN_TYPES = ("STRING",) + FUNCTION = "notify" + OUTPUT_NODE = True + OUTPUT_IS_LIST = (True,) + + CATEGORY = "Data Analysis" + + def notify(self, text, unique_id=None, extra_pnginfo=None): + if unique_id is not None and extra_pnginfo is not None: + if not isinstance(extra_pnginfo, list): + print("Error: extra_pnginfo is not a list") + elif ( + not isinstance(extra_pnginfo[0], dict) + or "workflow" not in extra_pnginfo[0] + ): + print("Error: extra_pnginfo[0] is not a dict or missing 'workflow' key") + else: + workflow = extra_pnginfo[0]["workflow"] + node = next( + (x for x in workflow["nodes"] if str(x["id"]) == str(unique_id[0])), + None, + ) + if node: + node["widgets_values"] = [text] + else: + print("unique_id or extra_pnginfo is None.") + return {"ui": {"text": text}, "result": (text,)} diff --git a/modules/pandas/pandas_sort.py b/modules/pandas_wrapper/pandas_sort.py similarity index 100% rename from modules/pandas/pandas_sort.py rename to modules/pandas_wrapper/pandas_sort.py diff --git a/modules/pandas/pandas_summary.py b/modules/pandas_wrapper/pandas_summary.py similarity index 100% rename from modules/pandas/pandas_summary.py rename to modules/pandas_wrapper/pandas_summary.py diff --git a/modules/pandas/pandas_to_string.py b/modules/pandas_wrapper/pandas_to_string.py similarity index 100% rename from modules/pandas/pandas_to_string.py rename to modules/pandas_wrapper/pandas_to_string.py diff --git a/modules/pandas_wrapper/utils.py b/modules/pandas_wrapper/utils.py new file mode 100644 index 0000000..7e95a27 --- /dev/null +++ b/modules/pandas_wrapper/utils.py @@ -0,0 +1,63 @@ +import json +import pandas as pd + +def series_to_jsons(series: pd.Series) -> str: + """ + Convert a pandas Series to a JSON string. + + Args: + series (pd.Series): The pandas Series to convert. + + Returns: + str: A JSON string representing the Series, including its data, index, and name. + """ + array = [x.item() if hasattr(x, "item") else x for x in series.tolist()] # Convert NumPy types to Python types + index = [x.item() if hasattr(x, "item") else x for x in series.index.tolist()] # Convert index to Python types + name = series.name + + d = {"array": array, "index": index, "name": name} + + return json.dumps(d) + +def jsons_to_series(s: str) -> pd.Series: + """ + Convert a JSON string to a pandas Series. + + Args: + s (str): The JSON string representing the Series. + + Returns: + pd.Series: The reconstructed pandas Series. + """ + d = json.loads(s) + return pd.Series(data=d["array"], index=d["index"], name=d["name"]) + +def index_to_jsons(index: pd.Index) -> str: + """ + Convert a pandas Index to a JSON string. + + Args: + index (pd.Index): The pandas Index to convert. + + Returns: + str: A JSON string representing the Index, including its values and name. + """ + index_list = [x.item() if hasattr(x, "item") else x for x in index.tolist()] # Convert NumPy types to Python types + index_name = index.name # Preserve index name + + d = {"index": index_list, "name": index_name} + + return json.dumps(d) + +def jsons_to_index(s: str) -> pd.Index: + """ + Convert a JSON string to a pandas Index. + + Args: + s (str): The JSON string representing the Index. + + Returns: + pd.Index: The reconstructed pandas Index. + """ + d = json.loads(s) + return pd.Index(d["index"], name=d.get("name")) # Restore index with name diff --git a/pyproject.toml b/pyproject.toml index 85a4f96..1c2ccd0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "data-analysis" description = "Data analysis custom modules for ComfyUI - Use Pandas & Matplotlib within ComfyUI" -version = "1.0.5" +version = "1.0.6" license = {file = "LICENSE"} [project.urls] diff --git a/tests/pandas/utils_index_json_test.py b/tests/pandas/utils_index_json_test.py new file mode 100644 index 0000000..7397364 --- /dev/null +++ b/tests/pandas/utils_index_json_test.py @@ -0,0 +1,51 @@ +import os +import sys +import unittest +import json +import pandas as pd + +PROJECT_ROOT = os.path.realpath(os.path.join(__file__, "..", "..", "..")) +MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules") +sys.path.append(MODULE_ROOT) +print(MODULE_ROOT) + +from pandas_wrapper.utils import index_to_jsons, jsons_to_index + +class TestIndexJsonConversion(unittest.TestCase): + def setUp(self): + """Set up test data""" + self.index_int = pd.Index([1, 2, 3], name="test_index_int") + self.index_float = pd.Index([1.1, 2.2, 3.3], name="test_index_float") + self.index_str = pd.Index(["a", "b", "c"], name="test_index_str") + self.index_mixed = pd.Index([1, "text", 3.5], name="test_index_mixed") + self.index_no_name = pd.Index([1, "text", 3.5]) + + def test_index_to_jsons_and_back(self): + """Test conversion from Index to JSON and back""" + for index in [self.index_int, self.index_float, self.index_str, self.index_mixed]: + with self.subTest(index=index): + json_str = index_to_jsons(index) + reconstructed_index = jsons_to_index(json_str) + self.assertTrue(index.equals(reconstructed_index), f"Failed for index: {index.name}") + + def test_json_format(self): + """Test if JSON output is correctly formatted""" + json_str = index_to_jsons(self.index_int) + data = json.loads(json_str) + + self.assertIn("index", data) + self.assertIn("name", data) + self.assertEqual(data["name"], "test_index_int") + + def test_json_format_no_name(self): + """Test if JSON output is correctly formatted""" + json_str = index_to_jsons(self.index_no_name) + data = json.loads(json_str) + + self.assertIn("index", data) + self.assertIn("name", data) + self.assertTrue(data["name"] is None) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pandas/utils_series_json_test.py b/tests/pandas/utils_series_json_test.py new file mode 100644 index 0000000..be5d8c1 --- /dev/null +++ b/tests/pandas/utils_series_json_test.py @@ -0,0 +1,41 @@ +import os +import sys +import unittest +import json +import pandas as pd + +PROJECT_ROOT = os.path.realpath(os.path.join(__file__, "..", "..", "..")) +MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules") +sys.path.append(MODULE_ROOT) +print(MODULE_ROOT) +from pandas_wrapper.utils import series_to_jsons +from pandas_wrapper.utils import jsons_to_series + +class TestSeriesJsonConversion(unittest.TestCase): + def setUp(self): + """Set up test data""" + self.series_int = pd.Series([10, 20, 30], index=["a", "b", "c"], name="test_series_int") + self.series_float = pd.Series([1.1, 2.2, 3.3], index=["x", "y", "z"], name="test_series_float") + self.series_str = pd.Series(["apple", "banana", "cherry"], index=[0, 1, 2], name="test_series_str") + self.series_mixed = pd.Series([1, "text", 3.5], index=["one", "two", "three"], name="test_series_mixed") + + def test_series_to_jsons_and_back(self): + """Test conversion from Series to JSON and back""" + for series in [self.series_int, self.series_float, self.series_str, self.series_mixed]: + with self.subTest(series=series): + json_str = series_to_jsons(series) + reconstructed_series = jsons_to_series(json_str) + self.assertTrue(series.equals(reconstructed_series), f"Failed for series: {series.name}") + + def test_json_format(self): + """Test if JSON output is correctly formatted""" + json_str = series_to_jsons(self.series_int) + data = json.loads(json_str) + + self.assertIn("array", data) + self.assertIn("index", data) + self.assertIn("name", data) + self.assertEqual(data["name"], "test_series_int") + +if __name__ == "__main__": + unittest.main() diff --git a/web/js/ShowText.js b/web/js/ShowText.js new file mode 100644 index 0000000..bd23863 --- /dev/null +++ b/web/js/ShowText.js @@ -0,0 +1,64 @@ +/* +Almost all of this code was taken from +https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/web/js/showText.js + +See credit/credit.md for the full license. +*/ + +import { app } from "../../../scripts/app.js"; +import { ComfyWidgets } from "../../../scripts/widgets.js"; + +// Displays input text on a node +app.registerExtension({ + name: "HowToSD.ShowText", + async beforeRegisterNodeDef(nodeType, nodeData, app) { + if (nodeData.name === "PandasShowText") { // This needs to match NODE_CLASS_MAPPINGS in Python + function populate(text) { + if (this.widgets) { + for (let i = 1; i < this.widgets.length; i++) { + this.widgets[i].onRemove?.(); + } + this.widgets.length = 1; + } + + const v = [...text]; + if (!v[0]) { + v.shift(); + } + for (const list of v) { + const w = ComfyWidgets["STRING"](this, "text2", ["STRING", { multiline: true }], app).widget; + w.inputEl.readOnly = true; + w.inputEl.style.opacity = 0.6; + w.value = list; + } + + requestAnimationFrame(() => { + const sz = this.computeSize(); + if (sz[0] < this.size[0]) { + sz[0] = this.size[0]; + } + if (sz[1] < this.size[1]) { + sz[1] = this.size[1]; + } + this.onResize?.(sz); + app.graph.setDirtyCanvas(true, false); + }); + } + + // When the node is executed we will be sent the input text, display this in the widget + const onExecuted = nodeType.prototype.onExecuted; + nodeType.prototype.onExecuted = function (message) { + onExecuted?.apply(this, arguments); + populate.call(this, message.text); + }; + + const onConfigure = nodeType.prototype.onConfigure; + nodeType.prototype.onConfigure = function () { + onConfigure?.apply(this, arguments); + if (this.widgets_values?.length) { + populate.call(this, this.widgets_values.slice(+this.widgets_values.length > 1)); + } + }; + } + }, +}); \ No newline at end of file