From a8aaa21ce0403c106977edeb0a10cfe94012f9ab Mon Sep 17 00:00:00 2001 From: HowToSD Date: Fri, 14 Mar 2025 02:53:10 -0700 Subject: [PATCH] Added Pandas Feature Split To Pt and Pandas Replace Advanced nodes. --- __init__.py | 6 ++ docs/reference/node_reference.md | 2 + docs/reference/pandas_feature_split_to_pt.md | 23 ++++++ docs/reference/pandas_replace_advanced.md | 22 ++++++ .../pandas_feature_split_to_pt.py | 78 +++++++++++++++++++ .../pandas_wrapper/pandas_replace_advanced.py | 61 +++++++++++++++ pyproject.toml | 2 +- .../pandas_feture_split_to_pt_test.py | 50 ++++++++++++ .../pandas_replace_advanced_test.py | 60 ++++++++++++++ 9 files changed, 303 insertions(+), 1 deletion(-) create mode 100644 docs/reference/pandas_feature_split_to_pt.md create mode 100644 docs/reference/pandas_replace_advanced.md create mode 100644 modules/pandas_wrapper/pandas_feature_split_to_pt.py create mode 100644 modules/pandas_wrapper/pandas_replace_advanced.py create mode 100644 tests/pandas_wrapper/pandas_feture_split_to_pt_test.py create mode 100644 tests/pandas_wrapper/pandas_replace_advanced_test.py diff --git a/__init__.py b/__init__.py index 3e295b0..0e3ceb0 100644 --- a/__init__.py +++ b/__init__.py @@ -73,6 +73,7 @@ from .modules.pandas_wrapper.pandas_eq_scalar_float import PandasEqScalarFloat from .modules.pandas_wrapper.pandas_eq_scalar_int import PandasEqScalarInt from .modules.pandas_wrapper.pandas_exp import PandasExp from .modules.pandas_wrapper.pandas_feature_split_to_numpy import PandasFeatureSplitToNumpy +from .modules.pandas_wrapper.pandas_feature_split_to_pt import PandasFeatureSplitToPt from .modules.pandas_wrapper.pandas_fillna_scalar_float import PandasFillNAScalarFloat from .modules.pandas_wrapper.pandas_fillna_scalar_int import PandasFillNAScalarInt from .modules.pandas_wrapper.pandas_ge import PandasGe @@ -138,6 +139,7 @@ from .modules.pandas_wrapper.pandas_pow_series import PandasPowSeries from .modules.pandas_wrapper.pandas_rename import PandasRename from .modules.pandas_wrapper.pandas_rename_advanced import PandasRenameAdvanced from .modules.pandas_wrapper.pandas_replace import PandasReplace +from .modules.pandas_wrapper.pandas_replace_advanced import PandasReplaceAdvanced from .modules.pandas_wrapper.pandas_save_csv import PandasSaveCSV from .modules.pandas_wrapper.pandas_save_json import PandasSaveJSON from .modules.pandas_wrapper.pandas_select_column_as_series import PandasSelectColumnAsSeries @@ -249,6 +251,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasEqScalarInt": PandasEqScalarInt, "PandasExp": PandasExp, "PandasFeatureSplitToNumpy": PandasFeatureSplitToNumpy, + "PandasFeatureSplitToPt": PandasFeatureSplitToPt, "PandasFillNAScalarFloat": PandasFillNAScalarFloat, "PandasFillNAScalarInt": PandasFillNAScalarInt, "PandasGe": PandasGe, @@ -314,6 +317,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasRename": PandasRename, "PandasRenameAdvanced": PandasRenameAdvanced, "PandasReplace": PandasReplace, + "PandasReplaceAdvanced": PandasReplaceAdvanced, "PandasSaveCSV": PandasSaveCSV, "PandasSaveJSON": PandasSaveJSON, "PandasSelectColumnAsSeries": PandasSelectColumnAsSeries, @@ -436,6 +440,7 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasEqScalarInt": "Pandas Eq Scalar Int", "PandasExp": "Pandas Exp", "PandasFeatureSplitToNumpy": "Pandas Feature Split To Numpy", + "PandasFeatureSplitToPt": "Pandas Feature Split To Pt", "PandasFillNAScalarFloat": "Pandas Fill NA Scalar Float", "PandasFillNAScalarInt": "Pandas Fill NA Scalar Int", "PandasGe": "Pandas Ge", @@ -501,6 +506,7 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasRename": "Pandas Rename", "PandasRenameAdvanced": "Pandas Rename Advanced", "PandasReplace": "Pandas Replace", + "PandasReplaceAdvanced": "Pandas Replace Advanced", "PandasSaveCSV": "Pandas Save CSV", "PandasSaveJSON": "Pandas Save JSON", "PandasSelectColumnAsSeries": "Pandas Select Column As Series", diff --git a/docs/reference/node_reference.md b/docs/reference/node_reference.md index 3a1e5ae..f0d0d08 100644 --- a/docs/reference/node_reference.md +++ b/docs/reference/node_reference.md @@ -47,6 +47,7 @@ | [Pandas Fill NA Scalar Int](pandas_fillna_scalar_int.md) | Replaces missing values in a Pandas DataFrame with an integer. | | [Pandas Is NA](pandas_isna.md) | Checks a pandas DataFrame for missing values. | | [Pandas Replace](pandas_replace.md) | Replaces part or all of a string in each DataFrame cell with a specified string using a wildcard for matching. If you want to remove some characters, then leave the replace_string field blank. | +| [Pandas Replace Advanced](pandas_replace_advanced.md) | Replaces strings in a DataFrame cell using a mapping specified in a dictionary. | ## Data subset selection | Node | Description | | --- | --- | @@ -214,6 +215,7 @@ | [Pandas At Set Int](pandas_at_set_int.md) | Sets a Python int data in a cell in a pandas DataFrame. | | [Pandas At Set String](pandas_at_set_string.md) | Sets a Python string data in a cell in a pandas DataFrame. | | [Pandas Feature Split To Numpy](pandas_feature_split_to_numpy.md) | Splits a Pandas DataFrame into feature Numpy ndarray and label Numpy ndarray. | +| [Pandas Feature Split To Pt](pandas_feature_split_to_pt.md) | Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor. | | [Pandas Horizontal Concat](pandas_horizontal_concat.md) | Horizontally concatenates two pandas DataFrames. | | [Pandas Horizontal Split](pandas_horizontal_split.md) | Horizontally splits a Pandas DataFrame into two pandas DataFrames. | | [Pandas Iat Set Datetime](pandas_iat_set_datetime.md) | Sets a Python datetime.datetime data in a cell in a pandas DataFrame. | diff --git a/docs/reference/pandas_feature_split_to_pt.md b/docs/reference/pandas_feature_split_to_pt.md new file mode 100644 index 0000000..c57ca81 --- /dev/null +++ b/docs/reference/pandas_feature_split_to_pt.md @@ -0,0 +1,23 @@ +# Pandas Feature Split To Pt +Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor. +If output_1d_label is set, label will be output as a 1d array instead of a 2d array. + +## Input +| Name | Data type | +|---|---| +| dataframe | DataFrame | +| label_integer_position | Int | +| output_1d_label | Boolean | +| feature_data_type | | +| label_data_type | | + +## Output +| Data type | +|---| +| Ndarray | +| Ndarray | + +
+Category: Transformation + +ComfyUI Data Analysis Node Reference. © 2025 Hide Inada (HowToSD.com). All rights reserved. diff --git a/docs/reference/pandas_replace_advanced.md b/docs/reference/pandas_replace_advanced.md new file mode 100644 index 0000000..6a66188 --- /dev/null +++ b/docs/reference/pandas_replace_advanced.md @@ -0,0 +1,22 @@ +# Pandas Replace Advanced +Replaces strings in a DataFrame cell using a mapping specified in a dictionary. +For example, if you want to replace "cat" with "dog" and replace "mountain" with "ocean", specify the mapping in Python dict format: +``` +{"cat":"dog", "mountain":"ocean"} +``` + +## Input +| Name | Data type | +|---|---| +| dataframe | DataFrame | +| replacement_dict | String | + +## Output +| Data type | +|---| +| DataFrame | + +
+Category: Data cleansing + +ComfyUI Data Analysis Node Reference. © 2025 Hide Inada (HowToSD.com). All rights reserved. diff --git a/modules/pandas_wrapper/pandas_feature_split_to_pt.py b/modules/pandas_wrapper/pandas_feature_split_to_pt.py new file mode 100644 index 0000000..d24c4c9 --- /dev/null +++ b/modules/pandas_wrapper/pandas_feature_split_to_pt.py @@ -0,0 +1,78 @@ +from typing import Any, Dict +import pandas as pd +import numpy as np +import torch + +class PandasFeatureSplitToPt: + """ + Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor. + If output_1d_label is set, label will be output as a 1d array instead of a 2d array. + + category: Transformation + """ + + @classmethod + def INPUT_TYPES(cls) -> Dict[str, Any]: + """ + Defines the input types for the function. + + Returns: + Dict[str, Any]: A dictionary specifying required input types. + """ + return { + "required": { + "dataframe": ("DATAFRAME", {}), + "label_integer_position": ("INT", {"default": -1, "min": -10, "max": 1024}), + "output_1d_label": ("BOOLEAN", {"default": True}), + "feature_data_type": (("float32","float16", "bloat16", "float64", "int8", "uint8", "int16", "int32", "int64"),), + "label_data_type": (("float32","float16", "bloat16", "float64", "int8", "uint8", "int16", "int32", "int64"),) + } + } + + RETURN_TYPES: tuple = ("TENSOR", "TENSOR") + RETURN_NAMES: tuple = ("feature", "label") + + FUNCTION: str = "f" + CATEGORY: str = "Data Analysis" + + def f(self, dataframe: pd.DataFrame, + label_integer_position: int, + output_1d_label: bool, + feature_data_type: str, + label_data_type: str) -> tuple: + """ + Splits a Pandas DataFrame into feature Numpy ndarray and label Numpy ndarray. + + Args: + dataframe (pd.DataFrame): The DataFrame to split. + label_integer_position (int): The column integer position of the label. + output_1d_label (bool): If output_1d_label is set, label will be output as a 1d array instead of a 2d array. + feature_data_type (str): Feature data type. + label_data_type (str): Label data type. + Returns: + tuple: A tuple containing the feature and label arrays. + """ + feature_dtype = getattr(torch, feature_data_type) + label_dtype = getattr(torch, label_data_type) + + num_columns = dataframe.shape[1] + + if label_integer_position < 0: + label_integer_position += num_columns + + # Validate label_integer_position + if label_integer_position >= num_columns: + raise ValueError(f"label_integer_position {label_integer_position} is out of range for dataframe with {num_columns} columns.") + + # Extract features + np_left = dataframe.iloc[:, :label_integer_position].to_numpy() + np_right = dataframe.iloc[:, label_integer_position + 1:].to_numpy() if label_integer_position < num_columns - 1 else np.empty((len(dataframe), 0)) + np_features = np.hstack([np_left, np_right]) if np_right.shape[1] > 0 else np_left + + # Extract label + np_label = dataframe.iloc[:, label_integer_position].to_numpy() + if output_1d_label: + np_label = np_label.ravel() + + return (torch.tensor(np_features,dtype=feature_dtype), + torch.tensor(np_label, dtype=label_dtype)) diff --git a/modules/pandas_wrapper/pandas_replace_advanced.py b/modules/pandas_wrapper/pandas_replace_advanced.py new file mode 100644 index 0000000..60d7856 --- /dev/null +++ b/modules/pandas_wrapper/pandas_replace_advanced.py @@ -0,0 +1,61 @@ +from typing import Any, Dict +import pandas as pd +import ast + +class PandasReplaceAdvanced: + """ + Replaces strings in a DataFrame cell using a mapping specified in a dictionary. + For example, if you want to replace "cat" with "dog" and replace "mountain" with "ocean", specify the mapping in Python dict format: + ``` + {"cat":"dog", "mountain":"ocean"} + ``` + category: Data cleansing + """ + + @classmethod + def INPUT_TYPES(cls) -> Dict[str, Any]: + """ + Defines the input types for the function. + + Returns: + Dict[str, Any]: A dictionary specifying required input types. + """ + return { + "required": { + "dataframe": ("DATAFRAME", {}), + "replacement_dict": ("STRING", {"default": "{}", "multiline":True}) + } + } + + RETURN_TYPES: tuple = ("DATAFRAME",) + FUNCTION: str = "f" + CATEGORY: str = "Data Analysis" + + def f(self, dataframe: pd.DataFrame, replacement_dict: str) -> tuple: + """ + Replaces part or all of a string in each DataFrame cell with a specified string using a wildcard for matching. + + Args: + dataframe (DataFrame): The target DataFrame. + replacement_dict (str): A dict containing the mapping. + + Returns: + tuple: A tuple containing the DataFrame. + """ + d = ast.literal_eval(replacement_dict) + if isinstance(d, dict) is False: + raise ValueError('You need to specify the mapping in Python dict format such as {"cat":"dog", "mountain":"ocean"}') + # check values + for k, v in d.items(): + if (isinstance(k, str) is False and + isinstance(k, float) is False and + isinstance(k, int) is False): + raise ValueError("You need to specify a string or a number for source values.") + + if (isinstance(v, str) is False and + isinstance(v, float) is False and + isinstance(v, int) is False): + raise ValueError("You need to specify a string or a number for target values.") + + df2 = dataframe.replace(to_replace=d) + return (df2,) diff --git a/pyproject.toml b/pyproject.toml index 77dc9da..375db79 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "data-analysis" description = "Data analysis custom modules for ComfyUI – Integrates Pandas, Seaborn and Matplotlib into ComfyUI." -version = "3.0.4" +version = "3.0.5" license = {file = "LICENSE"} [project.urls] diff --git a/tests/pandas_wrapper/pandas_feture_split_to_pt_test.py b/tests/pandas_wrapper/pandas_feture_split_to_pt_test.py new file mode 100644 index 0000000..d1de503 --- /dev/null +++ b/tests/pandas_wrapper/pandas_feture_split_to_pt_test.py @@ -0,0 +1,50 @@ +import os +import sys +import unittest +import numpy as np +import pandas as pd +import torch + +PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..")) +MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules") +sys.path.append(MODULE_ROOT) + +from pandas_wrapper.pandas_feature_split_to_pt import PandasFeatureSplitToPt + +class TestPandasFeatureSplitToPt(unittest.TestCase): + def setUp(self): + """Set up test data.""" + self.node = PandasFeatureSplitToPt() + + # Sample DataFrame + self.df = pd.DataFrame({ + 'A': [1, 2, 3], + 'B': [4, 5, 6], + 'C': [7, 8, 9] + }) + + def test_feature_split(self): + """Test splitting with different label positions.""" + test_cases = [ + (0, True, [[4, 7], [5, 8], [6, 9]], [1, 2, 3]), + (1, True, [[1, 7], [2, 8], [3, 9]], [4, 5, 6]), + (2, True, [[1, 4], [2, 5], [3, 6]], [7, 8, 9]), + (-1, True, [[1, 4], [2, 5], [3, 6]], [7, 8, 9]), + ] + + data_types = ["float32", "float16", "bfloat16", "float64", "int8", "uint8", "int16", "int32", "int64"] + for feature_dtype in data_types: + for label_dtype in data_types: + for label_pos, output_1d, expected_features, expected_labels in test_cases: + with self.subTest(label_pos=label_pos, output_1d=output_1d, feature_dtype=feature_dtype, label_dtype=label_dtype): + features, labels = self.node.f(self.df, label_pos, output_1d, feature_dtype, label_dtype) + torch.testing.assert_close(features, torch.tensor(expected_features, dtype=getattr(torch, feature_dtype))) + torch.testing.assert_close(labels, torch.tensor(expected_labels, dtype=getattr(torch, label_dtype))) + + def test_invalid_label_position(self): + """Test invalid label positions.""" + with self.assertRaises(ValueError): + self.node.f(self.df, 3, True, "float32", "int64") + +if __name__ == "__main__": + unittest.main() diff --git a/tests/pandas_wrapper/pandas_replace_advanced_test.py b/tests/pandas_wrapper/pandas_replace_advanced_test.py new file mode 100644 index 0000000..34346d1 --- /dev/null +++ b/tests/pandas_wrapper/pandas_replace_advanced_test.py @@ -0,0 +1,60 @@ +import os +import sys +import unittest +import pandas as pd +import ast +from typing import Any, Dict + +PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..")) +MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules") +sys.path.append(MODULE_ROOT) + +from pandas_wrapper.pandas_replace_advanced import PandasReplaceAdvanced + + +class TestPandasReplaceAdvanced(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.node = PandasReplaceAdvanced() + cls.df = pd.DataFrame( + { + "col1": ["cat", "dog", "fish"], + "col2": ["mountain", "ocean", "forest"], + } + ) + + def test_valid_replacement(self): + test_cases = [ + ('{"cat": 1, "dog": 2.34}', {"col1": [1, 2.34, "fish"], "col2": ["mountain", "ocean", "forest"]}), + ('{"mountain": "hill", "forest": "jungle"}', {"col1": ["cat", "dog", "fish"], "col2": ["hill", "ocean", "jungle"]}), + ] + for replacements, expected_data in test_cases: + with self.subTest(replacements=replacements): + result_df = self.node.f(self.df, replacements)[0] + expected_df = pd.DataFrame(expected_data) + pd.testing.assert_frame_equal(result_df, expected_df) + + def test_invalid_replacement_value_error(self): + invalid_cases = [ + '["cat", "lion"]', # List is not allowed + '{"cat": ["lion", "tiger"]}', # List is not allowed + '{"dog": {"wolf": "coyote"}}' # Nested dict is not allowed + ] + for invalid_pairs in invalid_cases: + with self.subTest(invalid_pairs=invalid_pairs): + with self.assertRaises(ValueError): + self.node.f(self.df, invalid_pairs) + + def test_invalid_replacement_syntax_error(self): + invalid_cases = [ + '{"mountain": ()"hill", "ocean")}', # Syntax error + ] + for invalid_pairs in invalid_cases: + with self.subTest(invalid_pairs=invalid_pairs): + with self.assertRaises(SyntaxError): + self.node.f(self.df, invalid_pairs) + + + +if __name__ == "__main__": + unittest.main()