nodes/HowToSD-ComfyUI-Data-Analysis
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added Pandas Feature Split To Pt and Pandas Replace Advanced nodes.

Browse Source
This commit is contained in:
HowToSD
2025-03-14 02:53:10 -07:00
parent 4d05b00f45
commit a8aaa21ce0
9 changed files with 303 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
__init__.py
+6
View File
@@ -73,6 +73,7 @@ from .modules.pandas_wrapper.pandas_eq_scalar_float import PandasEqScalarFloat
from .modules.pandas_wrapper.pandas_eq_scalar_int import PandasEqScalarInt
from .modules.pandas_wrapper.pandas_exp import PandasExp
from .modules.pandas_wrapper.pandas_feature_split_to_numpy import PandasFeatureSplitToNumpy
from .modules.pandas_wrapper.pandas_feature_split_to_pt import PandasFeatureSplitToPt
from .modules.pandas_wrapper.pandas_fillna_scalar_float import PandasFillNAScalarFloat
from .modules.pandas_wrapper.pandas_fillna_scalar_int import PandasFillNAScalarInt
from .modules.pandas_wrapper.pandas_ge import PandasGe
@@ -138,6 +139,7 @@ from .modules.pandas_wrapper.pandas_pow_series import PandasPowSeries
from .modules.pandas_wrapper.pandas_rename import PandasRename
from .modules.pandas_wrapper.pandas_rename_advanced import PandasRenameAdvanced
from .modules.pandas_wrapper.pandas_replace import PandasReplace
from .modules.pandas_wrapper.pandas_replace_advanced import PandasReplaceAdvanced
from .modules.pandas_wrapper.pandas_save_csv import PandasSaveCSV
from .modules.pandas_wrapper.pandas_save_json import PandasSaveJSON
from .modules.pandas_wrapper.pandas_select_column_as_series import PandasSelectColumnAsSeries
@@ -249,6 +251,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = {
"PandasEqScalarInt": PandasEqScalarInt,
"PandasExp": PandasExp,
"PandasFeatureSplitToNumpy": PandasFeatureSplitToNumpy,
"PandasFeatureSplitToPt": PandasFeatureSplitToPt,
"PandasFillNAScalarFloat": PandasFillNAScalarFloat,
"PandasFillNAScalarInt": PandasFillNAScalarInt,
"PandasGe": PandasGe,
@@ -314,6 +317,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = {
"PandasRename": PandasRename,
"PandasRenameAdvanced": PandasRenameAdvanced,
"PandasReplace": PandasReplace,
"PandasReplaceAdvanced": PandasReplaceAdvanced,
"PandasSaveCSV": PandasSaveCSV,
"PandasSaveJSON": PandasSaveJSON,
"PandasSelectColumnAsSeries": PandasSelectColumnAsSeries,
@@ -436,6 +440,7 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = {
"PandasEqScalarInt": "Pandas Eq Scalar Int",
"PandasExp": "Pandas Exp",
"PandasFeatureSplitToNumpy": "Pandas Feature Split To Numpy",
"PandasFeatureSplitToPt": "Pandas Feature Split To Pt",
"PandasFillNAScalarFloat": "Pandas Fill NA Scalar Float",
"PandasFillNAScalarInt": "Pandas Fill NA Scalar Int",
"PandasGe": "Pandas Ge",
@@ -501,6 +506,7 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = {
"PandasRename": "Pandas Rename",
"PandasRenameAdvanced": "Pandas Rename Advanced",
"PandasReplace": "Pandas Replace",
"PandasReplaceAdvanced": "Pandas Replace Advanced",
"PandasSaveCSV": "Pandas Save CSV",
"PandasSaveJSON": "Pandas Save JSON",
"PandasSelectColumnAsSeries": "Pandas Select Column As Series",
docs/reference/node_reference.md
+2
View File
@@ -47,6 +47,7 @@
| [Pandas Fill NA Scalar Int](pandas_fillna_scalar_int.md) | Replaces missing values in a Pandas DataFrame with an integer. |
| [Pandas Is NA](pandas_isna.md) | Checks a pandas DataFrame for missing values. |
| [Pandas Replace](pandas_replace.md) | Replaces part or all of a string in each DataFrame cell with a specified string using a wildcard for matching. If you want to remove some characters, then leave the replace_string field blank. |
| [Pandas Replace Advanced](pandas_replace_advanced.md) | Replaces strings in a DataFrame cell using a mapping specified in a dictionary. |
## Data subset selection
| Node | Description |
| --- | --- |
@@ -214,6 +215,7 @@
| [Pandas At Set Int](pandas_at_set_int.md) | Sets a Python int data in a cell in a pandas DataFrame. |
| [Pandas At Set String](pandas_at_set_string.md) | Sets a Python string data in a cell in a pandas DataFrame. |
| [Pandas Feature Split To Numpy](pandas_feature_split_to_numpy.md) | Splits a Pandas DataFrame into feature Numpy ndarray and label Numpy ndarray. |
| [Pandas Feature Split To Pt](pandas_feature_split_to_pt.md) | Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor. |
| [Pandas Horizontal Concat](pandas_horizontal_concat.md) | Horizontally concatenates two pandas DataFrames. |
| [Pandas Horizontal Split](pandas_horizontal_split.md) | Horizontally splits a Pandas DataFrame into two pandas DataFrames. |
| [Pandas Iat Set Datetime](pandas_iat_set_datetime.md) | Sets a Python datetime.datetime data in a cell in a pandas DataFrame. |
docs/reference/pandas_feature_split_to_pt.md
+23
View File
@@ -0,0 +1,23 @@
# Pandas Feature Split To Pt
Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor.
If output_1d_label is set, label will be output as a 1d array instead of a 2d array.
## Input
| Name | Data type |
|---|---|
| dataframe | DataFrame |
| label_integer_position | Int |
| output_1d_label | Boolean |
| feature_data_type | |
| label_data_type | |
## Output
| Data type |
|---|
| Ndarray |
| Ndarray |
<HR>
Category: Transformation
ComfyUI Data Analysis Node Reference. © 2025 Hide Inada (HowToSD.com). All rights reserved.
docs/reference/pandas_replace_advanced.md
+22
View File
@@ -0,0 +1,22 @@
# Pandas Replace Advanced
Replaces strings in a DataFrame cell using a mapping specified in a dictionary.
For example, if you want to replace "cat" with "dog" and replace "mountain" with "ocean", specify the mapping in Python dict format:
```
{"cat":"dog", "mountain":"ocean"}
```
## Input
| Name | Data type |
|---|---|
| dataframe | DataFrame |
| replacement_dict | String |
## Output
| Data type |
|---|
| DataFrame |
<HR>
Category: Data cleansing
ComfyUI Data Analysis Node Reference. © 2025 Hide Inada (HowToSD.com). All rights reserved.
modules/pandas_wrapper/pandas_feature_split_to_pt.py
+78
View File
@@ -0,0 +1,78 @@
from typing import Any, Dict
import pandas as pd
import numpy as np
import torch
class PandasFeatureSplitToPt:
"""
Splits a Pandas DataFrame into feature PyTorch tensor and label PyTorch tensor.
If output_1d_label is set, label will be output as a 1d array instead of a 2d array.
category: Transformation
"""
@classmethod
def INPUT_TYPES(cls) -> Dict[str, Any]:
"""
Defines the input types for the function.
Returns:
Dict[str, Any]: A dictionary specifying required input types.
"""
return {
"required": {
"dataframe": ("DATAFRAME", {}),
"label_integer_position": ("INT", {"default": -1, "min": -10, "max": 1024}),
"output_1d_label": ("BOOLEAN", {"default": True}),
"feature_data_type": (("float32","float16", "bloat16", "float64", "int8", "uint8", "int16", "int32", "int64"),),
"label_data_type": (("float32","float16", "bloat16", "float64", "int8", "uint8", "int16", "int32", "int64"),)
}
}
RETURN_TYPES: tuple = ("TENSOR", "TENSOR")
RETURN_NAMES: tuple = ("feature", "label")
FUNCTION: str = "f"
CATEGORY: str = "Data Analysis"
def f(self, dataframe: pd.DataFrame,
label_integer_position: int,
output_1d_label: bool,
feature_data_type: str,
label_data_type: str) -> tuple:
"""
Splits a Pandas DataFrame into feature Numpy ndarray and label Numpy ndarray.
Args:
dataframe (pd.DataFrame): The DataFrame to split.
label_integer_position (int): The column integer position of the label.
output_1d_label (bool): If output_1d_label is set, label will be output as a 1d array instead of a 2d array.
feature_data_type (str): Feature data type.
label_data_type (str): Label data type.
Returns:
tuple: A tuple containing the feature and label arrays.
"""
feature_dtype = getattr(torch, feature_data_type)
label_dtype = getattr(torch, label_data_type)
num_columns = dataframe.shape[1]
if label_integer_position < 0:
label_integer_position += num_columns
# Validate label_integer_position
if label_integer_position >= num_columns:
raise ValueError(f"label_integer_position {label_integer_position} is out of range for dataframe with {num_columns} columns.")
# Extract features
np_left = dataframe.iloc[:, :label_integer_position].to_numpy()
np_right = dataframe.iloc[:, label_integer_position + 1:].to_numpy() if label_integer_position < num_columns - 1 else np.empty((len(dataframe), 0))
np_features = np.hstack([np_left, np_right]) if np_right.shape[1] > 0 else np_left
# Extract label
np_label = dataframe.iloc[:, label_integer_position].to_numpy()
if output_1d_label:
np_label = np_label.ravel()
return (torch.tensor(np_features,dtype=feature_dtype),
torch.tensor(np_label, dtype=label_dtype))
modules/pandas_wrapper/pandas_replace_advanced.py
+61
View File
@@ -0,0 +1,61 @@
from typing import Any, Dict
import pandas as pd
import ast
class PandasReplaceAdvanced:
"""
Replaces strings in a DataFrame cell using a mapping specified in a dictionary.
For example, if you want to replace "cat" with "dog" and replace "mountain" with "ocean", specify the mapping in Python dict format:
```
{"cat":"dog", "mountain":"ocean"}
```
category: Data cleansing
"""
@classmethod
def INPUT_TYPES(cls) -> Dict[str, Any]:
"""
Defines the input types for the function.
Returns:
Dict[str, Any]: A dictionary specifying required input types.
"""
return {
"required": {
"dataframe": ("DATAFRAME", {}),
"replacement_dict": ("STRING", {"default": "{}", "multiline":True})
}
}
RETURN_TYPES: tuple = ("DATAFRAME",)
FUNCTION: str = "f"
CATEGORY: str = "Data Analysis"
def f(self, dataframe: pd.DataFrame, replacement_dict: str) -> tuple:
"""
Replaces part or all of a string in each DataFrame cell with a specified string using a wildcard for matching.
Args:
dataframe (DataFrame): The target DataFrame.
replacement_dict (str): A dict containing the mapping.
Returns:
tuple: A tuple containing the DataFrame.
"""
d = ast.literal_eval(replacement_dict)
if isinstance(d, dict) is False:
raise ValueError('You need to specify the mapping in Python dict format such as {"cat":"dog", "mountain":"ocean"}')
# check values
for k, v in d.items():
if (isinstance(k, str) is False and
isinstance(k, float) is False and
isinstance(k, int) is False):
raise ValueError("You need to specify a string or a number for source values.")
if (isinstance(v, str) is False and
isinstance(v, float) is False and
isinstance(v, int) is False):
raise ValueError("You need to specify a string or a number for target values.")
df2 = dataframe.replace(to_replace=d)
return (df2,)
pyproject.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[project]
name = "data-analysis"
description = "Data analysis custom modules for ComfyUI – Integrates Pandas, Seaborn and Matplotlib into ComfyUI."
version = "3.0.4"
version = "3.0.5"
license = {file = "LICENSE"}
[project.urls]
tests/pandas_wrapper/pandas_feture_split_to_pt_test.py
+50
View File
@@ -0,0 +1,50 @@
import os
import sys
import unittest
import numpy as np
import pandas as pd
import torch
PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", ".."))
MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules")
sys.path.append(MODULE_ROOT)
from pandas_wrapper.pandas_feature_split_to_pt import PandasFeatureSplitToPt
class TestPandasFeatureSplitToPt(unittest.TestCase):
def setUp(self):
"""Set up test data."""
self.node = PandasFeatureSplitToPt()
# Sample DataFrame
self.df = pd.DataFrame({
'A': [1, 2, 3],
'B': [4, 5, 6],
'C': [7, 8, 9]
})
def test_feature_split(self):
"""Test splitting with different label positions."""
test_cases = [
(0, True, [[4, 7], [5, 8], [6, 9]], [1, 2, 3]),
(1, True, [[1, 7], [2, 8], [3, 9]], [4, 5, 6]),
(2, True, [[1, 4], [2, 5], [3, 6]], [7, 8, 9]),
(-1, True, [[1, 4], [2, 5], [3, 6]], [7, 8, 9]),
]
data_types = ["float32", "float16", "bfloat16", "float64", "int8", "uint8", "int16", "int32", "int64"]
for feature_dtype in data_types:
for label_dtype in data_types:
for label_pos, output_1d, expected_features, expected_labels in test_cases:
with self.subTest(label_pos=label_pos, output_1d=output_1d, feature_dtype=feature_dtype, label_dtype=label_dtype):
features, labels = self.node.f(self.df, label_pos, output_1d, feature_dtype, label_dtype)
torch.testing.assert_close(features, torch.tensor(expected_features, dtype=getattr(torch, feature_dtype)))
torch.testing.assert_close(labels, torch.tensor(expected_labels, dtype=getattr(torch, label_dtype)))
def test_invalid_label_position(self):
"""Test invalid label positions."""
with self.assertRaises(ValueError):
self.node.f(self.df, 3, True, "float32", "int64")
if __name__ == "__main__":
unittest.main()
tests/pandas_wrapper/pandas_replace_advanced_test.py
+60
View File
@@ -0,0 +1,60 @@
import os
import sys
import unittest
import pandas as pd
import ast
from typing import Any, Dict
PROJECT_ROOT = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", ".."))
MODULE_ROOT = os.path.join(PROJECT_ROOT, "modules")
sys.path.append(MODULE_ROOT)
from pandas_wrapper.pandas_replace_advanced import PandasReplaceAdvanced
class TestPandasReplaceAdvanced(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.node = PandasReplaceAdvanced()
cls.df = pd.DataFrame(
{
"col1": ["cat", "dog", "fish"],
"col2": ["mountain", "ocean", "forest"],
}
)
def test_valid_replacement(self):
test_cases = [
('{"cat": 1, "dog": 2.34}', {"col1": [1, 2.34, "fish"], "col2": ["mountain", "ocean", "forest"]}),
('{"mountain": "hill", "forest": "jungle"}', {"col1": ["cat", "dog", "fish"], "col2": ["hill", "ocean", "jungle"]}),
]
for replacements, expected_data in test_cases:
with self.subTest(replacements=replacements):
result_df = self.node.f(self.df, replacements)[0]
expected_df = pd.DataFrame(expected_data)
pd.testing.assert_frame_equal(result_df, expected_df)
def test_invalid_replacement_value_error(self):
invalid_cases = [
'["cat", "lion"]', # List is not allowed
'{"cat": ["lion", "tiger"]}', # List is not allowed
'{"dog": {"wolf": "coyote"}}' # Nested dict is not allowed
]
for invalid_pairs in invalid_cases:
with self.subTest(invalid_pairs=invalid_pairs):
with self.assertRaises(ValueError):
self.node.f(self.df, invalid_pairs)
def test_invalid_replacement_syntax_error(self):
invalid_cases = [
'{"mountain": ()"hill", "ocean")}', # Syntax error
]
for invalid_pairs in invalid_cases:
with self.subTest(invalid_pairs=invalid_pairs):
with self.assertRaises(SyntaxError):
self.node.f(self.df, invalid_pairs)
if __name__ == "__main__":
unittest.main()