diff --git a/__init__.py b/__init__.py index 5222dcb..bc54f65 100644 --- a/__init__.py +++ b/__init__.py @@ -1,3 +1,4 @@ +# This file is auto-generated. Do not edit manually. from typing import Dict, Type, TypeVar from .modules.cda.cda_float_create import CDAFloatCreate from .modules.cda.cda_int_create import CDAIntCreate @@ -17,6 +18,7 @@ from .modules.pandas_wrapper.pandas_as_float import PandasAsFloat from .modules.pandas_wrapper.pandas_as_int import PandasAsInt from .modules.pandas_wrapper.pandas_as_string import PandasAsString from .modules.pandas_wrapper.pandas_columns import PandasColumns +from .modules.pandas_wrapper.pandas_cos import PandasCos from .modules.pandas_wrapper.pandas_count import PandasCount from .modules.pandas_wrapper.pandas_create import PandasCreate from .modules.pandas_wrapper.pandas_create_with_index import PandasCreateWithIndex @@ -28,12 +30,9 @@ from .modules.pandas_wrapper.pandas_div import PandasDiv from .modules.pandas_wrapper.pandas_div_scalar_float import PandasDivScalarFloat from .modules.pandas_wrapper.pandas_div_scalar_int import PandasDivScalarInt from .modules.pandas_wrapper.pandas_div_series import PandasDivSeries +from .modules.pandas_wrapper.pandas_drop_duplicates import PandasDropDuplicates from .modules.pandas_wrapper.pandas_dropna import PandasDropNA from .modules.pandas_wrapper.pandas_exp import PandasExp -from .modules.pandas_wrapper.pandas_log import PandasLog -from .modules.pandas_wrapper.pandas_sin import PandasSin -from .modules.pandas_wrapper.pandas_cos import PandasCos -from .modules.pandas_wrapper.pandas_tan import PandasTan from .modules.pandas_wrapper.pandas_group_by import PandasGroupBy from .modules.pandas_wrapper.pandas_head import PandasHead from .modules.pandas_wrapper.pandas_horizontal_concat import PandasHorizontalConcat @@ -51,6 +50,7 @@ from .modules.pandas_wrapper.pandas_load_html import PandasLoadHTML from .modules.pandas_wrapper.pandas_load_json import PandasLoadJSON from .modules.pandas_wrapper.pandas_loc_cell_str import PandasLocCellStr from .modules.pandas_wrapper.pandas_loc_row_series import PandasLocRowSeries +from .modules.pandas_wrapper.pandas_log import PandasLog from .modules.pandas_wrapper.pandas_max import PandasMax from .modules.pandas_wrapper.pandas_mean import PandasMean from .modules.pandas_wrapper.pandas_median import PandasMedian @@ -70,12 +70,13 @@ from .modules.pandas_wrapper.pandas_save_csv import PandasSaveCSV from .modules.pandas_wrapper.pandas_save_json import PandasSaveJSON from .modules.pandas_wrapper.pandas_select_columns import PandasSelectColumns from .modules.pandas_wrapper.pandas_select_rows import PandasSelectRows -from .modules.pandas_wrapper.pandas_series_to_string import PandasSeriesToString from .modules.pandas_wrapper.pandas_series_to_dataframe import PandasSeriesToDataFrame +from .modules.pandas_wrapper.pandas_series_to_string import PandasSeriesToString from .modules.pandas_wrapper.pandas_show_dataframe import PandasShowDataFrame from .modules.pandas_wrapper.pandas_show_index import PandasShowIndex from .modules.pandas_wrapper.pandas_show_series import PandasShowSeries from .modules.pandas_wrapper.pandas_show_text import PandasShowText +from .modules.pandas_wrapper.pandas_sin import PandasSin from .modules.pandas_wrapper.pandas_sort import PandasSort from .modules.pandas_wrapper.pandas_std import PandasStd from .modules.pandas_wrapper.pandas_sub import PandasSub @@ -84,14 +85,20 @@ from .modules.pandas_wrapper.pandas_sub_scalar_int import PandasSubScalarInt from .modules.pandas_wrapper.pandas_sub_series import PandasSubSeries from .modules.pandas_wrapper.pandas_sum import PandasSum from .modules.pandas_wrapper.pandas_summary import PandasSummary +from .modules.pandas_wrapper.pandas_tan import PandasTan from .modules.pandas_wrapper.pandas_to_string import PandasToString from .modules.pandas_wrapper.pandas_transpose import PandasTranspose from .modules.pandas_wrapper.pandas_var import PandasVar from .modules.pandas_wrapper.pandas_vertical_concat import PandasVerticalConcat from .modules.pandas_wrapper.pandas_vertical_split import PandasVerticalSplit - T = TypeVar("T") + +""" +NODE_CLASS_MAPPINGS (Dict[str, Type[T]]): + A dictionary mapping node names to their corresponding class implementations. +""" + NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "CDAFloatCreate": CDAFloatCreate, "CDAIntCreate": CDAIntCreate, @@ -111,6 +118,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasAsInt": PandasAsInt, "PandasAsString": PandasAsString, "PandasColumns": PandasColumns, + "PandasCos": PandasCos, "PandasCount": PandasCount, "PandasCreate": PandasCreate, "PandasCreateWithIndex": PandasCreateWithIndex, @@ -122,12 +130,9 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasDivScalarFloat": PandasDivScalarFloat, "PandasDivScalarInt": PandasDivScalarInt, "PandasDivSeries": PandasDivSeries, + "PandasDropDuplicates": PandasDropDuplicates, "PandasDropNA": PandasDropNA, "PandasExp": PandasExp, - "PandasLog": PandasLog, - "PandasSin": PandasSin, - "PandasCos": PandasCos, - "PandasTan": PandasTan, "PandasGroupBy": PandasGroupBy, "PandasHead": PandasHead, "PandasHorizontalConcat": PandasHorizontalConcat, @@ -145,6 +150,7 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasLoadJSON": PandasLoadJSON, "PandasLocCellStr": PandasLocCellStr, "PandasLocRowSeries": PandasLocRowSeries, + "PandasLog": PandasLog, "PandasMax": PandasMax, "PandasMean": PandasMean, "PandasMedian": PandasMedian, @@ -154,22 +160,23 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasMulScalarFloat": PandasMulScalarFloat, "PandasMulScalarInt": PandasMulScalarInt, "PandasMulSeries": PandasMulSeries, - "PandasRename": PandasRename, - "PandasReplace": PandasReplace, "PandasPow": PandasPow, "PandasPowScalarFloat": PandasPowScalarFloat, "PandasPowScalarInt": PandasPowScalarInt, "PandasPowSeries": PandasPowSeries, + "PandasRename": PandasRename, + "PandasReplace": PandasReplace, "PandasSaveCSV": PandasSaveCSV, "PandasSaveJSON": PandasSaveJSON, "PandasSelectColumns": PandasSelectColumns, "PandasSelectRows": PandasSelectRows, - "PandasSeriesToString": PandasSeriesToString, "PandasSeriesToDataFrame": PandasSeriesToDataFrame, + "PandasSeriesToString": PandasSeriesToString, "PandasShowDataFrame": PandasShowDataFrame, "PandasShowIndex": PandasShowIndex, "PandasShowSeries": PandasShowSeries, "PandasShowText": PandasShowText, + "PandasSin": PandasSin, "PandasSort": PandasSort, "PandasStd": PandasStd, "PandasSub": PandasSub, @@ -178,15 +185,18 @@ NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = { "PandasSubSeries": PandasSubSeries, "PandasSum": PandasSum, "PandasSummary": PandasSummary, + "PandasTan": PandasTan, "PandasToString": PandasToString, "PandasTranspose": PandasTranspose, "PandasVar": PandasVar, "PandasVerticalConcat": PandasVerticalConcat, - "PandasVerticalSplit": PandasVerticalSplit + "PandasVerticalSplit": PandasVerticalSplit, } + + """ -NODE_CLASS_MAPPINGS (Dict[str, Type[T]]): - A dictionary mapping node names to their corresponding class implementations. +NODE_DISPLAY_NAME_MAPPINGS (Dict[str, str]): + A dictionary mapping node names to user-friendly display names. """ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { @@ -202,12 +212,13 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "MPLScatter": "MPL Scatter Plot", "PandasAdd": "Pandas Add", "PandasAddScalarFloat": "Pandas Add Scalar Float", - "PandasAddScalarInt":" Pandas Add Scalar Int", + "PandasAddScalarInt": "Pandas Add Scalar Int", "PandasAddSeries": "Pandas Add Series", "PandasAsFloat": "Pandas As Float", "PandasAsInt": "Pandas As Int", "PandasAsString": "Pandas As String", "PandasColumns": "Pandas Columns", + "PandasCos": "Pandas Cos", "PandasCount": "Pandas Count", "PandasCreate": "Pandas Create", "PandasCreateWithIndex": "Pandas Create With Index", @@ -219,12 +230,9 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasDivScalarFloat": "Pandas Div Scalar Float", "PandasDivScalarInt": "Pandas Div Scalar Int", "PandasDivSeries": "Pandas Div Series", + "PandasDropDuplicates": "Pandas Drop Duplicates", "PandasDropNA": "Pandas Drop NA", "PandasExp": "Pandas Exp", - "PandasLog": "Pandas Log", - "PandasSin": "Pandas Sin", - "PandasCos": "Pandas Cos", - "PandasTan": "Pandas Tan", "PandasGroupBy": "Pandas Group By", "PandasHead": "Pandas Head", "PandasHorizontalConcat": "Pandas Horizontal Concat", @@ -242,6 +250,7 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasLoadJSON": "Pandas Load JSON", "PandasLocCellStr": "Pandas Loc Cell Str", "PandasLocRowSeries": "Pandas Loc Row Series", + "PandasLog": "Pandas Log", "PandasMax": "Pandas Max", "PandasMean": "Pandas Mean", "PandasMedian": "Pandas Median", @@ -261,12 +270,13 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasSaveJSON": "Pandas Save JSON", "PandasSelectColumns": "Pandas Select Columns", "PandasSelectRows": "Pandas Select Rows", - "PandasSeriesToString": "Pandas Series To String", "PandasSeriesToDataFrame": "Pandas Series To DataFrame", + "PandasSeriesToString": "Pandas Series To String", "PandasShowDataFrame": "Pandas Show DataFrame", "PandasShowIndex": "Pandas Show Index", "PandasShowSeries": "Pandas Show Series", "PandasShowText": "Pandas Show Text", + "PandasSin": "Pandas Sin", "PandasSort": "Pandas Sort", "PandasStd": "Pandas Std", "PandasSub": "Pandas Sub", @@ -275,21 +285,20 @@ NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = { "PandasSubSeries": "Pandas Sub Series", "PandasSum": "Pandas Sum", "PandasSummary": "Pandas Summary", + "PandasTan": "Pandas Tan", "PandasToString": "Pandas To String", "PandasTranspose": "Pandas Transpose", "PandasVar": "Pandas Var", "PandasVerticalConcat": "Pandas Vertical Concat", - "PandasVerticalSplit": "Pandas Vertical Split" + "PandasVerticalSplit": "Pandas Vertical Split", } -""" -NODE_DISPLAY_NAME_MAPPINGS (Dict[str, str]): - A dictionary mapping node names to user-friendly display names. -""" + """ Below two lines were taken from: https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/__init__.py See credit/credit.md for the full license. """ + WEB_DIRECTORY = "./web" __all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] diff --git a/dev/scripts/init_py_generate.py b/dev/scripts/init_py_generate.py new file mode 100644 index 0000000..c7b09f7 --- /dev/null +++ b/dev/scripts/init_py_generate.py @@ -0,0 +1,169 @@ +""" +Generate top level __init__.py. + +Run this at the project root if you add a new .py file. + +To skip the python file from included, put +``` +pragma: skip_doc +``` +in the docstring for the class. This pragma is used for both node reference as well as __init__.py. +``` +""" +import os +import ast +import pathlib +from typing import TypeVar + +T = TypeVar("T") +MODULES_DIR = "modules" + + +def extract_classes_from_file(file_path): + with open(file_path, "r", encoding="utf-8") as f: + tree = ast.parse(f.read(), filename=file_path) + + classes = [] + for node in ast.walk(tree): + if isinstance(node, ast.ClassDef): + docstring = ast.get_docstring(node) + if docstring and "pragma: skip_doc" in docstring: + continue + classes.append(node.name) + return classes + + +def format_display_name(class_name): + result = [] + first_lower_encountered = False + + i = 0 + while i < len(class_name): + if i > 0 and class_name[i].isupper(): + if first_lower_encountered or (i + 1 < len(class_name) and class_name[i + 1].islower()): + result.append(" ") + elif not first_lower_encountered and (i + 1 < len(class_name) and class_name[i + 1].isupper()): + if i + 2 < len(class_name) and class_name[i + 2].islower(): + result.append(" ") + + if class_name[i].islower(): + first_lower_encountered = True + + # Special case for JSON and CSV + if class_name[i:i+4] == "JSON": + if result and result[-1] != " ": + result.append(" ") + result.append("JSON") + if i + 4 < len(class_name) and class_name[i + 4].isupper(): + result.append(" ") + i += 3 # Skip processed letters + elif class_name[i:i+3] == "CSV": + if result and result[-1] != " ": + result.append(" ") + result.append("CSV") + if i + 3 < len(class_name) and class_name[i + 3].isupper(): + result.append(" ") + i += 2 # Skip processed letters + else: + result.append(class_name[i]) + i += 1 + + # Ensure acronyms at the start are not incorrectly split (like "CD A" instead of "CDA") + final_result = "".join(result) + + # TODO: Reduce hack below + replace_pairs = [ + ("CD A", "CDA"), + ("H T M L", "HTML"), + ("MP L", "MPL"), + (" N A", " NA"), + ("Data Frame", "DataFrame"), + ("MPL Bar", "MPL Bar Chart"), + ("MPL Line", "MPL Line Plot"), + ("MPL Scatter", "MPL Scatter Plot") + ] + for p in replace_pairs: + final_result = final_result.replace(p[0], p[1]) + + return " ".join(final_result.split()) # Remove unintended extra spaces + + +def discover_modules(): + module_entries = [] + for py_file in pathlib.Path(MODULES_DIR).rglob("*.py"): + relative_path = py_file.relative_to(MODULES_DIR).with_suffix("") + module_import_path = ".modules." + str(relative_path).replace(os.sep, ".") + class_names = extract_classes_from_file(py_file) + + for class_name in class_names: + display_name = format_display_name(class_name) + module_entries.append((module_import_path, class_name, display_name)) + + return module_entries + + +def get_class_mappings_doc(): + return """ +\"\"\" +NODE_CLASS_MAPPINGS (Dict[str, Type[T]]): + A dictionary mapping node names to their corresponding class implementations. +\"\"\" +""" + + +def get_display_name_mappings_doc(): + return """ +\"\"\" +NODE_DISPLAY_NAME_MAPPINGS (Dict[str, str]): + A dictionary mapping node names to user-friendly display names. +\"\"\" +""" + + +def get_license_doc(): + return """ +\"\"\" +Below two lines were taken from: +https://github.com/pythongosssss/ComfyUI-Custom-Scripts/blob/main/__init__.py +See credit/credit.md for the full license. +\"\"\" +""" + + +def generate_init_content(import_statements, node_class_mappings, node_display_name_mappings): + return f"""# This file is auto-generated. Do not edit manually. +from typing import Dict, Type, TypeVar +{os.linesep.join(import_statements)} +T = TypeVar("T") + +{get_class_mappings_doc()} +NODE_CLASS_MAPPINGS: Dict[str, Type[T]] = {{ +{os.linesep.join(node_class_mappings)} +}} + +{get_display_name_mappings_doc()} +NODE_DISPLAY_NAME_MAPPINGS: Dict[str, str] = {{ +{os.linesep.join(node_display_name_mappings)} +}} + +{get_license_doc()} +WEB_DIRECTORY = "./web" +__all__ = ["NODE_CLASS_MAPPINGS", "NODE_DISPLAY_NAME_MAPPINGS", "WEB_DIRECTORY"] +""" + + +def generate_init(): + module_entries = discover_modules() + + import_statements = sorted([f"from {entry[0]} import {entry[1]}" for entry in module_entries]) + node_class_mappings = sorted([f' "{entry[1]}": {entry[1]},' for entry in module_entries]) + node_display_name_mappings = sorted([f' "{entry[1]}": "{entry[2]}",' for entry in module_entries]) + + template = generate_init_content(import_statements, node_class_mappings, node_display_name_mappings) + + with open("__init__.py", "w", encoding="utf-8") as f: + f.write(template) + + +if __name__ == "__main__": + generate_init() diff --git a/docs/reference/node_reference.md b/docs/reference/node_reference.md index fc93bed..203c736 100644 --- a/docs/reference/node_reference.md +++ b/docs/reference/node_reference.md @@ -36,6 +36,7 @@ ## Data cleansing | Node | Description | | --- | --- | +| [Pandas Drop Duplicates](pandas_drop_duplicates.md) | Drops duplicate rows from a Pandas DataFrame. | | [Pandas Drop NA](pandas_dropna.md) | Drops missing values from a pandas DataFrame. | | [Pandas Is NA](pandas_isna.md) | Checks a pandas DataFrame for missing values. | | [Pandas Replace](pandas_replace.md) | Replaces part or all of a string in each DataFrame cell with a specified string using a wildcard for matching. If you want to remove some characters, then leave the replace_string field blank. | diff --git a/docs/reference/pandas_drop_duplicates.md b/docs/reference/pandas_drop_duplicates.md new file mode 100644 index 0000000..247cf36 --- /dev/null +++ b/docs/reference/pandas_drop_duplicates.md @@ -0,0 +1,17 @@ +# Pandas Drop Duplicates +Drops duplicate rows from a Pandas DataFrame. + +## Input +| Name | Data type | +|---|---| +| dataframe | DataFrame | + +## Output +| Data type | +|---| +| DataFrame | + +
+Category: Data cleansing + +ComfyUI Data Analysis Node Reference. © 2025 Hide Inada (HowToSD.com). All rights reserved. diff --git a/modules/pandas_wrapper/pandas_drop_duplicates.py b/modules/pandas_wrapper/pandas_drop_duplicates.py new file mode 100644 index 0000000..cb457de --- /dev/null +++ b/modules/pandas_wrapper/pandas_drop_duplicates.py @@ -0,0 +1,39 @@ +from typing import Any, Dict +import pandas as pd + +class PandasDropDuplicates: + """ + Drops duplicate rows from a Pandas DataFrame. + + category: Data cleansing + """ + + @classmethod + def INPUT_TYPES(cls) -> Dict[str, Any]: + """ + Defines the input types for the function. + + Returns: + Dict[str, Any]: A dictionary specifying required input types. + """ + return { + "required": { + "dataframe": ("DATAFRAME", {}) + } + } + + RETURN_TYPES: tuple = ("DATAFRAME",) + FUNCTION: str = "f" + CATEGORY: str = "Data Analysis" + + def f(self, dataframe: pd.DataFrame) -> tuple: + """ + Drops duplicate rows from a Pandas DataFrame. + + Args: + dataframe (DataFrame): The DataFrame. + + Returns: + tuple: A tuple containing the DataFrame. + """ + return (dataframe.drop_duplicates(),) diff --git a/modules/pandas_wrapper/pandas_dropna.py b/modules/pandas_wrapper/pandas_dropna.py index 6d9ce10..e4d680b 100644 --- a/modules/pandas_wrapper/pandas_dropna.py +++ b/modules/pandas_wrapper/pandas_dropna.py @@ -37,4 +37,4 @@ class PandasDropNA: Returns: tuple: A tuple containing the DataFrame. """ - return (dataframe.dropna().to_json(),) + return (dataframe.dropna(),) diff --git a/pyproject.toml b/pyproject.toml index d3696f2..23895be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "data-analysis" description = "Data analysis custom modules for ComfyUI - Use Pandas & Matplotlib within ComfyUI" -version = "1.1.12" +version = "1.1.13" license = {file = "LICENSE"} [project.urls]