rui40000-RUI-Nodes/utf8_converter_node.py

import os

def is_binary_file(path):
    try:
        with open(path, "rb") as f:
            chunk = f.read(1024)
            if b"\x00" in chunk:
                return True
    except:
        return True
    return False

def try_decode(data):
    """
    尝试多种编码格式解码二进制数据 (对齐 fix_encoding.py)
    """
    # 1. UTF-8
    try:
        return data.decode("utf-8"), "utf-8"
    except:
        pass

    # 2. UTF-16 with BOM
    if data.startswith(b'\xff\xfe') or data.startswith(b'\xfe\xff'):
        try:
            return data.decode("utf-16"), "utf-16"
        except:
            pass

    # 3. GB18030 (covers GBK and more)
    try:
        return data.decode("gb18030"), "gb18030"
    except:
        pass
        
    # 4. Fallback
    return data.decode("utf-8", errors="ignore"), "fallback-ignore"

def clean_text(text, remove_emoji=True):
    """
    严格清洗文本 (对齐 fix_encoding.py)：
    - 删除控制字符 (保留换行和制表)
    - 删除 surrogate 区
    - 删除 emoji (默认强制删除 > 0xFFFF 的所有 4 字节字符)
    - 强制统一换行符为 Unix 风格 (\n)
    """
    cleaned = []
    for c in text:
        code = ord(c)

        # 删除控制字符（保留换行和制表）
        if 0 <= code < 32 and c not in ("\n", "\t"):
            continue

        # 删除 surrogate 区
        if 0xD800 <= code <= 0xDFFF:
            continue

        # 删除 emoji（默认删除所有 > 0xFFFF 字符）
        if remove_emoji and code > 0xFFFF:
            continue

        cleaned.append(c)

    result_text = "".join(cleaned)
    # 强制统一换行符为 Unix 风格 (\n)，防止 \r 导致打标工具报错
    result_text = result_text.replace("\r\n", "\n").replace("\r", "\n")
    return result_text

class UTF8ConverterNode:
    """
    UTF-8 编码转换与文本清洗节点 (工业级强力重构版)：
    完全对齐 fix_encoding.py 的逻辑，使用鲁棒的解码策略并执行极其严格的字符清洗。
    """
    
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "input_text": ("STRING", {
                    "multiline": True,
                    "default": "",
                    "placeholder": "输入文本 / Input Text",
                    "tooltip": "待转码的文本。用于把从各处复制来的乱码文本\n"
                               "（GBK/UTF-16 等被误读的内容）修正为正常 UTF-8。\n"
                               "若下方填了文件路径，则以文件内容为准，本框会被忽略。"
                }),
                "remove_emoji": ("BOOLEAN", {
                    "default": True,
                    "label_on": "是 / Yes",
                    "label_off": "否 / No",
                    "tooltip": "是否强制删除所有 4 字节字符（包含绝大多数 Emoji）"
                }),
            },
            "optional": {
                "input_file": ("STRING", {
                    "multiline": False,
                    "default": "",
                    "placeholder": "文件路径 / File Path (优先读取 / Overrides input_text)",
                    "tooltip": "文本文件路径。填了它就优先读文件，忽略上面的输入框。\n"
                               "会依次尝试 UTF-8 → UTF-16(带BOM) → GB18030 解码，\n"
                               "都失败则忽略非法字节强行读出，尽量不让整个文件报废。"
                }),
            }
        }

    RETURN_TYPES = ("STRING", "STRING")
    RETURN_NAMES = ("filtered_text", "log")
    FUNCTION = "run"
    CATEGORY = "Rui-Node🐶/文本处理📝"

    def run(self, input_text, remove_emoji, input_file=""):
        log = []
        text_content = ""
        encoding_used = "none"

        # 1. 获取原始内容 (文件优先)
        if input_file and input_file.strip():
            if os.path.exists(input_file):
                log.append(f"正在读取文件: {input_file}")
                
                # 检查是否为二进制文件
                if is_binary_file(input_file):
                    log.append("警告: 检测到可能的二进制文件 (跳过)")
                    return ("", "\n".join(log))
                
                try:
                    with open(input_file, "rb") as f:
                        raw_data = f.read()
                    
                    text_content, encoding_used = try_decode(raw_data)
                    log.append(f"解码成功: 使用编码 {encoding_used}")
                    
                except Exception as e:
                    return ("", f"读取或解码文件时发生错误: {str(e)}")
            else:
                log.append(f"错误: 文件不存在 -> {input_file}")
                log.append("回退到使用 input_text")
                text_content = input_text
                encoding_used = "input_text"
        else:
            log.append("使用直接输入的文本")
            text_content = input_text
            encoding_used = "input_text"

        # 2. 清洗文本
        log.append("-" * 20)
        log.append("开始工业级强力清洗文本...")
        original_len = len(text_content)
        
        cleaned_content = clean_text(text_content, remove_emoji=remove_emoji)
        
        final_len = len(cleaned_content)
        removed_count = original_len - final_len
        
        # 3. 生成日志
        log.append(f"原始长度: {original_len}")
        log.append(f"输出长度: {final_len}")
        log.append(f"移除字符数: {removed_count}")
        
        if removed_count > 0:
            log.append(f"清洗详情: 已移除非法控制字符、代理对，并统一换行符为 \\n。")
            if remove_emoji:
                 log.append("已开启 remove_emoji: 删除了所有 4 字节字符。")
        else:
            log.append("结果: 文本已符合规范，无需修改。")

        return (cleaned_content, "\n".join(log))

# 节点映射字典
NODE_CLASS_MAPPINGS = {
    "UTF8Converter": UTF8ConverterNode
}

# 节点显示名称映射
NODE_DISPLAY_NAME_MAPPINGS = {
    "UTF8Converter": "转化为utf-8编码 / Convert to UTF-8"
}