import os def is_binary_file(path): try: with open(path, "rb") as f: chunk = f.read(1024) if b"\x00" in chunk: return True except: return True return False def try_decode(data): """ 尝试多种编码格式解码二进制数据 (对齐 fix_encoding.py) """ # 1. UTF-8 try: return data.decode("utf-8"), "utf-8" except: pass # 2. UTF-16 with BOM if data.startswith(b'\xff\xfe') or data.startswith(b'\xfe\xff'): try: return data.decode("utf-16"), "utf-16" except: pass # 3. GB18030 (covers GBK and more) try: return data.decode("gb18030"), "gb18030" except: pass # 4. Fallback return data.decode("utf-8", errors="ignore"), "fallback-ignore" def clean_text(text, remove_emoji=True): """ 严格清洗文本 (对齐 fix_encoding.py): - 删除控制字符 (保留换行和制表) - 删除 surrogate 区 - 删除 emoji (默认强制删除 > 0xFFFF 的所有 4 字节字符) - 强制统一换行符为 Unix 风格 (\n) """ cleaned = [] for c in text: code = ord(c) # 删除控制字符(保留换行和制表) if 0 <= code < 32 and c not in ("\n", "\t"): continue # 删除 surrogate 区 if 0xD800 <= code <= 0xDFFF: continue # 删除 emoji(默认删除所有 > 0xFFFF 字符) if remove_emoji and code > 0xFFFF: continue cleaned.append(c) result_text = "".join(cleaned) # 强制统一换行符为 Unix 风格 (\n),防止 \r 导致打标工具报错 result_text = result_text.replace("\r\n", "\n").replace("\r", "\n") return result_text class UTF8ConverterNode: """ UTF-8 编码转换与文本清洗节点 (工业级强力重构版): 完全对齐 fix_encoding.py 的逻辑,使用鲁棒的解码策略并执行极其严格的字符清洗。 """ @classmethod def INPUT_TYPES(cls): return { "required": { "input_text": ("STRING", { "multiline": True, "default": "", "placeholder": "输入文本 / Input Text", "tooltip": "待转码的文本。用于把从各处复制来的乱码文本\n" "(GBK/UTF-16 等被误读的内容)修正为正常 UTF-8。\n" "若下方填了文件路径,则以文件内容为准,本框会被忽略。" }), "remove_emoji": ("BOOLEAN", { "default": True, "label_on": "是 / Yes", "label_off": "否 / No", "tooltip": "是否强制删除所有 4 字节字符(包含绝大多数 Emoji)" }), }, "optional": { "input_file": ("STRING", { "multiline": False, "default": "", "placeholder": "文件路径 / File Path (优先读取 / Overrides input_text)", "tooltip": "文本文件路径。填了它就优先读文件,忽略上面的输入框。\n" "会依次尝试 UTF-8 → UTF-16(带BOM) → GB18030 解码,\n" "都失败则忽略非法字节强行读出,尽量不让整个文件报废。" }), } } RETURN_TYPES = ("STRING", "STRING") RETURN_NAMES = ("filtered_text", "log") FUNCTION = "run" CATEGORY = "Rui-Node🐶/文本处理📝" def run(self, input_text, remove_emoji, input_file=""): log = [] text_content = "" encoding_used = "none" # 1. 获取原始内容 (文件优先) if input_file and input_file.strip(): if os.path.exists(input_file): log.append(f"正在读取文件: {input_file}") # 检查是否为二进制文件 if is_binary_file(input_file): log.append("警告: 检测到可能的二进制文件 (跳过)") return ("", "\n".join(log)) try: with open(input_file, "rb") as f: raw_data = f.read() text_content, encoding_used = try_decode(raw_data) log.append(f"解码成功: 使用编码 {encoding_used}") except Exception as e: return ("", f"读取或解码文件时发生错误: {str(e)}") else: log.append(f"错误: 文件不存在 -> {input_file}") log.append("回退到使用 input_text") text_content = input_text encoding_used = "input_text" else: log.append("使用直接输入的文本") text_content = input_text encoding_used = "input_text" # 2. 清洗文本 log.append("-" * 20) log.append("开始工业级强力清洗文本...") original_len = len(text_content) cleaned_content = clean_text(text_content, remove_emoji=remove_emoji) final_len = len(cleaned_content) removed_count = original_len - final_len # 3. 生成日志 log.append(f"原始长度: {original_len}") log.append(f"输出长度: {final_len}") log.append(f"移除字符数: {removed_count}") if removed_count > 0: log.append(f"清洗详情: 已移除非法控制字符、代理对,并统一换行符为 \\n。") if remove_emoji: log.append("已开启 remove_emoji: 删除了所有 4 字节字符。") else: log.append("结果: 文本已符合规范,无需修改。") return (cleaned_content, "\n".join(log)) # 节点映射字典 NODE_CLASS_MAPPINGS = { "UTF8Converter": UTF8ConverterNode } # 节点显示名称映射 NODE_DISPLAY_NAME_MAPPINGS = { "UTF8Converter": "转化为utf-8编码 / Convert to UTF-8" }