bump: Update version to 1.3.1

Merge Dev

.env-cmdrc.json

@@ -0,0 +1,5 @@
+{
+  "test": {
+    "NODE_OPTIONS": "--experimental-vm-modules"
+  }
+}

package-lock.json

@@ -8,6 +8,7 @@
         "@babel/core": "^7.27.1",
         "@babel/preset-env": "^7.27.2",
         "babel-jest": "^29.7.0",
+        "env-cmd": "^10.1.0",
         "jest": "^29.7.0",
         "stylelint": "^16.19.1",
         "stylelint-config-idiomatic-order": "^10.0.0",
@@ -3018,6 +3019,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/commander": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
+      "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3266,6 +3277,23 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/env-cmd": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-10.1.0.tgz",
+      "integrity": "sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "commander": "^4.0.0",
+        "cross-spawn": "^7.0.0"
+      },
+      "bin": {
+        "env-cmd": "bin/env-cmd.js"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/env-paths": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",

package.json

@@ -1,12 +1,13 @@
 {
   "type": "module",
   "scripts": {
-    "test": "jest"
+    "test": "env-cmd -e test -- jest"
   },
   "devDependencies": {
     "@babel/core": "^7.27.1",
     "@babel/preset-env": "^7.27.2",
     "babel-jest": "^29.7.0",
+    "env-cmd": "^10.1.0",
     "jest": "^29.7.0",
     "stylelint": "^16.19.1",
     "stylelint-config-idiomatic-order": "^10.0.0",

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-autocomplete-plus"
 description = "Autocomplete and Related Tag display for ComfyUI"
-version = "1.3.0"
+version = "1.3.1"
 license = {file = "LICENSE"}
 dependencies = ["",]
 

tests/js/flexsearch.test.js

@@ -0,0 +1,324 @@
+
+import { 
+    createFlexSearchDocument,
+    __test__
+ } from "../../web/js/searchengine.js";
+
+const { createTagEncoder, createCJKEncoder } = __test__;
+
+function parseCSVLine(line) {
+    const result = [];
+    let current = '';
+    let inQuotes = false;
+
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+
+        if (char === '"') {
+            if (inQuotes && i + 1 < line.length && line[i + 1] === '"') {
+                current += '"';
+                i++;
+            } else {
+                inQuotes = !inQuotes;
+            }
+        } else if (char === ',' && !inQuotes) {
+            result.push(current);
+            current = '';
+        } else {
+            current += char;
+        }
+    }
+
+    result.push(current);
+
+    return result;
+}
+
+describe('FlexSearch Integration', () => {
+
+    const commonCSV = `
+1girl,0,6008644,"1girls,sole_female"
+highres,5,5256195,"high_res,high_resolution,hires"
+solo,0,5000954,"alone,female_solo,single,solo_female,solo_in_panel"
+long_hair,0,4350743,"/lh,longhair,very_long_hair"
+one_two_three,0,29389,
+`;
+
+    const cjkAliasCSV = `
+blue_hair,0,676176,"青髪,青い髪,水色髪"
+red_hair,0,413261,"赤髪,紅髪,红发,빨강머리,빨간머리"
+smile,0,2294308,"笑い,スマイル,笑顔,笑顏,守りたい、この笑顔,笑,笑容,微笑み,微笑,微笑む,미소,守りたいこの笑顔"
+gloves,0,1105296,"手袋,裸手袋,手袋コキ,てぶくろ,장갑,手套"
+dragon_girl,0,37930,"竜娘,ドラゴン娘,龍娘,龙娘,メスドラ,辰娘"
+double_bun,0,103538,"お団子頭,お団子"
+sanshoku_dango,0,2061,"三色団子,三色团子,花見団子,花见团子"
+`;
+
+    const specialCharCSV = `
+:d,0,436700,
+>:),0,11041,
+year:1999,0,1999,
+d.d.,0,1999,
+copyright_(series),2,1298,"copyright,コピーライト (シリーズ),コピーライト名,コピーライト,著作"
+`;
+
+    const ControlCSV = `
+__wildcard__,0,1000,
+<lora:my_lora1:1.0>,0,1000,
+Embedding: my_embedding,0,1000,
+`;
+
+    const mockCSV = [
+        commonCSV, cjkAliasCSV, specialCharCSV, ControlCSV
+    ].map(csv => csv.trim()).join('\n');
+    
+    let mockTags;
+
+    let tagEncoder, cjkEncoder;
+    let document;
+
+    let performSearch = function (query, limit = 100) {
+        const results = document.search(query, {
+            field: ["tag", "alias"],
+            limit: limit, 
+            suggest: false,
+            merge: true,
+        });
+
+        const ids = results.map(r => r.id);
+
+        return mockTags.filter(tag => ids.includes(tag.id)).map(tag => tag.tag);
+    }
+
+    beforeEach(() => {
+        mockTags = mockCSV.split('\n').map((line, id) => {
+            const [tag, category, count, alias] = parseCSVLine(line);
+            return { id, tag, category: parseInt(category), count: parseInt(count), alias };
+        });
+
+        tagEncoder = createTagEncoder();
+        cjkEncoder = createCJKEncoder();
+
+        document = createFlexSearchDocument();
+
+        mockTags.forEach(data => document.add(data));
+    });
+
+    describe('Encoder', () => {
+        test('should split underscore-separated tags', () => {
+            const encoded = tagEncoder.encode('sanshoku_dango');
+            expect(encoded).toEqual(['sanshoku', 'dango']);
+        });
+
+        test('should extract words from parentheses', () => {
+            const encoded = tagEncoder.encode('copyright_(series)');
+            expect(encoded).toEqual(['copyright', 'series']);
+        });
+        test('should preserve colon-separated special tags', () => {
+            const encoded = tagEncoder.encode('year:1234');
+            expect(encoded).toEqual(['year:1234']);
+        });
+        test('should preserve dot-separated tags', () => {
+            const encoded = tagEncoder.encode('d.d.');
+            expect(encoded).toEqual(['d.d.']);
+        });
+        test('should preserve double underscore wildcard tags', () => {
+            const encoded = tagEncoder.encode('__wildcard__');
+            expect(encoded).toEqual(['__wildcard__']);
+        });
+        test('should convert katakana to hiragana', () => {
+            const encoded = cjkEncoder.encode('ガーデン');
+            expect(encoded).toEqual(['がーでん']);
+        });
+        test('should remove trailing underscores when splitting', () => {
+            const encoded = tagEncoder.encode('one_two_');
+            expect(encoded).toEqual(['one', 'two']);
+        });
+    });
+
+    describe('Basic Search', () => {
+        test('should find a tag by exact match', () => {
+            const results = performSearch('1girl');
+            expect(results.length).toBeGreaterThan(0);
+            expect(results).toContain('1girl');
+        });
+
+        test('should find a tag by partial match (substring)', () => {
+            const results = performSearch('blue');
+            expect(results.length).toBeGreaterThan(0);
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should be case-insensitive', () => {
+            const results = performSearch('BLUE_HAIR');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by backward', () => {
+            const results = performSearch('gon');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('dragon_girl');
+        });
+
+        test('should find a tag for terms contain space', () => {
+            const results = performSearch('double ');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('double_bun');
+        });
+
+        test('should find a tag for terms contain underscore', () => {
+            const results = performSearch('double_');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('double_bun');
+        });
+    });
+
+    describe('Alias Search', () => {
+        test('should find a tag by its Japanese alias', () => {
+            const results = performSearch('髪');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by one of its multiple aliases', () => {
+            const results = performSearch('笑顔');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('smile');
+        });
+
+        test('should find a tag by its partial Japanese alias', () => {
+            const results = performSearch('青い');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by katakana', () => {
+            const results = performSearch('テブクロ');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('gloves');
+        });
+
+        test('should find a tag by hiragana', () => {
+            const results = performSearch('すまいる');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('smile');
+        });
+
+        test('should not find a tag by english alias substring', () => {
+            const results = performSearch('meg');
+            expect(results.length).toEqual(0);
+        });
+
+        test('should find a tag by japanese alias substring', () => {
+            const results = performSearch('団子');
+            expect(results.length).toEqual(2);
+
+            expect(results).toContain('sanshoku_dango');
+            expect(results).toContain('double_bun');
+        });
+    });
+
+    describe('Special Characters and Edge Cases', () => {
+        test('should find a tag with parentheses', () => {
+            const results = performSearch('copyright_(series)');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should find a tag by searching for content inside parentheses', () => {
+            const results = performSearch('series');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should find a tag by partial word', () => {
+            const results = performSearch('right');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should return an empty array for a non-existent tag', () => {
+            const results = performSearch('non_existent_tag_xyz');
+            expect(results).toEqual([]);
+        });
+
+        test('should match to special character only tag', () => {
+            const tag = '>:)';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag', () => {
+            const tag = ':d';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag2', () => {
+            const tag = 'year:1999';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag3', () => {
+            const tag = 'd.d.';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to wildcard tag', () => {
+            const tag = '__';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('__wildcard__');
+        });
+
+        test('should match to lora tag', () => {
+            const tag = '<lora';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain("<lora:my_lora1:1.0>");
+        });
+    });
+
+    describe('Search Options', () => {
+        test('should respect the limit option', () => {
+            const results = performSearch('hair', 1);
+
+            expect(results.length).toEqual(1);
+        });
+
+        test('should return all matches when limit is higher than results', () => {
+            const results = performSearch('hair', 5);
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toHaveLength(3);
+            expect(results).toContain('long_hair');
+            expect(results).toContain('blue_hair');
+            expect(results).toContain('red_hair');
+        });
+    });
+});

web/js/autocomplete.js

@@ -79,20 +79,15 @@ function matchWord(target, queries) {
  * @returns {Array<TagData>} The list of matching candidates.
  */
 function searchCompletionCandidates(textareaElement) {
-    const startTime = performance.now(); // Record start time for performance measurement
 
     const ESCAPE_SEQUENCE = ["#", "/"]; // If the first string is that character, autocomplete will not be displayed.
     const partialTag = getCurrentPartialTag(textareaElement);
-    if (!partialTag || partialTag.length <= 0 || 
-        ESCAPE_SEQUENCE.some(seq => partialTag.startsWith(seq)) || 
+    if (!partialTag || partialTag.length <= 0 ||
+        ESCAPE_SEQUENCE.some(seq => partialTag.startsWith(seq)) ||
         isLongText(partialTag)) {
         return []; // No valid input for autocomplete
     }
 
-    const exactMatches = [];
-    const partialMatches = [];
-    const addedTags = new Set();
-
     // Generate Hiragana/Katakana variations if applicable
     const queryVariations = new Set([partialTag, normalizeTagToSearch(partialTag)]);
     const kataQuery = hiraToKata(partialTag);
@@ -104,106 +99,78 @@ function searchCompletionCandidates(textareaElement) {
         queryVariations.add(hiraQuery);
     }
 
+    if (settingValues.useFastSearch) {
+        return searchWithFlexSearch(partialTag, queryVariations);
+    } else {
+        return sequentialSearch(partialTag, queryVariations);
+    }
+}
+
+/**
+ * Search completion candidates using sequential search.
+ * @param {string} partialTag 
+ * @param {Set<string>} queryVariations 
+ * @returns 
+ */
+function sequentialSearch(partialTag, queryVariations) {
+    const startTime = performance.now();
+
+    const exactMatches = [];
+    const partialMatches = [];
+    const addedTags = new Set();
+
     const sources = getEnabledTagSourceInPriorityOrder();
     for (const source of sources) {
-        // Use fast search if enabled and available for the source
-        if (settingValues.useFastSearch && autoCompleteData[source].flexSearchIndex) {
-            // Use the FlexSearch Index to search tag and alias IDs that match the partial tag.
-            const searchResults = autoCompleteData[source].flexSearchIndex.search(partialTag, {
-                limit: settingValues.maxSuggestions * autoCompleteData[source].flexSearchLimitMultiplier,
-                suggest: false,
-                cache: true,
-            });
+        // Search in sortedTags (already sorted by count)
+        for (const tagData of autoCompleteData[source].sortedTags) {
+            let matched = false;
+            let isExactMatch = false;
+            let matchedAlias = null;
 
-            // Get tag IDs from search results and filter duplicates
-            let result = searchResults.map((index) => {
-                return autoCompleteData[source].flexSearchMapping[index];
-            });
-            result = [...new Set(result)];
+            // Check primary tag against all variations for exact/partial match
+            const tagMatch = matchWord(tagData.tag, queryVariations);
+            matched = tagMatch.matched;
+            isExactMatch = tagMatch.isExactMatch;
 
-            // Sort results based on exact matches or id values (ID order is equal to tag count order)
-            result = result.sort((a, b) => {
-                const aTag = autoCompleteData[source].sortedTags[a];
-                const bTag = autoCompleteData[source].sortedTags[b];
-                if (matchWord(bTag.tag, queryVariations).isExactMatch) {
-                    return 999999999999;
+            // If primary tag didn't match, check aliases against all variations
+            if (!matched && tagData.alias && Array.isArray(tagData.alias) && tagData.alias.length > 0) {
+                for (const alias of tagData.alias) {
+                    const lowerAlias = alias.toLowerCase();
+                    const aliasMatch = matchWord(lowerAlias, queryVariations);
+                    if (aliasMatch.matched) {
+                        matched = true;
+                        isExactMatch = aliasMatch.isExactMatch;
+                        matchedAlias = alias;
+                        break;
+                    }
                 }
-                if (matchWord(aTag.tag, queryVariations).isExactMatch) {
-                    return -999999999999;
-                }
-                if (bTag.alias && bTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
-                    return 999999999999;
-                }
-                if (aTag.alias && aTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
-                    return -999999999999;
-                }
-                return a - b;
-            });
-
-            // Limit the results to maxSuggestions and map to TagData
-            result = result.slice(0, Math.min(result.length, settingValues.maxSuggestions));
-            result = result.map((index) => {
-                return autoCompleteData[source].sortedTags[index];
-            });
-
-            if (settingValues._logprocessingTime) {
-                const endTime = performance.now();
-                const duration = endTime - startTime;
-                console.debug(`[Autocomplete-Plus] Fast Search for "${partialTag}" in ${source} took ${duration.toFixed(2)}ms. Found ${result.length} candidates within ${searchResults.length} searches with aliases.`);
             }
-            return result;
-        } else {
-            // Search in sortedTags (already sorted by count)
-            for (const tagData of autoCompleteData[source].sortedTags) {
-                let matched = false;
-                let isExactMatch = false;
-                let matchedAlias = null;
 
-                // Check primary tag against all variations for exact/partial match
-                const tagMatch = matchWord(tagData.tag, queryVariations);
-                matched = tagMatch.matched;
-                isExactMatch = tagMatch.isExactMatch;
+            const tagSetKey = tagData.tag;
 
-                // If primary tag didn't match, check aliases against all variations
-                if (!matched && tagData.alias && Array.isArray(tagData.alias) && tagData.alias.length > 0) {
-                    for (const alias of tagData.alias) {
-                        const lowerAlias = alias.toLowerCase();
-                        const aliasMatch = matchWord(lowerAlias, queryVariations);
-                        if (aliasMatch.matched) {
-                            matched = true;
-                            isExactMatch = aliasMatch.isExactMatch;
-                            matchedAlias = alias;
-                            break;
-                        }
-                    }
+            // Add candidate if matched and not already added
+            if (matched && !addedTags.has(tagSetKey)) {
+                // Add to exact matches or partial matches based on match type
+                if (isExactMatch) {
+                    exactMatches.push(tagData);
+                } else {
+                    partialMatches.push(tagData);
                 }
 
-                const tagSetKey = tagData.tag;
+                addedTags.add(tagSetKey);
 
-                // Add candidate if matched and not already added
-                if (matched && !addedTags.has(tagSetKey)) {
-                    // Add to exact matches or partial matches based on match type
-                    if (isExactMatch) {
-                        exactMatches.push(tagData);
-                    } else {
-                        partialMatches.push(tagData);
+                // Check if we've reached the maximum suggestions limit combining both arrays
+                if (exactMatches.length + partialMatches.length >= settingValues.maxSuggestions) {
+                    // Return the combined results, prioritizing exact matches
+                    const result = [...exactMatches, ...partialMatches].slice(0, settingValues.maxSuggestions);
+
+                    if (settingValues._logprocessingTime) {
+                        const endTime = performance.now();
+                        const duration = endTime - startTime;
+                        console.debug(`[Autocomplete-Plus] Search for "${partialTag}" took ${duration.toFixed(2)}ms. Found ${result.length} candidates (max reached).`);
                     }
 
-                    addedTags.add(tagSetKey);
-
-                    // Check if we've reached the maximum suggestions limit combining both arrays
-                    if (exactMatches.length + partialMatches.length >= settingValues.maxSuggestions) {
-                        // Return the combined results, prioritizing exact matches
-                        const result = [...exactMatches, ...partialMatches].slice(0, settingValues.maxSuggestions);
-
-                        if (settingValues._logprocessingTime) {
-                            const endTime = performance.now();
-                            const duration = endTime - startTime;
-                            console.debug(`[Autocomplete-Plus] Search for "${partialTag}" took ${duration.toFixed(2)}ms. Found ${result.length} candidates (max reached).`);
-                        }
-
-                        return result; // Early exit
-                    }
+                    return result; // Early exit
                 }
             }
         }
@@ -221,6 +188,69 @@ function searchCompletionCandidates(textareaElement) {
     return candidates;
 }
 
+/**
+ * Search completion candidates using FlexSearch for fast matching.
+ * @param {string} partialTag 
+ * @param {Set<string>} queryVariations 
+ * @returns 
+ */
+function searchWithFlexSearch(partialTag, queryVariations) {
+    const startTime = performance.now();
+
+    let mergedResult = [];
+    let totalSearchCount = 0;
+
+    const sources = getEnabledTagSourceInPriorityOrder();
+    for (const source of sources) {
+        if (!autoCompleteData[source].flexSearchDocument) continue;
+        if (mergedResult.length >= settingValues.maxSuggestions) break;
+
+        // Use the FlexSearch Document to search
+        // NOTE: The limit param is reflected separately for "tag" and "alias".
+        let searchResult = autoCompleteData[source].flexSearchDocument.search(partialTag, {
+            field: ["tag", "alias"],
+            limit: Math.min(settingValues.maxSuggestions * 10, 500),
+            merge: true,
+            suggest: false,
+            cache: true,
+        });
+
+        if (!searchResult || searchResult.length <= 0) continue;
+
+        // Sort results based on exact matches and counts
+        searchResult = searchResult
+            .map(r => autoCompleteData[source].sortedTags[r.id])
+            .sort((aTag, bTag) => {
+                if (matchWord(bTag.tag, queryVariations).isExactMatch) {
+                    return 999999999999;
+                }
+                if (matchWord(aTag.tag, queryVariations).isExactMatch) {
+                    return -999999999999;
+                }
+                if (bTag.alias && bTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
+                    return 999999999999;
+                }
+                if (aTag.alias && aTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
+                    return -999999999999;
+                }
+                return bTag.count - aTag.count;
+            });
+
+        // Merge results into the final array
+        mergedResult = mergedResult.concat(searchResult.slice(0, settingValues.maxSuggestions - mergedResult.length));
+
+        totalSearchCount += searchResult.length;
+    }
+
+    if (settingValues._logprocessingTime) {
+        const endTime = performance.now();
+        const duration = endTime - startTime;
+        console.debug(`[Autocomplete-Plus] Fast Search for "${partialTag}" took ${duration.toFixed(2)}ms.Found ${mergedResult.length} candidates within ${totalSearchCount} searches from flexsearch.`);
+    }
+
+    return mergedResult;
+}
+
 /**
  * Extracts the current tag being typed before the cursor.
  * @param {HTMLTextAreaElement} inputElement

web/js/data.js

@@ -1,5 +1,5 @@
-import { Index } from './thirdparty/flexsearch.bundle.module.min.js'
 import { settingValues, updateMaxTagLength } from "./settings.js";
+import { createFlexSearchDocument } from "./searchengine.js";
 
 // --- Constants ---
 
@@ -64,15 +64,8 @@ export class TagData {
 
 class AutocompleteData {
     constructor() {
-        /** @type {Index} */
-        this.flexSearchIndex = null;
-
-        /** @type {number[]} */
-        this.flexSearchMapping = [];
-
-        /** @type {number} */
-        // The actual number will be calculated later when loading CSV files
-        this.flexSearchLimitMultiplier = 10;
+        /** @type {Document} */
+        this.flexSearchDocument = null;
 
         /** @type {TagData[]} */
         this.sortedTags = [];
@@ -91,7 +84,6 @@ class AutocompleteData {
 
         // Progress of "base" csv loading
         this.baseLoadingProgress = {
-            // tags: 0,
             cooccurrence: 0
         };
     }
@@ -210,39 +202,27 @@ async function buildFlexSearchIndex(siteName) {
             return;
         }
 
-        const index = new Index({
-            tokenize: "bidirectional",
-        });
+        const document = createFlexSearchDocument();
 
         let startIdx = 0;
-        let maxCountOfAlias = 0;
         const startTime = performance.now();
         function processChunkTasks() {
             const chunkSize = 1000;
             const end = Math.min(startIdx + chunkSize, autoCompleteData[siteName].sortedTags.length);
             for (; startIdx < end; startIdx++) {
                 const tagData = autoCompleteData[siteName].sortedTags[startIdx];
-
-                index.add(autoCompleteData[siteName].flexSearchMapping.length, tagData.tag);
-                autoCompleteData[siteName].flexSearchMapping.push(startIdx);
-
-                tagData.alias.forEach(alias => {
-                    index.add(autoCompleteData[siteName].flexSearchMapping.length, alias);
-                    autoCompleteData[siteName].flexSearchMapping.push(startIdx);
-                })
-
-                maxCountOfAlias = Math.max(maxCountOfAlias, tagData.alias.length);
+                document.add(startIdx, tagData);
             }
 
             if (startIdx < autoCompleteData[siteName].sortedTags.length) {
                 setTimeout(processChunkTasks, 0);
                 // console.log(`[Autocomplete-Plus] Current porcess: ${startIdx}`);
             } else {
+                autoCompleteData[siteName].flexSearchDocument = document;
+
                 const endTime = performance.now();
                 const duration = endTime - startTime;
-                autoCompleteData[siteName].flexSearchIndex = index;
-                autoCompleteData[siteName].flexSearchLimitMultiplier = Math.min(10, maxCountOfAlias + 1);
-                console.debug(`[Autocomplete-Plus] Building ${autoCompleteData[siteName].sortedTags.length} index for ${siteName} took ${duration.toFixed(2)}ms.`);
+                console.info(`[Autocomplete-Plus] Building ${autoCompleteData[siteName].sortedTags.length} index for ${siteName} took ${duration.toFixed(2)}ms.`);
             }
         }
         processChunkTasks();

web/js/searchengine.js

@@ -0,0 +1,86 @@
+import { Charset, Encoder, Document } from './thirdparty/flexsearch.bundle.module.min.js'
+import { kataToHira } from './utils.js';
+
+/**
+ * Creates an encoder optimized for processing English tag names.
+ * Handles tag formatting like underscores and parentheses commonly used in Danbooru tags.
+ * @returns {Encoder} FlexSearch encoder for English tags
+ */
+function createTagEncoder() {
+    return new Encoder({
+        normalize: true,
+        dedupe: false,
+        numeric: false,
+        cache: true,
+        // filter: new Set(['and', 'to', 'be', 'on']),
+        replacer: [/(?<=[a-zA-Z\)])_$/, ''],  // Remove trailing underscores after letters/parentheses
+        split: /(?<=[a-zA-Z\)])_(?=[a-zA-Z\(])|\((?=[a-zA-Z])|(?<=[a-zA-Z\)])\)|[ \n]/  // Split on underscores between words, parentheses, spaces, and newlines
+    });
+}
+
+/**
+ * Creates an encoder optimized for processing CJK (Chinese, Japanese, Korean) characters.
+ * Uses exact character matching and converts katakana to hiragana for better Japanese search.
+ * @returns {Encoder} FlexSearch encoder for CJK text
+ */
+function createCJKEncoder() {
+    return new Encoder(Charset.Exact, {
+        dedupe: true,
+        numeric: true,
+        cache: true,
+        filter: new Set(['(', ')']),  // Filter out parentheses characters
+        finalize: (term) => {         // Convert katakana to hiragana for better Japanese matching
+            return term.map(str => kataToHira(str));
+        }
+    });
+}
+
+/**
+ * Creates a FlexSearch Document instance optimized for tag searching.
+ * Configures separate encoders for English tags and CJK aliases with appropriate tokenization.
+ * @returns {Document} Configured FlexSearch document for tag indexing
+ */
+export function createFlexSearchDocument() {
+    const tagEncoder = createTagEncoder();
+    const cjkEncoder = createCJKEncoder();
+
+    // Custom encoding function for alias field that handles mixed language content
+    const encodeAlias = function (term) {
+        return term.split(",")
+            .flatMap(str => {
+                if (/[^\u0000-\u007f]/.test(str)) {
+                    // Contains non-ASCII characters (CJK text)
+                    return cjkEncoder.encode(str);
+                } else {
+                    // ASCII characters only (English text)
+                    return tagEncoder.encode(str);
+                }
+            })
+            .filter(Boolean);
+    }
+
+    // Configure the FlexSearch document with optimized indexing settings
+    const document = new Document({
+        document: {
+            id: "id",
+            index: [
+                {
+                    field: "tag",
+                    tokenize: "bidirectional",  // Allow partial matching from both ends
+                    encoder: tagEncoder,        // Use tag-optimized encoder
+                },
+                {
+                    field: "alias",             // Index the alias field for multi-language support
+                    tokenize: "full",           // Full tokenization for complete alias matching
+                    encode: encodeAlias,        // Use custom multi-language encoding function
+                }
+            ]
+        }
+    });
+
+    return document;
+}
+
+// Export functions for testing when in test environment
+const isTestEnvironment = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';
+export const __test__ = isTestEnvironment ? { createTagEncoder, createCJKEncoder } : undefined;