fix: Refactor tag loading logic to improve performance slightly

Feature/#26 embedding and lora autocompletion

.env-cmdrc.json

@@ -0,0 +1,5 @@
+{
+  "test": {
+    "NODE_OPTIONS": "--experimental-vm-modules"
+  }
+}

README.md

@@ -41,6 +41,7 @@ When you type in a text input area, tags that partially match the text are displ
 - Tags are color-coded by category. The color-coding rules are the same as Danbooru.
 - Tags that have already been entered are displayed grayed out.
 - You can display Danbooru and e621 tags at the same time. You can also change the priority from the settings.
+- Supports autocomplete for Lora and Embedding inputs. You can enable/disable this feature in the settings.
 
 ## Related Tags
 
@@ -132,6 +133,7 @@ When the browser is reloaded, you can check the list of loaded CSV files in the
 
 - **Enable Autocomplete**: Enable/disable the autocomplete feature.
 - **Max suggestions**: Maximum number of autocomplete suggestions to display.
+- **Enable Loras and Embeddings**: Display Lora and Embedding in the suggestions.
 - **Use Fast Search**: Switch autocomplete suggestions search to fast processing (see [About Fast Search for Autocomplete](#about-fast-search-for-autocomplete) for details).
 
 ### Related Tags

docs/README_jp.md

@@ -39,6 +39,7 @@
 - タグのカテゴリ毎に色分けされます。色分けのルールは Danbooru と同じです
 - 入力済みのタグはグレーアウトで表示されます
 - Danbooruとe621のタグを同時に表示出来ます。設定から優先順位を変更できます
+- LoraとEmbeddingの入力補完に対応しています。設定から有効・無効を切り替えられます
 
 ## 関連タグ
 
@@ -130,6 +131,7 @@ worst_quality,5,9999999,
 
 - **Enable Autocomplete**: オートコンプリート機能の有効化/無効化
 - **Max Suggestions**: オートコンプリート候補の最大表示件数
+- **Enable Loras and Embeddings**: LoraとEmbeddingを候補に表示する
 - **Use Fast Search**: オートコンプリート候補の検索を高速な処理に切り替える（詳細は [オートコンプリートの高速検索について](#オートコンプリートの高速検索について) を確認してください)
 
 ### 関連タグ

modules/api.py

@@ -1,7 +1,10 @@
-import os
 import json
+import os
+
+import folder_paths
 import server
 from aiohttp import web
+
 from . import downloader as dl
 
 # Get the absolute path to the 'data' directory
@@ -227,3 +230,21 @@ async def get_last_check_time(_request):
     except (IOError, json.JSONDecodeError) as e:
         print(f"[Autocomplete-Plus] Error reading csv_meta.json: {e}")
         return web.json_response({"last_check_time": None, "error": str(e)}, status=500)
+
+
+@server.PromptServer.instance.routes.get("/autocomplete-plus/embeddings")
+async def get_embeddings(request):
+    """
+    Returns a list of embedding files.
+    """
+    embeddings = folder_paths.get_filename_list("embeddings")
+    return web.json_response(list(map(lambda a: os.path.splitext(a)[0], embeddings)))
+
+
+@server.PromptServer.instance.routes.get("/autocomplete-plus/loras")
+async def get_loras(request):
+    """
+    Returns a list of lora files.
+    """
+    loras = folder_paths.get_filename_list("loras")
+    return web.json_response(list(map(lambda a: os.path.splitext(a)[0], loras)))

package-lock.json

@@ -8,6 +8,7 @@
         "@babel/core": "^7.27.1",
         "@babel/preset-env": "^7.27.2",
         "babel-jest": "^29.7.0",
+        "env-cmd": "^10.1.0",
         "jest": "^29.7.0",
         "stylelint": "^16.19.1",
         "stylelint-config-idiomatic-order": "^10.0.0",
@@ -3018,6 +3019,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/commander": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/commander/-/commander-4.1.1.tgz",
+      "integrity": "sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -3266,6 +3277,23 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/env-cmd": {
+      "version": "10.1.0",
+      "resolved": "https://registry.npmjs.org/env-cmd/-/env-cmd-10.1.0.tgz",
+      "integrity": "sha512-mMdWTT9XKN7yNth/6N6g2GuKuJTsKMDHlQFUDacb/heQRRWOTIZ42t1rMHnQu4jYxU1ajdTeJM+9eEETlqToMA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "commander": "^4.0.0",
+        "cross-spawn": "^7.0.0"
+      },
+      "bin": {
+        "env-cmd": "bin/env-cmd.js"
+      },
+      "engines": {
+        "node": ">=8.0.0"
+      }
+    },
     "node_modules/env-paths": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",

package.json

@@ -1,12 +1,13 @@
 {
   "type": "module",
   "scripts": {
-    "test": "jest"
+    "test": "env-cmd -e test -- jest"
   },
   "devDependencies": {
     "@babel/core": "^7.27.1",
     "@babel/preset-env": "^7.27.2",
     "babel-jest": "^29.7.0",
+    "env-cmd": "^10.1.0",
     "jest": "^29.7.0",
     "stylelint": "^16.19.1",
     "stylelint-config-idiomatic-order": "^10.0.0",

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "comfyui-autocomplete-plus"
 description = "Autocomplete and Related Tag display for ComfyUI"
-version = "1.3.0"
+version = "1.4.0"
 license = {file = "LICENSE"}
 dependencies = ["",]
 

tests/js/flexsearch.test.js

@@ -0,0 +1,387 @@
+
+import {
+    createFlexSearchDocument,
+    createFlexSearchDocumentForModel,
+    __test__
+} from "../../web/js/searchengine.js";
+
+const { createTagEncoder, createCJKEncoder, createModelEncoder } = __test__;
+
+function parseCSVLine(line) {
+    const result = [];
+    let current = '';
+    let inQuotes = false;
+
+    for (let i = 0; i < line.length; i++) {
+        const char = line[i];
+
+        if (char === '"') {
+            if (inQuotes && i + 1 < line.length && line[i + 1] === '"') {
+                current += '"';
+                i++;
+            } else {
+                inQuotes = !inQuotes;
+            }
+        } else if (char === ',' && !inQuotes) {
+            result.push(current);
+            current = '';
+        } else {
+            current += char;
+        }
+    }
+
+    result.push(current);
+
+    return result;
+}
+
+describe('FlexSearch Integration', () => {
+
+    const commonCSV = `
+1girl,0,6008644,"1girls,sole_female"
+highres,5,5256195,"high_res,high_resolution,hires"
+solo,0,5000954,"alone,female_solo,single,solo_female,solo_in_panel"
+long_hair,0,4350743,"/lh,longhair,very_long_hair"
+one_two_three,0,29389,
+`;
+
+    const cjkAliasCSV = `
+blue_hair,0,676176,"青髪,青い髪,水色髪"
+red_hair,0,413261,"赤髪,紅髪,红发,빨강머리,빨간머리"
+smile,0,2294308,"笑い,スマイル,笑顔,笑顏,守りたい、この笑顔,笑,笑容,微笑み,微笑,微笑む,미소,守りたいこの笑顔"
+gloves,0,1105296,"手袋,裸手袋,手袋コキ,てぶくろ,장갑,手套"
+dragon_girl,0,37930,"竜娘,ドラゴン娘,龍娘,龙娘,メスドラ,辰娘"
+double_bun,0,103538,"お団子頭,お団子"
+sanshoku_dango,0,2061,"三色団子,三色团子,花見団子,花见团子"
+`;
+
+    const specialCharCSV = `
+:d,0,436700,
+>:),0,11041,
+year:1999,0,1999,
+d.d.,0,1999,
+copyright_(series),2,1298,"copyright,コピーライト (シリーズ),コピーライト名,コピーライト,著作"
+__wildcard__,0,0,
+`;
+
+    const ModelCSV = `
+<lora:my_lora1>,0,0,
+<lora:日本語Lora_v1>,0,0,
+embedding: my_embedding,0,0,
+`;
+
+    const mockCSV = [
+        commonCSV, cjkAliasCSV, specialCharCSV
+    ].map(csv => csv.trim()).join('\n');
+
+    let mockTags, mockModelTags;
+
+    let tagEncoder, cjkEncoder, modelEncoder;
+    let document, modelDocument;
+
+    let performSearch = function (query, limit = 100) {
+        const ids1 = document.search(query, {
+            field: ["tag", "alias"],
+            limit: limit,
+            suggest: false,
+            merge: true,
+        }).map(r => r.id);
+
+        const result1 = mockTags.filter(tag => ids1.includes(tag.id)).map(tag => tag.tag);
+
+        const ids2 = modelDocument.search(query, {
+            field: ["tag", "alias"],
+            limit: limit,
+            suggest: false,
+            merge: true,
+        }).map(r => r.id);
+
+        const result2 = mockModelTags.filter(tag => ids2.includes(tag.id)).map(tag => tag.tag);
+
+        return [...result1, ...result2];
+    }
+
+    beforeEach(() => {
+        mockTags = mockCSV.split('\n').map((line, id) => {
+            const [tag, category, count, alias] = parseCSVLine(line);
+            return { id, tag, category: parseInt(category), count: parseInt(count), alias };
+        });
+
+        tagEncoder = createTagEncoder();
+        cjkEncoder = createCJKEncoder();
+
+        document = createFlexSearchDocument();
+
+        mockTags.forEach(data => document.add(data));
+
+        mockModelTags = ModelCSV.split('\n').map((line, id) => {
+            const [tag, category, count, alias] = parseCSVLine(line);
+            return { id, tag, category: parseInt(category), count: parseInt(count), alias };
+        });
+
+        modelEncoder = createModelEncoder();
+        modelDocument = createFlexSearchDocumentForModel();
+        mockModelTags.forEach(data => modelDocument.add(data));
+    });
+
+    describe('Encoder', () => {
+        test('should split underscore-separated tags', () => {
+            const encoded = tagEncoder.encode('sanshoku_dango');
+            expect(encoded).toEqual(['sanshoku', 'dango']);
+        });
+
+        test('should extract words from parentheses', () => {
+            const encoded = tagEncoder.encode('copyright_(series)');
+            expect(encoded).toEqual(['copyright', 'series']);
+        });
+        test('should preserve colon-separated special tags', () => {
+            const encoded = tagEncoder.encode('year:1234');
+            expect(encoded).toEqual(['year:1234']);
+        });
+        test('should preserve dot-separated tags', () => {
+            const encoded = tagEncoder.encode('d.d.');
+            expect(encoded).toEqual(['d.d.']);
+        });
+        test('should preserve double underscore wildcard tags', () => {
+            const encoded = tagEncoder.encode('__wildcard__');
+            expect(encoded).toEqual(['__wildcard__']);
+        });
+        test('should convert katakana to hiragana', () => {
+            const encoded = cjkEncoder.encode('ガーデン');
+            expect(encoded).toEqual(['がーでん']);
+        });
+        test('should remove trailing underscores when splitting', () => {
+            const encoded = tagEncoder.encode('one_two_');
+            expect(encoded).toEqual(['one', 'two']);
+        });
+        test('should properly encode embedding notation', () => {
+            let encoded = modelEncoder.encode('embedding:path/to/my_embed1');
+            expect(encoded).toEqual(['embedding:', 'path', 'to', 'my', 'embed1']);
+
+            expect(
+                modelEncoder.encode('embedding:path\\to\\my-embed1')
+            ).toEqual(['embedding:', 'path', 'to', 'my', 'embed1']);
+
+            expect(
+                modelEncoder.encode('embedding:path\\to\\this is my embed. my-negative01 (v1)__by me')
+            ).toEqual(['embedding:', 'path', 'to', 'this', 'is', 'my', 'embed', 'my', 'negative01', 'v1', 'by', 'me']);
+        });
+        test('should properly encode lora notation', () => {
+            expect(
+                modelEncoder.encode('<lora:path/to/my_lora1>')
+            ).toEqual(['lora:', 'path', 'to', 'my', 'lora1']);
+
+            expect(
+                modelEncoder.encode('<lora:path\\to\\my-lora1>')
+            ).toEqual(['lora:', 'path', 'to', 'my', 'lora1']);
+
+            expect(
+                modelEncoder.encode('<lora:path\\to\\this is my lora. my-style01 (v1)__by me>')
+            ).toEqual(['lora:', 'path', 'to', 'this', 'is', 'my', 'lora', 'my', 'style01', 'v1', 'by', 'me']);
+        });
+    });
+
+    describe('Basic Search', () => {
+        test('should find a tag by exact match', () => {
+            const results = performSearch('1girl');
+            expect(results.length).toBeGreaterThan(0);
+            expect(results).toContain('1girl');
+        });
+
+        test('should find a tag by partial match (substring)', () => {
+            const results = performSearch('blue');
+            expect(results.length).toBeGreaterThan(0);
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should be case-insensitive', () => {
+            const results = performSearch('BLUE_HAIR');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by backward', () => {
+            const results = performSearch('gon');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('dragon_girl');
+        });
+
+        test('should find a tag for terms contain space', () => {
+            const results = performSearch('double ');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('double_bun');
+        });
+
+        test('should find a tag for terms contain underscore', () => {
+            const results = performSearch('double_');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('double_bun');
+        });
+    });
+
+    describe('Alias Search', () => {
+        test('should find a tag by its Japanese alias', () => {
+            const results = performSearch('髪');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by one of its multiple aliases', () => {
+            const results = performSearch('笑顔');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('smile');
+        });
+
+        test('should find a tag by its partial Japanese alias', () => {
+            const results = performSearch('青い');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('blue_hair');
+        });
+
+        test('should find a tag by katakana', () => {
+            const results = performSearch('テブクロ');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('gloves');
+        });
+
+        test('should find a tag by hiragana', () => {
+            const results = performSearch('すまいる');
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('smile');
+        });
+
+        test('should not find a tag by english alias substring', () => {
+            const results = performSearch('meg');
+            expect(results.length).toEqual(0);
+        });
+
+        test('should find a tag by japanese alias substring', () => {
+            const results = performSearch('団子');
+            expect(results.length).toEqual(2);
+
+            expect(results).toContain('sanshoku_dango');
+            expect(results).toContain('double_bun');
+        });
+    });
+
+    describe('Special Characters and Edge Cases', () => {
+        test('should find a tag with parentheses', () => {
+            const results = performSearch('copyright_(series)');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should find a tag by searching for content inside parentheses', () => {
+            const results = performSearch('series');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should find a tag by partial word', () => {
+            const results = performSearch('right');
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toContain('copyright_(series)');
+        });
+
+        test('should return an empty array for a non-existent tag', () => {
+            const results = performSearch('non_existent_tag_xyz');
+            expect(results).toEqual([]);
+        });
+
+        test('should match to special character only tag', () => {
+            const tag = '>:)';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag', () => {
+            const tag = ':d';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag2', () => {
+            const tag = 'year:1999';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to contain special character tag3', () => {
+            const tag = 'd.d.';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain(tag);
+        });
+
+        test('should match to wildcard tag', () => {
+            const tag = '__';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain('__wildcard__');
+        });
+
+        test('should match to lora tag', () => {
+            const tag = '<lora';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(2);
+
+            expect(results).toContain("<lora:my_lora1>");
+            expect(results).toContain("<lora:日本語Lora_v1>");
+        });
+
+        test('should match to lora tag2', () => {
+            const tag = 'lora:';
+            const results = performSearch(tag);
+            expect(results.length).toEqual(2);
+
+            expect(results).toContain("<lora:my_lora1>");
+            expect(results).toContain("<lora:日本語Lora_v1>");
+        });
+
+        test('should match to lora that contain CJK characters', () => {
+            const word = 'lora: 日本語';
+            const results = performSearch(word);
+            expect(results.length).toEqual(1);
+
+            expect(results).toContain("<lora:日本語Lora_v1>");
+        });
+    });
+
+    describe('Search Options', () => {
+        test('should respect the limit option', () => {
+            const results = performSearch('hair', 1);
+
+            expect(results.length).toEqual(1);
+        });
+
+        test('should return all matches when limit is higher than results', () => {
+            const results = performSearch('hair', 5);
+            expect(results.length).toBeGreaterThan(0);
+
+            expect(results).toHaveLength(3);
+            expect(results).toContain('long_hair');
+            expect(results).toContain('blue_hair');
+            expect(results).toContain('red_hair');
+        });
+    });
+});

web/js/autocomplete.js

@@ -1,4 +1,5 @@
 import {
+    ModelTagSource,
     TagCategory,
     TagData,
     autoCompleteData,
@@ -9,6 +10,7 @@ import {
     hiraToKata,
     kataToHira,
     formatCountHumanReadable,
+    escapeHtml,
     isContainsLetterOrNumber,
     normalizeTagToInsert,
     normalizeTagToSearch,
@@ -79,22 +81,17 @@ function matchWord(target, queries) {
  * @returns {Array<TagData>} The list of matching candidates.
  */
 function searchCompletionCandidates(textareaElement) {
-    const startTime = performance.now(); // Record start time for performance measurement
 
     const ESCAPE_SEQUENCE = ["#", "/"]; // If the first string is that character, autocomplete will not be displayed.
     const partialTag = getCurrentPartialTag(textareaElement);
-    if (!partialTag || partialTag.length <= 0 || 
-        ESCAPE_SEQUENCE.some(seq => partialTag.startsWith(seq)) || 
+    if (!partialTag || partialTag.length <= 0 ||
+        ESCAPE_SEQUENCE.some(seq => partialTag.startsWith(seq)) ||
         isLongText(partialTag)) {
         return []; // No valid input for autocomplete
     }
 
-    const exactMatches = [];
-    const partialMatches = [];
-    const addedTags = new Set();
-
     // Generate Hiragana/Katakana variations if applicable
-    const queryVariations = new Set([partialTag, normalizeTagToSearch(partialTag)]);
+    const queryVariations = new Set([partialTag.toLowerCase(), normalizeTagToSearch(partialTag).toLowerCase()]);
     const kataQuery = hiraToKata(partialTag);
     if (kataQuery !== partialTag) {
         queryVariations.add(kataQuery);
@@ -104,106 +101,77 @@ function searchCompletionCandidates(textareaElement) {
         queryVariations.add(hiraQuery);
     }
 
+    if (settingValues.useFastSearch) {
+        return searchWithFlexSearch(partialTag, queryVariations);
+    } else {
+        return sequentialSearch(partialTag, queryVariations);
+    }
+}
+
+/**
+ * Search completion candidates using sequential search.
+ * @param {string} partialTag 
+ * @param {Set<string>} queryVariations 
+ * @returns 
+ */
+function sequentialSearch(partialTag, queryVariations) {
+    const startTime = performance.now();
+
+    const exactMatches = [];
+    const partialMatches = [];
+    const addedTags = new Set();
+
     const sources = getEnabledTagSourceInPriorityOrder();
     for (const source of sources) {
-        // Use fast search if enabled and available for the source
-        if (settingValues.useFastSearch && autoCompleteData[source].flexSearchIndex) {
-            // Use the FlexSearch Index to search tag and alias IDs that match the partial tag.
-            const searchResults = autoCompleteData[source].flexSearchIndex.search(partialTag, {
-                limit: settingValues.maxSuggestions * autoCompleteData[source].flexSearchLimitMultiplier,
-                suggest: false,
-                cache: true,
-            });
+        // Search in sortedTags (already sorted by count)
+        for (const tagData of autoCompleteData[source].sortedTags) {
+            let matched = false;
+            let isExactMatch = false;
+            let matchedAlias = null;
 
-            // Get tag IDs from search results and filter duplicates
-            let result = searchResults.map((index) => {
-                return autoCompleteData[source].flexSearchMapping[index];
-            });
-            result = [...new Set(result)];
+            // Check primary tag against all variations for exact/partial match
+            const tagMatch = matchWord(tagData.tag.toLowerCase(), queryVariations);
+            matched = tagMatch.matched;
+            isExactMatch = tagMatch.isExactMatch;
 
-            // Sort results based on exact matches or id values (ID order is equal to tag count order)
-            result = result.sort((a, b) => {
-                const aTag = autoCompleteData[source].sortedTags[a];
-                const bTag = autoCompleteData[source].sortedTags[b];
-                if (matchWord(bTag.tag, queryVariations).isExactMatch) {
-                    return 999999999999;
+            // If primary tag didn't match, check aliases against all variations
+            if (!matched && tagData.alias && Array.isArray(tagData.alias) && tagData.alias.length > 0) {
+                for (const alias of tagData.alias) {
+                    const aliasMatch = matchWord(alias.toLowerCase(), queryVariations);
+                    if (aliasMatch.matched) {
+                        matched = true;
+                        isExactMatch = aliasMatch.isExactMatch;
+                        matchedAlias = alias;
+                        break;
+                    }
                 }
-                if (matchWord(aTag.tag, queryVariations).isExactMatch) {
-                    return -999999999999;
-                }
-                if (bTag.alias && bTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
-                    return 999999999999;
-                }
-                if (aTag.alias && aTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
-                    return -999999999999;
-                }
-                return a - b;
-            });
-
-            // Limit the results to maxSuggestions and map to TagData
-            result = result.slice(0, Math.min(result.length, settingValues.maxSuggestions));
-            result = result.map((index) => {
-                return autoCompleteData[source].sortedTags[index];
-            });
-
-            if (settingValues._logprocessingTime) {
-                const endTime = performance.now();
-                const duration = endTime - startTime;
-                console.debug(`[Autocomplete-Plus] Fast Search for "${partialTag}" in ${source} took ${duration.toFixed(2)}ms. Found ${result.length} candidates within ${searchResults.length} searches with aliases.`);
             }
-            return result;
-        } else {
-            // Search in sortedTags (already sorted by count)
-            for (const tagData of autoCompleteData[source].sortedTags) {
-                let matched = false;
-                let isExactMatch = false;
-                let matchedAlias = null;
 
-                // Check primary tag against all variations for exact/partial match
-                const tagMatch = matchWord(tagData.tag, queryVariations);
-                matched = tagMatch.matched;
-                isExactMatch = tagMatch.isExactMatch;
+            const tagSetKey = tagData.tag;
 
-                // If primary tag didn't match, check aliases against all variations
-                if (!matched && tagData.alias && Array.isArray(tagData.alias) && tagData.alias.length > 0) {
-                    for (const alias of tagData.alias) {
-                        const lowerAlias = alias.toLowerCase();
-                        const aliasMatch = matchWord(lowerAlias, queryVariations);
-                        if (aliasMatch.matched) {
-                            matched = true;
-                            isExactMatch = aliasMatch.isExactMatch;
-                            matchedAlias = alias;
-                            break;
-                        }
-                    }
+            // Add candidate if matched and not already added
+            if (matched && !addedTags.has(tagSetKey)) {
+                // Add to exact matches or partial matches based on match type
+                if (isExactMatch) {
+                    exactMatches.push(tagData);
+                } else {
+                    partialMatches.push(tagData);
                 }
 
-                const tagSetKey = tagData.tag;
+                addedTags.add(tagSetKey);
 
-                // Add candidate if matched and not already added
-                if (matched && !addedTags.has(tagSetKey)) {
-                    // Add to exact matches or partial matches based on match type
-                    if (isExactMatch) {
-                        exactMatches.push(tagData);
-                    } else {
-                        partialMatches.push(tagData);
+                // Check if we've reached the maximum suggestions limit combining both arrays
+                if (exactMatches.length + partialMatches.length >= settingValues.maxSuggestions) {
+                    // Return the combined results, prioritizing exact matches
+                    const result = [...exactMatches, ...partialMatches].slice(0, settingValues.maxSuggestions);
+
+                    if (settingValues._logprocessingTime) {
+                        const endTime = performance.now();
+                        const duration = endTime - startTime;
+                        console.debug(`[Autocomplete-Plus] Search for "${partialTag}" took ${duration.toFixed(2)}ms. Found ${result.length} candidates (max reached).`);
                     }
 
-                    addedTags.add(tagSetKey);
-
-                    // Check if we've reached the maximum suggestions limit combining both arrays
-                    if (exactMatches.length + partialMatches.length >= settingValues.maxSuggestions) {
-                        // Return the combined results, prioritizing exact matches
-                        const result = [...exactMatches, ...partialMatches].slice(0, settingValues.maxSuggestions);
-
-                        if (settingValues._logprocessingTime) {
-                            const endTime = performance.now();
-                            const duration = endTime - startTime;
-                            console.debug(`[Autocomplete-Plus] Search for "${partialTag}" took ${duration.toFixed(2)}ms. Found ${result.length} candidates (max reached).`);
-                        }
-
-                        return result; // Early exit
-                    }
+                    return result; // Early exit
                 }
             }
         }
@@ -221,6 +189,69 @@ function searchCompletionCandidates(textareaElement) {
     return candidates;
 }
 
+/**
+ * Search completion candidates using FlexSearch for fast matching.
+ * @param {string} partialTag 
+ * @param {Set<string>} queryVariations 
+ * @returns 
+ */
+function searchWithFlexSearch(partialTag, queryVariations) {
+    const startTime = performance.now();
+
+    let mergedResult = [];
+    let totalSearchCount = 0;
+
+    const sources = getEnabledTagSourceInPriorityOrder();
+    for (const source of sources) {
+        if (!autoCompleteData[source].flexSearchDocument) continue;
+        if (mergedResult.length >= settingValues.maxSuggestions) break;
+
+        // Use the FlexSearch Document to search
+        // NOTE: The limit param is reflected separately for "tag" and "alias".
+        let searchResult = autoCompleteData[source].flexSearchDocument.search(partialTag, {
+            field: ["tag", "alias"],
+            limit: Math.min(settingValues.maxSuggestions * 10, 500),
+            merge: true,
+            suggest: false,
+            cache: true,
+        });
+
+        if (!searchResult || searchResult.length <= 0) continue;
+
+        // Sort results based on exact matches and counts
+        searchResult = searchResult
+            .map(r => autoCompleteData[source].sortedTags[r.id])
+            .sort((aTag, bTag) => {
+                if (matchWord(bTag.tag, queryVariations).isExactMatch) {
+                    return 999999999999;
+                }
+                if (matchWord(aTag.tag, queryVariations).isExactMatch) {
+                    return -999999999999;
+                }
+                if (bTag.alias && bTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
+                    return 999999999999;
+                }
+                if (aTag.alias && aTag.alias.some(alias => matchWord(alias, queryVariations).isExactMatch)) {
+                    return -999999999999;
+                }
+                return bTag.count - aTag.count;
+            });
+
+        // Merge results into the final array
+        mergedResult = mergedResult.concat(searchResult.slice(0, settingValues.maxSuggestions - mergedResult.length));
+
+        totalSearchCount += searchResult.length;
+    }
+
+    if (settingValues._logprocessingTime) {
+        const endTime = performance.now();
+        const duration = endTime - startTime;
+        console.debug(`[Autocomplete-Plus] Fast Search for "${partialTag}" took ${duration.toFixed(2)}ms.Found ${mergedResult.length} candidates within ${totalSearchCount} searches from flexsearch.`);
+    }
+
+    return mergedResult;
+}
+
 /**
  * Extracts the current tag being typed before the cursor.
  * @param {HTMLTextAreaElement} inputElement
@@ -269,9 +300,9 @@ function getCurrentPartialTag(inputElement) {
  * Inserts the selected tag into the textarea, replacing the partial tag,
  * making the change undoable.
  * @param {HTMLTextAreaElement} inputElement
- * @param {string} tagToInsert The raw tag string to insert.
+ * @param {TagData} tagDataToInsert The raw tag string to insert.
  */
-function insertTagToTextArea(inputElement, tagToInsert) {
+function insertTagToTextArea(inputElement, tagDataToInsert) {
     const text = inputElement.value;
     const cursorPos = inputElement.selectionStart;
 
@@ -279,7 +310,13 @@ function insertTagToTextArea(inputElement, tagToInsert) {
     const replaceStart = Math.min(cursorPos, tagStart);
     let replaceEnd = cursorPos;
 
-    const normalizedTag = normalizeTagToInsert(tagToInsert);
+    let normalizedTag;
+    if(Object.values(ModelTagSource).includes(tagDataToInsert.source)){
+        // If the tag is from a model tag source, don't want to normalize it
+        normalizedTag = tagDataToInsert.tag;
+    }else{
+        normalizedTag = normalizeTagToInsert(tagDataToInsert.tag);
+    }
 
     const currentTagAfterCursor = text.substring(cursorPos, tagEnd).trimEnd();
     if (normalizedTag.lastIndexOf(currentTagAfterCursor) !== -1) {
@@ -348,7 +385,7 @@ class AutocompleteUI {
         this.tagsList.addEventListener('mousedown', (e) => {
             const row = e.target.closest('.autocomplete-plus-item');
             if (row && row.dataset.tag) {
-                this.#insertTag(row.dataset.tag);
+                this.#insertTag(row.dataset);
                 e.preventDefault(); // Prevent focus loss from input
                 e.stopPropagation();
             }
@@ -413,10 +450,12 @@ class AutocompleteUI {
         this.#highlightItem();
     }
 
-    /** Selects the currently highlighted item */
+    /** Selects the currently highlighted item
+     * @returns {TagData|null} The selected tag data.
+     */
     getSelectedTag() {
         if (this.selectedIndex >= 0 && this.selectedIndex < this.candidates.length) {
-            return this.candidates[this.selectedIndex].tag;
+            return this.candidates[this.selectedIndex];
         }
 
         return null; // No valid selection
@@ -462,9 +501,10 @@ class AutocompleteUI {
         if (settingValues.tagSourceIconPosition == 'hidden') {
             tagName.textContent = tagData.tag;
         } else {
+            const escapedTag = escapeHtml(tagData.tag);
             tagName.innerHTML = settingValues.tagSourceIconPosition == 'left'
-                ? `${tagSourceIconHtml} ${tagData.tag}`
-                : `${tagData.tag} ${tagSourceIconHtml}`;
+                ? `${tagSourceIconHtml} ${escapedTag}`
+                : `${escapedTag} ${tagSourceIconHtml}`;
         }
 
         // grayout tag name if it already exists
@@ -605,10 +645,10 @@ class AutocompleteUI {
 
     /**
      * Handles the selection of an item
-     * @param {string} selectedTag The tag to insert.
+     * @param {TagData} selectedTag The tag to insert.
      */
     #insertTag(selectedTag) {
-        if (!this.target || !selectedTag || selectedTag.length <= 0) {
+        if (!this.target || !selectedTag) {
             this.hide();
             return;
         }

web/js/data.js

@@ -1,14 +1,20 @@
-import { Index } from './thirdparty/flexsearch.bundle.module.min.js'
 import { settingValues, updateMaxTagLength } from "./settings.js";
+import { createFlexSearchDocument, createFlexSearchDocumentForModel } from "./searchengine.js";
 
 // --- Constants ---
 
-// Tag data sources
+// Tag sources for booru-like tag data.
 export const TagSource = {
     Danbooru: 'danbooru',
     E621: 'e621',
 }
 
+// Tag sources for model based tag data.
+export const ModelTagSource = {
+    Embeddings: 'embeddings',
+    Lora: 'lora'
+}
+
 export const TagCategory = {
     'danbooru': [
         'general',
@@ -28,6 +34,12 @@ export const TagCategory = {
         'invalid',
         'meta',
         'lore',
+    ],
+    'embeddings': [
+        'embeddings'
+    ],
+    'lora': [
+        'lora'
     ]
 }
 
@@ -40,19 +52,19 @@ export class TagData {
     /**
      * Create a tag data object
      * @param {string} tag - The tag name
-     * @param {string[]} [alias=[]] - Array of aliases for the tag
-     * @param {string} [category='general'] - Category of the tag
+     * @param {number} [category] - Category index of the tag
      * @param {number} [count=0] - Frequency count/popularity of the tag
-     * @param {string} [source=TagSources.Danbooru] - The source of the tag data
+     * @param {string[]} [alias=[]] - Array of aliases for the tag
+     * @param {string} [source=TagSource.Danbooru] - The source of the tag data
      */
-    constructor(tag, alias = [], category = 'general', count = 0, source = TagSource.Danbooru) {
+    constructor(tag, category, count = 0, alias = [], source = TagSource.Danbooru) {
         /** @type {string} */
         this.tag = tag;
 
         /** @type {string[]} */
         this.alias = alias;
 
-        /** @type {string} */
+        /** @type {number} */
         this.category = category;
 
         /** @type {number} */
@@ -64,15 +76,8 @@ export class TagData {
 
 class AutocompleteData {
     constructor() {
-        /** @type {Index} */
-        this.flexSearchIndex = null;
-
-        /** @type {number[]} */
-        this.flexSearchMapping = [];
-
-        /** @type {number} */
-        // The actual number will be calculated later when loading CSV files
-        this.flexSearchLimitMultiplier = 10;
+        /** @type {Document} */
+        this.flexSearchDocument = null;
 
         /** @type {TagData[]} */
         this.sortedTags = [];
@@ -91,7 +96,6 @@ class AutocompleteData {
 
         // Progress of "base" csv loading
         this.baseLoadingProgress = {
-            // tags: 0,
             cooccurrence: 0
         };
     }
@@ -106,24 +110,32 @@ export const autoCompleteData = {};
 const TAGS_CSV_HEADER = 'tag,category,count,alias';
 const TAGS_CSV_HEADER_COLUMNS = TAGS_CSV_HEADER.split(',');
 const TAG_INDEX = TAGS_CSV_HEADER_COLUMNS.indexOf('tag');
-const ALIAS_INDEX = TAGS_CSV_HEADER_COLUMNS.indexOf('alias');
 const CATEGORY_INDEX = TAGS_CSV_HEADER_COLUMNS.indexOf('category');
 const COUNT_INDEX = TAGS_CSV_HEADER_COLUMNS.indexOf('count');
+const ALIAS_INDEX = TAGS_CSV_HEADER_COLUMNS.indexOf('alias');
 
 // --- Helder Functions ---
 
+
 /**
  * Get the available tag sources in priority order based on the current settings.
  * @returns {string[]} Array of available tag sources in priority order
  */
 export function getEnabledTagSourceInPriorityOrder() {
-    return Object.values(TagSource)
+    let enabledTagSources = Object.values(TagSource)
         .filter((s) => {
             return settingValues.tagSource === s || settingValues.tagSource === 'all';
         })
         .toSorted((a, b) => {
             return a === settingValues.primaryTagSource ? -1 : 1;
         });
+
+    // Append Loras and Embeddings if enabled
+    if (settingValues.enableModels) {
+        enabledTagSources = [...enabledTagSources, ...Object.values(ModelTagSource)];
+    }
+
+    return enabledTagSources;
 }
 
 // --- Data Loading Functions ---
@@ -142,7 +154,6 @@ async function loadTags(csvUrl, siteName) {
         }
         const csvText = await response.text();
         const lines = csvText.split('\n').filter(line => line.trim().length > 0);
-        const totalLines = lines.length;
 
         const startIndex = lines[0].toLowerCase().startsWith(TAGS_CSV_HEADER) ? 1 : 0;
 
@@ -167,23 +178,13 @@ async function loadTags(csvUrl, siteName) {
                 const aliases = aliasStr ? aliasStr.split(',').map(a => a.trim()).filter(a => a.length > 0) : [];
 
                 // Create a TagData instance instead of a plain object
-                const tagData = new TagData(tag, aliases, category, count, siteName);
+                const tagData = new TagData(tag, category, count, aliases, siteName);
 
                 updateMaxTagLength(tag.length);
 
                 autoCompleteData[siteName].sortedTags.push(tagData);
-            } else {
-                console.warn(`[Autocomplete-Plus] Invalid CSV format in line ${i + 1} of ${csvUrl}: ${line}. Expected ${TAGS_CSV_HEADER_COLUMNS.length} columns, but got ${columns.length}.`);
-                continue;
-            }
-        }
 
-        // Sort by count in descending order
-        autoCompleteData[siteName].sortedTags.sort((a, b) => b.count - a.count);
-
-        // Build maps as before, but ensure not to overwrite if already processed from extra files
-        autoCompleteData[siteName].sortedTags.forEach(tagData => {
-            if (!autoCompleteData[siteName].tagMap.has(tagData.tag)) {
+                // Set the tag and its alias in the maps
                 autoCompleteData[siteName].tagMap.set(tagData.tag, tagData);
                 if (tagData.alias && Array.isArray(tagData.alias)) {
                     tagData.alias.forEach(alias => {
@@ -192,8 +193,11 @@ async function loadTags(csvUrl, siteName) {
                         }
                     });
                 }
+            } else {
+                console.warn(`[Autocomplete-Plus] Invalid CSV format in line ${i + 1} of ${csvUrl}: ${line}. Expected ${TAGS_CSV_HEADER_COLUMNS.length} columns, but got ${columns.length}.`);
+                continue;
             }
-        });
+        }
 
     } catch (error) {
         console.error(`[Autocomplete-Plus] Failed to fetch or process tags from ${csvUrl}:`, error);
@@ -210,39 +214,34 @@ async function buildFlexSearchIndex(siteName) {
             return;
         }
 
-        const index = new Index({
-            tokenize: "bidirectional",
-        });
+        let document = null;
+        if (Object.values(TagSource).includes(siteName)) {
+            document = createFlexSearchDocument();
+        } else if (Object.values(ModelTagSource).includes(siteName)) {
+            document = createFlexSearchDocumentForModel();
+        } else {
+            throw new Error(`[Autocomplete-Plus] Invalid site name: ${siteName}`);
+        }
 
         let startIdx = 0;
-        let maxCountOfAlias = 0;
         const startTime = performance.now();
         function processChunkTasks() {
             const chunkSize = 1000;
             const end = Math.min(startIdx + chunkSize, autoCompleteData[siteName].sortedTags.length);
             for (; startIdx < end; startIdx++) {
                 const tagData = autoCompleteData[siteName].sortedTags[startIdx];
-
-                index.add(autoCompleteData[siteName].flexSearchMapping.length, tagData.tag);
-                autoCompleteData[siteName].flexSearchMapping.push(startIdx);
-
-                tagData.alias.forEach(alias => {
-                    index.add(autoCompleteData[siteName].flexSearchMapping.length, alias);
-                    autoCompleteData[siteName].flexSearchMapping.push(startIdx);
-                })
-
-                maxCountOfAlias = Math.max(maxCountOfAlias, tagData.alias.length);
+                document.add(startIdx, tagData);
             }
 
             if (startIdx < autoCompleteData[siteName].sortedTags.length) {
                 setTimeout(processChunkTasks, 0);
                 // console.log(`[Autocomplete-Plus] Current porcess: ${startIdx}`);
             } else {
+                autoCompleteData[siteName].flexSearchDocument = document;
+
                 const endTime = performance.now();
                 const duration = endTime - startTime;
-                autoCompleteData[siteName].flexSearchIndex = index;
-                autoCompleteData[siteName].flexSearchLimitMultiplier = Math.min(10, maxCountOfAlias + 1);
-                console.debug(`[Autocomplete-Plus] Building ${autoCompleteData[siteName].sortedTags.length} index for ${siteName} took ${duration.toFixed(2)}ms.`);
+                console.info(`[Autocomplete-Plus] Building ${autoCompleteData[siteName].sortedTags.length} index for ${siteName} took ${duration.toFixed(2)}ms.`);
             }
         }
         processChunkTasks();
@@ -269,17 +268,17 @@ async function loadCooccurrence(csvUrl, siteName) {
 
         const startIndex = lines[0].startsWith('tag_a,tag_b,count') ? 1 : 0;
 
-        await processInChunks(lines, startIndex, autoCompleteData[siteName].cooccurrenceMap, csvUrl, siteName);
+        await processCooccurrenceInChunks(lines, startIndex, autoCompleteData[siteName].cooccurrenceMap, csvUrl, siteName);
     } catch (error) {
         console.error(`[Autocomplete-Plus] Failed to fetch or process cooccurrence data from ${csvUrl}:`, error);
     }
 }
 
 /**
- * Process CSV data in chunks to avoid blocking the UI.
+ * Process Co-Occurrence CSV data in chunks to avoid blocking the UI.
  * Modifies the targetMap directly.
  */
-function processInChunks(lines, startIndex, targetMap, csvUrl, siteName) {
+function processCooccurrenceInChunks(lines, startIndex, targetMap, csvUrl, siteName) {
     return new Promise((resolve) => {
         const CHUNK_SIZE = 10000;
         let i = startIndex;
@@ -290,7 +289,7 @@ function processInChunks(lines, startIndex, targetMap, csvUrl, siteName) {
 
             for (; i < endIndex; i++) {
                 const line = lines[i];
-                const columns = parseCSVLine(line);
+                const columns = line.split(",");
 
                 if (columns.length >= 3) {
                     const tagA = columns[0].trim();
@@ -300,17 +299,19 @@ function processInChunks(lines, startIndex, targetMap, csvUrl, siteName) {
                     if (!tagA || !tagB || isNaN(count)) continue;
 
                     // Add tagA -> tagB relationship
-                    if (!targetMap.has(tagA)) {
-                        targetMap.set(tagA, new Map());
+                    let subMapA = targetMap.get(tagA);
+                    if (!subMapA) {
+                        subMapA = new Map();
+                        targetMap.set(tagA, subMapA);
                     }
-                    targetMap.get(tagA).set(tagB, count);
+                    subMapA.set(tagB, count);
 
-
-                    // Add tagB -> tagA relationship (bidirectional)
-                    if (!targetMap.has(tagB)) {
-                        targetMap.set(tagB, new Map());
+                    let subMapB = targetMap.get(tagB);
+                    if (!subMapB) {
+                        subMapB = new Map();
+                        targetMap.set(tagB, subMapB);
                     }
-                    targetMap.get(tagB).set(tagA, count);
+                    subMapB.set(tagA, count);
 
                     pairCount++;
                 }
@@ -361,7 +362,11 @@ function parseCSVLine(line) {
     return result;
 }
 
-export async function fetchCsvList() {
+/**
+ * Fetch the list of CSV files from the API endpoint
+ * @returns {Promise<void>}
+ */
+async function fetchCsvList() {
     try {
         const response = await fetch('/autocomplete-plus/csv');
         if (!response.ok) {
@@ -377,9 +382,8 @@ export async function fetchCsvList() {
 
 /**
  * Initializes the autocomplete data by fetching the list of CSV files and loading them.
- * This function is called when the extension is initialized.
  */
-export async function initializeData(csvListData, source) {
+async function initializeDataFromCSV(csvListData, source) {
     if (autoCompleteData.hasOwnProperty(source) === false) {
         autoCompleteData[source] = new AutocompleteData();
     }
@@ -440,6 +444,9 @@ export async function initializeData(csvListData, source) {
         await Promise.all([
             Promise.all(tagsLoadPromiseFactories.map(factory => factory()))
                 .then(() => {
+                    // Sort by count in descending order
+                    autoCompleteData[source].sortedTags.sort((a, b) => b.count - a.count);
+
                     // Build FlexSearch index after tags are loaded
                     return buildFlexSearchIndex(source);
                 })
@@ -464,3 +471,88 @@ export async function initializeData(csvListData, source) {
         autoCompleteData[source].isInitializing = false;
     }
 }
+
+/**
+ * Load Embeddings data from the API endpoint
+ * @returns {Promise<void>}
+ */
+async function loadEmbeddings() {
+    try {
+        const response = await fetch('/autocomplete-plus/embeddings', { cache: "no-store" });
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const embeddings = await response.json();
+        const source = ModelTagSource.Embeddings;
+
+        if (autoCompleteData.hasOwnProperty(source) === false) {
+            autoCompleteData[source] = new AutocompleteData();
+        }
+
+        embeddings.forEach(embedding => {
+            if (!autoCompleteData[source].tagMap.has(embedding)) {
+                const tagData = new TagData(`embedding:${embedding}`, 0, 0, [], source);
+                autoCompleteData[source].sortedTags.push(tagData);
+                autoCompleteData[source].tagMap.set(embedding, tagData);
+
+                updateMaxTagLength(embedding.length);
+            }
+        });
+
+        await buildFlexSearchIndex(ModelTagSource.Embeddings);
+
+        console.log(`[Autocomplete-Plus] Loaded ${embeddings.length} Embeddings`);
+    } catch (error) {
+        console.error(`[Autocomplete-Plus] Failed to fetch Embeddings data:`, error);
+    }
+}
+
+/**
+ * Load LoRA data from the API endpoint
+ * @returns {Promise<void>}
+ */
+async function loadLoras() {
+    try {
+        const response = await fetch('/autocomplete-plus/loras', { cache: "no-store" });
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const loraNames = await response.json();
+        const source = ModelTagSource.Lora;
+
+        if (autoCompleteData.hasOwnProperty(source) === false) {
+            autoCompleteData[source] = new AutocompleteData();
+        }
+
+        loraNames.forEach(loraName => {
+            if (!autoCompleteData[source].tagMap.has(loraName)) {
+                const tagData = new TagData(`<lora:${loraName}>`, 0, 0, [], source);
+                autoCompleteData[source].sortedTags.push(tagData);
+                autoCompleteData[source].tagMap.set(loraName, tagData);
+
+                updateMaxTagLength(loraName.length);
+            }
+        });
+
+        await buildFlexSearchIndex(ModelTagSource.Lora);
+
+        console.log(`[Autocomplete-Plus] Loaded ${loraNames.length} LoRA models`);
+    } catch (error) {
+        console.error(`[Autocomplete-Plus] Failed to fetch LoRA data:`, error);
+    }
+}
+
+/**
+ * Load all data sources in parallel.
+ */
+export async function loadDataAsync() {
+    return Promise.all([
+        fetchCsvList().then((csvList) => {
+            Object.values(TagSource).forEach((source) => {
+                initializeDataFromCSV(csvList, source);
+            });
+        }),
+        loadEmbeddings(),
+        loadLoras(),
+    ]);
+}

web/js/main.js

@@ -3,7 +3,7 @@ import { $el } from "/scripts/ui.js";
 import { ComfyWidgets } from "/scripts/widgets.js";
 import { settingValues } from "./settings.js";
 import { loadCSS } from "./utils.js";
-import { TagSource, fetchCsvList, initializeData } from "./data.js";
+import { TagSource, loadDataAsync } from "./data.js";
 import { AutocompleteEventHandler } from "./autocomplete.js";
 import { RelatedTagsEventHandler } from "./related-tags.js";
 
@@ -240,11 +240,7 @@ app.registerExtension({
         let rootPath = import.meta.url.replace("js/main.js", "");
         loadCSS(rootPath + "css/autocomplete-plus.css"); // Load CSS for autocomplete
 
-        fetchCsvList().then((csvList) => {
-            Object.values(TagSource).forEach((source) => {
-                initializeData(csvList, source);
-            });
-        });
+        loadDataAsync();
     },
 
     // One the Settings Screen, displays reverse order in same category
@@ -297,6 +293,17 @@ app.registerExtension({
                 settingValues.useFastSearch = newVal;
             }
         },
+        {
+            id: id + ".enable_models",
+            name: "Enable Loras and Embeddings",
+            tooltip: "Enable Lora and Embedding suggestions",
+            type: "boolean",
+            defaultValue: true,
+            category: [name, "Autocompletion", "Enable Loras and Embeddings"],
+            onChange: (newVal, oldVal) => {
+                settingValues.enableModels = newVal;
+            }
+        },
         {
             id: id + ".max_suggestions",
             name: "Max suggestions",

web/js/searchengine.js

@@ -0,0 +1,133 @@
+import { Charset, Encoder, Document } from './thirdparty/flexsearch.bundle.module.min.js'
+import { kataToHira } from './utils.js';
+
+/**
+ * Creates an encoder optimized for processing English tag names.
+ * Handles tag formatting like underscores and parentheses commonly used in Danbooru tags.
+ * @returns {Encoder} FlexSearch encoder for English tags
+ */
+function createTagEncoder() {
+    return new Encoder({
+        normalize: true,
+        dedupe: false,
+        numeric: false,
+        cache: true,
+        // filter: new Set(['and', 'to', 'be', 'on']),
+        replacer: [/(?<=[a-zA-Z\)])_$/, ''],  // Remove trailing underscores after letters/parentheses
+        split: /(?<=[a-zA-Z\)])_(?=[a-zA-Z\(])|\((?=[a-zA-Z])|(?<=[a-zA-Z\)])\)|[ \n]/  // Split on underscores between words, parentheses, spaces, and newlines
+    });
+}
+
+/**
+ * Creates an encoder optimized for processing CJK (Chinese, Japanese, Korean) characters.
+ * Uses exact character matching and converts katakana to hiragana for better Japanese search.
+ * @returns {Encoder} FlexSearch encoder for CJK text
+ */
+function createCJKEncoder() {
+    return new Encoder(Charset.Exact, {
+        dedupe: true,
+        numeric: true,
+        cache: true,
+        filter: new Set(['(', ')']),  // Filter out parentheses characters
+        finalize: (term) => {         // Convert katakana to hiragana for better Japanese matching
+            return term.map(str => kataToHira(str));
+        }
+    });
+}
+
+/**
+ * Creates an encoder optimized for processing Embedding or Lora notation.
+ * @returns {Encoder} FlexSearch encoder
+ */
+function createModelEncoder() {
+    return new Encoder({
+        normalize: true,
+        dedupe: false,
+        numeric: true,
+        cache: true,
+        prepare: function (str) {
+            return str.replace(/^<|>$/g, '').split(/(lora:|embedding:|[^\u0000-\u007f]+)/g).filter(Boolean).join(" ").trim();
+        },
+        split: /(?<=lora:.*|embedding:.*)[_./\(\)\-\s\\]+/
+    });
+}
+
+/**
+ * Creates a FlexSearch Document instance optimized for tag searching.
+ * Configures separate encoders for English tags and CJK aliases with appropriate tokenization.
+ * @returns {Document} Configured FlexSearch document for tag indexing
+ */
+export function createFlexSearchDocument() {
+    const tagEncoder = createTagEncoder();
+    const cjkEncoder = createCJKEncoder();
+
+    // Custom encoding function for alias field that handles mixed language content
+    const encodeAlias = function (word) {
+        return word.split(",")
+            .flatMap(str => {
+                if (/[^\u0000-\u007f]/.test(str)) {
+                    // Contains non-ASCII characters (CJK text)
+                    return cjkEncoder.encode(str);
+                } else {
+                    // ASCII characters only (English text)
+                    return tagEncoder.encode(str);
+                }
+            })
+            .filter(Boolean);
+    }
+
+    // Configure the FlexSearch document with optimized indexing settings
+    const document = new Document({
+        document: {
+            id: "id",
+            index: [
+                {
+                    field: "tag",
+                    tokenize: "bidirectional",  // Allow partial matching from both ends
+                    encoder: tagEncoder,        // Use tag-optimized encoder
+                },
+                {
+                    field: "alias",             // Index the alias field for multi-language support
+                    tokenize: "full",           // Full tokenization for complete alias matching
+                    encode: encodeAlias,        // Use custom multi-language encoding function
+                }
+            ]
+        }
+    });
+
+    return document;
+}
+
+/**
+ * Creates a FlexSearch Document instance optimized for lora or embedding searching.
+ * @returns {Document} Configured FlexSearch document
+ */
+export function createFlexSearchDocumentForModel() {
+    const modelEncoder = createModelEncoder();
+
+
+    // Configure the FlexSearch document with optimized indexing settings
+    // Note: alias field is not indexed for lora or embedding search
+    const document = new Document({
+        tokenize: "full",  // Allow partial matching from both ends
+        encoder: modelEncoder,
+        document: {
+            id: "id",
+            index: [
+                {
+                    field: "tag",
+
+                },
+                {
+                    field: "alias",
+                }
+            ]
+        }
+    });
+
+    return document;
+}
+
+// Export functions for testing when in test environment
+const isTestEnvironment = typeof process !== 'undefined' && process.env.NODE_ENV === 'test';
+export const __test__ = isTestEnvironment ? { createTagEncoder, createCJKEncoder, createModelEncoder } : undefined;

web/js/settings.js

@@ -7,6 +7,7 @@ export const settingValues = {
     // Autocomplete feature settings
     enabled: true,
     maxSuggestions: 10,
+    enableModels: true, // Enable Lora and Embedding suggestions
     useFastSearch: false,
 
     // Related tags feature settings

web/js/utils.js

@@ -109,6 +109,30 @@ export function formatCountHumanReadable(num) {
     return (num / si[i].value).toFixed(1).replace(rx, "$1") + si[i].symbol;
 }
 
+/**
+ * Escapes HTML special characters in a string.
+ * @param {string} str The input string.
+ * @returns {string} The escaped string.
+ */
+export function escapeHtml(str) {
+    if (typeof str !== 'string') {
+        return str;
+    }
+
+    const escapeMap = {
+        '&': '&',
+        '<': '&lt;',
+        '>': '&gt;',
+        '"': '&quot;',
+        "'": '&#x27;',
+        '`': '&#x60;',
+        '/': '&#x2F;'
+    };
+
+    return str.replace(/[&<>"'`/]/g, match =>
+        escapeMap[match]);
+}
+
 /**
  * Escapes parentheses in a string for use in prompts.
  * Replaces '(' with '\(' and ')' with '\)'.