From fd34e6cc4370a71a1d4a697fc7dc75f3a0d8bcd1 Mon Sep 17 00:00:00 2001 From: Weixuan Fu Date: Mon, 18 Mar 2024 01:15:25 +0800 Subject: [PATCH] indexdb.put models list even without filehash, always put models files list into indexdb --- ui/src/App.tsx | 3 - ui/src/components/DropdownTitle.tsx | 18 ++- ui/src/db-tables/UserSettingsTable.ts | 4 + ui/src/model-manager/hooks/useUpdateModels.ts | 6 +- .../models-list-drawer/ModelItem.tsx | 40 +------ .../topbar/ModelManagerTopbar.tsx | 43 ++++++- ui/src/model-manager/types.ts | 2 + ui/src/spacejson/DownloadSpaceJsonDialog.tsx | 105 ++++++++++++++++++ ui/src/spacejson/handleDownloadSpaceJson.ts | 65 +++++++++++ ui/src/topbar/Topbar.tsx | 13 ++- ui/src/types/dbTypes.ts | 8 +- ui/src/types/types.ts | 3 +- ui/src/utils/civitUtils.ts | 53 +++++++++ 13 files changed, 301 insertions(+), 62 deletions(-) create mode 100644 ui/src/spacejson/DownloadSpaceJsonDialog.tsx create mode 100644 ui/src/spacejson/handleDownloadSpaceJson.ts diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 7151ac8..20fc374 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -48,9 +48,6 @@ import { deepJsonDiffCheck } from "./utils/deepJsonDiffCheck"; const AppIsDirtyEventListener = React.lazy( () => import("./topbar/AppIsDirtyEventListener"), ); -const ModelsListDrawer = React.lazy( - () => import("./model-manager/models-list-drawer/ModelsListDrawer"), -); const usedWsEvents = [ // InstallProgress.tsx diff --git a/ui/src/components/DropdownTitle.tsx b/ui/src/components/DropdownTitle.tsx index 0f6f93b..8b2c5e4 100644 --- a/ui/src/components/DropdownTitle.tsx +++ b/ui/src/components/DropdownTitle.tsx @@ -46,6 +46,7 @@ const ShareDialog = lazy(() => import("../share/ShareDialog")); // @ts-ignore import { app } from "/scripts/app.js"; import { TOPBAR_BUTTON_HEIGHT } from "../const"; +import DownloadSpaceJsonDialog from "../spacejson/DownloadSpaceJsonDialog"; export default function DropdownTitle() { const { @@ -126,12 +127,6 @@ export default function DropdownTitle() { URL.revokeObjectURL(url); }, [curFlowID]); - const handleDownloadSpaceJson = useCallback(async () => { - console.log("app.graph", app.graph); - const graph = app.graph.serialize(); - console.log("graph serialize", graph); - }, []); - return ( <> File @@ -179,13 +174,13 @@ export default function DropdownTitle() { > Download - {/* setRoute("downloadSpaceJson")} icon={} iconSpacing={1} > Download .space.json - */} + setRoute("saveAsModal")} icon={} @@ -226,6 +221,7 @@ export default function DropdownTitle() { {route == "versionHistory" && ( setRoute("root")} /> )} + {route == "downloadSpaceJson" && } {route === "saveAsModal" && ( diff --git a/ui/src/db-tables/UserSettingsTable.ts b/ui/src/db-tables/UserSettingsTable.ts index b66d501..9bd1ef6 100644 --- a/ui/src/db-tables/UserSettingsTable.ts +++ b/ui/src/db-tables/UserSettingsTable.ts @@ -93,6 +93,10 @@ export class UserSettingsTable extends TableBase { await instance.getSetting("autoSave").then((res) => { instance._autoSave = res ?? true; }); + // overwrite legacy comfyspace.art + await instance.upsert({ + cloudHost: instance.defaultSettings.cloudHost, + }); return instance; } } diff --git a/ui/src/model-manager/hooks/useUpdateModels.ts b/ui/src/model-manager/hooks/useUpdateModels.ts index 27eed1d..3ee1dbc 100644 --- a/ui/src/model-manager/hooks/useUpdateModels.ts +++ b/ui/src/model-manager/hooks/useUpdateModels.ts @@ -1,11 +1,9 @@ import { useEffect, useState } from "react"; import { getAllModelsList } from "../../Api"; -import type { ModelsListRespItem } from "../types"; +import type { ModelsListRespItem, ModelsListRespItemFromApi } from "../types"; // @ts-ignore import { api } from "/scripts/api.js"; -type ModelsListRespItemFromApi = ModelsListRespItem & { date: number }; - export const useUpdateModels = () => { // all model types const [modelTypeList, setModelTypeList] = useState(["checkpoints"]); @@ -20,7 +18,7 @@ export const useUpdateModels = () => { initData(); api.addEventListener( "model_list", - (e: { detail: ModelsListRespItemFromApi[] }) => { + async (e: { detail: ModelsListRespItemFromApi[] }) => { updateModels(e.detail); }, ); diff --git a/ui/src/model-manager/models-list-drawer/ModelItem.tsx b/ui/src/model-manager/models-list-drawer/ModelItem.tsx index 1f33ee5..7576d12 100644 --- a/ui/src/model-manager/models-list-drawer/ModelItem.tsx +++ b/ui/src/model-manager/models-list-drawer/ModelItem.tsx @@ -6,24 +6,7 @@ import { Model } from "../../types/dbTypes"; import type { DragEvent } from "react"; import { userSettingsTable } from "../../db-tables/WorkspaceDB"; - -interface ResponsePartial { - id: number; - modelId: number; - name: string; - model: { - name: string; - type: string; - nsfw: boolean; - }; - images: { - url: string; - nsfw: "None" | "Soft" | "Mature" | "X"; - width: number; - height: number; - hash: string; - }[]; -} +import { fetchCivitModelFromHashKey } from "../../utils/civitUtils"; interface Props { data: ModelsListRespItem; @@ -59,19 +42,8 @@ export function ModelItem({ data }: Props) { } if (data.file_hash != null) { try { - const url = `https://civitai.com/api/v1/model-versions/by-hash/${data.file_hash}`; - const resp = await fetch(url); - const json: ResponsePartial = await resp.json(); - let image_url: string | undefined; - const showNsfwThumbnail = await userSettingsTable?.getSetting( - "showNsfwModelThumbnail", - ); - if (showNsfwThumbnail === true) { - image_url = json?.images?.[0]?.url; - } else if (!json.model.nsfw) { - const sfwImage = json.images.find((i) => i.nsfw === "None"); - image_url = sfwImage?.url; - } + const json = await fetchCivitModelFromHashKey(data.file_hash); + const image_url = json.imageUrl; image_url && setUrl(image_url); const newModel: Model = { @@ -79,9 +51,9 @@ export function ModelItem({ data }: Props) { fileHash: data.file_hash, fileFolder: data.model_type, fileName: data.model_name + data.model_extension, - modelName: json.model.name, - civitModelID: String(json.modelId), - civitModelVersionID: String(json.id), + modelName: json.modelName ?? null, + civitModelID: json.civitModelID, + civitModelVersionID: json.civitModelVersionID, imageUrl: image_url ?? null, }; indexdb.models.put(newModel); diff --git a/ui/src/model-manager/topbar/ModelManagerTopbar.tsx b/ui/src/model-manager/topbar/ModelManagerTopbar.tsx index 4683165..bc13273 100644 --- a/ui/src/model-manager/topbar/ModelManagerTopbar.tsx +++ b/ui/src/model-manager/topbar/ModelManagerTopbar.tsx @@ -1,11 +1,17 @@ import { Button, Stack } from "@chakra-ui/react"; -import { lazy, useEffect, DragEvent, useContext } from "react"; +import { lazy, useEffect, DragEvent, useContext, Suspense } from "react"; import ModelsListDrawer from "../models-list-drawer/ModelsListDrawer"; - // @ts-ignore import { app } from "/scripts/app.js"; import "./index.css"; import { WorkspaceContext } from "../../WorkspaceContext"; +// @ts-ignore +import { api } from "/scripts/api.js"; + +import { Model } from "../../types/dbTypes"; +import { fetchCivitModelFromHashKey } from "../../utils/civitUtils"; +import { indexdb } from "../../db-tables/indexdb"; +import type { ModelsListRespItemFromApi } from "../types"; const AddMissingModelsButton = lazy( () => import("./InstallMissingModelsButton"), @@ -31,6 +37,35 @@ export default function ModelManagerTopbar() { useEffect(() => { app.canvasEl.addEventListener("drop", handleModelDrop); + api.addEventListener( + "model_list", + async (e: { detail: ModelsListRespItemFromApi[] }) => { + console.log("model_list", e.detail); + const modelsPromises = e.detail?.map(async (item) => { + let newModel: Model = { + id: item.model_name + "@" + item.model_type, + modelName: null, + fileHash: item.file_hash ?? null, + fileFolder: item.model_type, + fileName: item.model_name + item.model_extension, + }; + if (!item.file_hash) return newModel; + const json = await fetchCivitModelFromHashKey(item.file_hash); + newModel = { + ...newModel, + modelName: json.modelName ?? null, + civitModelID: json.civitModelID, + civitModelVersionID: json.civitModelVersionID, + imageUrl: json.imageUrl ?? null, + }; + return newModel; + }); + const models = (await Promise.all(modelsPromises)).filter( + (model) => model != null, + ); + indexdb.models.bulkPut(models); + }, + ); return () => { app.canvasEl.removeEventListener("drop", handleModelDrop); }; @@ -48,7 +83,9 @@ export default function ModelManagerTopbar() { Models
- + + +
{route === "modelList" && ( setRoute("root")} /> diff --git a/ui/src/model-manager/types.ts b/ui/src/model-manager/types.ts index 9483b2e..b85ac10 100644 --- a/ui/src/model-manager/types.ts +++ b/ui/src/model-manager/types.ts @@ -135,3 +135,5 @@ export interface ModelsListRespItem { preview?: string; date: Date; } + +export type ModelsListRespItemFromApi = ModelsListRespItem & { date: number }; diff --git a/ui/src/spacejson/DownloadSpaceJsonDialog.tsx b/ui/src/spacejson/DownloadSpaceJsonDialog.tsx new file mode 100644 index 0000000..5855c35 --- /dev/null +++ b/ui/src/spacejson/DownloadSpaceJsonDialog.tsx @@ -0,0 +1,105 @@ +import { useContext, useEffect, useState } from "react"; +import { WorkspaceContext } from "../WorkspaceContext"; +import { + Box, + Flex, + Heading, + Input, + Modal, + ModalBody, + ModalContent, + ModalOverlay, + Stack, + Tag, +} from "@chakra-ui/react"; +import { + DepsResult, + ModelFile, + extractAndFetchFileNames, +} from "./handleDownloadSpaceJson"; +// @ts-ignore +import { app } from "/scripts/app.js"; +import { + IconAlertCircle, + IconEscalator, + IconQuestionMark, +} from "@tabler/icons-react"; + +export default function DownloadSpaceJsonDialog() { + const { route, setRoute } = useContext(WorkspaceContext); + const [result, setResult] = useState(); + useEffect(() => { + console.log("app.graph", app.graph); + const graph = app.graph.serialize(); + console.log("graph serialize", graph); + extractAndFetchFileNames(graph.nodes ?? []).then((result) => { + console.log(result); + setResult(result); + }); + }, []); + if (!result) { + return
Preparing...
; + } + + return ( + setRoute("root")} size={"xl"}> + {/* */} + + + + Workflow Resouce Dependencies + + + Models ({result.models.length}) + {result.models.map((modelFile) => ( + + ))} + + + {result.models.length > 0 && ( + + Images ({result.images.length}) + {result.images.map((image) => ( +

{image.filename}

+ ))} + + )} + + + + + ); +} + +function ModelDepsItem({ modelFile }: { modelFile: ModelFile }) { + if (!modelFile.models?.length) { + return ( + + +

{modelFile.filename}

+ + + + ); + } + if (modelFile.models.length === 1) { + const model = modelFile.models[0]; + return ( + + {model.fileFolder} +

{modelFile.filename}

+ + ); + } + return ( + +

{modelFile.filename}

+ {modelFile.models?.map((model) => ( + + {model.fileFolder} +

{model.modelName}

+ + ))} + + ); +} diff --git a/ui/src/spacejson/handleDownloadSpaceJson.ts b/ui/src/spacejson/handleDownloadSpaceJson.ts new file mode 100644 index 0000000..8eab5c1 --- /dev/null +++ b/ui/src/spacejson/handleDownloadSpaceJson.ts @@ -0,0 +1,65 @@ +import { indexdb } from "../db-tables/indexdb"; +import { Model } from "../types/dbTypes"; + +type LiteNode = { + id: number; + type: string; + widgets_values?: Array; +}; + +export type ModelFile = { + filename: string; + nodeType: string; + models?: Model[]; +}; + +type ImageFile = { + filename: string; + nodeType: string; +}; + +export type DepsResult = { + models: ModelFile[]; + images: ImageFile[]; +}; + +async function fetchModelData( + filename: string, + nodeType: string, +): Promise { + const res = + (await indexdb.models.where("fileName").equals(filename).toArray()) ?? []; + + const modelFile: ModelFile = { filename, nodeType, models: res }; + return modelFile; +} + +export async function extractAndFetchFileNames( + nodes: LiteNode[], +): Promise { + let modelPromises: Promise[] = []; + let images: ImageFile[] = []; + const modelFileExtensions = [".safetensors", ".bin"]; + const imageFileExtensions = [".jpeg", ".jpg", ".png", ".gif"]; + + nodes.forEach((node) => { + if (node.widgets_values) { + node.widgets_values.forEach((value) => { + if (typeof value != "string") return; + // Check if it's a model file + if (modelFileExtensions.some((ext) => value.endsWith(ext))) { + modelPromises.push(fetchModelData(value, node.type)); + } + // Check if it's an image file + if (imageFileExtensions.some((ext) => value.endsWith(ext))) { + images.push({ filename: value, nodeType: node.type }); + } + }); + } + }); + + // Wait for all the model data fetch operations to complete + const models = await Promise.all(modelPromises); + + return { models, images }; +} diff --git a/ui/src/topbar/Topbar.tsx b/ui/src/topbar/Topbar.tsx index 96e744c..68e2e17 100644 --- a/ui/src/topbar/Topbar.tsx +++ b/ui/src/topbar/Topbar.tsx @@ -10,7 +10,14 @@ import { IconLock, } from "@tabler/icons-react"; import DropdownTitle from "../components/DropdownTitle"; -import { lazy, useCallback, useContext, useEffect, useState } from "react"; +import { + Suspense, + lazy, + useCallback, + useContext, + useEffect, + useState, +} from "react"; import EditFlowName from "../components/EditFlowName"; import { WorkspaceContext } from "../WorkspaceContext"; import { PanelPosition } from "../types/dbTypes"; @@ -97,7 +104,9 @@ export function Topbar({ curFlowName, setCurFlowName }: Props) { - + }> + +