nodes/11cafe-comfyui-workspace-manager
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

indexdb.put models list even without filehash, always put models files list into indexdb

Browse Source
This commit is contained in:
Weixuan Fu
2024-03-18 01:15:25 +08:00
parent 5a9800b8e6
commit fd34e6cc43
13 changed files with 301 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ui/src/App.tsx
-3
View File
@@ -48,9 +48,6 @@ import { deepJsonDiffCheck } from "./utils/deepJsonDiffCheck";
const AppIsDirtyEventListener = React.lazy(
() => import("./topbar/AppIsDirtyEventListener"),
);
const ModelsListDrawer = React.lazy(
() => import("./model-manager/models-list-drawer/ModelsListDrawer"),
);
const usedWsEvents = [
// InstallProgress.tsx
ui/src/components/DropdownTitle.tsx
+7 -11
View File
@@ -46,6 +46,7 @@ const ShareDialog = lazy(() => import("../share/ShareDialog"));
// @ts-ignore
import { app } from "/scripts/app.js";
import { TOPBAR_BUTTON_HEIGHT } from "../const";
import DownloadSpaceJsonDialog from "../spacejson/DownloadSpaceJsonDialog";
export default function DropdownTitle() {
const {
@@ -126,12 +127,6 @@ export default function DropdownTitle() {
URL.revokeObjectURL(url);
}, [curFlowID]);
const handleDownloadSpaceJson = useCallback(async () => {
console.log("app.graph", app.graph);
const graph = app.graph.serialize();
console.log("graph serialize", graph);
}, []);
return (
<>
<HoverMenu
@@ -142,8 +137,8 @@ export default function DropdownTitle() {
height={TOPBAR_BUTTON_HEIGHT + "px"}
aria-label="menu"
size={"sm"}
// backgroundColor={"#4a4d6b"}
backgroundColor={"teal.600"}
backgroundColor={"#434554"}
// backgroundColor={"teal.600"}
>
File
<IconChevronDown size={20} />
@@ -179,13 +174,13 @@ export default function DropdownTitle() {
>
Download
</MenuItem>
{/* <MenuItem
onClick={handleDownloadSpaceJson}
<MenuItem
onClick={() => setRoute("downloadSpaceJson")}
icon={<IconDownload size={20} />}
iconSpacing={1}
>
Download .space.json
</MenuItem> */}
</MenuItem>
<MenuItem
onClick={() => setRoute("saveAsModal")}
icon={<IconDeviceFloppy size={20} />}
@@ -226,6 +221,7 @@ export default function DropdownTitle() {
{route == "versionHistory" && (
<VersionHistoryDrawer onClose={() => setRoute("root")} />
)}
{route == "downloadSpaceJson" && <DownloadSpaceJsonDialog />}
{route === "saveAsModal" && (
<Modal isOpen={true} onClose={handleOnCloseModal}>
<ModalOverlay />
ui/src/db-tables/UserSettingsTable.ts
+4
View File
@@ -93,6 +93,10 @@ export class UserSettingsTable extends TableBase<UserSettings> {
await instance.getSetting("autoSave").then((res) => {
instance._autoSave = res ?? true;
});
// overwrite legacy comfyspace.art
await instance.upsert({
cloudHost: instance.defaultSettings.cloudHost,
});
return instance;
}
}
ui/src/model-manager/hooks/useUpdateModels.ts
+2 -4
View File
@@ -1,11 +1,9 @@
import { useEffect, useState } from "react";
import { getAllModelsList } from "../../Api";
import type { ModelsListRespItem } from "../types";
import type { ModelsListRespItem, ModelsListRespItemFromApi } from "../types";
// @ts-ignore
import { api } from "/scripts/api.js";
type ModelsListRespItemFromApi = ModelsListRespItem & { date: number };
export const useUpdateModels = () => {
// all model types
const [modelTypeList, setModelTypeList] = useState<string[]>(["checkpoints"]);
@@ -20,7 +18,7 @@ export const useUpdateModels = () => {
initData();
api.addEventListener(
"model_list",
(e: { detail: ModelsListRespItemFromApi[] }) => {
async (e: { detail: ModelsListRespItemFromApi[] }) => {
updateModels(e.detail);
},
);
ui/src/model-manager/models-list-drawer/ModelItem.tsx
+6 -34
View File
@@ -6,24 +6,7 @@ import { Model } from "../../types/dbTypes";
import type { DragEvent } from "react";
import { userSettingsTable } from "../../db-tables/WorkspaceDB";
interface ResponsePartial {
id: number;
modelId: number;
name: string;
model: {
name: string;
type: string;
nsfw: boolean;
};
images: {
url: string;
nsfw: "None" | "Soft" | "Mature" | "X";
width: number;
height: number;
hash: string;
}[];
}
import { fetchCivitModelFromHashKey } from "../../utils/civitUtils";
interface Props {
data: ModelsListRespItem;
@@ -59,19 +42,8 @@ export function ModelItem({ data }: Props) {
}
if (data.file_hash != null) {
try {
const url = `https://civitai.com/api/v1/model-versions/by-hash/${data.file_hash}`;
const resp = await fetch(url);
const json: ResponsePartial = await resp.json();
let image_url: string | undefined;
const showNsfwThumbnail = await userSettingsTable?.getSetting(
"showNsfwModelThumbnail",
);
if (showNsfwThumbnail === true) {
image_url = json?.images?.[0]?.url;
} else if (!json.model.nsfw) {
const sfwImage = json.images.find((i) => i.nsfw === "None");
image_url = sfwImage?.url;
}
const json = await fetchCivitModelFromHashKey(data.file_hash);
const image_url = json.imageUrl;
image_url && setUrl(image_url);
const newModel: Model = {
@@ -79,9 +51,9 @@ export function ModelItem({ data }: Props) {
fileHash: data.file_hash,
fileFolder: data.model_type,
fileName: data.model_name + data.model_extension,
modelName: json.model.name,
civitModelID: String(json.modelId),
civitModelVersionID: String(json.id),
modelName: json.modelName ?? null,
civitModelID: json.civitModelID,
civitModelVersionID: json.civitModelVersionID,
imageUrl: image_url ?? null,
};
indexdb.models.put(newModel);
ui/src/model-manager/topbar/ModelManagerTopbar.tsx
+40 -3
View File
@@ -1,11 +1,17 @@
import { Button, Stack } from "@chakra-ui/react";
import { lazy, useEffect, DragEvent, useContext } from "react";
import { lazy, useEffect, DragEvent, useContext, Suspense } from "react";
import ModelsListDrawer from "../models-list-drawer/ModelsListDrawer";
// @ts-ignore
import { app } from "/scripts/app.js";
import "./index.css";
import { WorkspaceContext } from "../../WorkspaceContext";
// @ts-ignore
import { api } from "/scripts/api.js";
import { Model } from "../../types/dbTypes";
import { fetchCivitModelFromHashKey } from "../../utils/civitUtils";
import { indexdb } from "../../db-tables/indexdb";
import type { ModelsListRespItemFromApi } from "../types";
const AddMissingModelsButton = lazy(
() => import("./InstallMissingModelsButton"),
@@ -31,6 +37,35 @@ export default function ModelManagerTopbar() {
useEffect(() => {
app.canvasEl.addEventListener("drop", handleModelDrop);
api.addEventListener(
"model_list",
async (e: { detail: ModelsListRespItemFromApi[] }) => {
console.log("model_list", e.detail);
const modelsPromises = e.detail?.map(async (item) => {
let newModel: Model = {
id: item.model_name + "@" + item.model_type,
modelName: null,
fileHash: item.file_hash ?? null,
fileFolder: item.model_type,
fileName: item.model_name + item.model_extension,
};
if (!item.file_hash) return newModel;
const json = await fetchCivitModelFromHashKey(item.file_hash);
newModel = {
...newModel,
modelName: json.modelName ?? null,
civitModelID: json.civitModelID,
civitModelVersionID: json.civitModelVersionID,
imageUrl: json.imageUrl ?? null,
};
return newModel;
});
const models = (await Promise.all(modelsPromises)).filter(
(model) => model != null,
);
indexdb.models.bulkPut(models);
},
);
return () => {
app.canvasEl.removeEventListener("drop", handleModelDrop);
};
@@ -48,7 +83,9 @@ export default function ModelManagerTopbar() {
Models
</Button>
<div style={{ position: "absolute", top: "38px", left: "0px" }}>
<AddMissingModelsButton />
<Suspense>
<AddMissingModelsButton />
</Suspense>
</div>
{route === "modelList" && (
<ModelsListDrawer onClose={() => setRoute("root")} />
ui/src/model-manager/types.ts
+2
View File
@@ -135,3 +135,5 @@ export interface ModelsListRespItem {
preview?: string;
date: Date;
}
export type ModelsListRespItemFromApi = ModelsListRespItem & { date: number };
ui/src/spacejson/DownloadSpaceJsonDialog.tsx
+105
View File
@@ -0,0 +1,105 @@
import { useContext, useEffect, useState } from "react";
import { WorkspaceContext } from "../WorkspaceContext";
import {
Box,
Flex,
Heading,
Input,
Modal,
ModalBody,
ModalContent,
ModalOverlay,
Stack,
Tag,
} from "@chakra-ui/react";
import {
DepsResult,
ModelFile,
extractAndFetchFileNames,
} from "./handleDownloadSpaceJson";
// @ts-ignore
import { app } from "/scripts/app.js";
import {
IconAlertCircle,
IconEscalator,
IconQuestionMark,
} from "@tabler/icons-react";
export default function DownloadSpaceJsonDialog() {
const { route, setRoute } = useContext(WorkspaceContext);
const [result, setResult] = useState<DepsResult>();
useEffect(() => {
console.log("app.graph", app.graph);
const graph = app.graph.serialize();
console.log("graph serialize", graph);
extractAndFetchFileNames(graph.nodes ?? []).then((result) => {
console.log(result);
setResult(result);
});
}, []);
if (!result) {
return <div>Preparing...</div>;
}
return (
<Modal isOpen={true} onClose={() => setRoute("root")} size={"xl"}>
{/* <ModalOverlay /> */}
<ModalContent>
<ModalBody p={5}>
<Stack gap={5}>
<Heading size={"md"}>Workflow Resouce Dependencies</Heading>
<Stack>
<Heading size={"sm"}>Models ({result.models.length})</Heading>
{result.models.map((modelFile) => (
<ModelDepsItem modelFile={modelFile} />
))}
</Stack>
{result.models.length > 0 && (
<Stack>
<Heading size={"sm"}>Images ({result.images.length})</Heading>
{result.images.map((image) => (
<p key={image.filename}>{image.filename}</p>
))}
</Stack>
)}
</Stack>
</ModalBody>
</ModalContent>
</Modal>
);
}
function ModelDepsItem({ modelFile }: { modelFile: ModelFile }) {
if (!modelFile.models?.length) {
return (
<Stack>
<Flex>
<p>{modelFile.filename}</p>
</Flex>
<Input placeholder={"Please enter model download url"} />
</Stack>
);
}
if (modelFile.models.length === 1) {
const model = modelFile.models[0];
return (
<Flex key={model.id} gap={2}>
<Tag>{model.fileFolder}</Tag>
<p>{modelFile.filename}</p>
</Flex>
);
}
return (
<Stack ml={2}>
<p key={modelFile.filename}>{modelFile.filename}</p>
{modelFile.models?.map((model) => (
<Flex key={model.id} gap={2}>
<Tag>{model.fileFolder}</Tag>
<p>{model.modelName}</p>
</Flex>
))}
</Stack>
);
}
ui/src/spacejson/handleDownloadSpaceJson.ts
+65
View File
@@ -0,0 +1,65 @@
import { indexdb } from "../db-tables/indexdb";
import { Model } from "../types/dbTypes";
type LiteNode = {
id: number;
type: string;
widgets_values?: Array<string | number>;
};
export type ModelFile = {
filename: string;
nodeType: string;
models?: Model[];
};
type ImageFile = {
filename: string;
nodeType: string;
};
export type DepsResult = {
models: ModelFile[];
images: ImageFile[];
};
async function fetchModelData(
filename: string,
nodeType: string,
): Promise<ModelFile> {
const res =
(await indexdb.models.where("fileName").equals(filename).toArray()) ?? [];
const modelFile: ModelFile = { filename, nodeType, models: res };
return modelFile;
}
export async function extractAndFetchFileNames(
nodes: LiteNode[],
): Promise<DepsResult> {
let modelPromises: Promise<ModelFile>[] = [];
let images: ImageFile[] = [];
const modelFileExtensions = [".safetensors", ".bin"];
const imageFileExtensions = [".jpeg", ".jpg", ".png", ".gif"];
nodes.forEach((node) => {
if (node.widgets_values) {
node.widgets_values.forEach((value) => {
if (typeof value != "string") return;
// Check if it's a model file
if (modelFileExtensions.some((ext) => value.endsWith(ext))) {
modelPromises.push(fetchModelData(value, node.type));
}
// Check if it's an image file
if (imageFileExtensions.some((ext) => value.endsWith(ext))) {
images.push({ filename: value, nodeType: node.type });
}
});
}
});
// Wait for all the model data fetch operations to complete
const models = await Promise.all(modelPromises);
return { models, images };
}
ui/src/topbar/Topbar.tsx
+11 -2
View File
@@ -10,7 +10,14 @@ import {
IconLock,
} from "@tabler/icons-react";
import DropdownTitle from "../components/DropdownTitle";
import { lazy, useCallback, useContext, useEffect, useState } from "react";
import {
Suspense,
lazy,
useCallback,
useContext,
useEffect,
useState,
} from "react";
import EditFlowName from "../components/EditFlowName";
import { WorkspaceContext } from "../WorkspaceContext";
import { PanelPosition } from "../types/dbTypes";
@@ -97,7 +104,9 @@ export function Topbar({ curFlowName, setCurFlowName }: Props) {
<IconTriangleInvertedFilled size={8} />
</HStack>
</Button>
<ModelManagerTopbar />
<Suspense fallback={<div style={{ width: "60px" }} />}>
<ModelManagerTopbar />
</Suspense>
<Tooltip label="New workflow">
<Button
size={"sm"}
ui/src/types/dbTypes.ts
+4 -4
View File
@@ -46,11 +46,11 @@ export type LocalCache = {
export type Model = {
id: string;
fileName: string;
modelName: string;
fileFolder: string;
fileHash: string;
civitModelID: string;
civitModelVersionID: string;
fileHash: string | null; // files are not guaranteed to finish hashing
modelName: string | null;
civitModelID?: string;
civitModelVersionID?: string;
imageUrl?: string | null;
};
ui/src/types/types.ts
+2 -1
View File
@@ -5,4 +5,5 @@ export type WorkspaceRoute =
| "gallery"
| "versionHistory"
| "saveAsModal"
| "modelList";
| "modelList"
| "downloadSpaceJson";
ui/src/utils/civitUtils.ts
+53
View File
@@ -1,3 +1,6 @@
import { userSettingsTable } from "../db-tables/WorkspaceDB";
import { Model } from "../types/dbTypes";
const CIVIT_API_KEY_STORAGE_KEY = "WORKSPACE_CIVIT_API_KEY_STORAGE_KEY";
export function getCivitApiKey() {
return localStorage.getItem(CIVIT_API_KEY_STORAGE_KEY);
@@ -6,3 +9,53 @@ export function getCivitApiKey() {
export function setCivitApiKey(apiKey: string) {
localStorage.setItem(CIVIT_API_KEY_STORAGE_KEY, apiKey);
}
interface ResponsePartial {
id: number;
modelId: number;
name: string;
model: {
name: string;
type: string;
nsfw: boolean;
};
images: {
url: string;
nsfw: "None" | "Soft" | "Mature" | "X";
width: number;
height: number;
hash: string;
}[];
}
export async function fetchCivitModelFromHashKey(filehash: string): Promise<{
modelName?: string;
civitModelID?: string;
civitModelVersionID?: string;
imageUrl?: string;
}> {
try {
const url = `https://civitai.com/api/v1/model-versions/by-hash/${filehash}`;
const resp = await fetch(url);
const json: ResponsePartial = await resp.json();
let image_url: string | undefined;
const showNsfwThumbnail = await userSettingsTable?.getSetting(
"showNsfwModelThumbnail",
);
if (showNsfwThumbnail === true) {
image_url = json?.images?.[0]?.url;
} else if (!json.model.nsfw) {
const sfwImage = json.images.find((i) => i.nsfw === "None");
image_url = sfwImage?.url;
}
return {
modelName: json.model.name,
civitModelID: String(json.modelId),
civitModelVersionID: String(json.id),
imageUrl: image_url ?? undefined,
};
} catch (e) {
return {};
}
}