1188 lines
47 KiB
JavaScript
1188 lines
47 KiB
JavaScript
/**
|
|
* Analytical memory estimates, evidence applicability and candidate recommendations for cookbook
|
|
* deployments. Analytical results are not measurements and never mark a recipe as verified.
|
|
*
|
|
* Inputs use the static capacity catalog exported by build-cookbook-capacity.mjs:
|
|
* model.components is a mapping by role, hardware uses memory.nominal/memory.type (the
|
|
* normalizer also accepts legacy memory_gb/memory_type for custom callers), and examples
|
|
* preserve the native baseline. Execution settings are resolved from the serving configuration.
|
|
*/
|
|
|
|
const GIB = 1024 ** 3;
|
|
const isObject = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
const isCount = (value) => Number.isInteger(value) && value > 0;
|
|
|
|
/** NVFP4 stores two 4-bit values per byte plus a 1-byte scale for every 16 values. */
|
|
export const BYTES_PER_WEIGHT = { fp32: 4, fp16: 2, bf16: 2, fp8: 1, nvfp4: 0.5 + 1 / 16 };
|
|
|
|
export const DEFAULT_HEADROOM = { fraction: 0.1, reserveGiB: 1 };
|
|
/** Conservative application share for unified memory when the caller gives no budget. */
|
|
export const DEFAULT_UNIFIED_BUDGET_FRACTION = 0.75;
|
|
|
|
/**
|
|
* Wan working-memory terms. Tensor counts follow WanTransformerBlock.forward and the chunked
|
|
* AutoencoderKLWan decode with its causal feature cache. They are uncalibrated until measured.
|
|
*/
|
|
export const WAN_RULE = {
|
|
encodingWorkspaceGiB: 0.5,
|
|
attentionHiddenTensors: 10,
|
|
ffnHiddenTensors: 2,
|
|
ffnTensors: 2,
|
|
decodeFrameFeatures: 48,
|
|
};
|
|
|
|
const PHASES = ["startup", "encoding", "denoising", "decoding"];
|
|
const COMPONENT_PHASE = {
|
|
text_encoder: "encoding",
|
|
text_encoder_2: "encoding",
|
|
image_encoder: "encoding",
|
|
image_encoder_2: "encoding",
|
|
transformer: "denoising",
|
|
transformer_2: "denoising",
|
|
vae: "decoding",
|
|
audio_vae: "decoding",
|
|
};
|
|
const COMPONENT_KIND = {
|
|
transformer: "transformer",
|
|
transformer_2: "transformer",
|
|
text_encoder: "encoder",
|
|
text_encoder_2: "encoder",
|
|
image_encoder: "encoder",
|
|
image_encoder_2: "encoder",
|
|
vae: "vae",
|
|
audio_vae: "vae",
|
|
};
|
|
const SAMPLING_FIELDS = ["num_frames", "height", "width"];
|
|
|
|
/** Unset switches default on, matching OffloadConfig in fastvideo/api/schema.py. */
|
|
const OFFLOAD_DEFAULTS = {
|
|
dit: true,
|
|
dit_layerwise: true,
|
|
text_encoder: true,
|
|
image_encoder: true,
|
|
vae: true,
|
|
};
|
|
|
|
/** Streaming offload holds the running block and one prefetched block. */
|
|
const STREAMED_BLOCKS = 2;
|
|
|
|
/** Native pipeline precision defaults for a modeled family; explicit config values override them. */
|
|
export const PRECISION_DEFAULTS = {
|
|
wan: {
|
|
transformer: "bf16",
|
|
text_encoder: ["fp32"],
|
|
image_encoder: ["fp32"],
|
|
vae: "fp32",
|
|
vae_decode: "bf16",
|
|
vae_tiling: false,
|
|
},
|
|
};
|
|
|
|
const DENSE_ATTENTION_BACKENDS = new Set(["FLASH_ATTN", "TORCH_SDPA", "SAGE_ATTN", "SAGE_ATTN_THREE"]);
|
|
|
|
/** Settings that do not change a request's peak memory; other experimental keys do. */
|
|
const HARMLESS_EXPERIMENTAL = new Set([
|
|
"flow_shift",
|
|
"flow_shift_sr",
|
|
"dmd_denoising_steps",
|
|
"num_inference_steps",
|
|
"seed",
|
|
"fps",
|
|
]);
|
|
|
|
const MEMORY_ENV_KEYS = new Set([
|
|
"FASTVIDEO_ATTENTION_BACKEND",
|
|
"FASTVIDEO_LAYERWISE_OFFLOAD_BUFFERS",
|
|
"PYTORCH_CUDA_ALLOC_CONF",
|
|
]);
|
|
const MEMORY_ENV_PREFIXES = [
|
|
"FASTVIDEO_VSA_",
|
|
"FASTVIDEO_H3_",
|
|
"FASTVIDEO_LAYERWISE_",
|
|
"FASTVIDEO_NVFP4",
|
|
"FASTVIDEO_FP8",
|
|
"FASTVIDEO_COMPILE",
|
|
"FASTVIDEO_OFFLOAD",
|
|
"FASTVIDEO_TORCH",
|
|
];
|
|
|
|
const WORKING_MEMORY = {
|
|
wan: {
|
|
geometry: [
|
|
"temporal_compression",
|
|
"spatial_compression",
|
|
"patch_size",
|
|
"hidden_size",
|
|
"ffn_size",
|
|
"vae_decoder_channels",
|
|
],
|
|
estimate: wanWorkingMemory,
|
|
},
|
|
};
|
|
|
|
/** Resolve automatic degrees as FastVideoArgs does: tensor parallelism 1, sequence parallelism across every GPU. */
|
|
export function resolveTopology(engine = {}) {
|
|
const gpus = engine.num_gpus ?? 1;
|
|
const { tp_size: tp = -1, sp_size: sp = -1 } = engine.parallelism ?? {};
|
|
const tpSize = tp === -1 ? 1 : tp;
|
|
const spSize = sp === -1 ? gpus : sp;
|
|
return { gpus: Math.max(gpus, tpSize, spSize), tpSize, spSize };
|
|
}
|
|
|
|
function normalizeComponents(model) {
|
|
if (!isObject(model.components)) return null;
|
|
const components = [];
|
|
for (const [role, component] of Object.entries(model.components)) {
|
|
if (!isObject(component) || !COMPONENT_PHASE[role]) return null;
|
|
if (component.kind !== COMPONENT_KIND[role]) return null;
|
|
components.push({ role, ...component });
|
|
}
|
|
return components.length ? components : null;
|
|
}
|
|
|
|
function geometryFor(model, role) {
|
|
const geometry = model.components?.[role]?.geometry;
|
|
return isObject(geometry) ? geometry : null;
|
|
}
|
|
|
|
function asPrecisionList(value, fallback) {
|
|
if (typeof value === "string" && value) return [value];
|
|
if (Array.isArray(value) && value.length && value.every((item) => typeof item === "string")) return value;
|
|
return fallback;
|
|
}
|
|
|
|
function experimentalPrecision(experimental, name, fallback) {
|
|
const value = experimental[name];
|
|
return typeof value === "string" && value ? value : fallback;
|
|
}
|
|
|
|
/**
|
|
* Resolve the native execution settings that change memory. Values come from the serving config,
|
|
* then from family defaults keyed by memory_rules. The component catalog never fixes precision.
|
|
*/
|
|
export function resolveExecution({ model, config, env = {} }) {
|
|
const family = model.memory_rules;
|
|
const defaults = PRECISION_DEFAULTS[family] ?? {};
|
|
const engine = config.generator?.engine ?? {};
|
|
const pipeline = config.generator?.pipeline ?? {};
|
|
const experimental = isObject(pipeline.experimental) ? pipeline.experimental : {};
|
|
const textEncoderPrecisions = asPrecisionList(experimental.text_encoder_precisions, defaults.text_encoder ?? []);
|
|
const imageEncoderPrecisions = asPrecisionList(
|
|
experimental.image_encoder_precisions,
|
|
experimental.image_encoder_precision ? [experimental.image_encoder_precision] : (defaults.image_encoder ?? []),
|
|
);
|
|
const vae = experimentalPrecision(experimental, "vae_precision", defaults.vae);
|
|
const vaeDecode = experimentalPrecision(experimental, "vae_decode_precision", defaults.vae_decode ?? vae);
|
|
const transformer = experimentalPrecision(experimental, "dit_precision", defaults.transformer);
|
|
const offload = { ...OFFLOAD_DEFAULTS, ...engine.offload };
|
|
const quantization = isObject(engine.quantization) ? engine.quantization : {};
|
|
const attentionBackend = typeof env.FASTVIDEO_ATTENTION_BACKEND === "string" ? env.FASTVIDEO_ATTENTION_BACKEND : null;
|
|
const residentBlocks = Number.parseInt(env.FASTVIDEO_LAYERWISE_RESIDENT_BLOCKS ?? "0", 10);
|
|
const layerwiseResidentBlocks = Number.isInteger(residentBlocks) && residentBlocks >= 0 ? residentBlocks : 0;
|
|
const vsa =
|
|
(attentionBackend && attentionBackend.includes("SPARSE")) ||
|
|
Object.keys(experimental).some((key) => key.startsWith("VSA_")) ||
|
|
Object.keys(env).some((key) => key.startsWith("FASTVIDEO_VSA_"));
|
|
return {
|
|
family,
|
|
workload: pipeline.workload_type,
|
|
num_gpus: engine.num_gpus ?? 1,
|
|
topology: resolveTopology(engine),
|
|
use_fsdp: engine.use_fsdp_inference === true,
|
|
offload,
|
|
lazy_module_load: offload.lazy_module_load === true,
|
|
compile:
|
|
engine.compile?.enabled === true ||
|
|
engine.compile?.text_encoder_enabled === true ||
|
|
engine.compile?.vae_enabled === true,
|
|
quantization,
|
|
attentionBackend,
|
|
layerwise_resident_blocks: layerwiseResidentBlocks,
|
|
vae_feature_cache: experimental["vae_config.use_feature_cache"] !== false,
|
|
vsa,
|
|
vae_tiling: typeof pipeline.vae_tiling === "boolean" ? pipeline.vae_tiling : (defaults.vae_tiling ?? false),
|
|
vae_sp: pipeline.vae_sp === true,
|
|
precision: {
|
|
transformer,
|
|
text_encoder: textEncoderPrecisions,
|
|
image_encoder: imageEncoderPrecisions,
|
|
vae,
|
|
vae_decode: vaeDecode,
|
|
},
|
|
experimental,
|
|
};
|
|
}
|
|
|
|
function componentPrecision(component, execution) {
|
|
const { role } = component;
|
|
if (role === "transformer" || role === "transformer_2") return execution.precision.transformer;
|
|
if (role === "vae" || role === "audio_vae") return execution.precision.vae;
|
|
if (role === "image_encoder" || role === "image_encoder_2") {
|
|
const index = role === "image_encoder_2" ? 1 : 0;
|
|
return execution.precision.image_encoder[index];
|
|
}
|
|
const index = role === "text_encoder_2" ? 1 : 0;
|
|
return execution.precision.text_encoder[index];
|
|
}
|
|
|
|
function componentActivationPrecision(component, execution) {
|
|
if (component.role === "vae" || component.role === "audio_vae") return execution.precision.vae_decode;
|
|
return componentPrecision(component, execution);
|
|
}
|
|
|
|
function componentBytes(component, execution) {
|
|
const storage = componentPrecision(component, execution);
|
|
const perWeight = BYTES_PER_WEIGHT[storage];
|
|
return { storage, perWeight, full: component.parameter_count * perWeight };
|
|
}
|
|
|
|
/** Weights resident during a request phase after offload and layerwise streaming. */
|
|
function offloadFlag(role) {
|
|
if (role.startsWith("text_encoder")) return "text_encoder";
|
|
if (role.startsWith("image_encoder")) return "image_encoder";
|
|
return role;
|
|
}
|
|
|
|
function residentBytes(component, phase, execution, bytes) {
|
|
const full = bytes.full;
|
|
if (component.role === "transformer" || component.role === "transformer_2") {
|
|
const blocks = component.block_parameter_count * bytes.perWeight;
|
|
const outside = full - blocks;
|
|
if (execution.offload.dit_layerwise) {
|
|
const resident = execution.layerwise_resident_blocks ?? 0;
|
|
const share = component.block_count
|
|
? Math.min(1, (phase === "denoising" ? resident + STREAMED_BLOCKS : resident) / component.block_count)
|
|
: 1;
|
|
return outside + blocks * share;
|
|
}
|
|
return full;
|
|
}
|
|
const offloaded = execution.offload[offloadFlag(component.role)];
|
|
if (COMPONENT_PHASE[component.role] !== phase) return offloaded ? 0 : full;
|
|
return full;
|
|
}
|
|
|
|
/** Load-time residency: text/image encoders and the VAE rest on CPU when offloaded, the DiT loads on GPU. */
|
|
function startupBytes(component, execution, bytes) {
|
|
if (component.role === "transformer" || component.role === "transformer_2") return bytes.full;
|
|
return execution.offload[offloadFlag(component.role)] ? 0 : bytes.full;
|
|
}
|
|
|
|
function activationBytes(component, execution) {
|
|
return BYTES_PER_WEIGHT[componentActivationPrecision(component, execution)];
|
|
}
|
|
|
|
function wanWorkingMemory({ transformerGeometry, vaeGeometry, vaeComponent, sampling, components, execution }) {
|
|
const { temporal_compression, spatial_compression, vae_decoder_channels, output_channels } = vaeGeometry;
|
|
const { hidden_size, ffn_size, patch_size } = transformerGeometry;
|
|
const { num_frames, height, width } = sampling;
|
|
const latentFrames = Math.floor((num_frames - 1) / temporal_compression) + 1;
|
|
const tokens =
|
|
Math.ceil(latentFrames / patch_size[0]) *
|
|
Math.ceil(height / (spatial_compression * patch_size[1])) *
|
|
Math.ceil(width / (spatial_compression * patch_size[2]));
|
|
const transformer = components.find((component) => component.role === "transformer");
|
|
const vae = components.find((component) => component.role === "vae");
|
|
const dit = activationBytes(transformer, execution);
|
|
const vaeActivation = activationBytes(vae, execution);
|
|
const hidden = tokens * hidden_size * dit;
|
|
const ffn = tokens * ffn_size * dit;
|
|
const pixels = height * width;
|
|
const outputFrames = (latentFrames - 1) * temporal_compression + 1;
|
|
const decodeMultiplier = Number.isFinite(vaeComponent?.decode_feature_multiplier)
|
|
? vaeComponent.decode_feature_multiplier
|
|
: WAN_RULE.decodeFrameFeatures;
|
|
return {
|
|
startup: 0,
|
|
encoding: WAN_RULE.encodingWorkspaceGiB * GIB,
|
|
denoising: Math.max(
|
|
WAN_RULE.attentionHiddenTensors * hidden,
|
|
WAN_RULE.ffnHiddenTensors * hidden + WAN_RULE.ffnTensors * ffn,
|
|
),
|
|
decoding:
|
|
decodeMultiplier * vae_decoder_channels * pixels * vaeActivation +
|
|
outputFrames * (output_channels ?? 3) * pixels * (2 * vaeActivation + BYTES_PER_WEIGHT.fp32),
|
|
};
|
|
}
|
|
|
|
function quantityGiB(quantity) {
|
|
if (!isObject(quantity) || !Number.isFinite(quantity.value) || !["GB", "GiB"].includes(quantity.unit)) return null;
|
|
return (quantity.value * (quantity.unit === "GiB" ? 1.073741824 : 1) * 1e9) / GIB;
|
|
}
|
|
|
|
/**
|
|
* Normalize catalog or custom hardware. Catalog capacities are nominal; a dedicated
|
|
* device uses its nominal (or selected partition) capacity, while a unified device
|
|
* uses an explicit application budget or a labeled 75% default and never exposes
|
|
* system memory as dedicated VRAM.
|
|
*/
|
|
export function normalizeHardware(hardware, { memoryBudgetGiB, allocationMemoryGb, partitionId } = {}) {
|
|
if (!isObject(hardware)) return null;
|
|
const memory = isObject(hardware.memory) ? hardware.memory : null;
|
|
const memoryType = memory?.type ?? hardware.memory_type;
|
|
if (!["dedicated", "unified"].includes(memoryType)) return null;
|
|
const nominalGiB = memory?.nominal
|
|
? quantityGiB(memory.nominal)
|
|
: Number.isFinite(hardware.memory_gb)
|
|
? (hardware.memory_gb * 1e9) / GIB
|
|
: null;
|
|
if (!(nominalGiB > 0)) return null;
|
|
let allocationGiB = Number.isFinite(allocationMemoryGb) ? (allocationMemoryGb * 1e9) / GIB : null;
|
|
let partition = null;
|
|
if (partitionId !== undefined) {
|
|
partition = (hardware.partitions ?? []).find((item) => item.id === partitionId);
|
|
if (!partition) return null;
|
|
allocationGiB =
|
|
quantityGiB(partition.memory) ??
|
|
(Number.isFinite(partition.memory_gb) ? (partition.memory_gb * 1e9) / GIB : null);
|
|
if (!(allocationGiB > 0)) return null;
|
|
}
|
|
const catalogUsableGiB = memory?.usable
|
|
? quantityGiB(memory.usable)
|
|
: Number.isFinite(hardware.usable_memory_gb)
|
|
? (hardware.usable_memory_gb * 1e9) / GIB
|
|
: null;
|
|
const requestedBudget = Number.isFinite(memoryBudgetGiB) ? memoryBudgetGiB : null;
|
|
const catalogBudget = requestedBudget ?? catalogUsableGiB;
|
|
const assumedBudget =
|
|
memoryType === "unified" && catalogBudget === null ? nominalGiB * DEFAULT_UNIFIED_BUDGET_FRACTION : null;
|
|
const budgetGiB = catalogBudget ?? assumedBudget;
|
|
if (budgetGiB !== null && !(budgetGiB > 0)) return null;
|
|
if (budgetGiB !== null && budgetGiB > nominalGiB) return null;
|
|
if (allocationGiB !== null && allocationGiB > nominalGiB) return null;
|
|
const hostGiB =
|
|
quantityGiB(hardware.host_memory) ??
|
|
(Number.isFinite(hardware.host_memory_gb) ? (hardware.host_memory_gb * 1e9) / GIB : null);
|
|
const capacity = Math.min(allocationGiB ?? nominalGiB, budgetGiB ?? nominalGiB);
|
|
return {
|
|
...hardware,
|
|
id: hardware.id ?? (hardware.custom ? "custom" : "unknown"),
|
|
name: hardware.name ?? "Custom device",
|
|
memoryType,
|
|
nominalGiB,
|
|
catalogUsableGiB,
|
|
budgetGiB,
|
|
budgetSource:
|
|
requestedBudget !== null
|
|
? "user"
|
|
: catalogUsableGiB !== null
|
|
? "catalog_policy"
|
|
: assumedBudget !== null
|
|
? "assumed_fraction"
|
|
: "nominal",
|
|
budgetAssumed: assumedBudget !== null,
|
|
allocationGiB,
|
|
partition,
|
|
capacityGiB: capacity,
|
|
hostGiB,
|
|
backend: hardware.backend ?? "unknown",
|
|
supported_dtypes: hardware.supported_dtypes,
|
|
custom: hardware.custom === true,
|
|
};
|
|
}
|
|
|
|
function applyUnifiedMemoryPolicy(execution, device) {
|
|
if (device?.memoryType !== "unified") return execution;
|
|
return {
|
|
...execution,
|
|
offload: {
|
|
...execution.offload,
|
|
dit: false,
|
|
dit_layerwise: false,
|
|
text_encoder: false,
|
|
image_encoder: false,
|
|
vae: false,
|
|
},
|
|
};
|
|
}
|
|
|
|
function checkCoverage({ workload, model, device, execution, sampling, runtime = {}, components }) {
|
|
const unknown = (reason) => ({ status: "unknown", reason });
|
|
if (runtime.return_trajectory_latents || runtime.return_trajectory_decoded) {
|
|
return unknown("Trajectory capture keeps a tensor per denoising step; it is not modeled.");
|
|
}
|
|
if (!device) return unknown("Select a catalogued hardware variant or enter a custom device with capacity.");
|
|
if (device.backend !== "cuda") return unknown(`${device.name} is not a CUDA device.`);
|
|
if (device.gpu_count > 1) {
|
|
return unknown("Multi-GPU hardware records are not modeled as one memory pool.");
|
|
}
|
|
if (!(device.capacityGiB > 0)) return unknown(`${device.name} has no known memory size.`);
|
|
if (workload !== "t2v") return unknown(`${workload} workloads are not modeled yet.`);
|
|
if (execution.topology.gpus > 1 || execution.topology.tpSize > 1 || execution.topology.spSize > 1) {
|
|
return unknown("Multi-GPU layouts are not modeled as one memory pool yet.");
|
|
}
|
|
if (execution.use_fsdp) return unknown("FSDP inference sharding is not modeled yet.");
|
|
if (execution.lazy_module_load) return unknown("Lazy component loading is not modeled yet.");
|
|
if (execution.vsa) return unknown("VSA workspace is not modeled; a successful run alone is not a fit.");
|
|
if (execution.compile) return unknown("torch.compile buffers are not modeled yet.");
|
|
if (execution.vae_sp) return unknown("Parallel VAE decode is not modeled yet.");
|
|
if (execution.vae_feature_cache === false) {
|
|
return unknown("VAE feature caching is disabled; the full-clip decode path is not modeled.");
|
|
}
|
|
if (execution.quantization.transformer_quant) {
|
|
return unknown("Transformer quantization needs an explicit quantized parameter count.");
|
|
}
|
|
if (execution.quantization.transformer_retain_original_weights) {
|
|
return unknown("Retained original transformer weights are not modeled.");
|
|
}
|
|
if (execution.quantization.text_encoder_quant) return unknown("Text-encoder quantization is not modeled yet.");
|
|
if (execution.quantization.layer_profile) return unknown("Per-layer quantization profiles are not modeled.");
|
|
if (execution.attentionBackend && !DENSE_ATTENTION_BACKENDS.has(execution.attentionBackend)) {
|
|
return unknown(`Attention backend ${execution.attentionBackend} has no memory rule.`);
|
|
}
|
|
const rule = WORKING_MEMORY[model.memory_rules];
|
|
if (!rule) return unknown(`No working-memory rule for ${model.memory_rules}.`);
|
|
const transformerGeometry = geometryFor(model, "transformer");
|
|
const vaeGeometry = geometryFor(model, "vae");
|
|
const missing = rule.geometry.filter(
|
|
(field) => transformerGeometry?.[field] === undefined && vaeGeometry?.[field] === undefined,
|
|
);
|
|
if (missing.length) return unknown(`Model geometry lacks ${missing.join(", ")}.`);
|
|
if (!SAMPLING_FIELDS.every((field) => isCount(sampling[field]))) {
|
|
return unknown("Frames, height and width must be positive integers.");
|
|
}
|
|
if ((sampling.num_videos_per_prompt ?? 1) !== 1) return unknown("Only one video per prompt is modeled.");
|
|
if (sampling.batch_cfg === true) return unknown("Batch CFG is not modeled.");
|
|
if (!components) return unknown("Components must be a mapping by role.");
|
|
const unmodeled = components.find((component) => !COMPONENT_PHASE[component.role]);
|
|
if (unmodeled) return unknown(`The ${unmodeled.role} component is not modeled yet.`);
|
|
for (const role of ["transformer", "vae", "text_encoder"]) {
|
|
if (!components.some((component) => component.role === role)) {
|
|
return unknown(`The model needs a ${role} component for this rule.`);
|
|
}
|
|
}
|
|
const uncounted = components.find(
|
|
(component) =>
|
|
!isCount(component.parameter_count) ||
|
|
(component.block_count !== undefined &&
|
|
!(isCount(component.block_count) && isCount(component.block_parameter_count))),
|
|
);
|
|
if (uncounted) return unknown(`The ${uncounted.role} parameter counts are incomplete.`);
|
|
if (execution.offload.dit_layerwise) {
|
|
const missingBlocks = components.find(
|
|
(component) =>
|
|
component.role.startsWith("transformer") &&
|
|
(!isCount(component.block_count) || !isCount(component.block_parameter_count)),
|
|
);
|
|
if (missingBlocks) {
|
|
return unknown(`Layerwise offload needs block counts for ${missingBlocks.role}.`);
|
|
}
|
|
}
|
|
const unsized = components.find((component) => {
|
|
const storage = componentPrecision(component, execution);
|
|
const activation = componentActivationPrecision(component, execution);
|
|
return !BYTES_PER_WEIGHT[storage] || !BYTES_PER_WEIGHT[activation];
|
|
});
|
|
if (unsized) return unknown(`The ${unsized.role} precision is not modeled.`);
|
|
const dtypes = new Set(components.map((component) => componentPrecision(component, execution)));
|
|
if (!Array.isArray(device.supported_dtypes) || !device.supported_dtypes.length) {
|
|
const uncatalogued = [...dtypes].filter((dtype) => !["fp32", "fp16", "bf16"].includes(dtype));
|
|
if (uncatalogued.length) {
|
|
return unknown(`${device.name} does not catalog ${uncatalogued.join(", ")} weight support.`);
|
|
}
|
|
} else {
|
|
const unsupported = [...dtypes].find((dtype) => !device.supported_dtypes.includes(dtype));
|
|
if (unsupported) {
|
|
return { status: "unsupported", reason: `${device.name} does not support ${unsupported} weights.` };
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
const HOST_STAGING_GIB = 2;
|
|
|
|
/** Host bytes needed for CPU offload. Unified systems share one pool and are not double-counted. */
|
|
function hostRequirement(components, bytes, execution) {
|
|
if (execution.topology.gpus > 1) return 0;
|
|
let used = 0;
|
|
for (const [index, component] of components.entries()) {
|
|
const flag = offloadFlag(component.role);
|
|
if (component.role === "transformer" || component.role === "transformer_2") {
|
|
if (execution.offload.dit_layerwise) used += bytes[index].full;
|
|
} else if (execution.offload[flag]) used += bytes[index].full;
|
|
}
|
|
return used ? used / GIB + HOST_STAGING_GIB : 0;
|
|
}
|
|
|
|
/**
|
|
* Estimate one request's peak memory on one device from a resolved serving configuration.
|
|
* Startup (load) and request phases are compared separately. Uncovered execution returns unknown.
|
|
*/
|
|
export function estimateMemory({
|
|
config,
|
|
workload,
|
|
model,
|
|
hardware,
|
|
request,
|
|
env = {},
|
|
headroom = DEFAULT_HEADROOM,
|
|
memoryBudgetGiB,
|
|
allocationMemoryGb,
|
|
partitionId,
|
|
hostMemoryGb,
|
|
}) {
|
|
const resolvedExecution = resolveExecution({ model, config, env });
|
|
const sampling = request?.sampling ?? config.default_request?.sampling ?? {};
|
|
const runtime = request?.runtime ?? config.default_request?.runtime ?? {};
|
|
const components = normalizeComponents(model);
|
|
const device = normalizeHardware(hardware, { memoryBudgetGiB, allocationMemoryGb, partitionId });
|
|
const execution = applyUnifiedMemoryPolicy(resolvedExecution, device);
|
|
const gap = checkCoverage({
|
|
workload: workload ?? execution.workload,
|
|
model,
|
|
device,
|
|
execution,
|
|
sampling,
|
|
runtime,
|
|
components,
|
|
});
|
|
if (gap) return gap;
|
|
const bytes = components.map((component) => componentBytes(component, execution));
|
|
const transformerGeometry = geometryFor(model, "transformer");
|
|
const vaeGeometry = geometryFor(model, "vae");
|
|
const working = WORKING_MEMORY[model.memory_rules].estimate({
|
|
transformerGeometry,
|
|
vaeGeometry,
|
|
vaeComponent: model.components.vae,
|
|
sampling,
|
|
components,
|
|
execution,
|
|
});
|
|
const phases = Object.fromEntries(
|
|
PHASES.map((phase) => {
|
|
const weights = components.reduce((sum, component, index) => {
|
|
const resident =
|
|
phase === "startup"
|
|
? startupBytes(component, execution, bytes[index])
|
|
: residentBytes(component, phase, execution, bytes[index]);
|
|
return sum + resident;
|
|
}, 0);
|
|
const workingBytes = working[phase] ?? 0;
|
|
return [
|
|
phase,
|
|
{
|
|
weightsGiB: weights / GIB,
|
|
workingGiB: workingBytes / GIB,
|
|
totalGiB: (weights + workingBytes) / GIB,
|
|
},
|
|
];
|
|
}),
|
|
);
|
|
const peak = PHASES.reduce((best, phase) => (phases[phase].totalGiB > phases[best].totalGiB ? phase : best));
|
|
const peakGiB = phases[peak].totalGiB;
|
|
const requiredGiB = peakGiB * (1 + headroom.fraction) + headroom.reserveGiB;
|
|
const capacity = device.capacityGiB;
|
|
const hostRequiredGiB = device.memoryType === "unified" ? 0 : hostRequirement(components, bytes, execution);
|
|
const hostCapacityGiB =
|
|
device.memoryType === "unified"
|
|
? null
|
|
: Number.isFinite(hostMemoryGb)
|
|
? (hostMemoryGb * 1e9) / GIB
|
|
: device.hostGiB;
|
|
const host =
|
|
device.memoryType === "unified"
|
|
? { requiredGiB: 0, capacityGiB: null, status: "shared" }
|
|
: {
|
|
requiredGiB: hostRequiredGiB,
|
|
capacityGiB: hostCapacityGiB,
|
|
status:
|
|
hostRequiredGiB === 0
|
|
? "not_required"
|
|
: hostCapacityGiB === null
|
|
? "unknown"
|
|
: hostRequiredGiB <= hostCapacityGiB
|
|
? "fits"
|
|
: "exceeds",
|
|
};
|
|
let status = requiredGiB <= capacity ? "fits" : "exceeds";
|
|
if (host.status === "exceeds") status = "exceeds";
|
|
const notes = [];
|
|
if (device.memoryType === "unified") {
|
|
notes.push(
|
|
`Unified memory: ${capacity.toFixed(1)} GiB application budget, shared with the CPU and OS; not dedicated VRAM.`,
|
|
);
|
|
if (device.budgetAssumed) {
|
|
notes.push(
|
|
`Assumed ${Math.round(DEFAULT_UNIFIED_BUDGET_FRACTION * 100)}% of unified memory is available to the application; set a budget to override.`,
|
|
);
|
|
}
|
|
}
|
|
if (device.allocationGiB !== null && device.budgetGiB !== null && device.budgetGiB < device.allocationGiB) {
|
|
notes.push(
|
|
`The ${device.budgetGiB.toFixed(1)} GiB application budget is smaller than the ${device.allocationGiB.toFixed(1)} GiB allocation.`,
|
|
);
|
|
}
|
|
if (device.partition) {
|
|
notes.push(
|
|
device.allocationGiB < device.nominalGiB
|
|
? `Using the ${device.partition.label} partition; the full ${device.nominalGiB.toFixed(1)} GiB is not available.`
|
|
: `Using the ${device.partition.label} partition, which spans the full ${device.nominalGiB.toFixed(1)} GiB.`,
|
|
);
|
|
} else if (device.allocationGiB !== null) {
|
|
notes.push(
|
|
`Using a ${device.allocationGiB.toFixed(1)} GiB provisioned allocation; the full ${device.nominalGiB.toFixed(1)} GiB is not available.`,
|
|
);
|
|
}
|
|
if (device.custom) notes.push("Custom hardware specification; not catalog-verified.");
|
|
if (!Array.isArray(device.supported_dtypes) || !device.supported_dtypes.length) {
|
|
notes.push("Weight-format support is not cataloged; the estimate assumes fp32, fp16, or bf16 storage.");
|
|
}
|
|
if (execution.vae_tiling) notes.push("Decode memory assumes untiled VAE decoding; tiling may lower it.");
|
|
if (
|
|
device.memoryType === "unified" &&
|
|
["dit", "dit_layerwise", "text_encoder", "image_encoder", "vae"].some((key) => resolvedExecution.offload[key])
|
|
) {
|
|
notes.push("CPU offload is disabled on unified memory; selected components share one memory pool.");
|
|
}
|
|
if (host.status === "unknown") {
|
|
notes.push(`Host memory for offload is not specified; ${hostRequiredGiB.toFixed(1)} GiB is needed.`);
|
|
} else if (host.status === "exceeds") {
|
|
notes.push(`Host memory for offload exceeds the ${hostCapacityGiB.toFixed(1)} GiB available.`);
|
|
}
|
|
notes.push("Analytical estimate only; no memory was measured.");
|
|
return {
|
|
status,
|
|
basis: "analytical",
|
|
phases,
|
|
peakPhase: peak,
|
|
peakGiB,
|
|
requiredGiB,
|
|
capacityGiB: capacity,
|
|
headroom: { fraction: headroom.fraction, reserveGiB: headroom.reserveGiB },
|
|
memory: {
|
|
type: device.memoryType,
|
|
nominalGiB: device.nominalGiB,
|
|
budgetGiB: device.budgetGiB ?? null,
|
|
budgetSource: device.budgetSource,
|
|
partition: device.partition?.id ?? null,
|
|
},
|
|
host,
|
|
hardware: {
|
|
id: device.id,
|
|
name: device.name,
|
|
memory_type: device.memoryType,
|
|
capacity_gib: capacity,
|
|
custom: device.custom,
|
|
},
|
|
execution: {
|
|
topology: execution.topology,
|
|
offload: execution.offload,
|
|
precision: execution.precision,
|
|
attentionBackend: execution.attentionBackend,
|
|
vae_tiling: execution.vae_tiling,
|
|
layerwise_resident_blocks: execution.layerwise_resident_blocks,
|
|
},
|
|
notes,
|
|
};
|
|
}
|
|
|
|
function memoryEnv(env = {}) {
|
|
return Object.fromEntries(
|
|
Object.entries(env)
|
|
.filter(([key]) => MEMORY_ENV_KEYS.has(key) || MEMORY_ENV_PREFIXES.some((prefix) => key.startsWith(prefix)))
|
|
.sort(([left], [right]) => left.localeCompare(right)),
|
|
);
|
|
}
|
|
|
|
function normalizedExperimental(experimental = {}) {
|
|
return Object.fromEntries(
|
|
Object.entries(experimental)
|
|
.filter(([key]) => !HARMLESS_EXPERIMENTAL.has(key))
|
|
.sort(([left], [right]) => left.localeCompare(right)),
|
|
);
|
|
}
|
|
|
|
function difference(path, recorded, current) {
|
|
if (JSON.stringify(recorded) === JSON.stringify(current)) return null;
|
|
return { path, recorded, current };
|
|
}
|
|
|
|
const EXPERIMENTAL_PRECISION_KEYS = new Set([
|
|
"dit_precision",
|
|
"text_encoder_precisions",
|
|
"image_encoder_precision",
|
|
"image_encoder_precisions",
|
|
"vae_precision",
|
|
"vae_decode_precision",
|
|
]);
|
|
|
|
function executionFingerprint({ model, config, env = {} }) {
|
|
const execution = resolveExecution({ model, config, env });
|
|
const sampling = config.default_request?.sampling ?? {};
|
|
const components = config.generator?.pipeline?.components ?? {};
|
|
const quantization = Object.fromEntries(
|
|
Object.entries(execution.quantization).sort(([left], [right]) => left.localeCompare(right)),
|
|
);
|
|
const experimental = Object.fromEntries(
|
|
Object.entries(normalizedExperimental(execution.experimental))
|
|
.filter(([key]) => !EXPERIMENTAL_PRECISION_KEYS.has(key))
|
|
.sort(([left], [right]) => left.localeCompare(right)),
|
|
);
|
|
return {
|
|
"generator.model_path": config.generator?.model_path ?? null,
|
|
"generator.revision": config.generator?.revision ?? null,
|
|
"generator.engine.num_gpus": execution.num_gpus,
|
|
"generator.engine.use_fsdp_inference": execution.use_fsdp,
|
|
"generator.engine.parallelism.tp_size": execution.topology.tpSize,
|
|
"generator.engine.parallelism.sp_size": execution.topology.spSize,
|
|
"generator.engine.offload.dit": execution.offload.dit,
|
|
"generator.engine.offload.dit_layerwise": execution.offload.dit_layerwise,
|
|
"generator.engine.offload.text_encoder": execution.offload.text_encoder,
|
|
"generator.engine.offload.image_encoder": execution.offload.image_encoder,
|
|
"generator.engine.offload.vae": execution.offload.vae,
|
|
"generator.engine.offload.lazy_module_load": execution.lazy_module_load,
|
|
"generator.engine.compile": {
|
|
enabled: execution.compile,
|
|
text_encoder_enabled: config.generator?.engine?.compile?.text_encoder_enabled === true,
|
|
vae_enabled: config.generator?.engine?.compile?.vae_enabled === true,
|
|
},
|
|
"generator.engine.quantization": quantization,
|
|
"generator.pipeline.workload_type": execution.workload ?? null,
|
|
"generator.pipeline.vae_tiling": execution.vae_tiling,
|
|
"generator.pipeline.vae_sp": execution.vae_sp,
|
|
"generator.pipeline.precision": execution.precision,
|
|
"generator.pipeline.experimental": experimental,
|
|
"generator.pipeline.components.transformer_weights": components.transformer_weights ?? null,
|
|
"generator.pipeline.components.vae_weights": components.vae_weights ?? null,
|
|
"generator.pipeline.components.text_encoder_weights": components.text_encoder_weights ?? null,
|
|
"default_request.sampling.height": sampling.height ?? null,
|
|
"default_request.sampling.width": sampling.width ?? null,
|
|
"default_request.sampling.num_frames": sampling.num_frames ?? null,
|
|
"default_request.sampling.num_videos_per_prompt": sampling.num_videos_per_prompt ?? null,
|
|
"default_request.sampling.batch_cfg": sampling.batch_cfg ?? false,
|
|
};
|
|
}
|
|
|
|
function configDifferences(model, recorded = {}, current = {}) {
|
|
const recordedFingerprint = executionFingerprint({ model, config: recorded });
|
|
const currentFingerprint = executionFingerprint({ model, config: current });
|
|
const differences = [];
|
|
for (const [path, recordedValue] of Object.entries(recordedFingerprint)) {
|
|
const item = difference(path, recordedValue, currentFingerprint[path]);
|
|
if (item) differences.push(item);
|
|
}
|
|
return differences;
|
|
}
|
|
|
|
function revisionDifferences(recorded = {}, current) {
|
|
if (!isObject(current)) return { status: "unknown", differences: [] };
|
|
const differences = [];
|
|
for (const key of ["fastvideo", "model"]) {
|
|
const value = current[key];
|
|
if (value === undefined) return { status: "unknown", differences };
|
|
if (value !== recorded[key])
|
|
differences.push({ path: `revisions.${key}`, recorded: recorded[key], current: value });
|
|
}
|
|
const recordedComponents = recorded.components ?? {};
|
|
const currentComponents = current.components;
|
|
if (Object.keys(recordedComponents).length && currentComponents === undefined) {
|
|
return { status: "unknown", differences };
|
|
}
|
|
if (currentComponents !== undefined) {
|
|
for (const role of new Set([...Object.keys(recordedComponents), ...Object.keys(currentComponents)])) {
|
|
if (recordedComponents[role] !== currentComponents[role]) {
|
|
differences.push({
|
|
path: `revisions.components.${role}`,
|
|
recorded: recordedComponents[role],
|
|
current: currentComponents[role],
|
|
});
|
|
}
|
|
}
|
|
}
|
|
return { status: differences.length ? "mismatched" : "matched", differences };
|
|
}
|
|
|
|
/**
|
|
* Match completed-run evidence to the current deployment, configuration, request, environment and
|
|
* hardware. Non-memory settings (server port, prompt, seed, fps) do not invalidate evidence.
|
|
*/
|
|
export function matchEvidence({ model, example, config, request, env = example?.env ?? {}, hardware, revisions } = {}) {
|
|
const applicable = [];
|
|
const unverified = [];
|
|
const rejected = [];
|
|
const currentConfig = config ?? example?.base_config;
|
|
const currentRequest = request ?? { sampling: currentConfig?.default_request?.sampling ?? {} };
|
|
const currentEnv = env ?? example?.env ?? {};
|
|
for (const run of model.reference_runs ?? []) {
|
|
const differences = [];
|
|
if (run.hardware !== hardware?.id) {
|
|
differences.push({ path: "hardware", recorded: run.hardware, current: hardware?.id });
|
|
}
|
|
const partition = hardware?.partition?.id ?? hardware?.partition_id ?? null;
|
|
const recordedPartition = run.hardware_partition ?? null;
|
|
if (partition !== recordedPartition) {
|
|
differences.push({ path: "hardware_partition", recorded: recordedPartition, current: partition });
|
|
}
|
|
const allocationGiB = Number.isFinite(hardware?.allocationGiB)
|
|
? hardware.allocationGiB
|
|
: Number.isFinite(hardware?.allocationMemoryGb)
|
|
? (hardware.allocationMemoryGb * 1e9) / GIB
|
|
: null;
|
|
const recordedAllocationGb = run.hardware_allocation_gb ?? null;
|
|
if (partition === null && (allocationGiB !== null || recordedAllocationGb !== null)) {
|
|
const allocationGb = allocationGiB === null ? null : (allocationGiB * GIB) / 1e9;
|
|
if (
|
|
allocationGb === null ||
|
|
recordedAllocationGb === null ||
|
|
Math.abs(allocationGb - recordedAllocationGb) > 1e-6
|
|
) {
|
|
differences.push({
|
|
path: "hardware_allocation_gb",
|
|
recorded: recordedAllocationGb,
|
|
current: allocationGb,
|
|
});
|
|
}
|
|
} else if (partition !== null && recordedAllocationGb !== null && allocationGiB !== null) {
|
|
const allocationGb = (allocationGiB * GIB) / 1e9;
|
|
if (Math.abs(allocationGb - recordedAllocationGb) > 1e-6) {
|
|
differences.push({
|
|
path: "hardware_allocation_gb",
|
|
recorded: recordedAllocationGb,
|
|
current: allocationGb,
|
|
});
|
|
}
|
|
}
|
|
if (run.gpu_count !== (currentConfig?.generator?.engine?.num_gpus ?? 1)) {
|
|
differences.push({
|
|
path: "gpu_count",
|
|
recorded: run.gpu_count,
|
|
current: currentConfig?.generator?.engine?.num_gpus ?? 1,
|
|
});
|
|
}
|
|
if (example && run.context.deployment !== example.deployment_id) {
|
|
differences.push({
|
|
path: "context.deployment",
|
|
recorded: run.context.deployment,
|
|
current: example.deployment_id,
|
|
});
|
|
}
|
|
if (example && run.context.runtime !== example.runtime) {
|
|
differences.push({ path: "context.runtime", recorded: run.context.runtime, current: example.runtime });
|
|
}
|
|
const recordedConfig = { ...run.context.config, default_request: { sampling: run.context.request.sampling } };
|
|
const currentSamplingConfig = {
|
|
...currentConfig,
|
|
default_request: { sampling: currentRequest.sampling ?? currentConfig?.default_request?.sampling },
|
|
};
|
|
differences.push(...configDifferences(model, recordedConfig, currentSamplingConfig));
|
|
const trajectory = (runtime) => ({
|
|
latents: Boolean(runtime?.return_trajectory_latents),
|
|
decoded: Boolean(runtime?.return_trajectory_decoded),
|
|
});
|
|
const recordedTrajectory = trajectory(run.context.request.runtime);
|
|
const currentTrajectory = trajectory(currentRequest.runtime ?? currentConfig?.default_request?.runtime);
|
|
if (
|
|
recordedTrajectory.latents !== currentTrajectory.latents ||
|
|
recordedTrajectory.decoded !== currentTrajectory.decoded
|
|
) {
|
|
differences.push({
|
|
path: "request.runtime.trajectory",
|
|
recorded: recordedTrajectory,
|
|
current: currentTrajectory,
|
|
});
|
|
}
|
|
const envDifference = difference("context.env", memoryEnv(run.context.env), memoryEnv(currentEnv));
|
|
if (envDifference) differences.push(envDifference);
|
|
const revisionsMatch = revisionDifferences(run.context.revisions, revisions);
|
|
const entry = {
|
|
id: run.id,
|
|
date: run.date,
|
|
execution: run.execution,
|
|
evidence_url: run.evidence_url,
|
|
memory: run.memory ?? null,
|
|
revision_status: revisionsMatch.status,
|
|
revision_differences: revisionsMatch.differences,
|
|
applicable: differences.length === 0 && revisionsMatch.status === "matched",
|
|
differences,
|
|
};
|
|
if (entry.applicable) applicable.push(entry);
|
|
else if (differences.length === 0) unverified.push(entry);
|
|
else rejected.push(entry);
|
|
}
|
|
return { applicable, unverified, rejected };
|
|
}
|
|
|
|
function measuredDevicePeak(evidence, device) {
|
|
const capacity = device?.capacityGiB;
|
|
if (!(capacity > 0)) return null;
|
|
const measurements = [];
|
|
for (const run of evidence.applicable) {
|
|
if (run.memory?.scope !== "device" || !Array.isArray(run.memory.per_gpu) || !run.memory.per_gpu.length) continue;
|
|
const startupMeasured = run.memory.per_gpu.every((gpu) => Number.isFinite(gpu.startup_peak));
|
|
const requestPeak = Math.max(...run.memory.per_gpu.map((gpu) => gpu.request_peak));
|
|
const peak = Math.max(
|
|
requestPeak,
|
|
...run.memory.per_gpu.map((gpu) => (Number.isFinite(gpu.startup_peak) ? gpu.startup_peak : 0)),
|
|
);
|
|
measurements.push({
|
|
run: run.id,
|
|
metric: run.memory.metric,
|
|
scope: run.memory.scope,
|
|
execution: run.execution,
|
|
requestPeak,
|
|
peak,
|
|
startupMeasured,
|
|
});
|
|
}
|
|
if (!measurements.length) return null;
|
|
const highest = measurements.reduce((best, item) => (item.peak > best.peak ? item : best));
|
|
const base = {
|
|
basis: "measured",
|
|
peakGiB: highest.peak,
|
|
capacityGiB: capacity,
|
|
run: highest.run,
|
|
runs: measurements.map((item) => item.run),
|
|
metric: highest.metric,
|
|
scope: highest.scope,
|
|
execution: highest.execution,
|
|
};
|
|
const unmeasured = measurements.filter((item) => !item.startupMeasured);
|
|
if (unmeasured.length) {
|
|
return {
|
|
...base,
|
|
status: "unknown",
|
|
reason: `Startup peak was not measured for ${unmeasured.map((item) => item.run).join(", ")}; a request-only peak cannot establish device fit.`,
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
status: highest.peak <= capacity ? "fits" : "exceeds",
|
|
spreadGiB: Math.max(...measurements.map((item) => item.peak)) - Math.min(...measurements.map((item) => item.peak)),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Combine the analytical estimate with applicable completed-run evidence. A successful generation
|
|
* without a device-wide measurement establishes compatibility, not a memory minimum or fit.
|
|
*/
|
|
export function assessCapacity({
|
|
model,
|
|
example,
|
|
hardware,
|
|
config = example?.base_config,
|
|
request,
|
|
env = example?.env ?? {},
|
|
revisions,
|
|
headroom,
|
|
memoryBudgetGiB,
|
|
allocationMemoryGb,
|
|
partitionId,
|
|
hostMemoryGb,
|
|
} = {}) {
|
|
const sampling = request?.sampling ?? config?.default_request?.sampling ?? {};
|
|
const workload = example?.workload ?? config?.generator?.pipeline?.workload_type;
|
|
const device = normalizeHardware(hardware, { memoryBudgetGiB, allocationMemoryGb, partitionId });
|
|
const analytical = estimateMemory({
|
|
config,
|
|
workload,
|
|
model,
|
|
hardware: device ?? hardware,
|
|
request,
|
|
env,
|
|
headroom,
|
|
memoryBudgetGiB,
|
|
allocationMemoryGb,
|
|
partitionId,
|
|
hostMemoryGb,
|
|
});
|
|
const evidence = matchEvidence({
|
|
model,
|
|
example,
|
|
config,
|
|
request: { sampling, runtime: request?.runtime ?? config?.default_request?.runtime },
|
|
env,
|
|
hardware: device ?? hardware,
|
|
revisions,
|
|
});
|
|
const measured = analytical.status === "unsupported" ? null : measuredDevicePeak(evidence, device);
|
|
const compatibility = evidence.applicable.length
|
|
? "completed"
|
|
: evidence.unverified.length
|
|
? "completed_unverified_revision"
|
|
: "unverified";
|
|
if (measured) {
|
|
const hostFailure = analytical.host?.status === "exceeds";
|
|
return {
|
|
...measured,
|
|
status: hostFailure ? "exceeds" : measured.status,
|
|
deviceStatus: measured.status,
|
|
reason: hostFailure
|
|
? `Host memory for offload exceeds the ${analytical.host.capacityGiB.toFixed(1)} GiB available.`
|
|
: measured.reason,
|
|
analytical,
|
|
evidence,
|
|
compatibility,
|
|
hardware: analytical.hardware ?? device,
|
|
memory: analytical.memory ?? null,
|
|
host: analytical.host ?? null,
|
|
notes: [
|
|
`Measured ${measured.metric} (${measured.scope}) on ${analytical.hardware?.name ?? hardware?.name ?? "the selected device"} for run ${measured.run}.`,
|
|
hostFailure
|
|
? `Host memory for offload exceeds the ${analytical.host.capacityGiB.toFixed(1)} GiB available; a device measurement does not establish host fit.`
|
|
: null,
|
|
analytical.status === "unknown"
|
|
? `Analytical coverage remains unknown: ${analytical.reason}`
|
|
: "The analytical estimate is retained for comparison.",
|
|
].filter(Boolean),
|
|
};
|
|
}
|
|
return {
|
|
...analytical,
|
|
analytical,
|
|
evidence,
|
|
compatibility,
|
|
notes: [
|
|
...(analytical.notes ?? []),
|
|
...(evidence.applicable.length
|
|
? ["Applicable evidence records a completed generation but no device-wide memory peak."]
|
|
: []),
|
|
...(evidence.unverified.length
|
|
? ["A completed run matches the memory-relevant settings, but its revisions are not verified."]
|
|
: []),
|
|
...(evidence.rejected.length ? ["No evidence matches every memory-relevant setting."] : []),
|
|
],
|
|
};
|
|
}
|
|
|
|
function lowerMemoryConfigs({ model, example, hardware, config, request, env, revisions, headroom }) {
|
|
const alternatives = [];
|
|
const sampling = request?.sampling ?? config.default_request?.sampling ?? {};
|
|
const assess = (mutate, edit) => {
|
|
const changed = structuredClone(config);
|
|
const changedRequest = request ? structuredClone(request) : undefined;
|
|
mutate(changed, changedRequest);
|
|
const result = assessCapacity({
|
|
model,
|
|
example,
|
|
hardware,
|
|
config: changed,
|
|
request: changedRequest,
|
|
env,
|
|
revisions,
|
|
headroom,
|
|
});
|
|
alternatives.push({
|
|
changes: [edit],
|
|
status: result.status,
|
|
basis: result.basis ?? null,
|
|
required_gib: result.requiredGiB ?? null,
|
|
reason: result.reason ?? null,
|
|
});
|
|
};
|
|
const samplingChange = (key, value) =>
|
|
assess(
|
|
(changed, changedRequest) => {
|
|
if (changedRequest) {
|
|
changedRequest.sampling = { ...sampling, [key]: value };
|
|
} else {
|
|
changed.default_request = {
|
|
...(changed.default_request ?? {}),
|
|
sampling: { ...sampling, [key]: value },
|
|
};
|
|
}
|
|
},
|
|
{ path: request ? `request.sampling.${key}` : `default_request.sampling.${key}`, value },
|
|
);
|
|
if (isCount(sampling.num_frames) && sampling.num_frames > 1) {
|
|
const frames = Math.max(1, Math.floor((sampling.num_frames - 1) / 2) + 1);
|
|
if (frames < sampling.num_frames) samplingChange("num_frames", frames);
|
|
}
|
|
for (const key of ["height", "width"]) {
|
|
if (isCount(sampling[key]) && sampling[key] > 16) {
|
|
const value = Math.max(16, Math.floor(sampling[key] / 2 / 16) * 16);
|
|
if (value < sampling[key]) samplingChange(key, value);
|
|
}
|
|
}
|
|
const offload = config.generator?.engine?.offload ?? {};
|
|
const changes = {};
|
|
for (const key of ["dit_layerwise", "text_encoder", "vae"]) {
|
|
if (offload[key] === false) changes[`generator.engine.offload.${key}`] = true;
|
|
}
|
|
if (Object.keys(changes).length) {
|
|
assess(
|
|
(changed) => {
|
|
for (const [key, value] of Object.entries(changes)) {
|
|
const parts = key.split(".");
|
|
let cursor = changed;
|
|
for (const part of parts.slice(0, -1)) cursor = cursor[part] ??= {};
|
|
cursor[parts.at(-1)] = value;
|
|
}
|
|
},
|
|
{ path: "generator.engine.offload", value: changes },
|
|
);
|
|
}
|
|
return alternatives;
|
|
}
|
|
/**
|
|
* Rank maintained examples for one hardware/task choice. Ranking uses measured evidence first, then
|
|
* analytical memory fit. No speed or quality ordering is inferred from memory estimates.
|
|
*/
|
|
export function recommendCapacity({ models, hardware, task, request, env, revisions, headroom, limit = 3 } = {}) {
|
|
const candidates = [];
|
|
for (const model of models ?? []) {
|
|
for (const example of model.examples ?? []) {
|
|
if (task && example.workload !== task) continue;
|
|
const config = example.base_config;
|
|
const assessment = assessCapacity({
|
|
model,
|
|
example,
|
|
hardware,
|
|
config,
|
|
request,
|
|
env: env ?? example.env,
|
|
revisions,
|
|
headroom,
|
|
});
|
|
candidates.push({ model, example, assessment });
|
|
}
|
|
}
|
|
const rank = (candidate) => {
|
|
const { status, basis } = candidate.assessment;
|
|
if (status === "fits" && basis === "measured") return 0;
|
|
if (status === "fits") return 1;
|
|
if (status === "unknown") return 2;
|
|
if (status === "exceeds") return 3;
|
|
return 4;
|
|
};
|
|
candidates.sort((left, right) => {
|
|
const delta = rank(left) - rank(right);
|
|
if (delta) return delta;
|
|
const leftMemory = left.assessment.requiredGiB ?? left.assessment.peakGiB ?? Infinity;
|
|
const rightMemory = right.assessment.requiredGiB ?? right.assessment.peakGiB ?? Infinity;
|
|
return leftMemory - rightMemory;
|
|
});
|
|
return candidates.slice(0, limit).map(({ model, example, assessment }) => ({
|
|
model: { id: model.id, name: model.name, model_id: model.model_id },
|
|
example: {
|
|
id: example.id,
|
|
deployment_id: example.deployment_id,
|
|
workload: example.workload,
|
|
runtime: example.runtime,
|
|
},
|
|
assessment,
|
|
reasons: [
|
|
assessment.status === "fits"
|
|
? `${assessment.basis === "measured" ? "Measured" : "Estimated"} to fit ${hardware?.name ?? "the selected device"}.`
|
|
: assessment.status === "exceeds"
|
|
? `Estimated to exceed ${hardware?.name ?? "the selected device"}.`
|
|
: `Fit is unknown: ${assessment.reason ?? "coverage is incomplete"}.`,
|
|
assessment.compatibility === "completed"
|
|
? "Applicable evidence records a completed generation."
|
|
: "No applicable completed-run evidence.",
|
|
],
|
|
lower_memory:
|
|
assessment.status === "fits"
|
|
? []
|
|
: lowerMemoryConfigs({
|
|
model,
|
|
example,
|
|
hardware,
|
|
config: example.base_config,
|
|
request,
|
|
env: env ?? example.env,
|
|
revisions,
|
|
headroom,
|
|
}),
|
|
}));
|
|
}
|