Will Lin
1fe50c0092
The v2 top level had grown to ~27 dirs + 13 loose files — half of them
single-concept abstraction shells (memory/ 86 LOC, transport/ 222, parity/ 178,
extend/ 226) and half vendored fastvideo code sitting as peers to the actual v2
design. Regroup to mirror README section 3 "Planes & dependency order":
core/ enums+types, card, loop, program, parity, request, parallel
(the model-native contracts; no kernels)
runtime/ + folded-in substrate: cache, memory, transport, extend
(the import graph shows only runtime consumes them; compile/cudagraph
already lived here)
serving/ + deploy/ (products / fleet)
_vendor/ all copied fastvideo: models, layers, attention, configs, distributed,
platforms, api, hooks, logging_utils, third_party + fastvideo_args/
utils/logger/envs/forward_context — internal layout unchanged, still
mirrors upstream for diffing
28 dirs + 13 root files -> 8 dirs + 6 root files. Pure mechanical move: 947
absolute-import paths rewritten (v2.X -> v2.{core,runtime,serving,_vendor}.X),
boundary-anchored so platform/ (native dispatch) and platforms/ (vendored CUDA
detect) no longer collide and neither does hooks/ vs extend/. Deleted the empty
v2/loader/. README section 3 + 16 updated; stale test count corrected
(34 files/216 tests -> 22 files/143 tests).
Validated: `import v2` stays torch-free; `v2/run_tests.py` and `pytest v2/tests/`
-> 143 passed, 2 skipped on the numpy toy backend. (On a GPU box force the toy
backend with CUDA_VISIBLE_DEVICES="" or detect() picks cuda.) No external importer
changed — examples use the re-exported `from v2 import VideoGenerator`.
76 lines
2.8 KiB
Python
76 lines
2.8 KiB
Python
"""CacheManager — per-class pools with static budgets, behind one handle.
|
|
|
|
Static partitioning makes cross-class fragmentation impossible (jumbo slab traffic cannot
|
|
strand text-KV pages and vice versa). Each class gets a budget carved at init from the card's
|
|
CacheContracts. ``invalidate_weights`` implements the wholesale RL weight-epoch bump.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from v2.runtime.cache.classes import _Pool, make_pool
|
|
from v2.runtime.cache.keys import CachePolicy
|
|
|
|
|
|
class CacheManager:
|
|
|
|
def __init__(self, policies: list[CachePolicy] | None = None):
|
|
self._pools: dict[str, _Pool] = {}
|
|
for p in (policies or []):
|
|
self._pools[p.class_name] = make_pool(p)
|
|
|
|
@classmethod
|
|
def from_card(cls, card) -> CacheManager:
|
|
"""Build the per-class pools a card declares (KV pools materialize only if declared)."""
|
|
policies = []
|
|
for cc in card.caches.values():
|
|
policies.append(
|
|
CachePolicy(
|
|
class_name=cc.cache_class,
|
|
max_bytes=cc.max_bytes,
|
|
block_bytes=cc.block_bytes,
|
|
eviction=cc.eviction,
|
|
reuse_across_requests=cc.reuse_across_requests,
|
|
per_component=dict(cc.per_component),
|
|
training_mode_disables_recycle=cc.training_mode_disables_recycle,
|
|
))
|
|
return cls(policies)
|
|
|
|
def pool(self, class_name: str) -> _Pool:
|
|
if class_name not in self._pools:
|
|
raise KeyError(f"no cache pool for class {class_name!r}; card did not declare it")
|
|
return self._pools[class_name]
|
|
|
|
def has(self, class_name: str) -> bool:
|
|
return class_name in self._pools
|
|
|
|
def invalidate_weights(self, version: str) -> None:
|
|
for pool in self._pools.values():
|
|
if hasattr(pool, "invalidate_weights"):
|
|
pool.invalidate_weights(version)
|
|
|
|
def invalidate_components(self, components) -> None:
|
|
"""Component-scoped invalidation: only drop caches for changed components."""
|
|
comps = set(components)
|
|
for pool in self._pools.values():
|
|
if hasattr(pool, "invalidate_components"):
|
|
pool.invalidate_components(comps)
|
|
|
|
def clear_namespace(self, namespace: str) -> None:
|
|
"""Release a finished request's per-request caches (residual/slab)."""
|
|
for pool in self._pools.values():
|
|
if hasattr(pool, "clear_namespace"):
|
|
pool.clear_namespace(namespace)
|
|
if hasattr(pool, "free_namespace"):
|
|
pool.free_namespace(namespace)
|
|
|
|
def stats(self) -> dict[str, Any]:
|
|
return {
|
|
name: {
|
|
"used_bytes": p.used_bytes,
|
|
"hits": p.hits,
|
|
"misses": p.misses
|
|
}
|
|
for name, p in self._pools.items()
|
|
}
|