diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 62778ce6..95a52f3f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -153,7 +153,7 @@ jobs: pip install .[dev] - name: Run pytest - run: python -m pytest -q --timeout=300 --cov=mjr_am_backend --cov=mjr_am_shared --cov-fail-under=60 --cov-report=xml --cov-report=term-missing:skip-covered + run: python -m pytest -q --timeout=300 --cov=mjr_am_backend --cov=mjr_am_shared --cov-branch --cov-fail-under=60 --cov-report=xml --cov-report=term-missing:skip-covered - name: Upload coverage report if: matrix.python-version == '3.12' diff --git a/mjr_am_backend/adapters/db/schema.py b/mjr_am_backend/adapters/db/schema.py index aea8123a..03ddb6b7 100644 --- a/mjr_am_backend/adapters/db/schema.py +++ b/mjr_am_backend/adapters/db/schema.py @@ -610,6 +610,35 @@ async def _ensure_schema_fingerprint(db) -> Result[bool]: ) +def _parse_vec_table_columns(table_info: list[dict]) -> tuple[bool, bool, bool, bool]: + """Parse vec table_info rows into (has_id, id_pk, has_asset_id, asset_id_pk).""" + has_id = has_asset_id = False + id_pk = asset_id_pk = False + for row in table_info: + name = str(row.get("name") or "").strip().lower() + pk = int(row.get("pk") or 0) + if name == "id": + has_id = True + id_pk = pk == 1 + elif name == "asset_id": + has_asset_id = True + asset_id_pk = pk == 1 + return has_id, id_pk, has_asset_id, asset_id_pk + + +def _needs_rebuild_from_columns( + has_id: bool, id_pk: bool, has_asset_id: bool, asset_id_pk: bool +) -> bool: + """Decide if the vec table layout is legacy and needs a rebuild.""" + if not has_asset_id: + return False + if not has_id: + return True + if asset_id_pk: + return True + return not id_pk + + async def _vec_embeddings_needs_rebuild(db) -> bool: """Return True when vec.asset_embeddings still uses the legacy PK layout.""" try: @@ -625,27 +654,50 @@ async def _vec_embeddings_needs_rebuild(db) -> bool: if not info.ok or not info.data: return False - id_pk = False - asset_id_pk = False - has_id = False - has_asset_id = False - for row in info.data: - name = str(row.get("name") or "").strip().lower() - pk = int(row.get("pk") or 0) - if name == "id": - has_id = True - id_pk = pk == 1 - elif name == "asset_id": - has_asset_id = True - asset_id_pk = pk == 1 + has_id, id_pk, has_asset_id, asset_id_pk = _parse_vec_table_columns(info.data) + return _needs_rebuild_from_columns(has_id, id_pk, has_asset_id, asset_id_pk) - if not has_asset_id: - return False - if not has_id: - return True - if asset_id_pk: - return True - return not id_pk + +_VEC_REBUILD_SCRIPT = """ +CREATE TABLE IF NOT EXISTS vec.asset_embeddings__new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + asset_id INTEGER NOT NULL UNIQUE, + vector BLOB, + aesthetic_score REAL, + auto_tags TEXT DEFAULT '[]', + model_name TEXT DEFAULT '', + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +INSERT OR REPLACE INTO vec.asset_embeddings__new (asset_id, vector, aesthetic_score, auto_tags, model_name, updated_at) +SELECT asset_id, + vector, + aesthetic_score, + COALESCE(auto_tags, '[]'), + COALESCE(model_name, ''), + COALESCE(updated_at, CURRENT_TIMESTAMP) +FROM vec.asset_embeddings +WHERE asset_id IS NOT NULL; + +DROP TABLE vec.asset_embeddings; +ALTER TABLE vec.asset_embeddings__new RENAME TO asset_embeddings; +""" + + +async def _execute_vec_rebuild(db) -> Result[bool]: + """Run the vec embeddings rebuild inside a transaction.""" + async with db.atransaction(mode="immediate") as tx: + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") + rebuilt = await db.aexecutescript(_VEC_REBUILD_SCRIPT) + if not rebuilt.ok: + return Result.Err( + rebuilt.code or "DB_ERROR", + rebuilt.error or "Failed to rebuild vec.asset_embeddings", + ) + if not tx.ok: + return Result.Err("DB_ERROR", tx.error or "Commit failed") + return Result.Ok(True) async def _repair_vec_embeddings_layout(db) -> Result[bool]: @@ -659,40 +711,7 @@ async def _repair_vec_embeddings_layout(db) -> Result[bool]: logger.warning("Repairing vec.asset_embeddings layout (legacy PK -> id PK + asset_id UNIQUE)") try: - async with db.atransaction(mode="immediate") as tx: - if not tx.ok: - return Result.Err("DB_ERROR", tx.error or "Failed to begin transaction") - - repair_script = """ - CREATE TABLE IF NOT EXISTS vec.asset_embeddings__new ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - asset_id INTEGER NOT NULL UNIQUE, - vector BLOB, - aesthetic_score REAL, - auto_tags TEXT DEFAULT '[]', - model_name TEXT DEFAULT '', - updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP - ); - - INSERT OR REPLACE INTO vec.asset_embeddings__new (asset_id, vector, aesthetic_score, auto_tags, model_name, updated_at) - SELECT asset_id, - vector, - aesthetic_score, - COALESCE(auto_tags, '[]'), - COALESCE(model_name, ''), - COALESCE(updated_at, CURRENT_TIMESTAMP) - FROM vec.asset_embeddings - WHERE asset_id IS NOT NULL; - - DROP TABLE vec.asset_embeddings; - ALTER TABLE vec.asset_embeddings__new RENAME TO asset_embeddings; - """ - rebuilt = await db.aexecutescript(repair_script) - if not rebuilt.ok: - return Result.Err(rebuilt.code or "DB_ERROR", rebuilt.error or "Failed to rebuild vec.asset_embeddings") - - if not tx.ok: - return Result.Err("DB_ERROR", tx.error or "Commit failed") + return await _execute_vec_rebuild(db) except Exception as exc: logger.warning("Failed to repair vec.asset_embeddings layout: %s", exc) try: @@ -701,8 +720,6 @@ async def _repair_vec_embeddings_layout(db) -> Result[bool]: logger.debug("Cleanup of vec.asset_embeddings__new failed: %s", cleanup_exc) return Result.Err("SCHEMA_REPAIR_FAILED", str(exc)) - return Result.Ok(True) - async def _ensure_schema(db) -> Result[bool]: result = await ensure_tables_exist(db) diff --git a/pytest.ini b/pytest.ini index d9901712..f2c5e368 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,4 +6,4 @@ python_files = test_*.py asyncio_mode = auto norecursedirs = __old_version_zip .* build dist __pycache__ node_modules tests/__pytest_tmp__ tests/__pytest_tmp_* __pytest_tmp__ __pytest_tmp_* -addopts = -ra -p no:cacheprovider --cov-branch --basetemp=tests/__pytest_tmp__ +addopts = -ra -p no:cacheprovider --basetemp=tests/__pytest_tmp__