From 410f7073bed31983b5715f33bbef2a1da4253abb Mon Sep 17 00:00:00 2001 From: magicwang1111 Date: Wed, 26 Aug 2026 14:19:54 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96=E4=BA=91=E6=B8=85=E5=8D=95?= =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E4=B8=8E=E4=BA=BA=E5=B7=A5=E9=98=85=E8=AF=BB?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 ++- docs/OSS云归档使用文档.md | 6 +++--- tests/test_cloud_archive.py | 8 ++++++-- utils/cloud_archive.py | 14 ++++++++++---- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 9723b1f..29bb305 100644 --- a/README.md +++ b/README.md @@ -131,7 +131,8 @@ directly from the database. Start/end line numbers are included in `--status`. Completed and failed distributed jobs can be archived asynchronously to Aliyun OSS. The archive keeps public media objects separate from a private, gzip-encoded -`manifest.json` containing the prompt's related scheduler/history SQLite rows. +`manifest.json.gz` containing a pretty-printed `manifest.json` with the prompt's +related scheduler/history SQLite rows. It supports stable per-server IDs, cross-process SQLite leases, exactly three outer attempts, and controlled historical backfill: diff --git a/docs/OSS云归档使用文档.md b/docs/OSS云归档使用文档.md index 4a38b9a..c2a2e61 100644 --- a/docs/OSS云归档使用文档.md +++ b/docs/OSS云归档使用文档.md @@ -18,10 +18,10 @@ Goumee-ComfyUI-Server-Data/public/servers//YYYY/MM/DD///manifest.json +Goumee-ComfyUI-Server-Data/private/jobs//manifest.json.gz ``` -manifest 对象使用 `Content-Type: application/json`、`Content-Encoding: gzip`、`Cache-Control: no-store` 和 private object ACL。数据库只永久保存 `oss://bucket/key`，不保存会过期的签名 URL。 +manifest 对象使用 `Content-Type: application/gzip`、`Cache-Control: no-store` 和 private object ACL，不设置 `Content-Encoding`。从 OSS 控制台下载后会明确得到 `manifest.json.gz`；解压后是带两空格缩进和换行的 UTF-8 `manifest.json`。数据库只永久保存 `oss://bucket/key`，不保存会过期的签名 URL。 ## 2. 安全前置条件 @@ -215,7 +215,7 @@ SQLite BLOB 始终使用带原始类型、字节数、SHA-256 和 Base64 的 typ 1. public 媒体匿名 GET 返回 200； 2. private manifest 匿名 GET 返回 403； 3. 使用授权 SDK 或 `ossutil` 能读取 private manifest； -4. 根据 `Content-Encoding: gzip` 解压后可解析 JSON； +4. 下载的 `manifest.json.gz` 可以直接用 gzip 或 7-Zip 解压，解压后的 UTF-8 JSON 带缩进和换行并可正常解析； 5. manifest 记录数、自身内容哈希与 SQLite 查询一致； 6. 模拟 OSS 不可达时生成任务本身不被阻塞，第三次失败后 SQLite 显示 `cloud_failed`； 7. 本地 GPU 成功、API 成功、失败但有部分输出、失败且空输出四类任务都有 private manifest； diff --git a/tests/test_cloud_archive.py b/tests/test_cloud_archive.py index 4e01ea7..6fb123b 100644 --- a/tests/test_cloud_archive.py +++ b/tests/test_cloud_archive.py @@ -168,7 +168,7 @@ class CloudArchiveTests(unittest.TestCase): manifest = self.config.manifest_key("abc") self.assertIn("/public/servers/seetacloud-5090x2/", media) self.assertEqual( - "Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json", manifest + "Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json.gz", manifest ) self.assertNotIn("private", self.config.public_url(media)) @@ -195,9 +195,13 @@ class CloudArchiveTests(unittest.TestCase): manifest = json.loads(gzip.decompress(manifest_bytes)) self.assertEqual("prompt-1", manifest["export"]["prompt_id"]) self.assertEqual("uploaded", manifest["artifacts"][0]["upload_status"]) - self.assertEqual("gzip", backend.calls[-1][2]) + self.assertEqual("application/gzip", backend.calls[-1][1]) + self.assertIsNone(backend.calls[-1][2]) self.assertEqual("no-store", backend.calls[-1][3]) self.assertTrue(backend.calls[-1][5]) + decoded_text = gzip.decompress(manifest_bytes).decode("utf-8") + self.assertIn('\n "artifacts":', decoded_text) + self.assertTrue(decoded_text.endswith("\n")) def test_empty_failed_task_still_uploads_private_manifest(self): self.add_job(status="failed") diff --git a/utils/cloud_archive.py b/utils/cloud_archive.py index 6882492..d58a12e 100644 --- a/utils/cloud_archive.py +++ b/utils/cloud_archive.py @@ -75,7 +75,7 @@ class CloudArchiveConfig: ) def manifest_key(self, prompt_id: str) -> str: - return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json" + return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json.gz" def oss_uri(self, key: str) -> str: return f"oss://{self.bucket}/{key}" @@ -1148,7 +1148,14 @@ class ManifestExporter: def write_gzip(self, manifest: dict, path: str, max_bytes: int) -> dict: os.makedirs(os.path.dirname(path), exist_ok=True) with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as output: - json.dump(manifest, output, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + json.dump( + manifest, + output, + ensure_ascii=False, + sort_keys=True, + indent=2, + ) + output.write("\n") size = os.path.getsize(path) if size > int(max_bytes): raise ValueError("manifest_size_limit_exceeded") @@ -1446,8 +1453,7 @@ class CloudArchiveManager: self.backend.upload_file( task["manifest_object_key"], staging, - "application/json", - content_encoding="gzip", + "application/gzip", cache_control="no-store", sha256=file_result["sha256"], private=True,