优化云清单下载与人工阅读格式

This commit is contained in:
magicwang1111
2026-08-26 14:19:54 +08:00
parent 3d10902973
commit 410f7073be
4 changed files with 21 additions and 10 deletions
+2 -1
View File
@@ -131,7 +131,8 @@ directly from the database. Start/end line numbers are included in `--status`.
Completed and failed distributed jobs can be archived asynchronously to Aliyun
OSS. The archive keeps public media objects separate from a private, gzip-encoded
`manifest.json` containing the prompt's related scheduler/history SQLite rows.
`manifest.json.gz` containing a pretty-printed `manifest.json` with the prompt's
related scheduler/history SQLite rows.
It supports stable per-server IDs, cross-process SQLite leases, exactly three
outer attempts, and controlled historical backfill:
+3 -3
View File
@@ -18,10 +18,10 @@ Goumee-ComfyUI-Server-Data/public/servers/<server_id>/YYYY/MM/DD/<prompt_id>/<or
每个任务唯一的 gzip JSON 清单写入私有前缀:
```text
Goumee-ComfyUI-Server-Data/private/jobs/<prompt_id>/manifest.json
Goumee-ComfyUI-Server-Data/private/jobs/<prompt_id>/manifest.json.gz
```
manifest 对象使用 `Content-Type: application/json`、`Content-Encoding: gzip`、`Cache-Control: no-store` 和 private object ACL。数据库只永久保存 `oss://bucket/key`,不保存会过期的签名 URL。
manifest 对象使用 `Content-Type: application/gzip`、`Cache-Control: no-store` 和 private object ACL,不设置 `Content-Encoding`。从 OSS 控制台下载后会明确得到 `manifest.json.gz`;解压后是带两空格缩进和换行的 UTF-8 `manifest.json`。数据库只永久保存 `oss://bucket/key`,不保存会过期的签名 URL。
## 2. 安全前置条件
@@ -215,7 +215,7 @@ SQLite BLOB 始终使用带原始类型、字节数、SHA-256 和 Base64 的 typ
1. public 媒体匿名 GET 返回 200;
2. private manifest 匿名 GET 返回 403;
3. 使用授权 SDK 或 `ossutil` 能读取 private manifest;
4. 根据 `Content-Encoding: gzip` 解压后可解析 JSON;
4. 下载的 `manifest.json.gz` 可以直接用 gzip 或 7-Zip 解压,解压后的 UTF-8 JSON 带缩进和换行并可正常解析;
5. manifest 记录数、自身内容哈希与 SQLite 查询一致;
6. 模拟 OSS 不可达时生成任务本身不被阻塞,第三次失败后 SQLite 显示 `cloud_failed`;
7. 本地 GPU 成功、API 成功、失败但有部分输出、失败且空输出四类任务都有 private manifest;
+6 -2
View File
@@ -168,7 +168,7 @@ class CloudArchiveTests(unittest.TestCase):
manifest = self.config.manifest_key("abc")
self.assertIn("/public/servers/seetacloud-5090x2/", media)
self.assertEqual(
"Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json", manifest
"Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json.gz", manifest
)
self.assertNotIn("private", self.config.public_url(media))
@@ -195,9 +195,13 @@ class CloudArchiveTests(unittest.TestCase):
manifest = json.loads(gzip.decompress(manifest_bytes))
self.assertEqual("prompt-1", manifest["export"]["prompt_id"])
self.assertEqual("uploaded", manifest["artifacts"][0]["upload_status"])
self.assertEqual("gzip", backend.calls[-1][2])
self.assertEqual("application/gzip", backend.calls[-1][1])
self.assertIsNone(backend.calls[-1][2])
self.assertEqual("no-store", backend.calls[-1][3])
self.assertTrue(backend.calls[-1][5])
decoded_text = gzip.decompress(manifest_bytes).decode("utf-8")
self.assertIn('\n "artifacts":', decoded_text)
self.assertTrue(decoded_text.endswith("\n"))
def test_empty_failed_task_still_uploads_private_manifest(self):
self.add_job(status="failed")
+10 -4
View File
@@ -75,7 +75,7 @@ class CloudArchiveConfig:
)
def manifest_key(self, prompt_id: str) -> str:
return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json"
return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json.gz"
def oss_uri(self, key: str) -> str:
return f"oss://{self.bucket}/{key}"
@@ -1148,7 +1148,14 @@ class ManifestExporter:
def write_gzip(self, manifest: dict, path: str, max_bytes: int) -> dict:
os.makedirs(os.path.dirname(path), exist_ok=True)
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as output:
json.dump(manifest, output, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
json.dump(
manifest,
output,
ensure_ascii=False,
sort_keys=True,
indent=2,
)
output.write("\n")
size = os.path.getsize(path)
if size > int(max_bytes):
raise ValueError("manifest_size_limit_exceeded")
@@ -1446,8 +1453,7 @@ class CloudArchiveManager:
self.backend.upload_file(
task["manifest_object_key"],
staging,
"application/json",
content_encoding="gzip",
"application/gzip",
cache_control="no-store",
sha256=file_result["sha256"],
private=True,