优化云清单下载与人工阅读格式
This commit is contained in:
@@ -131,7 +131,8 @@ directly from the database. Start/end line numbers are included in `--status`.
|
||||
|
||||
Completed and failed distributed jobs can be archived asynchronously to Aliyun
|
||||
OSS. The archive keeps public media objects separate from a private, gzip-encoded
|
||||
`manifest.json` containing the prompt's related scheduler/history SQLite rows.
|
||||
`manifest.json.gz` containing a pretty-printed `manifest.json` with the prompt's
|
||||
related scheduler/history SQLite rows.
|
||||
It supports stable per-server IDs, cross-process SQLite leases, exactly three
|
||||
outer attempts, and controlled historical backfill:
|
||||
|
||||
|
||||
+3
-3
@@ -18,10 +18,10 @@ Goumee-ComfyUI-Server-Data/public/servers/<server_id>/YYYY/MM/DD/<prompt_id>/<or
|
||||
每个任务唯一的 gzip JSON 清单写入私有前缀:
|
||||
|
||||
```text
|
||||
Goumee-ComfyUI-Server-Data/private/jobs/<prompt_id>/manifest.json
|
||||
Goumee-ComfyUI-Server-Data/private/jobs/<prompt_id>/manifest.json.gz
|
||||
```
|
||||
|
||||
manifest 对象使用 `Content-Type: application/json`、`Content-Encoding: gzip`、`Cache-Control: no-store` 和 private object ACL。数据库只永久保存 `oss://bucket/key`,不保存会过期的签名 URL。
|
||||
manifest 对象使用 `Content-Type: application/gzip`、`Cache-Control: no-store` 和 private object ACL,不设置 `Content-Encoding`。从 OSS 控制台下载后会明确得到 `manifest.json.gz`;解压后是带两空格缩进和换行的 UTF-8 `manifest.json`。数据库只永久保存 `oss://bucket/key`,不保存会过期的签名 URL。
|
||||
|
||||
## 2. 安全前置条件
|
||||
|
||||
@@ -215,7 +215,7 @@ SQLite BLOB 始终使用带原始类型、字节数、SHA-256 和 Base64 的 typ
|
||||
1. public 媒体匿名 GET 返回 200;
|
||||
2. private manifest 匿名 GET 返回 403;
|
||||
3. 使用授权 SDK 或 `ossutil` 能读取 private manifest;
|
||||
4. 根据 `Content-Encoding: gzip` 解压后可解析 JSON;
|
||||
4. 下载的 `manifest.json.gz` 可以直接用 gzip 或 7-Zip 解压,解压后的 UTF-8 JSON 带缩进和换行并可正常解析;
|
||||
5. manifest 记录数、自身内容哈希与 SQLite 查询一致;
|
||||
6. 模拟 OSS 不可达时生成任务本身不被阻塞,第三次失败后 SQLite 显示 `cloud_failed`;
|
||||
7. 本地 GPU 成功、API 成功、失败但有部分输出、失败且空输出四类任务都有 private manifest;
|
||||
|
||||
@@ -168,7 +168,7 @@ class CloudArchiveTests(unittest.TestCase):
|
||||
manifest = self.config.manifest_key("abc")
|
||||
self.assertIn("/public/servers/seetacloud-5090x2/", media)
|
||||
self.assertEqual(
|
||||
"Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json", manifest
|
||||
"Goumee-ComfyUI-Server-Data/private/jobs/abc/manifest.json.gz", manifest
|
||||
)
|
||||
self.assertNotIn("private", self.config.public_url(media))
|
||||
|
||||
@@ -195,9 +195,13 @@ class CloudArchiveTests(unittest.TestCase):
|
||||
manifest = json.loads(gzip.decompress(manifest_bytes))
|
||||
self.assertEqual("prompt-1", manifest["export"]["prompt_id"])
|
||||
self.assertEqual("uploaded", manifest["artifacts"][0]["upload_status"])
|
||||
self.assertEqual("gzip", backend.calls[-1][2])
|
||||
self.assertEqual("application/gzip", backend.calls[-1][1])
|
||||
self.assertIsNone(backend.calls[-1][2])
|
||||
self.assertEqual("no-store", backend.calls[-1][3])
|
||||
self.assertTrue(backend.calls[-1][5])
|
||||
decoded_text = gzip.decompress(manifest_bytes).decode("utf-8")
|
||||
self.assertIn('\n "artifacts":', decoded_text)
|
||||
self.assertTrue(decoded_text.endswith("\n"))
|
||||
|
||||
def test_empty_failed_task_still_uploads_private_manifest(self):
|
||||
self.add_job(status="failed")
|
||||
|
||||
+10
-4
@@ -75,7 +75,7 @@ class CloudArchiveConfig:
|
||||
)
|
||||
|
||||
def manifest_key(self, prompt_id: str) -> str:
|
||||
return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json"
|
||||
return f"{self.prefix}/private/jobs/{prompt_id}/manifest.json.gz"
|
||||
|
||||
def oss_uri(self, key: str) -> str:
|
||||
return f"oss://{self.bucket}/{key}"
|
||||
@@ -1148,7 +1148,14 @@ class ManifestExporter:
|
||||
def write_gzip(self, manifest: dict, path: str, max_bytes: int) -> dict:
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
with gzip.open(path, "wt", encoding="utf-8", compresslevel=6) as output:
|
||||
json.dump(manifest, output, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
json.dump(
|
||||
manifest,
|
||||
output,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
indent=2,
|
||||
)
|
||||
output.write("\n")
|
||||
size = os.path.getsize(path)
|
||||
if size > int(max_bytes):
|
||||
raise ValueError("manifest_size_limit_exceeded")
|
||||
@@ -1446,8 +1453,7 @@ class CloudArchiveManager:
|
||||
self.backend.upload_file(
|
||||
task["manifest_object_key"],
|
||||
staging,
|
||||
"application/json",
|
||||
content_encoding="gzip",
|
||||
"application/gzip",
|
||||
cache_control="no-store",
|
||||
sha256=file_result["sha256"],
|
||||
private=True,
|
||||
|
||||
Reference in New Issue
Block a user