附件内容去重(hash)：上传改为内容寻址 + store 按 hash 去重 [tsk_fTul9G4eutVn]
- types: Attachment 增 hash?/size? 可选字段（向后兼容，老数据零迁移） - 上传端点 POST .../attachments 改为内容寻址：流式计算 sha256，写临时文件再 按 <sha256>.<ext> 原子 rename；同 hash 已存在则丢弃临时文件（真去重）； 超限/异常清理临时文件不留垃圾。磁盘名仅由内容 hash 决定，杜绝撞名覆盖丢数据。 - store.addAttachments 去重键改为 hash ?? path（later-wins，老数据回退 path）。 - DELETE 端点补注释：附件按 task 隔离，无需跨任务 refcount。 - 安全保持并固化：GET inline 仅硬白名单图片，其余强制 attachment+octet-stream +nosniff+CSP sandbox；路径越界防护；hash 命名不含路径可控字符。 - 测试：新增同内容去重/不同内容同名各留一条/响应含 hash+size 等用例。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
+98
-26
@@ -1,6 +1,6 @@
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync } from 'node:fs';
|
||||
import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readdirSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Store } from '../src/store/index.js';
|
||||
@@ -30,20 +30,46 @@ function placeFile(dir: string, taskId: string, name: string, mime: string, body
|
||||
|
||||
// ───────────────────────── Store 层 ─────────────────────────
|
||||
|
||||
test('addAttachments：按 path 去重（later-wins）', () => {
|
||||
test('addAttachments：按内容 hash 去重（later-wins，同内容不同名只一条）', () => {
|
||||
const { store, taskId } = freshTask();
|
||||
store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', path: `tasks/${taskId}/attachments/a.png` }]);
|
||||
// 重传同一 path（safeName 相同）→ 元数据只保留一条，且取最新 name/type
|
||||
const hA = 'a'.repeat(64);
|
||||
store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', hash: hA, size: 3, path: `tasks/${taskId}/attachments/${hA}.png` }]);
|
||||
// 重传相同内容（hash 相同）但改名 → 元数据只保留一条，且取最新 name/type
|
||||
const hB = 'b'.repeat(64);
|
||||
const next = store.addAttachments(taskId, [
|
||||
{ name: 'a-renamed.png', type: 'image/jpeg', path: `tasks/${taskId}/attachments/a.png` },
|
||||
{ name: 'b.pdf', type: 'application/pdf', path: `tasks/${taskId}/attachments/b.pdf` },
|
||||
{ name: 'a-renamed.png', type: 'image/jpeg', hash: hA, size: 3, path: `tasks/${taskId}/attachments/${hA}.png` },
|
||||
{ name: 'b.pdf', type: 'application/pdf', hash: hB, size: 7, path: `tasks/${taskId}/attachments/${hB}.pdf` },
|
||||
]);
|
||||
assert.equal(next.length, 2);
|
||||
const a = next.find((x) => x.path.endsWith('/a.png'));
|
||||
const a = next.find((x) => x.hash === hA);
|
||||
assert.equal(a?.name, 'a-renamed.png');
|
||||
assert.equal(a?.type, 'image/jpeg');
|
||||
// 插入顺序保持：a 在前、b 在后
|
||||
assert.deepEqual(next.map((x) => x.path.split('/').pop()), ['a.png', 'b.pdf']);
|
||||
// 插入顺序保持：A 在前、B 在后
|
||||
assert.deepEqual(next.map((x) => x.hash), [hA, hB]);
|
||||
store.close();
|
||||
});
|
||||
|
||||
test('addAttachments：不同内容（hash 不同）即便同显示名也各保留一条（不再静默覆盖丢数据）', () => {
|
||||
const { store, taskId } = freshTask();
|
||||
const h1 = '1'.repeat(64);
|
||||
const h2 = '2'.repeat(64);
|
||||
const next = store.addAttachments(taskId, [
|
||||
{ name: 'screenshot.png', type: 'image/png', hash: h1, size: 3, path: `tasks/${taskId}/attachments/${h1}.png` },
|
||||
{ name: 'screenshot.png', type: 'image/png', hash: h2, size: 3, path: `tasks/${taskId}/attachments/${h2}.png` },
|
||||
]);
|
||||
assert.equal(next.length, 2, '不同内容同名应各保留一条');
|
||||
assert.deepEqual(next.map((x) => x.hash), [h1, h2]);
|
||||
store.close();
|
||||
});
|
||||
|
||||
test('addAttachments：老数据无 hash 时回退按 path 去重', () => {
|
||||
const { store, taskId } = freshTask();
|
||||
store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', path: `tasks/${taskId}/attachments/a.png` }]);
|
||||
const next = store.addAttachments(taskId, [
|
||||
{ name: 'a-renamed.png', type: 'image/jpeg', path: `tasks/${taskId}/attachments/a.png` },
|
||||
]);
|
||||
assert.equal(next.length, 1);
|
||||
assert.equal(next[0].name, 'a-renamed.png');
|
||||
store.close();
|
||||
});
|
||||
|
||||
@@ -198,38 +224,84 @@ test('DELETE：元数据在但磁盘文件已被外部删除 → 仍成功清元
|
||||
store.close();
|
||||
});
|
||||
|
||||
test('上传→去重端到端：同名文件连传两次，元数据只一条', async () => {
|
||||
function multipart(filename: string, content: string): { payload: Buffer; headers: Record<string, string> } {
|
||||
const boundary = '----maestrotest';
|
||||
const body =
|
||||
`--${boundary}\r\n` +
|
||||
`Content-Disposition: form-data; name="files"; filename="${filename}"\r\n` +
|
||||
`Content-Type: image/png\r\n\r\n` +
|
||||
content + `\r\n` +
|
||||
`--${boundary}--\r\n`;
|
||||
return { payload: Buffer.from(body), headers: { 'content-type': `multipart/form-data; boundary=${boundary}` } };
|
||||
}
|
||||
|
||||
test('上传→内容去重端到端：同内容不同名连传 → 元数据 1 条、磁盘 1 份 <sha256>.png', async () => {
|
||||
const dir = tmpData();
|
||||
process.env.MAESTRO_DATA_DIR = dir;
|
||||
const { store, taskId } = freshTask();
|
||||
const app = buildServer({ store });
|
||||
|
||||
function multipart(filename: string, content: string): { payload: Buffer; headers: Record<string, string> } {
|
||||
const boundary = '----maestrotest';
|
||||
const body =
|
||||
`--${boundary}\r\n` +
|
||||
`Content-Disposition: form-data; name="files"; filename="${filename}"\r\n` +
|
||||
`Content-Type: image/png\r\n\r\n` +
|
||||
content + `\r\n` +
|
||||
`--${boundary}--\r\n`;
|
||||
return { payload: Buffer.from(body), headers: { 'content-type': `multipart/form-data; boundary=${boundary}` } };
|
||||
}
|
||||
|
||||
// 同内容（'aaa'），但文件名不同 → 内容寻址应去重为一条、磁盘一份
|
||||
const m1 = multipart('shot.png', 'aaa');
|
||||
const up1 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m1.payload, headers: m1.headers });
|
||||
assert.equal(up1.statusCode, 200);
|
||||
assert.equal(up1.json().attachments.length, 1);
|
||||
|
||||
const m2 = multipart('shot.png', 'bbb');
|
||||
const m2 = multipart('shot-renamed.png', 'aaa');
|
||||
const up2 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m2.payload, headers: m2.headers });
|
||||
assert.equal(up2.statusCode, 200);
|
||||
// 同名 safeName → path 相同 → 去重为一条
|
||||
const atts = up2.json().attachments;
|
||||
assert.equal(atts.length, 1, '同名重传应去重为一条');
|
||||
assert.equal(atts.length, 1, '同内容重传应去重为一条');
|
||||
|
||||
// 通过 GET /api/tasks/:id 复核
|
||||
// 磁盘上只有 1 个 <sha256>.png（外加可能的 .tmp-* 已清理）
|
||||
const attDir = join(dir, 'tasks', taskId, 'attachments');
|
||||
const files = readdirSync(attDir).filter((f) => !f.startsWith('.tmp-'));
|
||||
assert.equal(files.length, 1, '磁盘应只有一份内容文件');
|
||||
assert.match(files[0], /^[0-9a-f]{64}\.png$/, '磁盘名应为 <sha256>.png');
|
||||
|
||||
// GET /api/tasks/:id 复核：1 条，且含 hash(64 hex)/size 字段
|
||||
const t = await app.inject({ method: 'GET', url: `/api/tasks/${taskId}` });
|
||||
assert.equal(t.json().attachments.length, 1);
|
||||
const list = t.json().attachments;
|
||||
assert.equal(list.length, 1);
|
||||
assert.match(list[0].hash, /^[0-9a-f]{64}$/);
|
||||
assert.equal(list[0].size, 3);
|
||||
await app.close();
|
||||
store.close();
|
||||
});
|
||||
|
||||
test('上传→内容寻址端到端：不同内容同显示名连传 → 元数据 2 条、磁盘 2 份（旧的覆盖丢数据 bug 已消除）', async () => {
|
||||
const dir = tmpData();
|
||||
process.env.MAESTRO_DATA_DIR = dir;
|
||||
const { store, taskId } = freshTask();
|
||||
const app = buildServer({ store });
|
||||
|
||||
const m1 = multipart('screenshot.png', 'aaa');
|
||||
await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m1.payload, headers: m1.headers });
|
||||
const m2 = multipart('screenshot.png', 'bbb'); // 同名、不同内容
|
||||
const up2 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m2.payload, headers: m2.headers });
|
||||
assert.equal(up2.statusCode, 200);
|
||||
assert.equal(up2.json().attachments.length, 2, '不同内容同名应保留两条');
|
||||
|
||||
const attDir = join(dir, 'tasks', taskId, 'attachments');
|
||||
const files = readdirSync(attDir).filter((f) => !f.startsWith('.tmp-'));
|
||||
assert.equal(files.length, 2, '磁盘应有两份内容文件，不再静默覆盖');
|
||||
await app.close();
|
||||
store.close();
|
||||
});
|
||||
|
||||
test('上传：响应附件含 hash(64 hex) 与 size 字段', async () => {
|
||||
const dir = tmpData();
|
||||
process.env.MAESTRO_DATA_DIR = dir;
|
||||
const { store, taskId } = freshTask();
|
||||
const app = buildServer({ store });
|
||||
const m = multipart('pic.png', 'hello-bytes');
|
||||
const up = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m.payload, headers: m.headers });
|
||||
assert.equal(up.statusCode, 200);
|
||||
const att = up.json().attachments[0];
|
||||
assert.match(att.hash, /^[0-9a-f]{64}$/);
|
||||
assert.equal(att.size, 'hello-bytes'.length);
|
||||
// 磁盘名 = path 末段 = <hash>.png
|
||||
assert.equal(att.path, `tasks/${taskId}/attachments/${att.hash}.png`);
|
||||
await app.close();
|
||||
store.close();
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user