From f1ee537ab7bcf76696cad571042fbeaf055ca7aa Mon Sep 17 00:00:00 2001 From: wangjia <809946525@qq.com> Date: Tue, 30 Jun 2026 07:22:15 +0800 Subject: [PATCH] =?UTF-8?q?=E9=99=84=E4=BB=B6=E5=86=85=E5=AE=B9=E5=8E=BB?= =?UTF-8?q?=E9=87=8D(hash)=EF=BC=9A=E4=B8=8A=E4=BC=A0=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E5=AF=BB=E5=9D=80=20+=20store=20=E6=8C=89=20?= =?UTF-8?q?hash=20=E5=8E=BB=E9=87=8D=20[tsk=5FfTul9G4eutVn]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - types: Attachment 增 hash?/size? 可选字段（向后兼容，老数据零迁移） - 上传端点 POST .../attachments 改为内容寻址：流式计算 sha256，写临时文件再 按 . 原子 rename；同 hash 已存在则丢弃临时文件（真去重）； 超限/异常清理临时文件不留垃圾。磁盘名仅由内容 hash 决定，杜绝撞名覆盖丢数据。 - store.addAttachments 去重键改为 hash ?? path（later-wins，老数据回退 path）。 - DELETE 端点补注释：附件按 task 隔离，无需跨任务 refcount。 - 安全保持并固化：GET inline 仅硬白名单图片，其余强制 attachment+octet-stream +nosniff+CSP sandbox；路径越界防护；hash 命名不含路径可控字符。 - 测试：新增同内容去重/不同内容同名各留一条/响应含 hash+size 等用例。 Co-Authored-By: Claude Opus 4.8 --- src/api/server.ts | 48 ++++++++++++--- src/model/types.ts | 6 +- src/store/store.ts | 10 ++-- test/attachments.test.ts | 124 +++++++++++++++++++++++++++++++-------- 4 files changed, 147 insertions(+), 41 deletions(-) diff --git a/src/api/server.ts b/src/api/server.ts index d92eb13..caafd81 100644 --- a/src/api/server.ts +++ b/src/api/server.ts @@ -4,7 +4,7 @@ import { Store, StoreError, type ActiveRun, type PatchProjectInput, type PatchTa import type { Complexity } from '../model/complexity.js'; import { isComplexity } from '../model/complexity.js'; import type { TaskStatus } from '../model/status.js'; -import type { Project, Autonomy } from '../model/types.js'; +import type { Project, Autonomy, Attachment } from '../model/types.js'; import { syncProject, hasTodoJson } from '../sync/todo-sync.js'; import { resolvedExecutorModels } from '../executor/models.js'; import { classifyComplexity, type ClassifierFn } from '../executor/classify.js'; @@ -12,7 +12,8 @@ import { acceptAndMerge } from '../executor/exec-merge.js'; import { createWorktree, git } from '../executor/worktree.js'; import { resolveLogo, LOGO_MIME } from './logo.js'; import { readTranscript, TranscriptError } from '../executor/transcript.js'; -import { createReadStream, createWriteStream, mkdirSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs'; +import { createHash, randomUUID } from 'node:crypto'; import { transcriptDir } from '../executor/cc.js'; import { homedir } from 'node:os'; import { join, basename, resolve, sep } from 'node:path'; @@ -343,14 +344,40 @@ export function buildServer(opts: ApiOptions): FastifyInstance { if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` }); const dir = join(dataDir(), 'tasks', id, 'attachments'); mkdirSync(dir, { recursive: true }); - const saved: Array<{ name: string; type: string; path: string }> = []; - let seq = 0; + const saved: Attachment[] = []; for await (const part of req.files()) { - // 无名/无扩展名（如剪贴板粘贴 blob）→ 按 mimetype 生成安全文件名+扩展名 - const safe = safeAttachmentName(part.filename, part.mimetype, seq++); - await pipeline(part.file, createWriteStream(join(dir, safe))); - if (part.file.truncated) return reply.code(400).send({ error: `文件 ${part.filename || safe} 超过 25MB 上限` }); - saved.push({ name: part.filename || safe, type: part.mimetype, path: `tasks/${id}/attachments/${safe}` }); + // 内容寻址：先流式写临时文件并同步计算 sha256，再按 hash 重命名为最终磁盘名。 + // 磁盘名完全由内容 hash 决定（不再用文件名）：相同内容天然单份、不同内容永不撞名， + // 且十六进制 sha256 不含 `/`、`..`、脚本可控字符，从根上杜绝路径穿越/撞名覆盖。 + const ext = extFromMime(part.mimetype) || (hasExtension(part.filename ?? '') + ? (part.filename as string).slice((part.filename as string).lastIndexOf('.') + 1).replace(/[^A-Za-z0-9]/g, '') + : ''); + const tmp = join(dir, `.tmp-${randomUUID()}`); + const hash = createHash('sha256'); + let size = 0; + try { + await pipeline(part.file, async function* (src) { + for await (const c of src) { hash.update(c as Buffer); size += (c as Buffer).length; yield c; } + }, createWriteStream(tmp)); + } catch (e) { + rmSync(tmp, { force: true }); + throw e; + } + // 超限：清掉临时文件，绝不留垃圾；@fastify/multipart 在超过 fileSize 时置 truncated。 + if (part.file.truncated) { + rmSync(tmp, { force: true }); + return reply.code(400).send({ error: `文件 ${part.filename || 'attachment'} 超过 25MB 上限` }); + } + const digest = hash.digest('hex'); + const finalName = ext ? `${digest}.${ext}` : digest; + const finalAbs = join(dir, finalName); + // 相同内容已存在 → 丢弃临时文件（真去重）；否则原子 rename 同目录就位。 + if (existsSync(finalAbs)) rmSync(tmp, { force: true }); + else renameSync(tmp, finalAbs); + saved.push({ + name: part.filename || finalName, type: part.mimetype, + path: `tasks/${id}/attachments/${finalName}`, hash: digest, size, + }); } if (saved.length === 0) return reply.code(400).send({ error: '未收到文件' }); return { attachments: store.addAttachments(id, saved) }; @@ -392,6 +419,9 @@ export function buildServer(opts: ApiOptions): FastifyInstance { }); // 删除单个附件：先删元数据（拿到 removed），再尽力删磁盘文件 + // 前提：附件目录按 task 隔离（tasks//attachments/），内容寻址后同一 hash 在一个 task 内 + // 去重为一条元数据 → 删元数据即可安全删盘，无需跨任务引用计数。 + // ⚠️ 若将来改为跨任务共享存储（同一 文件被多任务引用），必须引入 refcount，本次不做。 app.delete('/api/tasks/:id/attachments/:name', (req, reply) => { const { id, name } = req.params as { id: string; name: string }; if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` }); diff --git a/src/model/types.ts b/src/model/types.ts index 8d54ade..3df9f17 100644 --- a/src/model/types.ts +++ b/src/model/types.ts @@ -114,9 +114,11 @@ export interface Task { /** 任务附件：图片/文件随任务提交，落盘于 /tasks//attachments/ */ export interface Attachment { - name: string; // 原始文件名 + name: string; // 原始文件名（展示用） type: string; // MIME 类型（如 image/png） - path: string; // 相对 data 根的存储路径（tasks//attachments/） + path: string; // 相对 data 根的存储路径，内容寻址：tasks//attachments/. + hash?: string; // 内容 sha256（hex）——去重键；老数据无此字段，回退按 path 去重 + size?: number; // 字节数；老数据可能缺省 } export type RunKind = 'planner' | 'executor' | 'reviewer' | 'security'; diff --git a/src/store/store.ts b/src/store/store.ts index 284eb0a..014339c 100644 --- a/src/store/store.ts +++ b/src/store/store.ts @@ -420,10 +420,12 @@ export class Store { const t = this.getTaskRow(taskId); if (!t) throw new StoreError(`任务不存在: ${taskId}`); const cur: Attachment[] = t.attachments ? JSON.parse(t.attachments) as Attachment[] : []; - // 按 path 去重（later-wins）：path 唯一对应一份磁盘文件，重传同名文件只保留最新一条元数据。 - const byPath = new Map(); - for (const a of [...cur, ...items]) byPath.set(a.path, a); - const next = [...byPath.values()]; + // 按内容 hash 去重（later-wins）：磁盘文件内容寻址，相同内容=同一份磁盘文件； + // 重传相同内容（即便改名）只保留最新一条元数据（更新展示名/type）。老数据无 hash → 回退按 path 去重。 + const keyOf = (a: Attachment) => a.hash ?? a.path; + const byKey = new Map(); + for (const a of [...cur, ...items]) byKey.set(keyOf(a), a); + const next = [...byKey.values()]; this.db.prepare(`UPDATE tasks SET attachments = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(next), now(), taskId); this.emit(t.project_id, taskId, 'task.updated', { field: 'attachments', count: next.length }); return next; diff --git a/test/attachments.test.ts b/test/attachments.test.ts index 3704510..2a59e93 100644 --- a/test/attachments.test.ts +++ b/test/attachments.test.ts @@ -1,6 +1,6 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdtempSync, mkdirSync, writeFileSync, existsSync } from 'node:fs'; +import { mkdtempSync, mkdirSync, writeFileSync, existsSync, readdirSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Store } from '../src/store/index.js'; @@ -30,20 +30,46 @@ function placeFile(dir: string, taskId: string, name: string, mime: string, body // ───────────────────────── Store 层 ───────────────────────── -test('addAttachments：按 path 去重（later-wins）', () => { +test('addAttachments：按内容 hash 去重（later-wins，同内容不同名只一条）', () => { const { store, taskId } = freshTask(); - store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', path: `tasks/${taskId}/attachments/a.png` }]); - // 重传同一 path（safeName 相同）→ 元数据只保留一条，且取最新 name/type + const hA = 'a'.repeat(64); + store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', hash: hA, size: 3, path: `tasks/${taskId}/attachments/${hA}.png` }]); + // 重传相同内容（hash 相同）但改名 → 元数据只保留一条，且取最新 name/type + const hB = 'b'.repeat(64); const next = store.addAttachments(taskId, [ - { name: 'a-renamed.png', type: 'image/jpeg', path: `tasks/${taskId}/attachments/a.png` }, - { name: 'b.pdf', type: 'application/pdf', path: `tasks/${taskId}/attachments/b.pdf` }, + { name: 'a-renamed.png', type: 'image/jpeg', hash: hA, size: 3, path: `tasks/${taskId}/attachments/${hA}.png` }, + { name: 'b.pdf', type: 'application/pdf', hash: hB, size: 7, path: `tasks/${taskId}/attachments/${hB}.pdf` }, ]); assert.equal(next.length, 2); - const a = next.find((x) => x.path.endsWith('/a.png')); + const a = next.find((x) => x.hash === hA); assert.equal(a?.name, 'a-renamed.png'); assert.equal(a?.type, 'image/jpeg'); - // 插入顺序保持：a 在前、b 在后 - assert.deepEqual(next.map((x) => x.path.split('/').pop()), ['a.png', 'b.pdf']); + // 插入顺序保持：A 在前、B 在后 + assert.deepEqual(next.map((x) => x.hash), [hA, hB]); + store.close(); +}); + +test('addAttachments：不同内容（hash 不同）即便同显示名也各保留一条（不再静默覆盖丢数据）', () => { + const { store, taskId } = freshTask(); + const h1 = '1'.repeat(64); + const h2 = '2'.repeat(64); + const next = store.addAttachments(taskId, [ + { name: 'screenshot.png', type: 'image/png', hash: h1, size: 3, path: `tasks/${taskId}/attachments/${h1}.png` }, + { name: 'screenshot.png', type: 'image/png', hash: h2, size: 3, path: `tasks/${taskId}/attachments/${h2}.png` }, + ]); + assert.equal(next.length, 2, '不同内容同名应各保留一条'); + assert.deepEqual(next.map((x) => x.hash), [h1, h2]); + store.close(); +}); + +test('addAttachments：老数据无 hash 时回退按 path 去重', () => { + const { store, taskId } = freshTask(); + store.addAttachments(taskId, [{ name: 'a.png', type: 'image/png', path: `tasks/${taskId}/attachments/a.png` }]); + const next = store.addAttachments(taskId, [ + { name: 'a-renamed.png', type: 'image/jpeg', path: `tasks/${taskId}/attachments/a.png` }, + ]); + assert.equal(next.length, 1); + assert.equal(next[0].name, 'a-renamed.png'); store.close(); }); @@ -198,38 +224,84 @@ test('DELETE：元数据在但磁盘文件已被外部删除 → 仍成功清元 store.close(); }); -test('上传→去重端到端：同名文件连传两次，元数据只一条', async () => { +function multipart(filename: string, content: string): { payload: Buffer; headers: Record } { + const boundary = '----maestrotest'; + const body = + `--${boundary}\r\n` + + `Content-Disposition: form-data; name="files"; filename="${filename}"\r\n` + + `Content-Type: image/png\r\n\r\n` + + content + `\r\n` + + `--${boundary}--\r\n`; + return { payload: Buffer.from(body), headers: { 'content-type': `multipart/form-data; boundary=${boundary}` } }; +} + +test('上传→内容去重端到端：同内容不同名连传 → 元数据 1 条、磁盘 1 份 .png', async () => { const dir = tmpData(); process.env.MAESTRO_DATA_DIR = dir; const { store, taskId } = freshTask(); const app = buildServer({ store }); - function multipart(filename: string, content: string): { payload: Buffer; headers: Record } { - const boundary = '----maestrotest'; - const body = - `--${boundary}\r\n` + - `Content-Disposition: form-data; name="files"; filename="${filename}"\r\n` + - `Content-Type: image/png\r\n\r\n` + - content + `\r\n` + - `--${boundary}--\r\n`; - return { payload: Buffer.from(body), headers: { 'content-type': `multipart/form-data; boundary=${boundary}` } }; - } - + // 同内容（'aaa'），但文件名不同 → 内容寻址应去重为一条、磁盘一份 const m1 = multipart('shot.png', 'aaa'); const up1 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m1.payload, headers: m1.headers }); assert.equal(up1.statusCode, 200); assert.equal(up1.json().attachments.length, 1); - const m2 = multipart('shot.png', 'bbb'); + const m2 = multipart('shot-renamed.png', 'aaa'); const up2 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m2.payload, headers: m2.headers }); assert.equal(up2.statusCode, 200); - // 同名 safeName → path 相同 → 去重为一条 const atts = up2.json().attachments; - assert.equal(atts.length, 1, '同名重传应去重为一条'); + assert.equal(atts.length, 1, '同内容重传应去重为一条'); - // 通过 GET /api/tasks/:id 复核 + // 磁盘上只有 1 个 .png（外加可能的 .tmp-* 已清理） + const attDir = join(dir, 'tasks', taskId, 'attachments'); + const files = readdirSync(attDir).filter((f) => !f.startsWith('.tmp-')); + assert.equal(files.length, 1, '磁盘应只有一份内容文件'); + assert.match(files[0], /^[0-9a-f]{64}\.png$/, '磁盘名应为 .png'); + + // GET /api/tasks/:id 复核：1 条，且含 hash(64 hex)/size 字段 const t = await app.inject({ method: 'GET', url: `/api/tasks/${taskId}` }); - assert.equal(t.json().attachments.length, 1); + const list = t.json().attachments; + assert.equal(list.length, 1); + assert.match(list[0].hash, /^[0-9a-f]{64}$/); + assert.equal(list[0].size, 3); + await app.close(); + store.close(); +}); + +test('上传→内容寻址端到端：不同内容同显示名连传 → 元数据 2 条、磁盘 2 份（旧的覆盖丢数据 bug 已消除）', async () => { + const dir = tmpData(); + process.env.MAESTRO_DATA_DIR = dir; + const { store, taskId } = freshTask(); + const app = buildServer({ store }); + + const m1 = multipart('screenshot.png', 'aaa'); + await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m1.payload, headers: m1.headers }); + const m2 = multipart('screenshot.png', 'bbb'); // 同名、不同内容 + const up2 = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m2.payload, headers: m2.headers }); + assert.equal(up2.statusCode, 200); + assert.equal(up2.json().attachments.length, 2, '不同内容同名应保留两条'); + + const attDir = join(dir, 'tasks', taskId, 'attachments'); + const files = readdirSync(attDir).filter((f) => !f.startsWith('.tmp-')); + assert.equal(files.length, 2, '磁盘应有两份内容文件，不再静默覆盖'); + await app.close(); + store.close(); +}); + +test('上传：响应附件含 hash(64 hex) 与 size 字段', async () => { + const dir = tmpData(); + process.env.MAESTRO_DATA_DIR = dir; + const { store, taskId } = freshTask(); + const app = buildServer({ store }); + const m = multipart('pic.png', 'hello-bytes'); + const up = await app.inject({ method: 'POST', url: `/api/tasks/${taskId}/attachments`, payload: m.payload, headers: m.headers }); + assert.equal(up.statusCode, 200); + const att = up.json().attachments[0]; + assert.match(att.hash, /^[0-9a-f]{64}$/); + assert.equal(att.size, 'hello-bytes'.length); + // 磁盘名 = path 末段 = .png + assert.equal(att.path, `tasks/${taskId}/attachments/${att.hash}.png`); await app.close(); store.close(); });