附件内容去重(hash):上传改为内容寻址 + store 按 hash 去重 [tsk_fTul9G4eutVn]

- types: Attachment 增 hash?/size? 可选字段(向后兼容,老数据零迁移)
- 上传端点 POST .../attachments 改为内容寻址:流式计算 sha256,写临时文件再
  按 <sha256>.<ext> 原子 rename;同 hash 已存在则丢弃临时文件(真去重);
  超限/异常清理临时文件不留垃圾。磁盘名仅由内容 hash 决定,杜绝撞名覆盖丢数据。
- store.addAttachments 去重键改为 hash ?? path(later-wins,老数据回退 path)。
- DELETE 端点补注释:附件按 task 隔离,无需跨任务 refcount。
- 安全保持并固化:GET inline 仅硬白名单图片,其余强制 attachment+octet-stream
  +nosniff+CSP sandbox;路径越界防护;hash 命名不含路径可控字符。
- 测试:新增同内容去重/不同内容同名各留一条/响应含 hash+size 等用例。

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
wangjia
2026-06-30 07:22:15 +08:00
parent d9d3de26b8
commit f1ee537ab7
4 changed files with 147 additions and 41 deletions
+39 -9
View File
@@ -4,7 +4,7 @@ import { Store, StoreError, type ActiveRun, type PatchProjectInput, type PatchTa
import type { Complexity } from '../model/complexity.js';
import { isComplexity } from '../model/complexity.js';
import type { TaskStatus } from '../model/status.js';
import type { Project, Autonomy } from '../model/types.js';
import type { Project, Autonomy, Attachment } from '../model/types.js';
import { syncProject, hasTodoJson } from '../sync/todo-sync.js';
import { resolvedExecutorModels } from '../executor/models.js';
import { classifyComplexity, type ClassifierFn } from '../executor/classify.js';
@@ -12,7 +12,8 @@ import { acceptAndMerge } from '../executor/exec-merge.js';
import { createWorktree, git } from '../executor/worktree.js';
import { resolveLogo, LOGO_MIME } from './logo.js';
import { readTranscript, TranscriptError } from '../executor/transcript.js';
import { createReadStream, createWriteStream, mkdirSync, readFileSync, rmSync, statSync } from 'node:fs';
import { createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs';
import { createHash, randomUUID } from 'node:crypto';
import { transcriptDir } from '../executor/cc.js';
import { homedir } from 'node:os';
import { join, basename, resolve, sep } from 'node:path';
@@ -343,14 +344,40 @@ export function buildServer(opts: ApiOptions): FastifyInstance {
if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` });
const dir = join(dataDir(), 'tasks', id, 'attachments');
mkdirSync(dir, { recursive: true });
const saved: Array<{ name: string; type: string; path: string }> = [];
let seq = 0;
const saved: Attachment[] = [];
for await (const part of req.files()) {
// 无名/无扩展名(如剪贴板粘贴 blob)→ 按 mimetype 生成安全文件名+扩展名
const safe = safeAttachmentName(part.filename, part.mimetype, seq++);
await pipeline(part.file, createWriteStream(join(dir, safe)));
if (part.file.truncated) return reply.code(400).send({ error: `文件 ${part.filename || safe} 超过 25MB 上限` });
saved.push({ name: part.filename || safe, type: part.mimetype, path: `tasks/${id}/attachments/${safe}` });
// 内容寻址:先流式写临时文件并同步计算 sha256,再按 hash 重命名为最终磁盘名。
// 磁盘名完全由内容 hash 决定(不再用文件名):相同内容天然单份、不同内容永不撞名,
// 且十六进制 sha256 不含 `/`、`..`、脚本可控字符,从根上杜绝路径穿越/撞名覆盖。
const ext = extFromMime(part.mimetype) || (hasExtension(part.filename ?? '')
? (part.filename as string).slice((part.filename as string).lastIndexOf('.') + 1).replace(/[^A-Za-z0-9]/g, '')
: '');
const tmp = join(dir, `.tmp-${randomUUID()}`);
const hash = createHash('sha256');
let size = 0;
try {
await pipeline(part.file, async function* (src) {
for await (const c of src) { hash.update(c as Buffer); size += (c as Buffer).length; yield c; }
}, createWriteStream(tmp));
} catch (e) {
rmSync(tmp, { force: true });
throw e;
}
// 超限:清掉临时文件,绝不留垃圾;@fastify/multipart 在超过 fileSize 时置 truncated。
if (part.file.truncated) {
rmSync(tmp, { force: true });
return reply.code(400).send({ error: `文件 ${part.filename || 'attachment'} 超过 25MB 上限` });
}
const digest = hash.digest('hex');
const finalName = ext ? `${digest}.${ext}` : digest;
const finalAbs = join(dir, finalName);
// 相同内容已存在 → 丢弃临时文件(真去重);否则原子 rename 同目录就位。
if (existsSync(finalAbs)) rmSync(tmp, { force: true });
else renameSync(tmp, finalAbs);
saved.push({
name: part.filename || finalName, type: part.mimetype,
path: `tasks/${id}/attachments/${finalName}`, hash: digest, size,
});
}
if (saved.length === 0) return reply.code(400).send({ error: '未收到文件' });
return { attachments: store.addAttachments(id, saved) };
@@ -392,6 +419,9 @@ export function buildServer(opts: ApiOptions): FastifyInstance {
});
// 删除单个附件:先删元数据(拿到 removed),再尽力删磁盘文件
// 前提:附件目录按 task 隔离(tasks/<id>/attachments/),内容寻址后同一 hash 在一个 task 内
// 去重为一条元数据 → 删元数据即可安全删盘,无需跨任务引用计数。
// ⚠️ 若将来改为跨任务共享存储(同一 <sha256> 文件被多任务引用),必须引入 refcount,本次不做。
app.delete('/api/tasks/:id/attachments/:name', (req, reply) => {
const { id, name } = req.params as { id: string; name: string };
if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` });
+4 -2
View File
@@ -114,9 +114,11 @@ export interface Task {
/** 任务附件:图片/文件随任务提交,落盘于 <MAESTRO_DATA_DIR>/tasks/<taskId>/attachments/ */
export interface Attachment {
name: string; // 原始文件名
name: string; // 原始文件名(展示用)
type: string; // MIME 类型(如 image/png
path: string; // 相对 data 根的存储路径tasks/<taskId>/attachments/<name>
path: string; // 相对 data 根的存储路径,内容寻址:tasks/<taskId>/attachments/<sha256>.<ext>
hash?: string; // 内容 sha256(hex)——去重键;老数据无此字段,回退按 path 去重
size?: number; // 字节数;老数据可能缺省
}
export type RunKind = 'planner' | 'executor' | 'reviewer' | 'security';
+6 -4
View File
@@ -420,10 +420,12 @@ export class Store {
const t = this.getTaskRow(taskId);
if (!t) throw new StoreError(`任务不存在: ${taskId}`);
const cur: Attachment[] = t.attachments ? JSON.parse(t.attachments) as Attachment[] : [];
// 按 path 去重(later-wins):path 唯一对应一份磁盘文件,重传同名文件只保留最新一条元数据。
const byPath = new Map<string, Attachment>();
for (const a of [...cur, ...items]) byPath.set(a.path, a);
const next = [...byPath.values()];
// 按内容 hash 去重(later-wins):磁盘文件内容寻址,相同内容=同一份磁盘文件;
// 重传相同内容(即便改名)只保留最新一条元数据(更新展示名/type)。老数据无 hash → 回退按 path 去重。
const keyOf = (a: Attachment) => a.hash ?? a.path;
const byKey = new Map<string, Attachment>();
for (const a of [...cur, ...items]) byKey.set(keyOf(a), a);
const next = [...byKey.values()];
this.db.prepare(`UPDATE tasks SET attachments = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(next), now(), taskId);
this.emit(t.project_id, taskId, 'task.updated', { field: 'attachments', count: next.length });
return next;