附件内容去重(hash):上传改为内容寻址 + store 按 hash 去重 [tsk_fTul9G4eutVn]
- types: Attachment 增 hash?/size? 可选字段(向后兼容,老数据零迁移) - 上传端点 POST .../attachments 改为内容寻址:流式计算 sha256,写临时文件再 按 <sha256>.<ext> 原子 rename;同 hash 已存在则丢弃临时文件(真去重); 超限/异常清理临时文件不留垃圾。磁盘名仅由内容 hash 决定,杜绝撞名覆盖丢数据。 - store.addAttachments 去重键改为 hash ?? path(later-wins,老数据回退 path)。 - DELETE 端点补注释:附件按 task 隔离,无需跨任务 refcount。 - 安全保持并固化:GET inline 仅硬白名单图片,其余强制 attachment+octet-stream +nosniff+CSP sandbox;路径越界防护;hash 命名不含路径可控字符。 - 测试:新增同内容去重/不同内容同名各留一条/响应含 hash+size 等用例。 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
+39
-9
@@ -4,7 +4,7 @@ import { Store, StoreError, type ActiveRun, type PatchProjectInput, type PatchTa
|
||||
import type { Complexity } from '../model/complexity.js';
|
||||
import { isComplexity } from '../model/complexity.js';
|
||||
import type { TaskStatus } from '../model/status.js';
|
||||
import type { Project, Autonomy } from '../model/types.js';
|
||||
import type { Project, Autonomy, Attachment } from '../model/types.js';
|
||||
import { syncProject, hasTodoJson } from '../sync/todo-sync.js';
|
||||
import { resolvedExecutorModels } from '../executor/models.js';
|
||||
import { classifyComplexity, type ClassifierFn } from '../executor/classify.js';
|
||||
@@ -12,7 +12,8 @@ import { acceptAndMerge } from '../executor/exec-merge.js';
|
||||
import { createWorktree, git } from '../executor/worktree.js';
|
||||
import { resolveLogo, LOGO_MIME } from './logo.js';
|
||||
import { readTranscript, TranscriptError } from '../executor/transcript.js';
|
||||
import { createReadStream, createWriteStream, mkdirSync, readFileSync, rmSync, statSync } from 'node:fs';
|
||||
import { createReadStream, createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync } from 'node:fs';
|
||||
import { createHash, randomUUID } from 'node:crypto';
|
||||
import { transcriptDir } from '../executor/cc.js';
|
||||
import { homedir } from 'node:os';
|
||||
import { join, basename, resolve, sep } from 'node:path';
|
||||
@@ -343,14 +344,40 @@ export function buildServer(opts: ApiOptions): FastifyInstance {
|
||||
if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` });
|
||||
const dir = join(dataDir(), 'tasks', id, 'attachments');
|
||||
mkdirSync(dir, { recursive: true });
|
||||
const saved: Array<{ name: string; type: string; path: string }> = [];
|
||||
let seq = 0;
|
||||
const saved: Attachment[] = [];
|
||||
for await (const part of req.files()) {
|
||||
// 无名/无扩展名(如剪贴板粘贴 blob)→ 按 mimetype 生成安全文件名+扩展名
|
||||
const safe = safeAttachmentName(part.filename, part.mimetype, seq++);
|
||||
await pipeline(part.file, createWriteStream(join(dir, safe)));
|
||||
if (part.file.truncated) return reply.code(400).send({ error: `文件 ${part.filename || safe} 超过 25MB 上限` });
|
||||
saved.push({ name: part.filename || safe, type: part.mimetype, path: `tasks/${id}/attachments/${safe}` });
|
||||
// 内容寻址:先流式写临时文件并同步计算 sha256,再按 hash 重命名为最终磁盘名。
|
||||
// 磁盘名完全由内容 hash 决定(不再用文件名):相同内容天然单份、不同内容永不撞名,
|
||||
// 且十六进制 sha256 不含 `/`、`..`、脚本可控字符,从根上杜绝路径穿越/撞名覆盖。
|
||||
const ext = extFromMime(part.mimetype) || (hasExtension(part.filename ?? '')
|
||||
? (part.filename as string).slice((part.filename as string).lastIndexOf('.') + 1).replace(/[^A-Za-z0-9]/g, '')
|
||||
: '');
|
||||
const tmp = join(dir, `.tmp-${randomUUID()}`);
|
||||
const hash = createHash('sha256');
|
||||
let size = 0;
|
||||
try {
|
||||
await pipeline(part.file, async function* (src) {
|
||||
for await (const c of src) { hash.update(c as Buffer); size += (c as Buffer).length; yield c; }
|
||||
}, createWriteStream(tmp));
|
||||
} catch (e) {
|
||||
rmSync(tmp, { force: true });
|
||||
throw e;
|
||||
}
|
||||
// 超限:清掉临时文件,绝不留垃圾;@fastify/multipart 在超过 fileSize 时置 truncated。
|
||||
if (part.file.truncated) {
|
||||
rmSync(tmp, { force: true });
|
||||
return reply.code(400).send({ error: `文件 ${part.filename || 'attachment'} 超过 25MB 上限` });
|
||||
}
|
||||
const digest = hash.digest('hex');
|
||||
const finalName = ext ? `${digest}.${ext}` : digest;
|
||||
const finalAbs = join(dir, finalName);
|
||||
// 相同内容已存在 → 丢弃临时文件(真去重);否则原子 rename 同目录就位。
|
||||
if (existsSync(finalAbs)) rmSync(tmp, { force: true });
|
||||
else renameSync(tmp, finalAbs);
|
||||
saved.push({
|
||||
name: part.filename || finalName, type: part.mimetype,
|
||||
path: `tasks/${id}/attachments/${finalName}`, hash: digest, size,
|
||||
});
|
||||
}
|
||||
if (saved.length === 0) return reply.code(400).send({ error: '未收到文件' });
|
||||
return { attachments: store.addAttachments(id, saved) };
|
||||
@@ -392,6 +419,9 @@ export function buildServer(opts: ApiOptions): FastifyInstance {
|
||||
});
|
||||
|
||||
// 删除单个附件:先删元数据(拿到 removed),再尽力删磁盘文件
|
||||
// 前提:附件目录按 task 隔离(tasks/<id>/attachments/),内容寻址后同一 hash 在一个 task 内
|
||||
// 去重为一条元数据 → 删元数据即可安全删盘,无需跨任务引用计数。
|
||||
// ⚠️ 若将来改为跨任务共享存储(同一 <sha256> 文件被多任务引用),必须引入 refcount,本次不做。
|
||||
app.delete('/api/tasks/:id/attachments/:name', (req, reply) => {
|
||||
const { id, name } = req.params as { id: string; name: string };
|
||||
if (!store.getTask(id)) return reply.code(404).send({ error: `任务不存在: ${id}` });
|
||||
|
||||
+4
-2
@@ -114,9 +114,11 @@ export interface Task {
|
||||
|
||||
/** 任务附件:图片/文件随任务提交,落盘于 <MAESTRO_DATA_DIR>/tasks/<taskId>/attachments/ */
|
||||
export interface Attachment {
|
||||
name: string; // 原始文件名
|
||||
name: string; // 原始文件名(展示用)
|
||||
type: string; // MIME 类型(如 image/png)
|
||||
path: string; // 相对 data 根的存储路径(tasks/<taskId>/attachments/<name>)
|
||||
path: string; // 相对 data 根的存储路径,内容寻址:tasks/<taskId>/attachments/<sha256>.<ext>
|
||||
hash?: string; // 内容 sha256(hex)——去重键;老数据无此字段,回退按 path 去重
|
||||
size?: number; // 字节数;老数据可能缺省
|
||||
}
|
||||
|
||||
export type RunKind = 'planner' | 'executor' | 'reviewer' | 'security';
|
||||
|
||||
+6
-4
@@ -420,10 +420,12 @@ export class Store {
|
||||
const t = this.getTaskRow(taskId);
|
||||
if (!t) throw new StoreError(`任务不存在: ${taskId}`);
|
||||
const cur: Attachment[] = t.attachments ? JSON.parse(t.attachments) as Attachment[] : [];
|
||||
// 按 path 去重(later-wins):path 唯一对应一份磁盘文件,重传同名文件只保留最新一条元数据。
|
||||
const byPath = new Map<string, Attachment>();
|
||||
for (const a of [...cur, ...items]) byPath.set(a.path, a);
|
||||
const next = [...byPath.values()];
|
||||
// 按内容 hash 去重(later-wins):磁盘文件内容寻址,相同内容=同一份磁盘文件;
|
||||
// 重传相同内容(即便改名)只保留最新一条元数据(更新展示名/type)。老数据无 hash → 回退按 path 去重。
|
||||
const keyOf = (a: Attachment) => a.hash ?? a.path;
|
||||
const byKey = new Map<string, Attachment>();
|
||||
for (const a of [...cur, ...items]) byKey.set(keyOf(a), a);
|
||||
const next = [...byKey.values()];
|
||||
this.db.prepare(`UPDATE tasks SET attachments = ?, updated_at = ? WHERE id = ?`).run(JSON.stringify(next), now(), taskId);
|
||||
this.emit(t.project_id, taskId, 'task.updated', { field: 'attachments', count: next.length });
|
||||
return next;
|
||||
|
||||
Reference in New Issue
Block a user