root/goodbuddy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(storage): move desktop SQLite ownership into a storage utility process

Browse Source 
Main no longer opens desktop business SQLite. A single utility-process
storage owner holds the Assistant, Knowledge, attachment, document-result,
remote-binding and native model-ledger databases and runs whole domain
transactions; Main and Runtime callers use explicit async domain operations.

- Remove the Main readWithFallback path and the supervision worker's
  separate business write connection; manifests are scanned read-only and
  committed by the sole writer.
- Remote events are ACKed only after commit; process loss reports
  unconfirmed writes instead of replaying them.
- Use native utilityProcess messaging with one in-flight bound (32) and
  ordered waiting instead of capacity rejection and retries.
- Start the storage process before the window so startup overlaps.
- Keep short tool output in memory and store full output once in owned
  storage; paging, branching, restart and deletion reuse that copy.
- Put required runtime launch files under goodbuddy-prefixed owned temp
  directories and reclaim confirmed stale ones; stop writing duplicate
  parsed.md.
- Add lint guards for SQLite ownership and architecture, concurrency
  acceptance fixtures, and maintenance/architecture documentation.

Validation: typecheck and lint pass. Alternating baseline (edf019aa) vs
candidate runs show no regression on single-session, C01, C02, C04 and a
5-minute C06; Main p99 drops from ~21 ms to ~16.7 ms under concurrency.
Fresh portable copy (1.75M events) upgrade content unchanged; real-model
output paging passes with 4 external requests. Full npm test runs had
1-9 different timing-sensitive failures per run that pass when rerun.
Not covered: C03 review publication, C07 source body, app-level C05.
This commit is contained in:
mesalogo
2026-10-06 15:40:18 +08:00
parent bc677a3718
commit d80d42b032
229 changed files with 13632 additions and 3800 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/conversation-attachment-storage.ts
+136 -25
View File
@@ -2,12 +2,17 @@ import { DatabaseSync } from 'node:sqlite'
import { randomUUID } from 'node:crypto'
import { cpSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'
import { extname, join, relative } from 'node:path'
import { appendFile, open, stat } from 'node:fs/promises'
import type { ContextAttachment } from '../shared/contracts'
import type { ParsedDocument } from './knowledge/document-parser'
import type { DocumentResultStorage } from './document-result-storage'
type AssetRow = { id: string; path: string; name: string; result_id: string | null; metadata: string }
// Released databases contain both Windows and POSIX relative paths.
function portablePath(path: string): string { return path.replaceAll('\\', '/') }
export type AttachmentReferenceKind = 'draft' | 'message' | 'queue' | 'parsing'
function metadata(context: ContextAttachment): string {
const { thumbnailUrl: _thumbnail, contentUrl: _content, ...descriptor } = context
void _thumbnail
@@ -19,6 +24,7 @@ function originalFileName(context: Pick<ContextAttachment, 'name' | 'originalMim
return `original${context.originalMime ? `.${context.originalMime === 'image/jpeg' ? 'jpg' : context.originalMime.slice(6)}` : extname(context.name).toLowerCase()}`
}
/** Construct only in the desktop storage host. */
export class ConversationAttachmentStorage {
private readonly database: DatabaseSync
constructor(private readonly root: string, private readonly results: DocumentResultStorage) {
@@ -42,20 +48,21 @@ export class ConversationAttachmentStorage {
PRAGMA user_version = 1; COMMIT;`)
results.setPersistentLookup((id) => {
const row = this.database.prepare('SELECT * FROM attachments WHERE result_id = ?').get(id) as AssetRow | undefined
return row ? { directory: join(root, row.path), original: join(root, row.path, originalFileName({ ...JSON.parse(row.metadata), name: row.name })) } : undefined
return row ? { directory: join(root, portablePath(row.path)), original: join(root, portablePath(row.path), originalFileName({ ...JSON.parse(row.metadata), name: row.name })) } : undefined
})
}
private row(id: string): AssetRow {
const row = this.database.prepare('SELECT * FROM attachments WHERE id = ?').get(id) as AssetRow | undefined
if (!row) throw new Error('附件资源不存在，请重新添加')
return row
return { ...row, path: portablePath(row.path) }
}
has(id: string): boolean { return Boolean(this.database.prepare('SELECT 1 FROM attachments WHERE id = ?').get(id)) }
save(context: ContextAttachment, serialized: string, original: Buffer): void {
const path = join('temp', 'document-parsing', context.id)
if (this.has(context.id)) throw new Error('Attachment already exists')
const path = portablePath(join('temp', 'document-parsing', context.id))
const directory = join(this.root, path)
try {
mkdirSync(directory, { recursive: true })
@@ -76,14 +83,14 @@ export class ConversationAttachmentStorage {
discardResult(id: string): Promise<void> { return this.results.release(id) }
adoptDocument(context: ContextAttachment, serialized: string): void {
const location = this.results.detach(context.resultId!)
try {
writeFileSync(join(location.directory, 'request.json'), serialized)
this.database.prepare('INSERT INTO attachments VALUES (?, ?, ?, ?, ?)').run(
context.id, relative(this.root, location.directory), context.originalName ?? context.name,
context.resultId!, metadata(context)
)
} catch (error) { rmSync(location.directory, { recursive: true, force: true }); throw error }
if (!this.results.isTemporaryResult(context.resultId!)) throw new Error('Document result is already owned')
const location = this.results.location(context.resultId!)
writeFileSync(join(location.directory, 'request.json'), serialized)
this.database.prepare('INSERT INTO attachments VALUES (?, ?, ?, ?, ?)').run(
context.id, portablePath(relative(this.root, location.directory)), context.originalName ?? context.name,
context.resultId!, metadata(context)
)
this.results.detach(context.resultId!)
}
get(id: string): ContextAttachment {
@@ -103,7 +110,6 @@ export class ConversationAttachmentStorage {
const target = this.row(targetId)
const from = join(this.root, source.path), to = join(this.root, target.path)
const manifest = JSON.parse(readFileSync(join(from, 'manifest.json'), 'utf8'))
writeFileSync(join(to, 'parsed.md'), readFileSync(join(from, 'parsed.md')))
cpSync(join(from, 'images'), join(to, 'images'), { recursive: true })
writeFileSync(join(to, 'manifest.json'), JSON.stringify({ ...manifest, id: targetId }))
this.database.prepare('UPDATE attachments SET result_id = ? WHERE id = ?').run(targetId, targetId)
@@ -116,11 +122,30 @@ export class ConversationAttachmentStorage {
return { name: row.name, data: readFileSync(path), path }
}
reference(conversationId: string, kind: 'draft' | 'message' | 'queue' | 'parsing', ownerId: string, ids: string[]): void {
reference(conversationId: string, kind: AttachmentReferenceKind, ownerId: string, ids: string[], outputHandles: string[] = []): void {
const requiredOutputs = new Map<string, AssetRow>()
for (const handle of new Set(outputHandles)) {
const row = this.row(handle.slice(handle.lastIndexOf(':') + 1))
const descriptor = JSON.parse(row.metadata)
if (descriptor.outputHandle !== handle || !descriptor.outputFinished) throw new Error('Full output is missing or unfinished')
requiredOutputs.set(row.id, row)
}
const assets: AssetRow[] = []
for (const id of new Set(ids)) {
const asset = this.database.prepare('SELECT * FROM attachments WHERE id = ?').get(id) as AssetRow | undefined
if (asset) assets.push(asset)
if (asset) assets.push({ ...asset, path: portablePath(asset.path) })
}
// Renderer snapshots can lag owner adoption. Only explicit history release or
// reconciliation may remove these refs; replacing the attachment list cannot.
const retainedOutputs = this.database.prepare(`SELECT a.* FROM attachments a
JOIN attachment_refs r ON r.attachment_id = a.id
WHERE r.kind = ? AND r.owner_id = ? AND json_extract(a.metadata, '$.outputHandle') IS NOT NULL
ORDER BY r.rowid`).all(kind, ownerId) as AssetRow[]
const included = new Set(assets.map(asset => asset.id))
for (const asset of [...requiredOutputs.values(), ...retainedOutputs]) {
if (included.has(asset.id)) continue
included.add(asset.id)
assets.push({ ...asset, path: portablePath(asset.path) })
}
const refs = this.database.prepare(`SELECT attachment_id, conversation_id FROM attachment_refs
WHERE kind = ? AND owner_id = ? ORDER BY rowid`).all(kind, ownerId) as { attachment_id: string; conversation_id: string }[]
@@ -131,7 +156,7 @@ export class ConversationAttachmentStorage {
for (const asset of assets) {
if (!asset.path.startsWith(`temp`)) continue
const context = JSON.parse(asset.metadata) as ContextAttachment
const target = join('conversation-assets', conversationId, context.kind === 'image' ? 'images' : 'documents', asset.id)
const target = portablePath(join('conversation-assets', conversationId, context.kind === 'image' ? 'images' : 'documents', asset.id))
mkdirSync(join(this.root, 'conversation-assets', conversationId, context.kind === 'image' ? 'images' : 'documents'), { recursive: true })
renameSync(join(this.root, asset.path), join(this.root, target))
try {
@@ -157,6 +182,7 @@ export class ConversationAttachmentStorage {
}
release(kind: 'draft' | 'message' | 'queue' | 'parsing', ownerId: string): void {
// Authoritative owner deletion/cancellation, including adopted full outputs.
this.database.prepare('DELETE FROM attachment_refs WHERE kind = ? AND owner_id = ?').run(kind, ownerId)
}
@@ -165,8 +191,18 @@ export class ConversationAttachmentStorage {
this.collect(protectedIds)
}
deleteProject(conversationIds: string[], protectedIds: readonly string[] = []): void {
this.database.exec('BEGIN IMMEDIATE')
try {
const remove = this.database.prepare('DELETE FROM attachment_refs WHERE conversation_id = ?')
for (const id of new Set(conversationIds)) remove.run(id)
this.database.exec('COMMIT')
} catch (error) { this.database.exec('ROLLBACK'); throw error }
this.collect(protectedIds)
}
private collectOrphanDirectories(): void {
const owned = new Set((this.database.prepare('SELECT path FROM attachments').all() as { path: string }[]).map((row) => row.path))
const owned = new Set((this.database.prepare('SELECT path FROM attachments').all() as { path: string }[]).map((row) => portablePath(row.path)))
const directories = (path: string): string[] => {
try { return readdirSync(path, { withFileTypes: true }).filter((entry) => entry.isDirectory() && /^[0-9a-f-]{36}$/iu.test(entry.name)).map((entry) => entry.name) }
catch (error) { if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; throw error }
@@ -178,7 +214,7 @@ export class ConversationAttachmentStorage {
}
}
for (const path of candidates) {
if (owned.has(path) || this.results.isTemporaryResult(path.split(/[\\/]/u).at(-1)!)) continue
if (owned.has(portablePath(path)) || this.results.isTemporaryResult(path.split(/[\\/]/u).at(-1)!)) continue
rmSync(join(this.root, path), { recursive: true, force: true })
}
}
@@ -189,20 +225,24 @@ export class ConversationAttachmentStorage {
(SELECT 1 FROM attachment_refs r WHERE r.attachment_id = a.id)`).all() as AssetRow[]
for (const row of rows) {
if (protectedSet.has(row.id)) continue
try {
rmSync(join(this.root, row.path), { recursive: true, force: true })
this.database.prepare('DELETE FROM attachments WHERE id = ?').run(row.id)
} catch {
// The unreferenced row remains authoritative for the next cleanup attempt.
console.warn('Attachment cleanup deferred; unreferenced files will be retried on the next cleanup')
}
this.collectRow(row)
}
}
private collectRow(row: AssetRow): void {
try {
rmSync(join(this.root, portablePath(row.path)), { recursive: true, force: true })
this.database.prepare('DELETE FROM attachments WHERE id = ?').run(row.id)
} catch {
// The unreferenced row remains authoritative for the next cleanup attempt.
console.warn('Attachment cleanup deferred; unreferenced files will be retried on the next cleanup')
}
}
reconcile(hasOwner: (conversationId: string, kind: string, ownerId: string) => boolean, startup = false, protectedIds: readonly string[] = []): void {
const owners = this.database.prepare('SELECT DISTINCT conversation_id, kind, owner_id FROM attachment_refs').all() as { conversation_id: string; kind: string; owner_id: string }[]
for (const owner of owners) {
if (!hasOwner(owner.conversation_id, owner.kind, owner.owner_id)) {
if (owner.kind === 'output-call' ? startup : !hasOwner(owner.conversation_id, owner.kind, owner.owner_id)) {
this.database.prepare('DELETE FROM attachment_refs WHERE conversation_id = ? AND kind = ? AND owner_id = ?').run(owner.conversation_id, owner.kind, owner.owner_id)
}
}
@@ -229,5 +269,76 @@ export class ConversationAttachmentStorage {
WHERE r.kind = 'parsing' AND r.conversation_id = ? AND json_extract(a.metadata, '$.parsingState') IN ('failed', 'interrupted')`).all(conversationId) as { id: string }[]).map((row) => this.get(row.id))
}
private output(ownerId: string, handle: string, requireCall = true): AssetRow {
const row = this.row(handle.slice(handle.lastIndexOf(':') + 1))
const descriptor = JSON.parse(row.metadata)
const call = this.database.prepare("SELECT 1 FROM attachment_refs WHERE attachment_id = ? AND kind = 'output-call' AND owner_id = ?").get(row.id, ownerId)
const history = !requireCall && this.database.prepare('SELECT 1 FROM attachment_refs WHERE attachment_id = ? AND conversation_id = ?').get(row.id, ownerId)
if (descriptor.outputHandle !== handle || (!call && !history)) throw new Error('Output does not belong to this owner')
return row
}
outputCreate(ownerId: string, handle: string): void {
const id = handle.slice(handle.lastIndexOf(':') + 1)
if (!/^[0-9a-f-]{36}$/iu.test(id)) throw new Error('Invalid output handle')
const context = { id, name: 'output.txt', kind: 'text' as const, size: 0, preview: '', outputHandle: handle, outputFinished: false }
this.save(context, JSON.stringify(context), Buffer.alloc(0))
try {
this.database.prepare('INSERT INTO attachment_refs VALUES (?, ?, ?, ?)').run(id, '', 'output-call', ownerId)
} catch (error) { this.collectRow(this.row(id)); throw error }
}
async outputAppend(ownerId: string, handle: string, bytes: Uint8Array): Promise<void> {
if (bytes.byteLength > 64 * 1024) throw new Error('Output append exceeds 64 KiB')
const row = this.output(ownerId, handle)
if (JSON.parse(row.metadata).outputFinished) throw new Error('Output is already finished')
await appendFile(join(this.root, row.path, 'original.txt'), bytes)
}
async outputFinish(ownerId: string, handle: string, totalBytes: number): Promise<void> {
const row = this.output(ownerId, handle)
const file = await stat(join(this.root, row.path, 'original.txt'))
if (!Number.isSafeInteger(totalBytes) || totalBytes < 0 || file.size !== totalBytes) throw new Error('Incomplete stored output')
this.database.prepare('UPDATE attachments SET metadata = ? WHERE id = ?').run(
JSON.stringify({ ...JSON.parse(row.metadata), size: totalBytes, outputFinished: true }), row.id)
}
async outputRead(ownerId: string, handle: string, cursor: number, length: number): Promise<Uint8Array> {
if (!Number.isSafeInteger(cursor) || cursor < 0 || !Number.isSafeInteger(length) || length < 0 || length > 32 * 1024 + 3) throw new Error('Invalid output page')
const row = this.output(ownerId, handle, false)
const file = await open(join(this.root, row.path, 'original.txt'), 'r')
try {
const bytes = Buffer.alloc(length)
const { bytesRead } = await file.read(bytes, 0, length, cursor)
return bytes.subarray(0, bytesRead)
} finally { await file.close() }
}
outputAdopt(ownerId: string, handle: string, conversationId: string, kind: AttachmentReferenceKind, referenceId: string): string {
const row = this.output(ownerId, handle)
if (!JSON.parse(row.metadata).outputFinished) throw new Error('Output capture has not finished')
const existing = this.database.prepare('SELECT attachment_id FROM attachment_refs WHERE kind = ? AND owner_id = ? ORDER BY rowid').all(kind, referenceId) as { attachment_id: string }[]
this.reference(conversationId, kind, referenceId, [...existing.map(item => item.attachment_id), row.id])
return row.id
}
outputRelease(ownerId: string, handle: string): void {
const id = handle.slice(handle.lastIndexOf(':') + 1)
if (!this.has(id)) return
const row = this.row(id)
if (JSON.parse(row.metadata).outputHandle !== handle) throw new Error('Invalid output handle')
this.database.prepare("DELETE FROM attachment_refs WHERE attachment_id = ? AND kind = 'output-call' AND owner_id = ?").run(id, ownerId)
// Releasing one call must not collect Main's not-yet-referenced attachments.
if (!this.database.prepare('SELECT 1 FROM attachment_refs WHERE attachment_id = ?').get(id)) this.collectRow(row)
}
releaseOutputs(): void {
const calls = this.database.prepare("SELECT attachment_id, owner_id FROM attachment_refs WHERE kind = 'output-call'").all() as { attachment_id: string; owner_id: string }[]
for (const call of calls) {
const descriptor = JSON.parse(this.row(call.attachment_id).metadata)
this.outputRelease(call.owner_id, descriptor.outputHandle)
}
}
close(): void { this.database.close() }
}