feat(storage): move desktop SQLite ownership into a storage utility process

Main no longer opens desktop business SQLite. A single utility-process storage owner holds the Assistant, Knowledge, attachment, document-result, remote-binding and native model-ledger databases and runs whole domain transactions; Main and Runtime callers use explicit async domain operations. - Remove the Main readWithFallback path and the supervision worker's separate business write connection; manifests are scanned read-only and committed by the sole writer. - Remote events are ACKed only after commit; process loss reports unconfirmed writes instead of replaying them. - Use native utilityProcess messaging with one in-flight bound (32) and ordered waiting instead of capacity rejection and retries. - Start the storage process before the window so startup overlaps. - Keep short tool output in memory and store full output once in owned storage; paging, branching, restart and deletion reuse that copy. - Put required runtime launch files under goodbuddy-prefixed owned temp directories and reclaim confirmed stale ones; stop writing duplicate parsed.md. - Add lint guards for SQLite ownership and architecture, concurrency acceptance fixtures, and maintenance/architecture documentation. Validation: typecheck and lint pass. Alternating baseline (edf019aa) vs candidate runs show no regression on single-session, C01, C02, C04 and a 5-minute C06; Main p99 drops from ~21 ms to ~16.7 ms under concurrency. Fresh portable copy (1.75M events) upgrade content unchanged; real-model output paging passes with 4 external requests. Full npm test runs had 1-9 different timing-sensitive failures per run that pass when rerun. Not covered: C03 review publication, C07 source body, app-level C05.
2026-10-06 15:40:18 +08:00
parent bc677a3718
commit d80d42b032
229 changed files with 13632 additions and 3800 deletions
@@ -1,22 +1,20 @@
-import { mkdtemp, readdir, rm } from 'node:fs/promises'
-import { tmpdir } from 'node:os'
-import { join } from 'node:path'
-import { afterEach, describe, expect, it } from 'vitest'
+import { afterEach, describe, expect, it, vi } from 'vitest'
+import { FileOutputBacking } from '../../../tests/support/paged-output-backing'
 import { PagedOutputStore, PAGED_OUTPUT_MAX_PAGE_BYTES } from './paged-output-store'

 const stores: PagedOutputStore[] = []
-const directories: string[] = []
+const backings: FileOutputBacking[] = []
 afterEach(async () => {
  await Promise.all(stores.splice(0).map((store) => store.dispose()))
-  await Promise.all(directories.splice(0).map((directory) => rm(directory, { recursive: true, force: true })))
+  await Promise.all(backings.splice(0).map((backing) => backing.dispose()))
 })

 async function createStore() {
-  const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-output-test-'))
-  directories.push(directory)
-  const store = new PagedOutputStore('test', { temporaryParentDirectory: directory })
+  const backing = new FileOutputBacking()
+  backings.push(backing)
+  const store = new PagedOutputStore('test', { backingStore: backing })
  stores.push(store)
-  return { store, directory }
+  return { store, backing }
 }

 describe('PagedOutputStore', () => {
@@ -24,7 +22,7 @@ describe('PagedOutputStore', () => {
    const { store } = await createStore()
    const text = 'a\u4f60\ud83d\ude00\ufffd\u597dZ'
    const writer = await store.create('owner', 2)
-    for (const byte of Buffer.from(text)) writer.append(Buffer.from([byte]))
+    for (const byte of Buffer.from(text)) await writer.append(Buffer.from([byte]))
    const saved = await writer.finish()
    expect(saved.text).toBe('a')
    let output = saved.text
@@ -51,7 +49,7 @@ describe('PagedOutputStore', () => {
  it('keeps literal replacement characters in previews', async () => {
    const { store } = await createStore()
    const writer = await store.create('owner', 3)
-    writer.append('\ufffdrest')
+    await writer.append('\ufffdrest')
    const saved = await writer.finish()
    expect(saved.text).toBe('\ufffd')
    expect(saved.reference?.nextCursor).toBe(3)
@@ -60,7 +58,7 @@ describe('PagedOutputStore', () => {
  it('retains output omitted to keep JSON-escaped previews bounded', async () => {
    const { store } = await createStore()
    const writer = await store.create('owner', 12)
-    writer.append('\u0000'.repeat(12))
+    await writer.append('\u0000'.repeat(12))
    const saved = await writer.finish()
    expect(saved.text).toBe('\u0000\u0000')
    expect(saved.reference?.nextCursor).toBe(2)
@@ -69,36 +67,116 @@ describe('PagedOutputStore', () => {
  })

  it('removes small, aborted, released and disposed output files', async () => {
-    const { store, directory } = await createStore()
+    const { store, backing } = await createStore()
    const small = await store.create('owner', 10)
-    small.append('small')
+    await small.append('small')
    expect(await small.finish()).toEqual({ text: 'small', truncated: false })
+    const empty = await store.create('owner', 10)
+    expect(await empty.finish()).toEqual({ text: '', truncated: false })
+    expect(backing.createdDirectory).toBe(false)
    const retained = await store.create('owner', 1)
-    retained.append('retained')
+    await retained.append('retained')
    await retained.finish()
    const active = await store.create('owner', 1)
-    active.append('active')
+    await active.append('active')
    const other = await store.create('other', 1)
-    other.append('other')
+    await other.append('other')
    await other.finish()
-    const [childDirectory] = await readdir(directory)
    await store.releaseOwner('owner')
-    expect(await readdir(join(directory, childDirectory!))).toHaveLength(1)
+    expect(backing.count).toBe(1)
    await expect(active.finish()).rejects.toThrow()
    await expect(store.read('owner', retained.handle)).rejects.toThrow()
    await expect(store.read('other', other.handle)).resolves.toMatchObject({ content: 'other' })
    await store.create('other', 1)
    await store.dispose()
-    expect(await readdir(directory)).toEqual([])
+    expect(backing.count).toBe(0)
    await expect(store.create('owner', 1)).rejects.toThrow()
  })

-  it('cleans up when disposed during directory creation', async () => {
-    const { store, directory } = await createStore()
-    const pending = store.create('owner', 1)
-    const rejected = expect(pending).rejects.toThrow()
+  it('waits for in-flight creation and writes before disposal', async () => {
+    const { store, backing } = await createStore()
+    let resume!: () => void
+    const gate = new Promise<void>((resolve) => { resume = resolve })
+    const create = backing.create.bind(backing)
+    const creating = vi.spyOn(backing, 'create').mockImplementation(async (...args) => {
+      await gate
+      await create(...args)
+    })
+    const writer = await store.create('owner', 1)
+    const pending = writer.append('complete output')
+    await vi.waitFor(() => expect(creating).toHaveBeenCalledOnce())
+    let disposed = false
+    const disposal = store.dispose().then(() => { disposed = true })
+    await Promise.resolve()
+    expect(disposed).toBe(false)
+    resume()
+    await pending
+    await disposal
+    expect(backing.count).toBe(0)
+  })
+
+  it.each(['create', 'append', 'finish'] as const)('cleans partial backing content after %s failure', async (method) => {
+    const { store, backing } = await createStore()
+    const original = backing[method].bind(backing)
+    vi.spyOn(backing, method).mockImplementationOnce(async (...args: unknown[]) => {
+      await (original as (...args: unknown[]) => Promise<void>)(...args)
+      throw new Error('disk failure')
+    })
+    const writer = await store.create('owner', 1)
+    await writer.append('complete output').catch(() => undefined)
+    await expect(writer.finish()).rejects.toThrow()
+    expect(backing.count).toBe(0)
+  })
+
+  it('does not return a successful partial result when the owner is missing', async () => {
+    const store = new PagedOutputStore('unconfigured')
+    stores.push(store)
+    const writer = await store.create('owner', 2)
+    await writer.append('ok')
+    expect(() => writer.append('lost')).toThrow('not configured')
+    await expect(writer.finish()).rejects.toThrow()
+  })
+
+  it('bounds capture across stores and releases admission after settlement', async () => {
+    const { store, backing } = await createStore()
+    const second = new PagedOutputStore('second', { backingStore: backing })
+    stores.push(second)
+    const writer = await store.create('owner', 8 * 1024 * 1024)
+    await expect(second.create('other', 1)).rejects.toThrow('busy')
+    await writer.abort()
+    const next = await second.create('other', 1)
+    await next.append('recovered')
+    await next.finish()
+  })
+
+  it('rejects an unbounded producer and retries failed cleanup', async () => {
+    const { store, backing } = await createStore()
+    const writer = await store.create('owner', 1)
+    expect(() => writer.append(Buffer.alloc(8 * 1024 * 1024))).toThrow('busy')
+    await expect(writer.finish()).rejects.toThrow()
+    const next = await store.create('owner', 1)
+    await next.append('retained')
+    await next.finish()
+    vi.spyOn(backing, 'release').mockRejectedValueOnce(new Error('file busy'))
+    await expect(store.dispose()).rejects.toThrow('file busy')
    await store.dispose()
-    await rejected
-    expect(await readdir(directory)).toEqual([])
+    expect(backing.count).toBe(0)
+  })
+
+  it('streams more than the memory budget to one backing object in bounded writes', async () => {
+    const { store, backing } = await createStore()
+    const writer = await store.create('owner', 1024)
+    const chunk = 'x'.repeat(64 * 1024)
+    for (let i = 0; i < 160; i += 1) await writer.append(chunk)
+    const saved = await writer.finish()
+    expect(saved.reference?.totalBytes).toBe(10 * 1024 * 1024)
+    expect(backing.count).toBe(1)
+    expect(backing.maximumWriteBytes).toBeLessThanOrEqual(64 * 1024)
+    let cursor = 0
+    while (cursor < saved.reference!.totalBytes) {
+      const page = await store.read('owner', writer.handle, cursor)
+      expect(page.content).toBe('x'.repeat(page.nextCursor - cursor))
+      cursor = page.nextCursor
+    }
  })
 })