fix: retry transient supervisor model failures once

src/main/assistant/supervision-model-retry.test.ts

@@ -0,0 +1,231 @@
+import { afterEach, expect, it, vi } from 'vitest'
+import type { AgentRuntime } from '../agent/runtime'
+import { ModelAgentRuntime, noModelTools } from '../agent/model-runtime'
+import { AssistantDatabase } from './assistant-database'
+import { SupervisionModelPool } from './supervision-model-pool'
+import { createProductionSuggestionPhraser } from './supervision-production'
+
+const cleanups: Array<() => void> = []
+afterEach(() => {
+  for (const cleanup of cleanups.splice(0)) cleanup()
+  vi.useRealTimers()
+})
+
+function fixture(run: AgentRuntime['run'], timeoutSeconds = 30) {
+  vi.useFakeTimers()
+  const db = new AssistantDatabase(':memory:')
+  db.initialize(process.cwd())
+  const pool = new SupervisionModelPool()
+  const release = vi.fn<NonNullable<AgentRuntime['releaseConversation']>>(async () => {})
+  const dispose = vi.fn(async () => {})
+  const runtime = { run: vi.fn(run), releaseConversation: release, dispose } as unknown as AgentRuntime
+  const statuses = vi.spyOn(db, 'updateTaskStatus')
+  const phrase = createProductionSuggestionPhraser(db, async () => ({ supervisorModelConcurrency: 1,
+    supervisorOrganizeTimeoutSeconds: timeoutSeconds }), async () => runtime, pool)
+  cleanups.push(() => { pool.dispose(); db.close() })
+  return { db, pool, runtime, release, dispose, statuses,
+    start: () => phrase({ systemInstruction: 'Suggest', outputContract: '{}', candidates: [] }) }
+}
+
+it.each([
+  Object.assign(new Error('Slow down'), { status: 429 }),
+  Object.assign(new Error('Gateway unavailable'), { statusCode: 502 }),
+  new TypeError('fetch failed', { cause: Object.assign(new Error('socket closed'), { code: 'ECONNRESET' }) }),
+  new TypeError('terminated'),
+  '模型接口请求失败（HTTP 503）',
+  'Gateway Timeout (HTTP 504)',
+  'TypeError: terminated',
+  '模型接口流式响应意外中断',
+  'Rate limit exceeded'
+])('retries a transient failure once with fresh text, usage identities and cleanup: %s', async failure => {
+  let calls = 0
+  const finalized = vi.fn()
+  const f = fixture(async function* (input, signal) {
+    calls++
+    try {
+      yield { type: 'model-usage', requestId: input.requestId, callId: 'provider-call', runtime: 'model', provider: 'openai',
+        model: 'fake', inputTokens: 10, outputTokens: 2, cacheReadTokens: 0, cacheWriteTokens: 0 }
+      if (calls === 1) {
+        yield { type: 'text', requestId: input.requestId, delta: 'discard partial {' }
+        if (typeof failure === 'string') yield { type: 'error', status: 'failed', requestId: input.requestId, message: failure }
+        else throw failure
+      } else {
+        expect(f.release).toHaveBeenCalledTimes(1)
+        expect(signal.aborted).toBe(false)
+        yield { type: 'text', requestId: input.requestId, delta: 'clean result' }
+        yield { type: 'done', requestId: input.requestId }
+      }
+    } finally { finalized() }
+  })
+  const result = f.start()
+  await vi.advanceTimersByTimeAsync(499)
+  expect(calls).toBe(1)
+  expect(f.release).toHaveBeenCalledTimes(1)
+  await vi.advanceTimersByTimeAsync(1)
+  await expect(result).resolves.toBe('clean result')
+  expect(calls).toBe(2)
+  expect(finalized).toHaveBeenCalledTimes(2)
+  expect(f.release).toHaveBeenCalledTimes(2)
+  expect(new Set(f.release.mock.calls.map(args => args[0])).size).toBe(2)
+  expect(f.dispose).toHaveBeenCalledOnce()
+  expect(f.statuses.mock.calls.map(args => args[1])).toEqual(['failed', 'completed'])
+  expect(f.db.getTokenUsageSummary().systemTotals).toMatchObject({ callCount: 2, input: 20, output: 4 })
+  expect(f.db.listTasks()).toEqual([])
+  await expect(f.pool.run(async () => 'released')).resolves.toBe('released')
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it('makes exactly two calls on persistent transient failure and releases the pool', async () => {
+  const failure = Object.assign(new Error('Unavailable'), { status: 503 })
+  const f = fixture(async function* (input) { yield { type: 'status', requestId: input.requestId, message: 'starting' }; throw failure })
+  const result = expect(f.start()).rejects.toBe(failure)
+  await vi.runAllTimersAsync()
+  await result
+  expect(f.runtime.run).toHaveBeenCalledTimes(2)
+  expect(f.release).toHaveBeenCalledTimes(2)
+  expect(f.dispose).toHaveBeenCalledOnce()
+  await expect(f.pool.run(async () => true)).resolves.toBe(true)
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it.each([
+  Object.assign(new Error('Unauthorized'), { status: 401 }),
+  Object.assign(new Error('Forbidden'), { status: 403 }),
+  Object.assign(new Error('Not implemented'), { status: 501 }),
+  Object.assign(new Error('Insufficient quota'), { status: 429 }),
+  Object.assign(new Error('Context length exceeded'), { status: 503 }),
+  Object.assign(new Error('Invalid API key'), { status: 429 }),
+  new Error('模型接口请求超时', { cause: new TypeError('terminated') }),
+  new DOMException('aborted', 'AbortError'),
+  new Error('模型接口返回了无效的流式 JSON'),
+  new Error('Provider unavailable'),
+  new Error('Configuration invalid'),
+  new TypeError('fetch failed', { cause: Object.assign(new Error('bad host'), { code: 'ENOTFOUND' }) }),
+  new TypeError('fetch failed', { cause: Object.assign(new Error('bad certificate'), { code: 'CERT_HAS_EXPIRED' }) }),
+  new Error('单次模型响应超过 100 KiB'),
+  new Error('Unrelated error mentioning 429')
+])('does not retry nontransient errors, thrown or emitted: %s', async failure => {
+  for (const emitted of [false, true]) {
+    const f = fixture(async function* (input) {
+      if (emitted && !failure.cause) yield { type: 'error', status: 'failed', requestId: input.requestId, message: failure.message }
+      else throw failure
+    })
+    await expect(f.start()).rejects.toThrow(failure.message)
+    expect(f.runtime.run).toHaveBeenCalledOnce()
+    expect(f.release).toHaveBeenCalledOnce()
+    expect(f.dispose).toHaveBeenCalledOnce()
+    expect(vi.getTimerCount()).toBe(0)
+  }
+})
+
+it('cancels during the retry wait without dispatching another request', async () => {
+  const f = fixture(async function* (input) { yield { type: 'error', status: 'failed', requestId: input.requestId, message: 'terminated' } })
+  const result = expect(f.start()).rejects.toThrow()
+  await vi.advanceTimersByTimeAsync(100)
+  expect(f.release).toHaveBeenCalledOnce()
+  f.pool.cancelAll(new Error('User cancelled'))
+  await result
+  await vi.runAllTimersAsync()
+  expect(f.runtime.run).toHaveBeenCalledOnce()
+  expect(f.dispose).toHaveBeenCalledOnce()
+  await expect(f.pool.run(async () => true)).resolves.toBe(true)
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it('does not retry cancellation disguised as a transient stream error', async () => {
+  const f = fixture(async function* (input) {
+    yield { type: 'status', requestId: input.requestId, message: 'starting' }
+    f.pool.cancelAll(new Error('User cancelled'))
+    throw new TypeError('terminated')
+  })
+  await expect(f.start()).rejects.toThrow('User cancelled')
+  expect(f.runtime.run).toHaveBeenCalledOnce()
+  expect(f.statuses).toHaveBeenLastCalledWith(expect.any(String), 'cancelled', 'User cancelled')
+})
+
+it.each([false, true])('keeps a per-attempt timeout and never retries timeout (transient first: %s)', async transientFirst => {
+  let calls = 0
+  const f = fixture(async function* (input, signal) {
+    calls++
+    if (transientFirst && calls === 1) {
+      await new Promise(resolve => setTimeout(resolve, 900))
+      throw new TypeError('terminated')
+    }
+    await new Promise<void>((_, reject) => signal.addEventListener('abort', () => reject(new TypeError('terminated')), { once: true }))
+    yield { type: 'done', requestId: input.requestId }
+  }, 1)
+  const result = expect(f.start()).rejects.toThrow('超过 1 秒')
+  await vi.advanceTimersByTimeAsync(transientFirst ? 2300 : 900)
+  expect(calls).toBe(transientFirst ? 2 : 1)
+  expect(f.dispose).not.toHaveBeenCalled()
+  await vi.advanceTimersByTimeAsync(100)
+  await result
+  expect(f.release).toHaveBeenCalledTimes(transientFirst ? 2 : 1)
+  expect(f.dispose).toHaveBeenCalledOnce()
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it('does not retry a cancelled runtime error event with a transient-looking message', async () => {
+  const f = fixture(async function* (input) {
+    yield { type: 'error', status: 'cancelled', requestId: input.requestId, message: 'terminated' }
+  })
+  await expect(f.start()).rejects.toThrow('terminated')
+  expect(f.runtime.run).toHaveBeenCalledOnce()
+  expect(f.statuses).toHaveBeenLastCalledWith(expect.any(String), 'cancelled', 'terminated')
+  expect(f.release).toHaveBeenCalledOnce()
+})
+
+it('does not dispatch a retry if conversation cleanup fails', async () => {
+  const f = fixture(async function* (input) {
+    yield { type: 'error', status: 'failed', requestId: input.requestId, message: 'terminated' }
+  })
+  f.release.mockRejectedValueOnce(new Error('Cleanup failed'))
+  await expect(f.start()).rejects.toThrow('Cleanup failed')
+  expect(f.runtime.run).toHaveBeenCalledOnce()
+  expect(f.dispose).toHaveBeenCalledOnce()
+  await expect(f.pool.run(async () => true)).resolves.toBe(true)
+  expect(vi.getTimerCount()).toBe(0)
+})
+
+it.each([429, 500, 502, 503, 504])('retries actual direct-model HTTP %i with provider detail intact', async status => {
+  const detail = status === 504 ? 'Gateway Timeout' : 'Provider detail'
+  const fetcher = vi.fn<typeof fetch>()
+    .mockResolvedValueOnce(new Response(JSON.stringify({ error: { message: detail } }), { status }))
+    .mockResolvedValueOnce(new Response('data: {"choices":[{"delta":{"content":"success"}}]}\n\ndata: [DONE]\n\n'))
+  const runtime = new ModelAgentRuntime({ baseUrl: 'https://model.invalid', model: 'fake', authentication: 'none',
+    protocol: 'openai-chat-completions', toolProvider: noModelTools, fetcher })
+  const f = fixture(runtime.run.bind(runtime))
+  f.release.mockImplementation(runtime.releaseConversation.bind(runtime))
+  f.dispose.mockImplementation(runtime.dispose.bind(runtime))
+  const result = expect(f.start()).resolves.toBe('success')
+  await vi.runAllTimersAsync()
+  await result
+  expect(fetcher).toHaveBeenCalledTimes(2)
+  expect(f.statuses.mock.calls[0]?.[2]).toBe(detail)
+  expect(f.release).toHaveBeenCalledTimes(2)
+  expect(f.dispose).toHaveBeenCalledOnce()
+})
+
+it('retries an actual interrupted direct-model stream and retains its reported usage', async () => {
+  let reads = 0
+  const stream = new ReadableStream<Uint8Array>({ pull(controller) {
+    if (reads++ === 0) controller.enqueue(new TextEncoder().encode(
+      'data: {"choices":[{"delta":{"content":"discard me"}}],"usage":{"prompt_tokens":12,"completion_tokens":3}}\n\n'))
+    else controller.error(new TypeError('terminated'))
+  } })
+  const fetcher = vi.fn<typeof fetch>().mockResolvedValueOnce(new Response(stream))
+    .mockResolvedValueOnce(new Response('data: {"choices":[{"delta":{"content":"success"}}],"usage":{"prompt_tokens":14,"completion_tokens":4}}\n\ndata: [DONE]\n\n'))
+  const runtime = new ModelAgentRuntime({ baseUrl: 'https://model.invalid', model: 'fake', authentication: 'none',
+    protocol: 'openai-chat-completions', toolProvider: noModelTools, fetcher })
+  const f = fixture(runtime.run.bind(runtime))
+  f.release.mockImplementation(runtime.releaseConversation.bind(runtime))
+  f.dispose.mockImplementation(runtime.dispose.bind(runtime))
+  const result = expect(f.start()).resolves.toBe('success')
+  await vi.runAllTimersAsync()
+  await result
+  expect(fetcher).toHaveBeenCalledTimes(2)
+  expect(stream.locked).toBe(false)
+  expect(f.db.getTokenUsageSummary().systemTotals).toMatchObject({ callCount: 2, input: 26, output: 7 })
+  expect(f.release).toHaveBeenCalledTimes(2)
+  expect(f.dispose).toHaveBeenCalledOnce()
+})

src/main/assistant/supervision-production.test.ts

@@ -181,6 +181,7 @@ it('stops oversized runtime responses with a readable error while preserving res
   const f = fixture()
   f.respond.mockReturnValue({ ...empty, summary: 'x'.repeat(110_000) })
   await expect(f.service(100).run(f.request)).rejects.toThrow('单次模型响应超过 100 KiB')
+  expect(f.respond).toHaveBeenCalledOnce()
   expect(f.db.listSupervisionResults()).toHaveLength(0)
   expect(f.db.listSupervisionActivity()[0]!.reviewProgress).toMatchObject({ batches: 0, remainingSources: 1 })
   const runId = f.db.listSupervisionActivity()[0]!.id
@@ -263,7 +264,7 @@ it('production factory normalizes two absent identities, then reuses exact candi
   expect(f.respond).toHaveBeenCalledTimes(2)
 })
 
-it('retains candidate mappings across leaf batches, failed merge and resume without replaying leaves', async () => {
+it.each([false, true])('retains candidate mappings across leaf batches, failed merge and resume without replaying leaves (transient: %s)', async transient => {
   const f = fixture('Atlas and Beacon. '.repeat(100))
   f.db.saveSupervisionResult({ request: f.request, evidence: [], output: { ...empty,
     entities: ['Atlas', 'Beacon'].map(label => ({ id: label, label, description: label, sourceReferenceIds: [] })) } })
@@ -275,13 +276,15 @@ it('retains candidate mappings across leaf batches, failed merge and resume with
     if (prompt.includes('These inputs are navigation summaries')) {
       expect(prompt).toContain('KNOWN ENTITIES:\n\n[]')
       expect(prompt).toContain('"entities":[]')
-      if (failMerge) { failMerge = false; throw new Error('Merge interrupted') }
+      if (failMerge) throw transient ? Object.assign(new Error('Merge interrupted'), { status: 503 }) : new Error('Merge interrupted')
       return empty
     }
     const output = original(prompt) as { entities: Array<{ label: string }> }
     return { ...output, entities: output.entities.map(entity => ({ ...entity, persistedId: candidates.find(candidate => candidate.label === entity.label)!.id })) }
   })
   await expect(f.service().run(f.request)).rejects.toThrow('Merge interrupted')
+  expect(f.respond.mock.calls.filter(([prompt]) => prompt.includes('These inputs are navigation summaries'))).toHaveLength(transient ? 2 : 1)
+  failMerge = false
   const runId = f.db.listSupervisionActivity().find(item => item.status === 'failed')!.id
   const saved = f.db.supervisionReviewStore().batches(runId)
   expect(saved).toHaveLength(2)

src/main/assistant/supervision-production.ts

@@ -20,7 +20,33 @@ type SupervisionModelDependencies = {
   persistUsage: (event: RuntimeModelUsageEvent) => void
 }
 
-/** One bounded, tool-free text call that shares the supervisor concurrency pool. */
+function isTransientSupervisionError(error: unknown): boolean {
+  let transient = false
+  const seen = new Set<unknown>()
+  for (let current = error; current && typeof current === 'object' && !seen.has(current);) {
+    seen.add(current)
+    const record = current as Record<string, unknown>
+    const message = typeof record.message === 'string' ? record.message : ''
+    const code = typeof record.code === 'string' ? record.code : ''
+    // Direct-model errors retain status; runtime events carry only the public message.
+    const status = record.status ?? record.statusCode ?? /\bHTTP\s+(\d{3})\b/iu.exec(message)?.[1]
+    const failureText = Number(status) === 504 ? message.replace(/\bgateway (?:timeout|timed out)\b/giu, '') : message
+    if (record.name === 'AbortError' || record.name === 'TimeoutError' ||
+      /^(?:ENOTFOUND|ERR_INVALID_URL|CERT_.*|.*CERTIFICATE.*|DEPTH_ZERO_SELF_SIGNED_CERT)$/u.test(code) ||
+      /abort|cancel|timed?\s*out|timeout|超时|取消|容量|context[_ ](?:length|window)|max[_ ]tokens|quota|billing|insufficient|capacity|api[_ -]?key|unauthori[sz]ed|forbidden|authentication|configuration/iu.test(`${code} ${failureText}`)) return false
+    if (status !== undefined) {
+      if (![429, 500, 502, 503, 504].includes(Number(status))) return false
+      transient = true
+    }
+    if (/^(?:ECONNRESET|ECONNREFUSED|EPIPE|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|UND_ERR_SOCKET)$/u.test(code) ||
+      /^(?:TypeError:\s*)?(?:terminated|fetch failed|network error|socket hang up)$/iu.test(message) ||
+      message === '模型接口流式响应意外中断' || /^rate limit (?:exceeded|reached)\b/iu.test(message)) transient = true
+    current = record.cause
+  }
+  return transient
+}
+
+/** One bounded, tool-free text call, with one transient retry in the supervisor pool. */
 async function runSupervisionModel(dependencies: SupervisionModelDependencies, request: {
   prompt: string
   title: string
@@ -36,49 +62,64 @@ async function runSupervisionModel(dependencies: SupervisionModelDependencies, r
   return pool.run(async signal => {
     const timeoutSeconds = request.timeoutSeconds ?? settings?.supervisorOrganizeTimeoutSeconds ?? defaultSupervisionTimeoutSeconds
     const runtime = await resolveRuntime()
-    const controller = new AbortController()
-    const modelSignal = AbortSignal.any([signal, controller.signal, ...(request.signal ? [request.signal] : [])])
-    modelSignal.throwIfAborted()
-    let timeoutError: Error | undefined
-    const timeout = setTimeout(() => {
-      timeoutError = new Error(request.timeoutMessage(timeoutSeconds))
-      controller.abort(timeoutError)
-    }, timeoutSeconds * 1000)
-    const responseKiB = settings?.supervisionReview?.responseKiB ?? 1024
-    let output = '', completed = false
-    const requestId = randomUUID()
-    const conversationId = `supervision:${requestId}`
-    // Hidden task row so model usage can be attributed to supervision.
-    database.createTask({ id: requestId, conversationId, title: request.title, instructions: request.instructions,
-      origin: 'assistant', visible: false })
-    try {
-      for await (const event of runtime.run({ requestId, conversationId, prompt: request.prompt },
-        modelSignal, async approval => { await request.authorizeTool(approval.toolName ?? approval.scopeKey); return 'deny' })) {
-        if (event.type === 'text') {
-          output += event.delta
-          if (Buffer.byteLength(output) > responseKiB * 1024) {
-            controller.abort(new Error(`单次模型响应超过 ${responseKiB} KiB。请在监督者设置中调高响应容量后继续；已保存批次会保留。`))
-            modelSignal.throwIfAborted()
-          }
-        }
-        if (event.type === 'model-usage') persistUsage(event)
-        if (event.type === 'tool') throw new Error('监督者只允许只读模型摘要，不允许工具调用')
-        if (event.type === 'error') throw new Error(event.message)
-        if (event.type === 'done') completed = true
-      }
-      modelSignal.throwIfAborted()
-      if (!completed) throw new Error('监督者模型未报告完成')
-      database.updateTaskStatus(requestId, 'completed')
-      return output
-    } catch (error) {
-      const failure = timeoutError ?? error
+    for (let attempt = 0; ; attempt++) {
+      signal.throwIfAborted()
+      const controller = new AbortController()
+      const modelSignal = AbortSignal.any([signal, controller.signal])
+      let timeoutError: Error | undefined
+      const timeout = setTimeout(() => {
+        timeoutError = new Error(request.timeoutMessage(timeoutSeconds))
+        controller.abort(timeoutError)
+      }, timeoutSeconds * 1000)
+      const responseKiB = settings?.supervisionReview?.responseKiB ?? 1024
+      let output = '', completed = false
+      const requestId = randomUUID()
+      const conversationId = `supervision:${requestId}`
       try {
-        database.updateTaskStatus(requestId, modelSignal.aborted ? 'cancelled' : 'failed',
-          failure instanceof Error ? failure.message.slice(0, 2_000) : '监督者模型调用失败')
-      } catch { /* status bookkeeping must not mask the model failure */ }
-      throw failure
+        // Each attempt has its own hidden task so reported usage survives retries.
+        database.createTask({ id: requestId, conversationId, title: request.title, instructions: request.instructions,
+          origin: 'assistant', visible: false })
+        for await (const event of runtime.run({ requestId, conversationId, prompt: request.prompt },
+          modelSignal, async approval => { await request.authorizeTool(approval.toolName ?? approval.scopeKey); return 'deny' })) {
+          if (event.type === 'text') {
+            output += event.delta
+            if (Buffer.byteLength(output) > responseKiB * 1024) {
+              controller.abort(new Error(`单次模型响应超过 ${responseKiB} KiB。请在监督者设置中调高响应容量后继续；已保存批次会保留。`))
+              modelSignal.throwIfAborted()
+            }
+          }
+          if (event.type === 'model-usage') persistUsage(event)
+          if (event.type === 'tool') throw new Error('监督者只允许只读模型摘要，不允许工具调用')
+          if (event.type === 'error') {
+            const error = new Error(event.message)
+            if (event.status === 'cancelled') controller.abort(error)
+            throw error
+          }
+          if (event.type === 'done') completed = true
+        }
+        modelSignal.throwIfAborted()
+        if (!completed) throw new Error('监督者模型未报告完成')
+        database.updateTaskStatus(requestId, 'completed')
+        return output
+      } catch (error) {
+        const failure = timeoutError ?? (modelSignal.aborted ? modelSignal.reason : error)
+        try {
+          database.updateTaskStatus(requestId, modelSignal.aborted ? 'cancelled' : 'failed',
+            failure instanceof Error ? failure.message.slice(0, 2_000) : '监督者模型调用失败')
+        } catch { /* status bookkeeping must not mask the model failure */ }
+        if (attempt > 0 || modelSignal.aborted || !isTransientSupervisionError(failure)) throw failure
+      } finally {
+        clearTimeout(timeout)
+        controller.abort()
+        await runtime.releaseConversation?.(conversationId)
+      }
+      signal.throwIfAborted()
+      await new Promise<void>((resolve, reject) => {
+        const abort = () => { clearTimeout(wait); reject(signal.reason) }
+        const wait = setTimeout(() => { signal.removeEventListener('abort', abort); resolve() }, 500)
+        signal.addEventListener('abort', abort, { once: true })
+      })
     }
-    finally { clearTimeout(timeout); await runtime.releaseConversation?.(conversationId) }
   }, request.signal)
 }