fix: retry unusable model output in smaller review batches

Leaf batches retry at half and a quarter of the text budget, navigation merges retry once; provider errors are not retried. Adds the long-session acceptance script and records the results.
This commit is contained in:
mesalogo
2026-10-02 21:48:29 +08:00
parent 7aad78752b
commit 0fd2f7e3c8
4 changed files with 120 additions and 15 deletions
@@ -249,6 +249,43 @@ it('reopens saved batches after failure, resumes exact Unicode offsets and only
expect(f.summarize).toHaveBeenCalledTimes(calls)
})
it('retries an unusable leaf answer on a smaller part of the same text without skipping any of it', async () => {
const f = await fixture([['y'.repeat(2000)]], { concurrency: 1 })
// Invalid JSON, then an answer citing a source that is not in the batch, then usable answers.
f.summarize.mockResolvedValueOnce('not json' as never).mockImplementationOnce(async () => ({ ...empty,
events: [{ title: 'Bad', description: 'x', occurredAt: '2026-09-21T00:00:00.000Z', eventType: 'discussion' as const, entityIds: [], sourceReferenceIds: ['missing'] }] }))
const result = await f.service().run(request)
expect(result.status).toBe('completed')
const evidence = f.db.supervisionReviewStore().batches(result.runId!, 100).flatMap(row => row.evidence)
// Third attempt (a quarter of 1000, at least 500) succeeded; the rest of the text continued at full size.
expect(evidence.map(item => item.content.length)).toEqual([500, 1000, 500])
expect(evidence.map(item => item.content).join('')).toBe('y'.repeat(2000))
})
it('asks a navigation merge again once when its answer is unusable', async () => {
const f = await fixture([['n'.repeat(1500)]], { concurrency: 1 })
// Two leaves succeed; the merge first returns invalid JSON, then a usable answer.
f.summarize.mockImplementation(async (input: SupervisorSummarizerRequest) => input.evidence[0]?.sourceType === 'note' ? empty
: { ...empty, events: input.evidence.map(source => ({ title: 'Fact', description: 'x', occurredAt: source.occurredAt, eventType: 'discussion' as const, entityIds: [], sourceReferenceIds: [source.id] })) })
let navigation = 0
const base = f.summarize.getMockImplementation()!
f.summarize.mockImplementation(async input => input.evidence[0]?.sourceType === 'note' && navigation++ === 0 ? 'not json' as never : base(input))
const result = await f.service().run(request)
expect(result.status).toBe('completed')
expect(navigation).toBe(2)
})
it('fails after the retries and does not retry provider errors', async () => {
const f = await fixture([['z'.repeat(1500)]], { concurrency: 1 })
f.summarize.mockResolvedValue('not json' as never)
await expect(f.service().run(request)).rejects.toThrow('无效 JSON')
expect(f.summarize).toHaveBeenCalledTimes(3)
const g = await fixture([['z'.repeat(1500)]], { concurrency: 1 })
g.summarize.mockRejectedValueOnce(new Error('Provider down'))
await expect(g.service().run(request)).rejects.toThrow('Provider down')
expect(g.summarize).toHaveBeenCalledTimes(1)
})
it('continues extraction and navigation beyond the saved execution budget', async () => {
const f = await fixture([['x'.repeat(2500)]], { concurrency: 1, executionSeconds: 30 })
const clock = vi.spyOn(Date, 'now').mockReturnValue(time)