fix: reduce supervisor stalls and preserve task delivery
Load Supervisor graphs only when opened, preserve source details on demand, reuse indexed story digests and window selection lists without losing keyboard or focus behavior. Route heavy UI reads through the existing worker, batch scoped queries and construct each publication snapshot once. Keep remote cancellation responsive during model-route cleanup, remove late routes and retain reported usage after response-cap cancellation. Preserve Telegram pending replies through polling conflicts, show delivery errors and honor the final rate-limit cooldown. Schema 62 adds lookup indexes without rewriting user content; older clients cannot open upgraded data. Large review publication still blocks Main and remains documented follow-up work. Record the future storage-service direction without performing the architecture migration. Validation: exact staged code passed isolated full tests (6212 passed, 87 skipped), typecheck and lint. Three-round source performance comparisons, SQLite migration equivalence, native Electron interaction checks and current-source Linux x64 Host validation completed; eight external text-model requests. Release note: Supervisor opens reviews without loading the entire graph, keeps long selection lists responsive and preserves details and keyboard navigation. Remote cancellation and Telegram reply recovery are more reliable; remote fixes require an Agent build containing this change.
This commit is contained in:
@@ -203,6 +203,12 @@ describe('ChannelManager', () => {
|
||||
})
|
||||
status({ state: 'running' })
|
||||
deliveryFailure(new Error('send failed'))
|
||||
expect((await manager.snapshot()).telegram.status).toEqual({ state: 'running', lastError: 'Telegram sending failed: send failed' })
|
||||
status({ state: 'running' })
|
||||
expect((await manager.snapshot()).telegram.status.lastError).toBe('Telegram sending failed: send failed')
|
||||
deliveryFailure(new Error('status-test-secret token=synthetic-secret failed'))
|
||||
expect(JSON.stringify((await manager.snapshot()).telegram.status)).not.toMatch(/status-test-secret|synthetic-secret/)
|
||||
deliverySuccess()
|
||||
expect((await manager.snapshot()).telegram.status).toEqual({ state: 'running' })
|
||||
await manager.stopAll()
|
||||
})
|
||||
|
||||
@@ -488,12 +488,16 @@ export class ChannelManager {
|
||||
settings: ResolvedChannelSettings,
|
||||
generation?: symbol
|
||||
): Promise<{ service: ManagedChannelService; driver: TestableChannelDriver }> {
|
||||
let connectionStatus: ChannelRuntimeStatus = { state: 'starting' }
|
||||
let deliveryError: string | undefined
|
||||
const onStatus = (status: ChannelRuntimeStatus): void => {
|
||||
if (generation === undefined || this.serviceGenerations.get(settings.channel) !== generation) return
|
||||
connectionStatus = status
|
||||
const lastError = status.lastError ?? deliveryError
|
||||
this.setStatus(settings.channel, {
|
||||
state: status.state,
|
||||
...(status.lastError ? {
|
||||
lastError: redactManagerError(status.lastError, [
|
||||
...(lastError ? {
|
||||
lastError: redactManagerError(lastError, [
|
||||
settings.channel === 'weixin' ? settings.token : settings.secret
|
||||
])
|
||||
} : {})
|
||||
@@ -506,7 +510,11 @@ export class ChannelManager {
|
||||
dedupStore: this.dedupStore,
|
||||
outbox: this.outbox,
|
||||
onDeliveryFailure: (error) => {
|
||||
if (settings.channel === 'telegram') return
|
||||
if (settings.channel === 'telegram') {
|
||||
deliveryError = `Telegram sending failed: ${redactManagerError(error, [settings.secret])}`
|
||||
onStatus(connectionStatus)
|
||||
return
|
||||
}
|
||||
onStatus({
|
||||
state: 'error',
|
||||
lastError: redactManagerError(error, [
|
||||
@@ -517,7 +525,11 @@ export class ChannelManager {
|
||||
})
|
||||
},
|
||||
onDeliverySuccess: () => {
|
||||
if (settings.channel === 'telegram') return
|
||||
if (settings.channel === 'telegram') {
|
||||
deliveryError = undefined
|
||||
onStatus(connectionStatus)
|
||||
return
|
||||
}
|
||||
onStatus({ state: 'running' })
|
||||
}
|
||||
})
|
||||
|
||||
@@ -318,6 +318,18 @@ describe('TelegramChannelDriver', () => {
|
||||
expect(onStatus.mock.calls.at(-1)?.[0]).toEqual({ state: 'stopped' })
|
||||
})
|
||||
|
||||
it('rechecks a transient webhook failure before polling after identifying the Bot', async () => {
|
||||
const { driver, fetch, calls, onStatus } = setup()
|
||||
fetch.mockResolvedValueOnce(ok({ id: 42, is_bot: true }))
|
||||
.mockResolvedValueOnce(error(503))
|
||||
await driver.start(vi.fn())
|
||||
await vi.waitFor(() => expect(onStatus).toHaveBeenCalledWith(expect.objectContaining({ state: 'starting', lastError: expect.stringContaining('503') })))
|
||||
expect(calls.some(call => call.method === 'getUpdates')).toBe(false)
|
||||
await vi.waitFor(() => expect(calls.some(call => call.method === 'getUpdates')).toBe(true), { timeout: 2500 })
|
||||
expect(fetch.mock.calls.map(([input]) => String(input).split('/').at(-1)))
|
||||
.toEqual(['getMe', 'getWebhookInfo', 'getWebhookInfo', 'getUpdates'])
|
||||
})
|
||||
|
||||
it('caller cancellation releases an identity waiter without cancelling startup', async () => {
|
||||
const { driver, fetch, onStatus } = setup()
|
||||
let validationSignal: AbortSignal | undefined
|
||||
|
||||
@@ -95,7 +95,11 @@ export class TelegramChannelDriver implements ChannelDriver {
|
||||
}
|
||||
|
||||
testConnection(): Promise<BotIdentity> {
|
||||
return this.track(this.validate(this.lifecycle.signal))
|
||||
return this.track((async () => {
|
||||
const identity = await this.identify(this.lifecycle.signal)
|
||||
await this.checkWebhook(this.lifecycle.signal)
|
||||
return identity
|
||||
})())
|
||||
}
|
||||
|
||||
updateAllowedSenderIds(ids: readonly string[]): void {
|
||||
@@ -184,23 +188,32 @@ export class TelegramChannelDriver implements ChannelDriver {
|
||||
}
|
||||
}
|
||||
|
||||
private async validate(signal: AbortSignal): Promise<BotIdentity> {
|
||||
private async identify(signal: AbortSignal): Promise<BotIdentity> {
|
||||
const me = await this.request<{ id: number; is_bot: boolean; username?: string }>('getMe', {}, signal)
|
||||
if (!positiveId(me?.id) || me.is_bot !== true) throw new Error('Telegram returned an invalid Bot identity.')
|
||||
return { botId: String(me.id), ...(me.username ? { botUsername: me.username } : {}) }
|
||||
}
|
||||
|
||||
private async checkWebhook(signal: AbortSignal): Promise<void> {
|
||||
const webhook = await this.request<{ url: string }>('getWebhookInfo', {}, signal)
|
||||
if (typeof webhook?.url !== 'string') throw new Error('Telegram returned invalid webhook information.')
|
||||
if (webhook.url) throw new TelegramError(409)
|
||||
return { botId: String(me.id), ...(me.username ? { botUsername: me.username } : {}) }
|
||||
}
|
||||
|
||||
private async poll(handler: ChannelInboundHandler, signal: AbortSignal): Promise<void> {
|
||||
let failures = 0
|
||||
let pollingReady = false
|
||||
while (!signal.aborted) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
if (!this.identity) {
|
||||
this.identity = await this.validate(signal)
|
||||
// Outgoing replies need the Bot identity, not permission to long-poll.
|
||||
this.identity = await this.identify(signal)
|
||||
checkSignal(signal)
|
||||
}
|
||||
if (!pollingReady) {
|
||||
await this.checkWebhook(signal)
|
||||
pollingReady = true
|
||||
for (const notify of this.readinessWaiters) notify()
|
||||
failures = 0
|
||||
this.status({ state: 'running' })
|
||||
@@ -317,8 +330,10 @@ export class TelegramChannelDriver implements ChannelDriver {
|
||||
} finally {
|
||||
clearTimeout(timer)
|
||||
}
|
||||
if (failure.code !== 429 || !retryRateLimit || attempt >= 3) throw failure
|
||||
if (failure.code !== 429 || !retryRateLimit) throw failure
|
||||
// The outbox may retry as soon as this call rejects, including the final attempt.
|
||||
await wait(failure.retryAfter * 1000, signal)
|
||||
if (attempt >= 3) throw failure
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
// @vitest-environment node
|
||||
import { createServer } from 'node:http'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { performance } from 'node:perf_hooks'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import { AssistantDatabase } from '../assistant/assistant-database'
|
||||
import { ChannelManager } from './channel-manager'
|
||||
import { ChannelSettingsStore } from './channel-settings-store'
|
||||
import { SqliteChannelOutbox } from './sqlite-channel-state'
|
||||
import { TelegramChannelDriver } from './telegram-channel-driver'
|
||||
import type { ChannelRuntimeStatusChange } from '../../shared/channel-settings-contracts'
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'goodbuddy-telegram-delivery-'))
|
||||
const database = new AssistantDatabase(':memory:')
|
||||
database.initialize(root)
|
||||
const outbox = new SqliteChannelOutbox(database)
|
||||
const settings = new ChannelSettingsStore(join(root, 'channels.json'), {
|
||||
isAvailable: () => true, encrypt: value => Buffer.from(value), decrypt: value => value.toString()
|
||||
}, {})
|
||||
const secret = 'synthetic-telegram-delivery-token'
|
||||
await settings.apply({ telegram: { enabled: true, secret: { action: 'replace', value: secret },
|
||||
allowedSenderIds: ['7'], allowGroupMessages: false } })
|
||||
const state = { webhook: '', pollingCode: 0, polls: 0,
|
||||
updates: [] as object[], sends: [] as { text: string; time: number }[],
|
||||
sendErrors: [] as { code: number; retryAfter?: number }[] }
|
||||
const server = createServer(async (request, response) => {
|
||||
let body = ''
|
||||
for await (const chunk of request) body += String(chunk)
|
||||
const method = request.url!.slice(1)
|
||||
let result: unknown
|
||||
let failure: { code: number; retryAfter?: number } | undefined
|
||||
if (method === 'getMe') result = { id: 42, is_bot: true }
|
||||
else if (method === 'getWebhookInfo') result = { url: state.webhook }
|
||||
else if (method === 'getUpdates') {
|
||||
state.polls++
|
||||
if (state.pollingCode) failure = { code: state.pollingCode }
|
||||
else result = state.updates.splice(0)
|
||||
} else if (method === 'sendMessage') {
|
||||
state.sends.push({ text: JSON.parse(body).text, time: performance.now() })
|
||||
failure = state.sendErrors.shift()
|
||||
result = { message_id: state.sends.length }
|
||||
} else if (method === 'sendChatAction') result = true
|
||||
else throw new Error('Unexpected fixture method')
|
||||
response.setHeader('content-type', 'application/json')
|
||||
response.statusCode = failure?.code ?? 200
|
||||
response.end(JSON.stringify(failure
|
||||
? { ok: false, error_code: failure.code, parameters: { retry_after: failure.retryAfter },
|
||||
description: `token=${secret} https://api.telegram.org/bot${secret}/sendMessage` }
|
||||
: { ok: true, result }))
|
||||
})
|
||||
await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
|
||||
const address = server.address() as { port: number }
|
||||
const execute = vi.fn(async () => ({ status: 'completed', output: 'synthetic result' }))
|
||||
const changes: ChannelRuntimeStatusChange[] = []
|
||||
const manager = new ChannelManager(settings, execute, {
|
||||
outbox, onStatusChanged: change => changes.push(change),
|
||||
createDriver: (resolved, onStatus) => new TelegramChannelDriver({
|
||||
secret: resolved.channel === 'telegram' ? resolved.secret! : '',
|
||||
allowedSenderIds: resolved.allowedSenderIds, onStatus,
|
||||
fetch: (input, init) => fetch(`http://127.0.0.1:${address.port}/${String(input).split('/').at(-1)}`, init)
|
||||
})
|
||||
})
|
||||
return { state, outbox, manager, execute, changes, secret,
|
||||
enqueue: (eventId: string, bot = '42') => outbox.enqueue({ channel: 'telegram', eventId,
|
||||
conversationId: `${bot}:7`, recipientId: '7', status: 'completed', output: eventId }),
|
||||
inbound: (id: number) => state.updates.push({ update_id: id, message: { message_id: id,
|
||||
date: 1234, from: { id: 7, is_bot: false }, chat: { id: 7, type: 'private' }, text: 'request' } }),
|
||||
async close() {
|
||||
await manager.stopAll()
|
||||
server.closeAllConnections()
|
||||
await new Promise<void>(resolve => server.close(() => resolve()))
|
||||
database.close()
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
describe('Telegram delivery through loopback HTTP and SQLite', () => {
|
||||
it('delivers identity-matched pending replies during a startup webhook conflict without sending another Bot results', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
f.state.webhook = 'https://example.invalid/webhook'
|
||||
f.enqueue('same-bot')
|
||||
f.enqueue('other-bot', '41')
|
||||
await f.manager.initialize()
|
||||
await vi.waitFor(async () => expect((await f.manager.snapshot()).telegram.status).toMatchObject({
|
||||
state: 'error', lastError: expect.stringContaining('polling conflict')
|
||||
}))
|
||||
await vi.waitFor(() => expect(f.outbox.listUndelivered().every(entry => entry.state === 'terminal')).toBe(true))
|
||||
expect(f.state.sends.map(send => send.text)).toEqual(['same-bot'])
|
||||
expect(f.outbox.listUndelivered().map(entry => [entry.message.eventId, entry.state])).toEqual([['other-bot', 'terminal']])
|
||||
expect(f.state.polls).toBe(0)
|
||||
f.state.webhook = ''
|
||||
await f.manager.reload('telegram')
|
||||
await vi.waitFor(async () => expect((await f.manager.snapshot()).telegram.status.state).toBe('running'))
|
||||
expect(f.state.sends.map(send => send.text)).toEqual(['same-bot'])
|
||||
expect(f.execute).not.toHaveBeenCalled()
|
||||
} finally { await f.close() }
|
||||
})
|
||||
|
||||
it('surfaces sanitized sending failures without changing polling state or clearing them on healthy polls', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
f.state.sendErrors.push({ code: 403 })
|
||||
f.inbound(1)
|
||||
await f.manager.initialize()
|
||||
await vi.waitFor(() => expect(f.outbox.listUndelivered()[0]).toMatchObject({ state: 'terminal', attempts: 1 }))
|
||||
await vi.waitFor(() => expect(f.state.polls).toBeGreaterThanOrEqual(2))
|
||||
expect((await f.manager.snapshot()).telegram.status).toEqual({ state: 'running',
|
||||
lastError: expect.stringMatching(/sending failed.*forbidden/i) })
|
||||
expect(JSON.stringify(f.changes)).not.toContain(f.secret)
|
||||
expect(JSON.stringify(f.changes)).not.toContain('api.telegram.org')
|
||||
f.inbound(2)
|
||||
await vi.waitFor(() => expect(f.state.sends).toHaveLength(2))
|
||||
await vi.waitFor(async () => expect((await f.manager.snapshot()).telegram.status).toEqual({ state: 'running' }))
|
||||
expect(f.execute).toHaveBeenCalledTimes(2)
|
||||
} finally { await f.close() }
|
||||
})
|
||||
|
||||
it('honors the latest retry_after when an exhausted driver attempt passes to the outbox', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
f.state.sendErrors.push({ code: 429, retryAfter: 1 }, { code: 429, retryAfter: 1 },
|
||||
{ code: 429, retryAfter: 1 }, { code: 429, retryAfter: 3 })
|
||||
f.inbound(1)
|
||||
await f.manager.initialize()
|
||||
await vi.waitFor(() => expect(f.state.sends).toHaveLength(5), { timeout: 10_000 })
|
||||
expect(f.state.sends[4]!.time - f.state.sends[3]!.time).toBeGreaterThanOrEqual(2995)
|
||||
await vi.waitFor(() => expect(f.outbox.listUndelivered()).toEqual([]))
|
||||
expect(f.state.sends.every(send => send.text === 'synthetic result')).toBe(true)
|
||||
expect(f.execute).toHaveBeenCalledOnce()
|
||||
} finally { await f.close() }
|
||||
}, 15_000)
|
||||
|
||||
it('cancels the final rate-limit wait without consuming or duplicating the pending reply', async () => {
|
||||
const f = await fixture()
|
||||
try {
|
||||
f.state.sendErrors.push({ code: 429, retryAfter: 1 }, { code: 429, retryAfter: 1 },
|
||||
{ code: 429, retryAfter: 1 }, { code: 429, retryAfter: 86400 })
|
||||
f.enqueue('cancelled-wait')
|
||||
await f.manager.initialize()
|
||||
await vi.waitFor(() => expect(f.state.sends).toHaveLength(4), { timeout: 5000 })
|
||||
await f.manager.stopAll()
|
||||
expect(f.outbox.listUndelivered()[0]).toMatchObject({ state: 'pending', attempts: 0 })
|
||||
expect(f.state.sends).toHaveLength(4)
|
||||
await f.manager.initialize()
|
||||
await vi.waitFor(() => expect(f.outbox.listUndelivered()).toEqual([]))
|
||||
expect(f.state.sends).toHaveLength(5)
|
||||
expect(f.execute).not.toHaveBeenCalled()
|
||||
} finally { await f.close() }
|
||||
}, 10_000)
|
||||
})
|
||||
Reference in New Issue
Block a user