feat: separate system task token usage and add usage refresh

Add a System tasks tab to run history that splits background usage (supervisor, auto-supervision, magic notes, knowledge base, delegation) from conversation usage, with a manual refresh button and automatic refresh when usage is recorded. Record previously dropped usage from supervisor reviews, image regeneration, knowledge graph extraction, and remote embeddings.
This commit is contained in:
mesalogo
2026-09-29 10:38:37 +08:00
parent 3c9229ac7a
commit 9a9170441e
24 changed files with 991 additions and 286 deletions
@@ -96,6 +96,25 @@ describe('createModelGraphExtractor', () => {
}
)
it.each([
['anthropic-messages', { content: [{ type: 'text', text: '{}' }], model: 'claude-x',
usage: { input_tokens: 11, output_tokens: 3, cache_read_input_tokens: 2, cache_creation_input_tokens: 1 } },
{ provider: 'anthropic', model: 'claude-x', input: 11, output: 3, cacheRead: 2, cacheWrite: 1 }],
['openai-chat-completions', { choices: [{ message: { content: '{}' } }],
usage: { prompt_tokens: 9, completion_tokens: 4, prompt_tokens_details: { cached_tokens: 5 } } },
{ provider: 'openai', model: 'intranet-model', input: 9, output: 4, cacheRead: 5, cacheWrite: 0 }]
] as const)('reports %s token usage', async (protocol, payload, expected) => {
const onUsage = vi.fn()
const extract = createModelGraphExtractor(
store({ modelProtocol: protocol }),
vi.fn<typeof fetch>(async () => jsonResponse(payload)),
undefined,
onUsage
)
await expect(extract('extract this')).resolves.toEqual({})
expect(onUsage).toHaveBeenCalledWith(expected)
})
it('uses an unauthenticated Anthropic endpoint with its path and query', async () => {
const fetcher = vi.fn(async () =>
jsonResponse({
+67 -1
View File
@@ -158,10 +158,64 @@ function completionDetail(payload: unknown): string | undefined {
: undefined
}
export type GraphExtractionUsage = {
provider: 'anthropic' | 'openai'
model: string
input: number
output: number
cacheRead: number
cacheWrite: number
}
function tokenCount(value: unknown): number {
return Number.isSafeInteger(value) && (value as number) >= 0
? value as number
: 0
}
/** Reads provider-reported token usage from a non-streaming response. */
export function readGraphExtractionUsage(
payload: unknown,
protocol: string,
fallbackModel: string
): GraphExtractionUsage | undefined {
const response = record(payload)
const usage = record(response?.usage)
if (!usage) {
return undefined
}
const model =
typeof response?.model === 'string' && response.model.length > 0
? response.model.slice(0, 500)
: fallbackModel.slice(0, 500)
if (protocol === 'anthropic-messages') {
return {
provider: 'anthropic',
model,
input: tokenCount(usage.input_tokens),
output: tokenCount(usage.output_tokens),
cacheRead: tokenCount(usage.cache_read_input_tokens),
cacheWrite: tokenCount(usage.cache_creation_input_tokens)
}
}
const details = record(
usage.prompt_tokens_details ?? usage.input_tokens_details
)
return {
provider: 'openai',
model,
input: tokenCount(usage.prompt_tokens ?? usage.input_tokens),
output: tokenCount(usage.completion_tokens ?? usage.output_tokens),
cacheRead: tokenCount(details?.cached_tokens),
cacheWrite: 0
}
}
export function createModelGraphExtractor(
settingsStore: RuntimeSettingsStore,
fetcher: typeof fetch = fetch,
requestTimeoutMilliseconds = defaultGraphRequestTimeoutMilliseconds
requestTimeoutMilliseconds = defaultGraphRequestTimeoutMilliseconds,
onUsage?: (usage: GraphExtractionUsage) => void
): ExtractStructured {
return async (prompt, signal) => {
const settings = await settingsStore.getResolvedSettings()
@@ -289,6 +343,18 @@ export function createModelGraphExtractor(
{ cause: error }
)
}
const usage = readGraphExtractionUsage(
payload,
protocol,
settings.modelName
)
if (usage && onUsage) {
try {
onUsage(usage)
} catch {
// Usage accounting must never fail a successful extraction.
}
}
const text =
protocol === 'anthropic-messages'
? anthropicText(payload)
@@ -42,6 +42,16 @@ describe('OpenAIEmbeddingClient', () => {
})
})
it('reports provider token usage per batch without failing on observer errors', async () => {
const transport = vi.fn<typeof fetch>(async () =>
new Response(JSON.stringify({ data: [{ index: 0, embedding: [1, 2] }], usage: { prompt_tokens: 7, total_tokens: 7 } }))
)
const onUsage = vi.fn(() => { throw new Error('observer failed') })
const client = new OpenAIEmbeddingClient({ endpoint: 'http://127.0.0.1/v1/embeddings', model: 'embed', fetch: transport, onUsage })
await expect(client.embed(['alpha'])).resolves.toEqual([[1, 2]])
expect(onUsage).toHaveBeenCalledWith({ model: 'embed', inputTokens: 7 })
})
it('accepts unauthenticated endpoints and restores response index order', async () => {
const transport = vi.fn<typeof fetch>(async () =>
new Response(
+27 -4
View File
@@ -22,6 +22,8 @@ export interface OpenAIEmbeddingClientOptions {
batchSize?: number
timeoutMs?: number
fetch?: typeof fetch
/** Receives provider-reported input tokens for each successful batch. */
onUsage?: (usage: { model: string; inputTokens: number }) => void
}
export type EmbeddingInputRole = 'query' | 'document'
@@ -268,8 +270,10 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
private readonly batchSize: number
private readonly timeoutMs: number
private readonly transport: typeof fetch
private readonly onUsage?: OpenAIEmbeddingClientOptions['onUsage']
constructor(options: OpenAIEmbeddingClientOptions) {
this.onUsage = options.onUsage
this.endpoint = normalizedEndpoint(options.endpoint)
this.model = requiredString(options.model, 'model', MAX_MODEL_LENGTH)
this.apiKey = options.apiKey?.trim() || undefined
@@ -395,6 +399,26 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
return embeddings
}
private reportUsage(payload: unknown): void {
if (!this.onUsage || typeof payload !== 'object' || payload === null) {
return
}
const usage = (payload as { usage?: unknown }).usage
if (typeof usage !== 'object' || usage === null) {
return
}
const values = usage as Record<string, unknown>
const tokens = values.prompt_tokens ?? values.total_tokens
if (!Number.isSafeInteger(tokens) || (tokens as number) <= 0) {
return
}
try {
this.onUsage({ model: this.model, inputTokens: tokens as number })
} catch {
// Usage accounting must never fail an embedding request.
}
}
private async embedBatch(
input: readonly string[],
signal?: AbortSignal
@@ -439,10 +463,9 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
`Embedding request failed with HTTP ${response.status}`
)
}
const vectors = validateEmbeddings(
await readBoundedJson(response),
input.length
)
const payload = await readBoundedJson(response)
const vectors = validateEmbeddings(payload, input.length)
this.reportUsage(payload)
if (
this.dimensions !== undefined &&
vectors.some((vector) => vector.length !== this.dimensions)