feat: separate system task token usage and add usage refresh
Add a System tasks tab to run history that splits background usage (supervisor, auto-supervision, magic notes, knowledge base, delegation) from conversation usage, with a manual refresh button and automatic refresh when usage is recorded. Record previously dropped usage from supervisor reviews, image regeneration, knowledge graph extraction, and remote embeddings.
This commit is contained in:
@@ -96,6 +96,25 @@ describe('createModelGraphExtractor', () => {
|
||||
}
|
||||
)
|
||||
|
||||
it.each([
|
||||
['anthropic-messages', { content: [{ type: 'text', text: '{}' }], model: 'claude-x',
|
||||
usage: { input_tokens: 11, output_tokens: 3, cache_read_input_tokens: 2, cache_creation_input_tokens: 1 } },
|
||||
{ provider: 'anthropic', model: 'claude-x', input: 11, output: 3, cacheRead: 2, cacheWrite: 1 }],
|
||||
['openai-chat-completions', { choices: [{ message: { content: '{}' } }],
|
||||
usage: { prompt_tokens: 9, completion_tokens: 4, prompt_tokens_details: { cached_tokens: 5 } } },
|
||||
{ provider: 'openai', model: 'intranet-model', input: 9, output: 4, cacheRead: 5, cacheWrite: 0 }]
|
||||
] as const)('reports %s token usage', async (protocol, payload, expected) => {
|
||||
const onUsage = vi.fn()
|
||||
const extract = createModelGraphExtractor(
|
||||
store({ modelProtocol: protocol }),
|
||||
vi.fn<typeof fetch>(async () => jsonResponse(payload)),
|
||||
undefined,
|
||||
onUsage
|
||||
)
|
||||
await expect(extract('extract this')).resolves.toEqual({})
|
||||
expect(onUsage).toHaveBeenCalledWith(expected)
|
||||
})
|
||||
|
||||
it('uses an unauthenticated Anthropic endpoint with its path and query', async () => {
|
||||
const fetcher = vi.fn(async () =>
|
||||
jsonResponse({
|
||||
|
||||
@@ -158,10 +158,64 @@ function completionDetail(payload: unknown): string | undefined {
|
||||
: undefined
|
||||
}
|
||||
|
||||
export type GraphExtractionUsage = {
|
||||
provider: 'anthropic' | 'openai'
|
||||
model: string
|
||||
input: number
|
||||
output: number
|
||||
cacheRead: number
|
||||
cacheWrite: number
|
||||
}
|
||||
|
||||
function tokenCount(value: unknown): number {
|
||||
return Number.isSafeInteger(value) && (value as number) >= 0
|
||||
? value as number
|
||||
: 0
|
||||
}
|
||||
|
||||
/** Reads provider-reported token usage from a non-streaming response. */
|
||||
export function readGraphExtractionUsage(
|
||||
payload: unknown,
|
||||
protocol: string,
|
||||
fallbackModel: string
|
||||
): GraphExtractionUsage | undefined {
|
||||
const response = record(payload)
|
||||
const usage = record(response?.usage)
|
||||
if (!usage) {
|
||||
return undefined
|
||||
}
|
||||
const model =
|
||||
typeof response?.model === 'string' && response.model.length > 0
|
||||
? response.model.slice(0, 500)
|
||||
: fallbackModel.slice(0, 500)
|
||||
if (protocol === 'anthropic-messages') {
|
||||
return {
|
||||
provider: 'anthropic',
|
||||
model,
|
||||
input: tokenCount(usage.input_tokens),
|
||||
output: tokenCount(usage.output_tokens),
|
||||
cacheRead: tokenCount(usage.cache_read_input_tokens),
|
||||
cacheWrite: tokenCount(usage.cache_creation_input_tokens)
|
||||
}
|
||||
}
|
||||
const details = record(
|
||||
usage.prompt_tokens_details ?? usage.input_tokens_details
|
||||
)
|
||||
return {
|
||||
provider: 'openai',
|
||||
model,
|
||||
input: tokenCount(usage.prompt_tokens ?? usage.input_tokens),
|
||||
output: tokenCount(usage.completion_tokens ?? usage.output_tokens),
|
||||
cacheRead: tokenCount(details?.cached_tokens),
|
||||
cacheWrite: 0
|
||||
}
|
||||
}
|
||||
|
||||
export function createModelGraphExtractor(
|
||||
settingsStore: RuntimeSettingsStore,
|
||||
fetcher: typeof fetch = fetch,
|
||||
requestTimeoutMilliseconds = defaultGraphRequestTimeoutMilliseconds
|
||||
requestTimeoutMilliseconds = defaultGraphRequestTimeoutMilliseconds,
|
||||
onUsage?: (usage: GraphExtractionUsage) => void
|
||||
): ExtractStructured {
|
||||
return async (prompt, signal) => {
|
||||
const settings = await settingsStore.getResolvedSettings()
|
||||
@@ -289,6 +343,18 @@ export function createModelGraphExtractor(
|
||||
{ cause: error }
|
||||
)
|
||||
}
|
||||
const usage = readGraphExtractionUsage(
|
||||
payload,
|
||||
protocol,
|
||||
settings.modelName
|
||||
)
|
||||
if (usage && onUsage) {
|
||||
try {
|
||||
onUsage(usage)
|
||||
} catch {
|
||||
// Usage accounting must never fail a successful extraction.
|
||||
}
|
||||
}
|
||||
const text =
|
||||
protocol === 'anthropic-messages'
|
||||
? anthropicText(payload)
|
||||
|
||||
@@ -42,6 +42,16 @@ describe('OpenAIEmbeddingClient', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('reports provider token usage per batch without failing on observer errors', async () => {
|
||||
const transport = vi.fn<typeof fetch>(async () =>
|
||||
new Response(JSON.stringify({ data: [{ index: 0, embedding: [1, 2] }], usage: { prompt_tokens: 7, total_tokens: 7 } }))
|
||||
)
|
||||
const onUsage = vi.fn(() => { throw new Error('observer failed') })
|
||||
const client = new OpenAIEmbeddingClient({ endpoint: 'http://127.0.0.1/v1/embeddings', model: 'embed', fetch: transport, onUsage })
|
||||
await expect(client.embed(['alpha'])).resolves.toEqual([[1, 2]])
|
||||
expect(onUsage).toHaveBeenCalledWith({ model: 'embed', inputTokens: 7 })
|
||||
})
|
||||
|
||||
it('accepts unauthenticated endpoints and restores response index order', async () => {
|
||||
const transport = vi.fn<typeof fetch>(async () =>
|
||||
new Response(
|
||||
|
||||
@@ -22,6 +22,8 @@ export interface OpenAIEmbeddingClientOptions {
|
||||
batchSize?: number
|
||||
timeoutMs?: number
|
||||
fetch?: typeof fetch
|
||||
/** Receives provider-reported input tokens for each successful batch. */
|
||||
onUsage?: (usage: { model: string; inputTokens: number }) => void
|
||||
}
|
||||
|
||||
export type EmbeddingInputRole = 'query' | 'document'
|
||||
@@ -268,8 +270,10 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
|
||||
private readonly batchSize: number
|
||||
private readonly timeoutMs: number
|
||||
private readonly transport: typeof fetch
|
||||
private readonly onUsage?: OpenAIEmbeddingClientOptions['onUsage']
|
||||
|
||||
constructor(options: OpenAIEmbeddingClientOptions) {
|
||||
this.onUsage = options.onUsage
|
||||
this.endpoint = normalizedEndpoint(options.endpoint)
|
||||
this.model = requiredString(options.model, 'model', MAX_MODEL_LENGTH)
|
||||
this.apiKey = options.apiKey?.trim() || undefined
|
||||
@@ -395,6 +399,26 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
|
||||
return embeddings
|
||||
}
|
||||
|
||||
private reportUsage(payload: unknown): void {
|
||||
if (!this.onUsage || typeof payload !== 'object' || payload === null) {
|
||||
return
|
||||
}
|
||||
const usage = (payload as { usage?: unknown }).usage
|
||||
if (typeof usage !== 'object' || usage === null) {
|
||||
return
|
||||
}
|
||||
const values = usage as Record<string, unknown>
|
||||
const tokens = values.prompt_tokens ?? values.total_tokens
|
||||
if (!Number.isSafeInteger(tokens) || (tokens as number) <= 0) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.onUsage({ model: this.model, inputTokens: tokens as number })
|
||||
} catch {
|
||||
// Usage accounting must never fail an embedding request.
|
||||
}
|
||||
}
|
||||
|
||||
private async embedBatch(
|
||||
input: readonly string[],
|
||||
signal?: AbortSignal
|
||||
@@ -439,10 +463,9 @@ export class OpenAIEmbeddingClient implements EmbeddingProvider {
|
||||
`Embedding request failed with HTTP ${response.status}`
|
||||
)
|
||||
}
|
||||
const vectors = validateEmbeddings(
|
||||
await readBoundedJson(response),
|
||||
input.length
|
||||
)
|
||||
const payload = await readBoundedJson(response)
|
||||
const vectors = validateEmbeddings(payload, input.length)
|
||||
this.reportUsage(payload)
|
||||
if (
|
||||
this.dimensions !== undefined &&
|
||||
vectors.some((vector) => vector.length !== this.dimensions)
|
||||
|
||||
Reference in New Issue
Block a user