149 lines
8.5 KiB
JavaScript
149 lines
8.5 KiB
JavaScript
// Knowledge retrieval benchmark (PERF-05). Bundles the production
|
|
// KnowledgeDatabase with esbuild, seeds synthetic libraries of increasing size
|
|
// in a temporary SQLite file and times the synchronous search calls that the
|
|
// Electron Main process runs inline during retrieval. Because these calls are
|
|
// synchronous, their duration is the time Main is blocked per query.
|
|
const { buildSync } = require('esbuild')
|
|
const { createHash } = require('node:crypto')
|
|
const { mkdtempSync, rmSync, writeFileSync } = require('node:fs')
|
|
const { tmpdir } = require('node:os')
|
|
const { join, resolve } = require('node:path')
|
|
const { monitorEventLoopDelay, performance } = require('node:perf_hooks')
|
|
|
|
const root = resolve(__dirname, '..')
|
|
|
|
// Measure on the project's Electron (the production runtime). Older runtimes
|
|
// (Node 24.18 / Electron 43) stall worker threads for hundreds of ms on many
|
|
// small ArrayBuffer allocations (SQLite BLOB rows); GB_KPERF_NODE=1 opts out.
|
|
if (process.env.GB_KPERF_NODE !== '1') {
|
|
let electron
|
|
let version
|
|
try { electron = require('electron'); version = require('electron/package.json').version } catch { electron = undefined }
|
|
if (typeof electron === 'string' && process.versions.electron !== version) {
|
|
const { status } = require('node:child_process').spawnSync(electron, [__filename], {
|
|
stdio: 'inherit', env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }
|
|
})
|
|
process.exit(status ?? 1)
|
|
}
|
|
}
|
|
const sizes = (process.env.GB_KPERF_SIZES || '5000,20000,50000').split(',').map(Number)
|
|
const dimensions = Number(process.env.GB_KPERF_DIMENSIONS || 384)
|
|
const chunksPerDocument = 100
|
|
const queries = Number(process.env.GB_KPERF_QUERIES || 15)
|
|
|
|
const directory = mkdtempSync(join(tmpdir(), 'goodbuddy-knowledge-perf-'))
|
|
const bundle = join(directory, 'knowledge-database.cjs')
|
|
const workerBundle = join(directory, 'readonly-query-worker.cjs')
|
|
for (const [entry, outfile] of [
|
|
['src/main/knowledge/knowledge-database.ts', bundle],
|
|
['src/main/readonly-query-worker.ts', workerBundle]
|
|
]) {
|
|
buildSync({
|
|
entryPoints: [join(root, entry)],
|
|
bundle: true,
|
|
platform: 'node',
|
|
format: 'cjs',
|
|
target: 'node22',
|
|
outfile,
|
|
logLevel: 'error',
|
|
// The worker reuses AssistantDatabase, which imports Electron-only modules lazily.
|
|
external: ['electron']
|
|
})
|
|
}
|
|
const { KnowledgeDatabase } = require(bundle)
|
|
|
|
// Deterministic pseudo-random generator so runs are comparable.
|
|
let seed = 42
|
|
const random = () => ((seed = (seed * 1664525 + 1013904223) >>> 0) / 4294967296)
|
|
const words = 'lighthouse harbor policy engine renderer stream vector index budget latency schedule archive network storage contract invoice release window memory review'.split(' ')
|
|
const sentence = n => Array.from({ length: n }, () => words[Math.floor(random() * words.length)]).join(' ')
|
|
const vector = () => Array.from({ length: dimensions }, () => random() * 2 - 1)
|
|
const checksum = value => createHash('sha256').update(value).digest('hex')
|
|
|
|
function percentile(values, p) {
|
|
const sorted = [...values].sort((a, b) => a - b)
|
|
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))]
|
|
}
|
|
function time(body) {
|
|
const samples = []
|
|
body() // warm statement cache and page cache
|
|
for (let index = 0; index < queries; index += 1) {
|
|
const started = performance.now()
|
|
body()
|
|
samples.push(performance.now() - started)
|
|
}
|
|
return { p50: round(percentile(samples, 50)), p95: round(percentile(samples, 95)), max: round(Math.max(...samples)) }
|
|
}
|
|
const round = value => Math.round(value * 10) / 10
|
|
|
|
// Main event-loop delay while `queries` searches run back to back. A sync
|
|
// search blocks the loop for its whole duration; a worker search should not.
|
|
async function loopDelay(body) {
|
|
const histogram = monitorEventLoopDelay({ resolution: 5 })
|
|
await body() // warm worker / caches outside the measurement
|
|
// Let the warm-up's blocked interval drain before sampling.
|
|
await new Promise(resolve => setTimeout(resolve, 50))
|
|
histogram.enable()
|
|
const started = performance.now()
|
|
for (let index = 0; index < queries; index += 1) {
|
|
await body()
|
|
// Yield like a real IPC handler, so the histogram samples between queries.
|
|
await new Promise(resolve => setImmediate(resolve))
|
|
}
|
|
const wallMs = performance.now() - started
|
|
await new Promise(resolve => setTimeout(resolve, 20))
|
|
histogram.disable()
|
|
const ms = value => round(value / 1e6)
|
|
return { p50: ms(histogram.percentile(50)), p99: ms(histogram.percentile(99)), max: ms(histogram.max), perQueryMs: round(wallMs / queries) }
|
|
}
|
|
|
|
const results = []
|
|
async function main() {
|
|
console.log(`[kperf] runtime: node ${process.version}${process.versions.electron ? ` (electron ${process.versions.electron})` : ''}`)
|
|
for (const size of sizes) {
|
|
const path = join(directory, `knowledge-${size}.sqlite`)
|
|
const database = new KnowledgeDatabase(path)
|
|
database.initialize()
|
|
const library = database.createKnowledgeBase({ name: `Perf ${size}`, storageMode: 'reference' })
|
|
const sqlite = database.database
|
|
const seedStarted = performance.now()
|
|
for (let documentIndex = 0; documentIndex < size / chunksPerDocument; documentIndex += 1) {
|
|
const source = database.upsertSource({ knowledgeBaseId: library.id, type: 'file', location: `C:\\perf\\doc-${documentIndex}.md`, displayName: `doc-${documentIndex}.md`, status: 'ready' })
|
|
const chunks = Array.from({ length: chunksPerDocument }, (_, ordinal) => ({ id: `d${documentIndex}-c${ordinal}`, ordinal, content: sentence(80), location: `line ${ordinal + 1}` }))
|
|
const document = database.upsertDocument({ knowledgeBaseId: library.id, sourceId: source.id, externalId: `doc-${documentIndex}`, title: `doc-${documentIndex}`, sourceLocation: source.location }, chunks)
|
|
const stored = sqlite.prepare("SELECT id, index_content FROM chunks WHERE document_id = ? AND enabled = 1 AND role <> 'parent'").all(document.id)
|
|
database.replaceDocumentEmbeddings(document.id, 'perf', 'perf-model', stored.map(row => ({ chunkId: row.id, contentChecksum: checksum(row.index_content), vector: vector() })))
|
|
}
|
|
const seedMs = performance.now() - seedStarted
|
|
database.close()
|
|
|
|
// Reopen so timings start from a fresh connection, like an app restart.
|
|
const reopened = new KnowledgeDatabase(path)
|
|
reopened.initialize()
|
|
const query = vector()
|
|
const entry = {
|
|
chunks: size,
|
|
dimensions,
|
|
seedSeconds: round(seedMs / 1000),
|
|
ftsMs: time(() => reopened.search({ knowledgeBaseId: library.id, query: 'lighthouse budget', limit: 40 })),
|
|
vectorMs: time(() => reopened.vectorSearch({ knowledgeBaseId: library.id, provider: 'perf', model: 'perf-model', vector: query, limit: 40 })),
|
|
hybridMs: time(() => reopened.hybridSearchWithDiagnostics(hybridOptions(library.id, query)))
|
|
}
|
|
// Main event-loop delay: synchronous search on Main vs the readonly worker.
|
|
entry.mainLoopDelaySyncMs = await loopDelay(async () => reopened.hybridSearchWithDiagnostics(hybridOptions(library.id, query)))
|
|
reopened.enableReadonlyWorker(workerBundle)
|
|
entry.mainLoopDelayWorkerMs = await loopDelay(() => reopened.hybridSearchWithDiagnosticsAsync(hybridOptions(library.id, query)))
|
|
reopened.close()
|
|
results.push(entry)
|
|
console.log(`[kperf] ${size} chunks: fts p95 ${entry.ftsMs.p95} ms, vector p95 ${entry.vectorMs.p95} ms, hybrid p95 ${entry.hybridMs.p95} ms (seed ${entry.seedSeconds} s)`)
|
|
console.log(`[kperf] ${size} chunks: Main loop delay max sync ${entry.mainLoopDelaySyncMs.max} ms vs worker ${entry.mainLoopDelayWorkerMs.max} ms (p99 ${entry.mainLoopDelaySyncMs.p99} / ${entry.mainLoopDelayWorkerMs.p99} ms; per query ${entry.mainLoopDelaySyncMs.perQueryMs} / ${entry.mainLoopDelayWorkerMs.perQueryMs} ms)`)
|
|
}
|
|
const output = process.env.GB_KPERF_OUTPUT
|
|
if (output) writeFileSync(output, JSON.stringify({ kind: 'goodbuddy-knowledge-retrieval-benchmark', node: process.version, results }, null, 2))
|
|
console.table(results.map(r => ({ chunks: r.chunks, 'fts p50/p95': `${r.ftsMs.p50} / ${r.ftsMs.p95}`, 'vector p50/p95': `${r.vectorMs.p50} / ${r.vectorMs.p95}`, 'hybrid p50/p95': `${r.hybridMs.p50} / ${r.hybridMs.p95}`, 'loop max sync/worker': `${r.mainLoopDelaySyncMs.max} / ${r.mainLoopDelayWorkerMs.max}` })))
|
|
}
|
|
const hybridOptions = (knowledgeBaseId, vector) => ({ knowledgeBaseId, query: 'lighthouse budget', limit: 40, provider: 'perf', model: 'perf-model', vector, graphEnabled: false, candidateMultiplier: 1 })
|
|
main().catch(error => { process.exitCode = 1; console.error(error) }).finally(() => {
|
|
rmSync(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
|
|
})
|