perf: parse, chunk and hash documents in a worker thread and add startup and document-parse benchmarks
This commit is contained in:
mesalogo
@@ -0,0 +1,109 @@
|
||||
// Document parse benchmark (PERF-15/16, P7). Bundles the production parser
|
||||
// client and worker with esbuild, then imports synthetic large documents
|
||||
// (hash -> parse -> chunk, as Knowledge import does) twice per round: inline on
|
||||
// the main thread, then through the parse worker. Records the main event-loop
|
||||
// max block with perf_hooks.monitorEventLoopDelay.
|
||||
//
|
||||
// node build/run-document-parse-perf.cjs
|
||||
// GB_DOCPERF_ROUNDS=3 GB_DOCPERF_NODE=1 (skip Electron) GB_DOCPERF_OUTPUT=file.json
|
||||
const { buildSync } = require('esbuild')
|
||||
const { mkdirSync, rmSync, writeFileSync } = require('node:fs')
|
||||
const { join, resolve } = require('node:path')
|
||||
const { monitorEventLoopDelay, performance } = require('node:perf_hooks')
|
||||
|
||||
const root = resolve(__dirname, '..')
|
||||
|
||||
// Measure on the project's Electron (the production runtime), like perf:knowledge.
|
||||
if (process.env.GB_DOCPERF_NODE !== '1') {
|
||||
let electron
|
||||
let version
|
||||
try { electron = require('electron'); version = require('electron/package.json').version } catch { electron = undefined }
|
||||
if (typeof electron === 'string' && process.versions.electron !== version) {
|
||||
const { status } = require('node:child_process').spawnSync(electron, [__filename], {
|
||||
stdio: 'inherit', env: { ...process.env, ELECTRON_RUN_AS_NODE: '1' }
|
||||
})
|
||||
process.exit(status ?? 1)
|
||||
}
|
||||
}
|
||||
|
||||
const rounds = Number(process.env.GB_DOCPERF_ROUNDS || 3)
|
||||
// Under node_modules so the external pdfjs-dist resolves from the repository.
|
||||
const directory = join(root, 'node_modules', '.cache', `goodbuddy-document-parse-perf-${process.pid}`)
|
||||
mkdirSync(directory, { recursive: true })
|
||||
const harness = join(directory, 'harness.cjs')
|
||||
const workerBundle = join(directory, 'document-parse-worker.cjs')
|
||||
const harnessSource = join(directory, 'harness.ts')
|
||||
writeFileSync(harnessSource, [
|
||||
`export * from ${JSON.stringify(join(root, 'src/main/document-parse-client.ts'))}`,
|
||||
`export * from ${JSON.stringify(join(root, 'tests/support/document-parse-worker-fixture.ts'))}`
|
||||
].join('\n'))
|
||||
for (const [entry, outfile] of [[harnessSource, harness], [join(root, 'src/main/document-parse-worker.ts'), workerBundle]]) {
|
||||
buildSync({
|
||||
entryPoints: [entry], bundle: true, platform: 'node', format: 'cjs', target: 'node22',
|
||||
outfile, logLevel: 'error', external: ['pdfjs-dist', 'electron', 'esbuild']
|
||||
})
|
||||
}
|
||||
const {
|
||||
DocumentParseWorkerClient, chunkDocumentOffMain, parseDocumentOffMain, sha256OffMain,
|
||||
createDocx, createLargePdf, sentenceGenerator
|
||||
} = require(harness)
|
||||
|
||||
const settings = {
|
||||
version: 1, mode: 'structure', targetCharacters: 1600, overlapCharacters: 100,
|
||||
parentCharacters: 1600, childCharacters: 400, contextualIndexingEnabled: false
|
||||
}
|
||||
const round = value => Math.round(value * 10) / 10
|
||||
|
||||
async function importOnce(name, buffer, client) {
|
||||
await sha256OffMain(buffer, client)
|
||||
const parsed = await parseDocumentOffMain(name, buffer, undefined, client)
|
||||
return (await chunkDocumentOffMain(parsed, settings, client)).length
|
||||
}
|
||||
|
||||
async function measure(name, buffer, client) {
|
||||
await importOnce(name, buffer, client) // warm modules / worker
|
||||
await new Promise(resolve => setTimeout(resolve, 50))
|
||||
const histogram = monitorEventLoopDelay({ resolution: 5 })
|
||||
histogram.enable()
|
||||
// The histogram only measures from its first timer tick on; a block that
|
||||
// starts before that tick (the inline path is all microtasks) is not seen.
|
||||
await new Promise(resolve => setTimeout(resolve, 30))
|
||||
const started = performance.now()
|
||||
const chunks = await importOnce(name, buffer, client)
|
||||
const wallMs = performance.now() - started
|
||||
await new Promise(resolve => setTimeout(resolve, 20))
|
||||
histogram.disable()
|
||||
return { maxMs: round(histogram.max / 1e6), p99Ms: round(histogram.percentile(99) / 1e6), wallMs: round(wallMs), chunks }
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log(`[docperf] runtime: node ${process.version}${process.versions.electron ? ` (electron ${process.versions.electron})` : ''}`)
|
||||
const sentence = sentenceGenerator(11)
|
||||
const inputs = [
|
||||
{ name: 'large.pdf', buffer: createLargePdf(500, 60) },
|
||||
{ name: 'large.docx', buffer: createDocx(Array.from({ length: 4000 }, () => sentence(25))) },
|
||||
{ name: 'large.md', buffer: Buffer.from(Array.from({ length: 4000 }, (_, index) => `${index % 50 === 0 ? '## Section\n' : ''}${sentence(60)}.`).join('\n')) }
|
||||
]
|
||||
const client = new DocumentParseWorkerClient(workerBundle)
|
||||
const results = []
|
||||
try {
|
||||
for (const input of inputs) {
|
||||
for (let index = 0; index < rounds; index += 1) {
|
||||
// Alternate inline / worker within each round (P7).
|
||||
const inline = await measure(input.name, input.buffer, undefined)
|
||||
const worker = await measure(input.name, input.buffer, client)
|
||||
if (worker.chunks !== inline.chunks) throw new Error(`chunk count mismatch for ${input.name}`)
|
||||
results.push({ input: input.name, bytes: input.buffer.byteLength, round: index + 1, inline, worker })
|
||||
console.log(`[docperf] ${input.name} round ${index + 1}: Main loop max ${inline.maxMs} -> ${worker.maxMs} ms; wall ${inline.wallMs} / ${worker.wallMs} ms; ${inline.chunks} chunks`)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client.close()
|
||||
}
|
||||
const output = process.env.GB_DOCPERF_OUTPUT
|
||||
if (output) writeFileSync(output, JSON.stringify({ kind: 'goodbuddy-document-parse-benchmark', node: process.version, results }, null, 2))
|
||||
console.table(results.map(r => ({ input: r.input, round: r.round, 'loop max inline/worker ms': `${r.inline.maxMs} / ${r.worker.maxMs}`, 'wall inline/worker ms': `${r.inline.wallMs} / ${r.worker.wallMs}` })))
|
||||
}
|
||||
main().catch(error => { process.exitCode = 1; console.error(error) }).finally(() => {
|
||||
rmSync(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
|
||||
})
|
||||
@@ -0,0 +1,360 @@
|
||||
// Cold-start benchmark supervisor (PERF-01).
|
||||
// Launches the current production build (out/, from `npm run build:bundle`)
|
||||
// GB_PERF_STARTUP_RUNS times (default 5) per variant through
|
||||
// tests/support/startup-perf-driver.mjs, each launch in a fresh temporary
|
||||
// user-data directory, and reports median / p95 per startup phase.
|
||||
//
|
||||
// Variants (interleaved each round, per docs P7):
|
||||
// first-run empty user-data dir (creates databases, shows release notes)
|
||||
// empty copy of a prepared profile: release notes seen, startup update
|
||||
// check off, 0 conversations
|
||||
// seeded-300 same as empty plus GB_PERF_STARTUP_SEED (default 300)
|
||||
// synthetic conversations, reusing the PERF-11 seed shape
|
||||
//
|
||||
// Isolation: temporary profile/session/logs, a loopback fake model on an
|
||||
// ephemeral port (GB_PERF_MODEL_PORT to pin one; 11434 is left alone), all
|
||||
// non-loopback requests blocked in the driver. Output has phase names, times
|
||||
// and counts only.
|
||||
//
|
||||
// Env: GB_PERF_STARTUP_RUNS, GB_PERF_STARTUP_SEED, GB_PERF_STARTUP_VARIANTS
|
||||
// (comma list), GB_PERF_STARTUP_SKIP_BUILD=1, GB_PERF_STARTUP_PROFILE=1 (one
|
||||
// extra CPU-profiled launch per variant, excluded from the table),
|
||||
// GB_PERF_OUTPUT (report directory), GB_PERF_KEEP_PROFILE=1,
|
||||
// GB_PERF_STARTUP_APP_ROOT (checkout/worktree to build and launch; default .).
|
||||
const { spawn, spawnSync, execFileSync } = require('node:child_process')
|
||||
const { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } = require('node:fs')
|
||||
const { createServer } = require('node:http')
|
||||
const { cpus, tmpdir, totalmem } = require('node:os')
|
||||
const { join, resolve } = require('node:path')
|
||||
const { npmInvocation } = require('./npm-invocation.cjs')
|
||||
|
||||
const root = resolve(__dirname, '..')
|
||||
// The app under test. Defaults to this checkout; point it at a temporary
|
||||
// worktree to measure a baseline or a clean HEAD while this tree is mid-edit.
|
||||
const appRoot = resolve(process.env.GB_PERF_STARTUP_APP_ROOT || root)
|
||||
const runs = Math.max(1, Number(process.env.GB_PERF_STARTUP_RUNS || 5))
|
||||
const seedCount = Number(process.env.GB_PERF_STARTUP_SEED || 300)
|
||||
const seededName = `seeded-${seedCount}`
|
||||
const variants = (process.env.GB_PERF_STARTUP_VARIANTS || `first-run,empty,${seededName}`).split(',').map(v => v.trim()).filter(Boolean)
|
||||
const requestedPort = Number(process.env.GB_PERF_MODEL_PORT || 0)
|
||||
const launchTimeoutMs = Number(process.env.GB_PERF_STARTUP_LAUNCH_TIMEOUT_MS || 150_000)
|
||||
const keepProfile = process.env.GB_PERF_KEEP_PROFILE === '1'
|
||||
const cpuProfile = process.env.GB_PERF_STARTUP_PROFILE === '1'
|
||||
|
||||
// ------------------------------------------------------------------ build
|
||||
|
||||
function newestMtime(path) {
|
||||
let newest = 0
|
||||
const stack = [path]
|
||||
while (stack.length) {
|
||||
const current = stack.pop()
|
||||
let info
|
||||
try { info = statSync(current) } catch { continue }
|
||||
if (info.isDirectory()) {
|
||||
for (const entry of readdirSync(current)) if (entry !== 'node_modules') stack.push(join(current, entry))
|
||||
} else if (!/\.test\.tsx?$/.test(current)) {
|
||||
newest = Math.max(newest, info.mtimeMs)
|
||||
}
|
||||
}
|
||||
return newest
|
||||
}
|
||||
|
||||
function ensureBuild() {
|
||||
const outputs = ['out/main/index.js', 'out/preload/index.cjs', 'out/renderer/index.html'].map(file => join(appRoot, file))
|
||||
const missing = outputs.some(file => !existsSync(file))
|
||||
const builtAt = missing ? 0 : Math.min(...outputs.map(file => statSync(file).mtimeMs))
|
||||
const sourceAt = Math.max(...['src/main', 'src/preload', 'src/renderer', 'src/shared', 'electron.vite.config.ts', 'package.json'].map(p => newestMtime(join(appRoot, p))))
|
||||
const stale = missing || sourceAt > builtAt
|
||||
if (!stale) return { rebuilt: false }
|
||||
if (process.env.GB_PERF_STARTUP_SKIP_BUILD === '1') {
|
||||
if (missing) throw new Error('Missing production build and GB_PERF_STARTUP_SKIP_BUILD=1')
|
||||
console.log('[startup] build is older than src/, measuring it anyway (GB_PERF_STARTUP_SKIP_BUILD=1)')
|
||||
return { rebuilt: false, stale: true }
|
||||
}
|
||||
console.log(`[startup] ${missing ? 'no production build' : 'production build is older than src/'}; running npm run build:bundle`)
|
||||
const npm = npmInvocation(appRoot)
|
||||
const result = spawnSync(npm.command, [...npm.prefixArgs, 'run', 'build:bundle'], { cwd: appRoot, stdio: 'inherit', shell: npm.prefixArgs.length === 0 && process.platform === 'win32' })
|
||||
if (result.status !== 0) throw new Error(`npm run build:bundle failed (${result.status})`)
|
||||
return { rebuilt: true }
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- fake model
|
||||
|
||||
function startModelServer(ledger) {
|
||||
const server = createServer((request, response) => {
|
||||
ledger.push({ method: request.method, path: request.url })
|
||||
if (request.method === 'GET' && request.url?.endsWith('/models')) {
|
||||
response.writeHead(200, { 'content-type': 'application/json' })
|
||||
response.end(JSON.stringify({ object: 'list', data: [{ id: 'qwen3', object: 'model' }] }))
|
||||
return
|
||||
}
|
||||
if (request.method === 'POST' && request.url?.endsWith('/chat/completions')) {
|
||||
request.resume()
|
||||
request.on('end', () => {
|
||||
response.writeHead(200, { 'content-type': 'application/json' })
|
||||
response.end(JSON.stringify({ id: 'perf', object: 'chat.completion', model: 'qwen3', choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content: 'ok' } }], usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 } }))
|
||||
})
|
||||
return
|
||||
}
|
||||
response.writeHead(404).end()
|
||||
})
|
||||
return new Promise((resolvePromise, reject) => {
|
||||
server.once('error', reject)
|
||||
server.listen(requestedPort, '127.0.0.1', () => resolvePromise(server))
|
||||
})
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ launch
|
||||
|
||||
function baseEnvironment(modelPort, workspace) {
|
||||
const environment = { ...process.env, GB_PERF_ROOT: appRoot, GB_PERF_STARTUP_MARKS: '1' }
|
||||
delete environment.ELECTRON_RUN_AS_NODE
|
||||
delete environment.ELECTRON_RENDERER_URL
|
||||
for (const key of ['GOODBUDDY_MODEL_API_KEY', 'GOODBUDDY_MODEL_BASE_URL', 'GOODBUDDY_MODEL_NAME', 'GOODBUDDY_BIGTOKEN_API_KEY', 'GOODBUDDY_BIGTOKEN_BASE_URL', 'GOODBUDDY_BIGTOKEN_MODEL']) delete environment[key]
|
||||
environment.GOODBUDDY_MODEL_API_KEY = 'perf-loopback'
|
||||
environment.GOODBUDDY_MODEL_BASE_URL = `http://127.0.0.1:${modelPort}/v1`
|
||||
environment.GOODBUDDY_MODEL_NAME = 'qwen3'
|
||||
environment.GOODBUDDY_WORKSPACE = workspace
|
||||
return environment
|
||||
}
|
||||
|
||||
async function launch({ directory, environment, mode = 'measure', seed = 0, expectConversations = 0, profilePath }) {
|
||||
const resultPath = join(directory, 'startup-result.json')
|
||||
const env = { ...environment, GB_STARTUP_DIR: directory, GB_STARTUP_RESULT: resultPath, GB_STARTUP_MODE: mode, GB_STARTUP_SEED: String(seed), GB_STARTUP_EXPECT_CONVERSATIONS: String(expectConversations) }
|
||||
if (profilePath) env.GB_STARTUP_CPU_PROFILE = profilePath
|
||||
const stderr = []
|
||||
const spawnAt = Date.now()
|
||||
const child = spawn(require('electron'), [join(root, 'tests', 'support', 'startup-perf-driver.mjs')], { cwd: root, env, stdio: ['ignore', 'ignore', 'pipe'], windowsHide: false })
|
||||
child.stderr.on('data', chunk => { if (stderr.length < 200) stderr.push(String(chunk)) })
|
||||
let timedOut = false
|
||||
const timer = setTimeout(() => {
|
||||
timedOut = true
|
||||
if (process.platform === 'win32') spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' })
|
||||
else child.kill('SIGKILL')
|
||||
}, launchTimeoutMs)
|
||||
const exit = await new Promise(resolvePromise => child.once('exit', code => resolvePromise(code)))
|
||||
const exitAt = Date.now()
|
||||
clearTimeout(timer)
|
||||
const report = existsSync(resultPath) ? JSON.parse(readFileSync(resultPath, 'utf8')) : { status: 'failed', error: timedOut ? 'launch timeout' : 'driver exited without a report' }
|
||||
if (exit !== 0 || timedOut) report.status = 'failed'
|
||||
report.spawnAt = spawnAt
|
||||
report.exitMs = exitAt - spawnAt
|
||||
// stderr is kept only for failures, and only its first line (no user data exists in these profiles).
|
||||
if (report.status !== 'passed') report.stderrHead = stderr.join('').split(/\r?\n/).filter(Boolean).slice(0, 8)
|
||||
return report
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------- analysis
|
||||
|
||||
function phases(report) {
|
||||
const t = report.times
|
||||
const r = report.renderer
|
||||
const s = report.spawnAt
|
||||
const marks = Object.fromEntries((report.startupMarks ?? []).map(m => [m.name, m.at]))
|
||||
const d = (from, to) => (from === undefined || from === null || to === undefined || to === null) ? null : to - from
|
||||
const result = {
|
||||
'spawn -> driver start (Electron boot)': d(s, t.driverStart),
|
||||
'driver start -> Main import done': d(t.driverStart, t.mainImportEnd),
|
||||
'spawn -> app ready': d(s, t.appReady),
|
||||
'app ready -> BrowserWindow created': d(t.appReady, t.windowCreated),
|
||||
'window created -> loadMainWindow': d(t.windowCreated, marks['main:load-main-window']),
|
||||
'loadMainWindow -> ready-to-show': d(marks['main:load-main-window'], t.readyToShow),
|
||||
'window created -> ready-to-show': d(t.windowCreated, t.readyToShow),
|
||||
'ready-to-show -> first visible frame': d(t.readyToShow, r.firstVisibleFrame),
|
||||
'ready-to-show -> composer interactive': d(t.readyToShow, r.composerInteractive),
|
||||
'spawn -> ready-to-show': d(s, t.readyToShow),
|
||||
'spawn -> first visible frame': d(s, r.firstVisibleFrame),
|
||||
// Only reported by Chromium when the first paint happens while visible.
|
||||
'spawn -> first contentful paint': d(s, r.firstContentfulPaint),
|
||||
'spawn -> composer present': d(s, r.composerPresent),
|
||||
'spawn -> composer interactive': d(s, r.composerInteractive),
|
||||
'spawn -> sidebar populated (seeded)': d(s, r.sidebarPopulated),
|
||||
'composer interactive -> sidebar populated': d(r.composerInteractive, r.sidebarPopulated)
|
||||
}
|
||||
// Main phases (GB_PERF_STARTUP_MARKS spans; parallel spans overlap).
|
||||
const main = {
|
||||
'main: module import (index.js eval)': d(t.mainImportStart, marks['main:module-evaluated']),
|
||||
'main: whenReady -> window created': d(marks['main:when-ready'], marks['main:create-window:end']),
|
||||
'main: createMainWindow()': d(marks['main:create-window:start'], marks['main:create-window:end']),
|
||||
'main: tray': d(marks['main:create-window:end'], marks['main:tray-built'])
|
||||
}
|
||||
for (const m of report.startupMarks ?? []) {
|
||||
if (!m.name.endsWith(':start')) continue
|
||||
const base = m.name.slice(0, -':start'.length)
|
||||
if (base === 'main:create-window') continue
|
||||
main[`${base.replace(/^main:/, 'main: ')}`] = d(m.at, marks[`${base}:end`])
|
||||
// Synchronous part of the span: from the call until the first await yields.
|
||||
if (marks[`${base}:sync-end`] !== undefined) main[`${base.replace(/^main:/, 'main: ')} (sync part)`] = d(m.at, marks[`${base}:sync-end`])
|
||||
}
|
||||
main['main: whenReady -> loadMainWindow (total)'] = d(marks['main:when-ready'], marks['main:load-main-window'])
|
||||
// Critical path in Main: consecutive top-level marks (spans that run in
|
||||
// parallel inside the prerequisites are excluded), so each gap is the time
|
||||
// Main spent between two points of the sequential startup function.
|
||||
const nested = /^main:(knowledge-init|knowledge-gateway-start|runtime-hydrate|assistant-db-init)/
|
||||
const sequence = (report.startupMarks ?? []).filter(m => !nested.test(m.name) && !m.name.endsWith(':sync-end')).sort((a, b) => a.at - b.at)
|
||||
for (let index = 1; index < sequence.length; index += 1) {
|
||||
main[`gap: ${sequence[index - 1].name} -> ${sequence[index].name}`] = sequence[index].at - sequence[index - 1].at
|
||||
}
|
||||
const renderer = {
|
||||
'renderer: navigation start -> DOMContentLoaded': d(r.navigationStart, r.domContentLoaded),
|
||||
'renderer: DOMContentLoaded -> composer present': d(r.domContentLoaded, r.composerPresent),
|
||||
'renderer: navigation start -> composer present': d(r.navigationStart, r.composerPresent)
|
||||
}
|
||||
return { ...result, ...main, ...renderer }
|
||||
}
|
||||
|
||||
function percentile(values, p) {
|
||||
if (!values.length) return null
|
||||
const sorted = [...values].sort((a, b) => a - b)
|
||||
// Nearest rank; with 5 samples p95 is the maximum.
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1))]
|
||||
}
|
||||
|
||||
function summarize(reportsByVariant) {
|
||||
const rows = new Map()
|
||||
for (const [variant, reports] of Object.entries(reportsByVariant)) {
|
||||
const passed = reports.filter(r => r.status === 'passed')
|
||||
const perRun = passed.map(phases)
|
||||
for (const name of new Set(perRun.flatMap(Object.keys))) {
|
||||
const values = perRun.map(p => p[name]).filter(v => typeof v === 'number' && Number.isFinite(v))
|
||||
const row = rows.get(name) ?? {}
|
||||
row[variant] = values.length ? { median: percentile(values, 50), p95: percentile(values, 95), n: values.length } : null
|
||||
rows.set(name, row)
|
||||
}
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
function sourceInfo() {
|
||||
try {
|
||||
const commit = execFileSync('git', ['rev-parse', '--short', 'HEAD'], { cwd: appRoot, encoding: 'utf8' }).trim()
|
||||
const dirtyFiles = execFileSync('git', ['status', '--porcelain'], { cwd: appRoot, encoding: 'utf8' }).split('\n').filter(Boolean).length
|
||||
return { commit, dirty: dirtyFiles > 0, dirtyFiles, appRoot: appRoot === root ? '.' : 'GB_PERF_STARTUP_APP_ROOT' }
|
||||
} catch {
|
||||
return { commit: null }
|
||||
}
|
||||
}
|
||||
|
||||
const fmt = v => (v === null || v === undefined ? '-' : Math.round(v).toString())
|
||||
|
||||
// ------------------------------------------------------------------- main
|
||||
|
||||
async function main() {
|
||||
const build = ensureBuild()
|
||||
const runDirectory = mkdtempSync(join(tmpdir(), 'goodbuddy-startup-perf-'))
|
||||
const outputDirectory = resolve(process.env.GB_PERF_OUTPUT || join(runDirectory, 'artifacts'))
|
||||
mkdirSync(outputDirectory, { recursive: true })
|
||||
const workspace = join(runDirectory, 'workspace')
|
||||
mkdirSync(workspace, { recursive: true })
|
||||
const ledger = []
|
||||
let server
|
||||
try {
|
||||
server = await startModelServer(ledger)
|
||||
} catch (error) {
|
||||
throw new Error(`Cannot listen on 127.0.0.1:${requestedPort} for the fake model (${error.code ?? error.message}); set GB_PERF_MODEL_PORT to a free port or leave it unset for an ephemeral port`, { cause: error })
|
||||
}
|
||||
const modelPort = server.address().port
|
||||
const environment = baseEnvironment(modelPort, workspace)
|
||||
const templates = {}
|
||||
const prepareReports = {}
|
||||
try {
|
||||
// Templates are prepared once and copied per launch, so every measured
|
||||
// launch starts from an identical, never-used-before directory.
|
||||
for (const variant of variants) {
|
||||
if (variant === 'first-run') continue
|
||||
const seed = variant === 'empty' ? 0 : variant === seededName ? seedCount : null
|
||||
if (seed === null) throw new Error(`Unknown variant ${variant}`)
|
||||
const directory = join(runDirectory, `template-${variant}`)
|
||||
mkdirSync(directory, { recursive: true })
|
||||
console.log(`[startup] preparing ${variant} profile${seed ? ` (${seed} conversations)` : ''}`)
|
||||
const report = await launch({ directory, environment, mode: 'prepare', seed })
|
||||
prepareReports[variant] = { status: report.status, error: report.error, seeded: report.seeded, exitMs: report.exitMs, stderrHead: report.stderrHead }
|
||||
if (report.status !== 'passed') throw new Error(`Preparing ${variant} failed: ${report.error ?? 'unknown'}\n${(report.stderrHead ?? []).join('\n')}`)
|
||||
rmSync(join(directory, 'startup-result.json'), { force: true })
|
||||
templates[variant] = directory
|
||||
}
|
||||
|
||||
const fresh = (variant, label) => {
|
||||
const directory = join(runDirectory, `${label}-${variant}`)
|
||||
if (templates[variant]) {
|
||||
cpSync(templates[variant], directory, { recursive: true })
|
||||
} else {
|
||||
mkdirSync(directory, { recursive: true })
|
||||
}
|
||||
return directory
|
||||
}
|
||||
|
||||
const expectedFor = variant => (variant === seededName ? seedCount : 0)
|
||||
const results = Object.fromEntries(variants.map(v => [v, []]))
|
||||
// A discarded warm-up launch per invocation so the first measured launch
|
||||
// does not also pay for loading Electron binaries from a cold disk cache.
|
||||
console.log('[startup] warm-up launch (discarded)')
|
||||
await launch({ directory: fresh(variants.includes('empty') ? 'empty' : variants[0], 'warmup'), environment })
|
||||
for (let round = 0; round < runs; round += 1) {
|
||||
for (const variant of variants) {
|
||||
const directory = fresh(variant, `run${round + 1}`)
|
||||
const report = await launch({ directory, environment, expectConversations: expectedFor(variant) })
|
||||
report.round = round + 1
|
||||
results[variant].push(report)
|
||||
const p = report.status === 'passed' ? phases(report) : {}
|
||||
console.log(`[startup] round ${round + 1} ${variant}: ${report.status}, ready ${fmt(p['spawn -> app ready'])} ms, ready-to-show ${fmt(p['spawn -> ready-to-show'])} ms, first frame ${fmt(p['spawn -> first visible frame'])} ms, composer interactive ${fmt(p['spawn -> composer interactive'])} ms${report.error ? `, error ${report.error}` : ''}`)
|
||||
if (!keepProfile) rmSync(directory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
|
||||
}
|
||||
}
|
||||
|
||||
const profiles = {}
|
||||
if (cpuProfile) {
|
||||
for (const variant of variants) {
|
||||
const directory = fresh(variant, 'profiled')
|
||||
const profilePath = join(outputDirectory, `main-startup-${variant}.cpuprofile`)
|
||||
const report = await launch({ directory, environment, profilePath, expectConversations: expectedFor(variant) })
|
||||
profiles[variant] = { status: report.status, cpuProfile: report.cpuProfile, phases: report.status === 'passed' ? phases(report) : null, file: profilePath }
|
||||
console.log(`[startup] CPU-profiled ${variant}: ${report.status} -> ${profilePath}`)
|
||||
}
|
||||
}
|
||||
|
||||
const table = summarize(results)
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
kind: 'goodbuddy-startup-benchmark',
|
||||
source: { ...sourceInfo(), build },
|
||||
environment: { platform: process.platform, arch: process.arch, electron: require('electron/package.json').version, cpu: `${cpus()[0]?.model?.trim()} x${cpus().length}`, memoryGB: Math.round(totalmem() / 1073741824) },
|
||||
config: { runs, variants, seedConversations: seedCount, percentile: 'nearest-rank' },
|
||||
prepare: prepareReports,
|
||||
phases: Object.fromEntries(table),
|
||||
runs: Object.fromEntries(Object.entries(results).map(([variant, reports]) => [variant, reports.map(r => ({
|
||||
round: r.round, status: r.status, error: r.error, stderrHead: r.stderrHead, exitMs: r.exitMs,
|
||||
phases: r.status === 'passed' ? phases(r) : null,
|
||||
renderer: r.renderer ? { domNodes: r.renderer.domNodes, conversationRows: r.renderer.conversationRows, scriptResources: r.renderer.scriptResources, scriptBytes: r.renderer.scriptBytes, jsHeapMB: r.renderer.jsHeapMB, startupDialog: r.renderer.startupDialog !== undefined, composerPresentBeforeProbe: r.renderer.alreadyAtInstall === true, longAnimationFrames: r.renderer.longAnimationFrames && { count: r.renderer.longAnimationFrames.count, totalMs: r.renderer.longAnimationFrames.totalMs, maxMs: r.renderer.longAnimationFrames.maxMs, scriptMs: r.renderer.longAnimationFrames.scriptMs } } : null,
|
||||
mainEventLoopDelay: r.mainEventLoopDelay,
|
||||
blockedRequests: r.blockedRequests && { fetch: r.blockedRequests.fetch.length, renderer: r.blockedRequests.renderer.length, hosts: [...new Set([...r.blockedRequests.fetch, ...r.blockedRequests.renderer])] }
|
||||
}))])),
|
||||
cpuProfiles: profiles,
|
||||
modelRequests: { boundary: 'loopback fake model only; non-loopback requests blocked', total: ledger.length, paths: [...new Set(ledger.map(e => `${e.method} ${e.path}`))] }
|
||||
}
|
||||
const resultPath = join(outputDirectory, 'startup-result.json')
|
||||
writeFileSync(resultPath, JSON.stringify(report, null, 2))
|
||||
|
||||
console.log(`\nGoodBuddy cold-start benchmark: ${runs} runs per variant, ms (median / p95, nearest rank)`)
|
||||
console.log(`source ${report.source.commit ?? '?'}${report.source.dirty ? ` (dirty, ${report.source.dirtyFiles} files)` : ''}, Electron ${report.environment.electron}, ${report.environment.cpu}, ${report.environment.memoryGB} GB`)
|
||||
console.table(Object.fromEntries([...table].map(([name, row]) => [name, Object.fromEntries(variants.map(v => [v, row[v] ? `${fmt(row[v].median)} / ${fmt(row[v].p95)}${row[v].n < runs ? ` (n=${row[v].n})` : ''}` : '-']))])))
|
||||
const failed = Object.values(results).flat().filter(r => r.status !== 'passed').length
|
||||
if (failed) console.log(`${failed} launch(es) failed; see report`)
|
||||
const blocked = Object.values(results).flat().flatMap(r => [...(r.blockedRequests?.fetch ?? []), ...(r.blockedRequests?.renderer ?? [])])
|
||||
console.log(`blocked non-loopback requests: ${blocked.length}${blocked.length ? ` (${[...new Set(blocked)].join(', ')})` : ''}; fake model requests: ${ledger.length}`)
|
||||
console.log(`\nreport: ${resultPath}`)
|
||||
process.exitCode = failed ? 1 : 0
|
||||
} finally {
|
||||
await new Promise(resolvePromise => server.close(() => resolvePromise()))
|
||||
if (!keepProfile) {
|
||||
for (const entry of readdirSync(runDirectory)) {
|
||||
if (entry !== 'artifacts') rmSync(join(runDirectory, entry), { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main().catch(error => {
|
||||
console.error(error instanceof Error ? error.message : error)
|
||||
process.exitCode = 1
|
||||
})
|
||||
@@ -134,6 +134,7 @@ export default defineConfig({
|
||||
'assistant-storage-worker': resolve('src/main/assistant-storage-worker.ts'),
|
||||
'execution-stats-worker': resolve('src/main/execution-stats-worker.ts'),
|
||||
'readonly-query-worker': resolve('src/main/readonly-query-worker.ts'),
|
||||
'document-parse-worker': resolve('src/main/document-parse-worker.ts'),
|
||||
'wechat-sidecar': resolve(
|
||||
'src/main/channels/wechat-sidecar.ts'
|
||||
),
|
||||
|
||||
@@ -27,8 +27,10 @@
|
||||
"test:watch": "vitest",
|
||||
"perf:app": "node build/run-app-perf.cjs",
|
||||
"perf:check": "node build/check-perf-thresholds.cjs",
|
||||
"perf:document-parse": "node build/run-document-parse-perf.cjs",
|
||||
"perf:knowledge": "node build/run-knowledge-perf.cjs",
|
||||
"perf:main": "node build/run-main-perf.cjs",
|
||||
"perf:startup": "node build/run-startup-perf.cjs",
|
||||
"eval:retrieval": "vitest run --config tests/support/knowledge-retrieval-evaluation.ts tests/knowledge-retrieval-metrics.test.ts tests/knowledge-retrieval-evaluation.test.ts",
|
||||
"build": "npm run typecheck && npm run build:bundle",
|
||||
"build:bundle": "electron-vite build && node build/check-control-plane-package-installer.cjs",
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { Worker } from 'node:worker_threads'
|
||||
import type { KnowledgeChunkingSettings } from '../shared/knowledge-contracts'
|
||||
import {
|
||||
assertDocumentBuffer,
|
||||
chunkDocumentAdvanced,
|
||||
DocumentTextUnavailableError,
|
||||
extractPdfTextPages,
|
||||
parseDocument,
|
||||
type DocumentChunk,
|
||||
type ParsedDocument,
|
||||
type PdfTextExtraction,
|
||||
type PdfTextExtractionOptions
|
||||
} from './knowledge/document-parser'
|
||||
import { extractPptxPages, type PptxPage } from './knowledge/pptx-parser'
|
||||
import type {
|
||||
DocumentParseOperation,
|
||||
DocumentParseRequest,
|
||||
DocumentParseResponse,
|
||||
SerializedDocumentParseError
|
||||
} from './document-parse-errors'
|
||||
|
||||
export type { DocumentParseOperation } from './document-parse-errors'
|
||||
|
||||
// PERF-15/16: pure document work (PDF / Office / text parsing, chunking,
|
||||
// hashing) runs in a single worker thread with a FIFO queue, so Main only
|
||||
// orchestrates. Database writes stay on the caller's side, in the same order.
|
||||
// Same fallback rules as readonly-query-reader: if the worker cannot start or
|
||||
// crashes, the call runs inline (the previous behaviour) and the worker is
|
||||
// retried after a backoff. Aborting a running task terminates the worker.
|
||||
|
||||
/** Worker infrastructure failure; callers fall back to the inline path. */
|
||||
export class DocumentParseWorkerUnavailableError extends Error {
|
||||
constructor(message: string, options?: { cause?: unknown }) {
|
||||
super(message, options)
|
||||
this.name = 'DocumentParseWorkerUnavailableError'
|
||||
}
|
||||
}
|
||||
|
||||
type Task = {
|
||||
id: number
|
||||
op: DocumentParseOperation
|
||||
args: unknown[]
|
||||
transfer: ArrayBuffer[]
|
||||
resolve: (value: unknown) => void
|
||||
reject: (error: unknown) => void
|
||||
cleanup: () => void
|
||||
}
|
||||
|
||||
const MAX_QUEUED = 32
|
||||
const RESTART_BACKOFF_MS = 30_000
|
||||
|
||||
type PdfExceptionConstructor = new (message: string) => Error
|
||||
type PdfExceptionClasses = Partial<Record<string, PdfExceptionConstructor>>
|
||||
const pdfExceptionNames = [
|
||||
'AbortException',
|
||||
'InvalidPDFException',
|
||||
'PasswordException',
|
||||
'ResponseException',
|
||||
'UnknownErrorException'
|
||||
] as const
|
||||
|
||||
let pdfExceptions: Promise<PdfExceptionClasses> | undefined
|
||||
|
||||
/** PDF.js exception classes, loaded only once a PDF error crosses the boundary. */
|
||||
function loadPdfExceptions(): Promise<PdfExceptionClasses> {
|
||||
pdfExceptions ??= import('pdfjs-dist/legacy/build/pdf.mjs').then(
|
||||
(pdfjs) => {
|
||||
const module = pdfjs as unknown as Record<string, unknown>
|
||||
return Object.fromEntries(pdfExceptionNames.flatMap((name) =>
|
||||
typeof module[name] === 'function' ? [[name, module[name] as PdfExceptionConstructor]] : []))
|
||||
},
|
||||
() => ({})
|
||||
)
|
||||
return pdfExceptions
|
||||
}
|
||||
|
||||
/**
|
||||
* Rebuilds an error with the same class, name, message, cause and primitive
|
||||
* fields as the one thrown inline (stack traces differ by nature).
|
||||
*/
|
||||
export async function deserializeDocumentParseError(error: SerializedDocumentParseError): Promise<Error> {
|
||||
let result: Error
|
||||
if (error.name === 'DocumentTextUnavailableError') {
|
||||
result = new DocumentTextUnavailableError(error.message)
|
||||
} else if ((pdfExceptionNames as readonly string[]).includes(error.name)) {
|
||||
const Constructor = (await loadPdfExceptions())[error.name]
|
||||
result = Constructor ? new Constructor(error.message) : new Error(error.message)
|
||||
} else {
|
||||
const Constructor = error.name === 'RangeError' ? RangeError
|
||||
: error.name === 'TypeError' ? TypeError : Error
|
||||
result = new Constructor(error.message)
|
||||
}
|
||||
if (result.name !== error.name) result.name = error.name
|
||||
if (error.cause) {
|
||||
Object.defineProperty(result, 'cause', {
|
||||
value: await deserializeDocumentParseError(error.cause), writable: true, configurable: true
|
||||
})
|
||||
}
|
||||
for (const [key, value] of Object.entries(error.extra ?? {})) {
|
||||
Object.defineProperty(result, key, { value, writable: true, configurable: true, enumerable: true })
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
export class DocumentParseWorkerClient {
|
||||
private worker?: Worker
|
||||
private nextId = 0
|
||||
private disabledUntil = 0
|
||||
private closed = false
|
||||
private active?: Task
|
||||
private readonly queue: Task[] = []
|
||||
|
||||
constructor(
|
||||
private readonly workerPath: string,
|
||||
private readonly now: () => number = Date.now
|
||||
) {}
|
||||
|
||||
/** False while closed, backing off after a crash, or when the queue is full. */
|
||||
get available(): boolean {
|
||||
return !this.closed && this.now() >= this.disabledUntil &&
|
||||
this.queue.length < MAX_QUEUED
|
||||
}
|
||||
|
||||
get pendingCount(): number {
|
||||
return this.queue.length + (this.active ? 1 : 0)
|
||||
}
|
||||
|
||||
call<T>(
|
||||
op: DocumentParseOperation,
|
||||
args: unknown[],
|
||||
transfer: ArrayBuffer[] = [],
|
||||
signal?: AbortSignal
|
||||
): Promise<T> {
|
||||
if (this.closed) return Promise.reject(new DocumentParseWorkerUnavailableError('Document parse worker closed'))
|
||||
if (signal?.aborted) return Promise.reject(signal.reason)
|
||||
if (this.now() < this.disabledUntil) {
|
||||
return Promise.reject(new DocumentParseWorkerUnavailableError('Document parse worker is backing off'))
|
||||
}
|
||||
if (this.queue.length >= MAX_QUEUED) {
|
||||
return Promise.reject(new DocumentParseWorkerUnavailableError('Document parse worker is busy'))
|
||||
}
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const task: Task = {
|
||||
id: ++this.nextId,
|
||||
op,
|
||||
args,
|
||||
transfer,
|
||||
resolve: resolve as (value: unknown) => void,
|
||||
reject,
|
||||
cleanup: () => signal?.removeEventListener('abort', onAbort)
|
||||
}
|
||||
const onAbort = (): void => {
|
||||
if (this.active === task) {
|
||||
// The task is CPU-bound in the worker; stop it by terminating the worker.
|
||||
this.active = undefined
|
||||
const worker = this.worker
|
||||
this.worker = undefined
|
||||
if (worker) void worker.terminate()
|
||||
} else {
|
||||
const index = this.queue.indexOf(task)
|
||||
if (index < 0) return
|
||||
this.queue.splice(index, 1)
|
||||
}
|
||||
task.cleanup()
|
||||
reject(signal!.reason)
|
||||
this.pump()
|
||||
}
|
||||
signal?.addEventListener('abort', onAbort, { once: true })
|
||||
this.queue.push(task)
|
||||
this.pump()
|
||||
})
|
||||
}
|
||||
|
||||
private pump(): void {
|
||||
if (this.active || this.closed) return
|
||||
const task = this.queue.shift()
|
||||
if (!task) {
|
||||
this.worker?.unref()
|
||||
return
|
||||
}
|
||||
let worker: Worker
|
||||
try {
|
||||
worker = this.ensureWorker()
|
||||
} catch (error) {
|
||||
this.disabledUntil = this.now() + RESTART_BACKOFF_MS
|
||||
this.failAll(task, new DocumentParseWorkerUnavailableError('Document parse worker failed to start', { cause: error }))
|
||||
return
|
||||
}
|
||||
this.active = task
|
||||
// Keep the process alive only while a task is outstanding.
|
||||
worker.ref()
|
||||
try {
|
||||
worker.postMessage(
|
||||
{ id: task.id, op: task.op, args: task.args } satisfies DocumentParseRequest,
|
||||
task.transfer
|
||||
)
|
||||
} catch (error) {
|
||||
this.active = undefined
|
||||
task.cleanup()
|
||||
task.reject(new DocumentParseWorkerUnavailableError('Document parse task could not be posted', { cause: error }))
|
||||
this.pump()
|
||||
}
|
||||
}
|
||||
|
||||
private failAll(first: Task | undefined, error: Error): void {
|
||||
const tasks = [...(first ? [first] : []), ...this.queue.splice(0)]
|
||||
for (const task of tasks) {
|
||||
task.cleanup()
|
||||
task.reject(error)
|
||||
}
|
||||
}
|
||||
|
||||
private ensureWorker(): Worker {
|
||||
if (this.worker) return this.worker
|
||||
const worker = new Worker(this.workerPath)
|
||||
this.worker = worker
|
||||
worker.on('message', (message: DocumentParseResponse) => {
|
||||
const task = this.active
|
||||
if (this.worker !== worker || !task || task.id !== message.id) return
|
||||
this.active = undefined
|
||||
task.cleanup()
|
||||
if ('error' in message) {
|
||||
void deserializeDocumentParseError(message.error).then(task.reject)
|
||||
} else {
|
||||
task.resolve(message.result)
|
||||
}
|
||||
this.pump()
|
||||
})
|
||||
const failed = (error: Error): void => {
|
||||
// Ignore exits of workers that were terminated on purpose (abort, close).
|
||||
if (this.worker !== worker) return
|
||||
this.worker = undefined
|
||||
this.disabledUntil = this.now() + RESTART_BACKOFF_MS
|
||||
const active = this.active
|
||||
this.active = undefined
|
||||
this.failAll(active, new DocumentParseWorkerUnavailableError(error.message, { cause: error }))
|
||||
void worker.terminate()
|
||||
}
|
||||
worker.once('error', failed)
|
||||
worker.once('exit', (code) => failed(new Error(`Document parse worker exited (${code})`)))
|
||||
worker.unref()
|
||||
return worker
|
||||
}
|
||||
|
||||
/** Test hook: simulate a crash by terminating the worker. */
|
||||
async terminateWorkerForTest(): Promise<void> {
|
||||
await this.worker?.terminate()
|
||||
}
|
||||
|
||||
/** Test hook: end the post-crash backoff. */
|
||||
resetBackoffForTest(): void {
|
||||
this.disabledUntil = 0
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.closed = true
|
||||
const worker = this.worker
|
||||
this.worker = undefined
|
||||
const active = this.active
|
||||
this.active = undefined
|
||||
this.failAll(active, new DocumentParseWorkerUnavailableError('Document parse worker closed'))
|
||||
if (worker) void worker.terminate()
|
||||
}
|
||||
}
|
||||
|
||||
let sharedClient: DocumentParseWorkerClient | undefined
|
||||
|
||||
/** Enables the parse worker for this process (Main calls it once at startup). */
|
||||
export function configureDocumentParseWorker(workerPath: string | undefined): void {
|
||||
sharedClient?.close()
|
||||
sharedClient = workerPath ? new DocumentParseWorkerClient(workerPath) : undefined
|
||||
}
|
||||
|
||||
export function closeDocumentParseWorker(): void {
|
||||
sharedClient?.close()
|
||||
sharedClient = undefined
|
||||
}
|
||||
|
||||
/** A private copy of the bytes that can be transferred without detaching the caller's buffer. */
|
||||
function transferableCopy(buffer: Uint8Array): ArrayBuffer {
|
||||
return Uint8Array.prototype.slice.call(buffer).buffer as ArrayBuffer
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs `op` in the worker when available, otherwise inline. Worker
|
||||
* infrastructure failures fall back to the inline path; operation errors and
|
||||
* aborts propagate unchanged. An already-aborted signal also runs inline, so
|
||||
* validation and abort errors surface in exactly the inline order.
|
||||
*/
|
||||
export async function runOffMain<T>(
|
||||
client: DocumentParseWorkerClient | undefined,
|
||||
op: DocumentParseOperation,
|
||||
request: () => { args: unknown[]; transfer?: ArrayBuffer[] },
|
||||
signal: AbortSignal | undefined,
|
||||
inline: () => T | Promise<T>
|
||||
): Promise<T> {
|
||||
if (!client?.available || signal?.aborted) return inline()
|
||||
const { args, transfer } = request()
|
||||
try {
|
||||
return await client.call<T>(op, args, transfer, signal)
|
||||
} catch (error) {
|
||||
if (error instanceof DocumentParseWorkerUnavailableError && !signal?.aborted) return inline()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
export function parseDocumentOffMain(
|
||||
name: string,
|
||||
buffer: Buffer,
|
||||
signal?: AbortSignal,
|
||||
client: DocumentParseWorkerClient | undefined = sharedClient
|
||||
): Promise<ParsedDocument> {
|
||||
// Cheap size checks stay inline so oversized input is never copied.
|
||||
try {
|
||||
assertDocumentBuffer(buffer)
|
||||
} catch (error) {
|
||||
return Promise.reject(error)
|
||||
}
|
||||
return runOffMain(client, 'parseDocument', () => {
|
||||
const bytes = transferableCopy(buffer)
|
||||
return { args: [name, bytes], transfer: [bytes] }
|
||||
}, signal, () => parseDocument(name, buffer, signal))
|
||||
}
|
||||
|
||||
export function extractPdfTextPagesOffMain(
|
||||
buffer: Buffer,
|
||||
options: PdfTextExtractionOptions = {},
|
||||
client: DocumentParseWorkerClient | undefined = sharedClient
|
||||
): Promise<PdfTextExtraction> {
|
||||
const { signal, ...limits } = options
|
||||
return runOffMain(client, 'extractPdfTextPages', () => {
|
||||
const bytes = transferableCopy(buffer)
|
||||
return { args: [bytes, limits], transfer: [bytes] }
|
||||
}, signal, () => extractPdfTextPages(buffer, options))
|
||||
}
|
||||
|
||||
export function extractPptxPagesOffMain(
|
||||
buffer: Buffer,
|
||||
client: DocumentParseWorkerClient | undefined = sharedClient
|
||||
): Promise<PptxPage[]> {
|
||||
return runOffMain(client, 'extractPptxPages', () => {
|
||||
const bytes = transferableCopy(buffer)
|
||||
return { args: [bytes], transfer: [bytes] }
|
||||
}, undefined, () => extractPptxPages(buffer))
|
||||
}
|
||||
|
||||
/**
|
||||
* Not cancellable on purpose: the inline call it replaces was synchronous, so
|
||||
* an abort during chunking keeps surfacing at the caller's next checkpoint.
|
||||
*/
|
||||
export function chunkDocumentOffMain(
|
||||
document: ParsedDocument,
|
||||
settings: KnowledgeChunkingSettings,
|
||||
client: DocumentParseWorkerClient | undefined = sharedClient
|
||||
): Promise<DocumentChunk[]> {
|
||||
return runOffMain(client, 'chunkDocument', () => ({
|
||||
// Only the fields chunking reads; images (Buffers) would otherwise be cloned.
|
||||
args: [{ title: document.title, sourceFormat: document.sourceFormat, content: '', sections: document.sections, warnings: [] }, settings]
|
||||
}), undefined, () => chunkDocumentAdvanced(document, settings))
|
||||
}
|
||||
|
||||
/** Hex SHA-256 of `buffer`, computed in the worker for large inputs. */
|
||||
export function sha256OffMain(
|
||||
buffer: Uint8Array,
|
||||
client: DocumentParseWorkerClient | undefined = sharedClient
|
||||
): Promise<string> {
|
||||
const inline = (): string => createHash('sha256').update(buffer).digest('hex')
|
||||
// Small inputs hash faster than a round trip.
|
||||
if (buffer.byteLength < 1024 * 1024) return Promise.resolve().then(inline)
|
||||
return runOffMain(client, 'sha256', () => {
|
||||
const bytes = transferableCopy(buffer)
|
||||
return { args: [bytes], transfer: [bytes] }
|
||||
}, undefined, inline)
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Wire format shared by document-parse-client (Main) and document-parse-worker.
|
||||
|
||||
export type DocumentParseOperation =
|
||||
| 'parseDocument'
|
||||
| 'extractPdfTextPages'
|
||||
| 'extractPptxPages'
|
||||
| 'chunkDocument'
|
||||
| 'sha256'
|
||||
|
||||
export type DocumentParseRequest = {
|
||||
id: number
|
||||
op: DocumentParseOperation
|
||||
args: unknown[]
|
||||
}
|
||||
|
||||
export type SerializedDocumentParseError = {
|
||||
name: string
|
||||
message: string
|
||||
/** Primitive own properties some error types carry (PDF.js, Node). */
|
||||
extra?: Record<string, unknown>
|
||||
cause?: SerializedDocumentParseError
|
||||
}
|
||||
|
||||
export type DocumentParseResponse =
|
||||
| { id: number; result: unknown }
|
||||
| { id: number; error: SerializedDocumentParseError }
|
||||
|
||||
export const documentParseErrorExtraKeys = ['code', 'status', 'missing', 'details', 'extraDelay'] as const
|
||||
@@ -0,0 +1,284 @@
|
||||
// @vitest-environment node
|
||||
import { createHash } from 'node:crypto'
|
||||
import { isDeepStrictEqual } from 'node:util'
|
||||
import { afterAll, beforeAll, describe, expect, it, onTestFinished } from 'vitest'
|
||||
import { strToU8, zipSync } from 'fflate'
|
||||
import type { KnowledgeChunkingSettings } from '../shared/knowledge-contracts'
|
||||
import {
|
||||
buildDocumentParseWorker,
|
||||
createDocx,
|
||||
createLargePdf,
|
||||
createPptx,
|
||||
createTextPdf,
|
||||
createXlsx,
|
||||
sentenceGenerator
|
||||
} from '../../tests/support/document-parse-worker-fixture'
|
||||
import { createImagePptx } from '../../tests/support/pptx-fixture'
|
||||
import {
|
||||
chunkDocumentOffMain,
|
||||
DocumentParseWorkerClient,
|
||||
DocumentParseWorkerUnavailableError,
|
||||
extractPdfTextPagesOffMain,
|
||||
extractPptxPagesOffMain,
|
||||
parseDocumentOffMain,
|
||||
sha256OffMain
|
||||
} from './document-parse-client'
|
||||
import {
|
||||
chunkDocumentAdvanced,
|
||||
DocumentTextUnavailableError,
|
||||
extractPdfTextPages,
|
||||
parseDocument
|
||||
} from './knowledge/document-parser'
|
||||
import { extractPptxPages } from './knowledge/pptx-parser'
|
||||
|
||||
let workerPath: string
|
||||
let dispose: () => Promise<void>
|
||||
let client: DocumentParseWorkerClient
|
||||
|
||||
beforeAll(async () => {
|
||||
({ workerPath, dispose } = await buildDocumentParseWorker())
|
||||
client = new DocumentParseWorkerClient(workerPath)
|
||||
}, 60_000)
|
||||
afterAll(async () => {
|
||||
client?.close()
|
||||
await dispose?.()
|
||||
})
|
||||
|
||||
const sentence = sentenceGenerator(3)
|
||||
const markdown = [
|
||||
'# GoodBuddy',
|
||||
'知识内容。'.repeat(400),
|
||||
'## 安装',
|
||||
...Array.from({ length: 40 }, (_, index) => `Step ${index}: ${sentence(30)}.`),
|
||||
'### 配置\r\n' + sentence(200),
|
||||
'# 附录',
|
||||
sentence(600)
|
||||
].join('\n')
|
||||
|
||||
/** Fixture corpus: every format the parser accepts, plus edge cases. */
|
||||
const fixtures: Array<{ name: string; buffer: Buffer }> = [
|
||||
{ name: 'notes.md', buffer: Buffer.from(markdown) },
|
||||
{ name: 'plain.txt', buffer: Buffer.from(` ${sentence(5_000)} `) },
|
||||
{ name: 'data.json', buffer: Buffer.from(JSON.stringify({ items: Array.from({ length: 200 }, (_, id) => ({ id, text: sentence(8) })) }, null, 2)) },
|
||||
{ name: 'page.html', buffer: Buffer.from('<main><h1>安全标题</h1><p>网页正文</p><table><tr><td>a</td><td>b</td></tr></table></main><script>恶意脚本</script><style>x{}</style>') },
|
||||
{ name: 'large.txt', buffer: Buffer.from('x'.repeat(5_000_100)) },
|
||||
{ name: 'sample.pdf', buffer: createTextPdf([['PDF body text']]) },
|
||||
{ name: 'multi.pdf', buffer: createTextPdf([['first line', 'second (line)'], [], ['third page', 'ends.']]) },
|
||||
{ name: 'big.pdf', buffer: createLargePdf(40, 40) },
|
||||
{ name: 'sample.docx', buffer: createDocx(['文档正文', '第二段', ''], [[['h1', 'h2'], ['v1', 'v2']]]) },
|
||||
{ name: 'sample.xlsx', buffer: createXlsx([[['name', 'qty', 'note'], ['apple', '3', 'fresh']], [['b', '7', 'inline']]]) },
|
||||
{ name: 'shared.xlsx', buffer: Buffer.from(zipSync({ 'xl/sharedStrings.xml': strToU8('<sst><si><t>表格内容</t></si></sst>') })) },
|
||||
{ name: 'sample.pptx', buffer: createPptx(['幻灯片内容', 'second slide', 'third']) },
|
||||
{ name: 'image.pptx', buffer: createPptx(['with image', 'two'], true) }
|
||||
]
|
||||
|
||||
const failureFixtures: Array<{ name: string; buffer: Buffer }> = [
|
||||
{ name: 'archive.zip', buffer: Buffer.from('not supported') },
|
||||
{ name: 'empty.txt', buffer: Buffer.alloc(0) },
|
||||
{ name: 'large.txt', buffer: Buffer.alloc(20 * 1024 * 1024 + 1) },
|
||||
{ name: 'expanded.docx', buffer: Buffer.from(zipSync({ 'word/document.xml': new Uint8Array(11 * 1024 * 1024) })) },
|
||||
{ name: 'invalid.txt', buffer: Buffer.from([0xc3, 0x28]) },
|
||||
{ name: 'nulls.txt', buffer: Buffer.from('a\0b\0c\0d') },
|
||||
{ name: 'blank.md', buffer: Buffer.from(' \n\n ') },
|
||||
{ name: 'broken.pdf', buffer: Buffer.from('%PDF-1.4 not really') },
|
||||
{ name: 'empty.pdf', buffer: createTextPdf([[]]) },
|
||||
{ name: 'corrupt.xlsx', buffer: Buffer.from('PK not a zip') }
|
||||
]
|
||||
|
||||
const chunkingModes: KnowledgeChunkingSettings[] = [
|
||||
{ version: 1, mode: 'fixed', targetCharacters: 500, overlapCharacters: 50, parentCharacters: 4_800, childCharacters: 900, contextualIndexingEnabled: false },
|
||||
{ version: 1, mode: 'structure', targetCharacters: 800, overlapCharacters: 80, parentCharacters: 4_800, childCharacters: 900, contextualIndexingEnabled: true },
|
||||
{ version: 1, mode: 'parent-child', targetCharacters: 1_600, overlapCharacters: 100, parentCharacters: 1_600, childCharacters: 400, contextualIndexingEnabled: false }
|
||||
]
|
||||
|
||||
/** Byte-level comparison: JSON (key order) plus deep equality (undefined fields, typed arrays). */
|
||||
function expectIdentical(actual: unknown, expected: unknown): void {
|
||||
expect(JSON.stringify(actual)).toBe(JSON.stringify(expected))
|
||||
expect(isDeepStrictEqual(normalizeBytes(actual), normalizeBytes(expected))).toBe(true)
|
||||
}
|
||||
|
||||
function normalizeBytes(value: unknown): unknown {
|
||||
if (value instanceof Uint8Array) return { bytes: Buffer.from(value).toString('hex') }
|
||||
if (Array.isArray(value)) return value.map(normalizeBytes)
|
||||
if (value && typeof value === 'object') {
|
||||
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, normalizeBytes(item)]))
|
||||
}
|
||||
return value
|
||||
}
|
||||
|
||||
async function settle<T>(run: () => Promise<T>): Promise<{ ok: true; value: T } | { ok: false; error: unknown }> {
|
||||
try {
|
||||
return { ok: true, value: await run() }
|
||||
} catch (error) {
|
||||
return { ok: false, error }
|
||||
}
|
||||
}
|
||||
|
||||
function describeError(error: unknown): unknown {
|
||||
return error instanceof Error
|
||||
? { constructor: error.constructor, name: error.name, message: error.message,
|
||||
own: Object.entries(error).filter(([key]) => key !== 'stack'),
|
||||
unavailable: error instanceof DocumentTextUnavailableError,
|
||||
cause: describeError(error.cause) }
|
||||
: error
|
||||
}
|
||||
|
||||
describe('document parse worker equivalence', () => {
|
||||
it('parses every fixture to output identical to the inline parser', async () => {
|
||||
for (const fixture of fixtures) {
|
||||
const inline = await parseDocument(fixture.name, fixture.buffer)
|
||||
const before = Buffer.from(fixture.buffer)
|
||||
const offMain = await parseDocumentOffMain(fixture.name, fixture.buffer, undefined, client)
|
||||
expectIdentical(offMain, inline)
|
||||
// The caller's buffer is not detached or modified by the transfer.
|
||||
expect(fixture.buffer.equals(before)).toBe(true)
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it('chunks every fixture identically in all chunking modes', async () => {
|
||||
// large.txt (5M chars, no break points) is excluded: splitNatural's backwards
|
||||
// boundary scan makes it take minutes on either path.
|
||||
for (const fixture of fixtures.filter((item) => item.name !== 'large.txt')) {
|
||||
const parsed = await parseDocument(fixture.name, fixture.buffer)
|
||||
for (const settings of chunkingModes) {
|
||||
const inline = await settle(async () => chunkDocumentAdvanced(parsed, settings))
|
||||
const offMain = await settle(() => chunkDocumentOffMain(parsed, settings, client))
|
||||
expect(offMain.ok).toBe(inline.ok)
|
||||
if (inline.ok && offMain.ok) expectIdentical(offMain.value, inline.value)
|
||||
else expect(describeError(offMain.ok ? undefined : offMain.error)).toEqual(describeError(inline.ok ? undefined : inline.error))
|
||||
}
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it('extracts PDF pages and PPTX pages identically', async () => {
|
||||
// Pre-existing PDF.js behaviour on the inline path: cancelling a text stream
|
||||
// at the character limit can leave a stray "Controller is already closed".
|
||||
const strayPdfRejection = (reason: unknown): void => {
|
||||
if ((reason as { code?: string })?.code !== 'ERR_INVALID_STATE') throw reason
|
||||
}
|
||||
process.on('unhandledRejection', strayPdfRejection)
|
||||
onTestFinished(async () => {
|
||||
await new Promise((resolve) => setTimeout(resolve, 50))
|
||||
process.removeListener('unhandledRejection', strayPdfRejection)
|
||||
})
|
||||
for (const fixture of fixtures.filter((item) => item.name.endsWith('.pdf'))) {
|
||||
for (const options of [{}, { maximumCharacters: 50 }, { maximumPages: 1 }]) {
|
||||
const inline = await settle(() => extractPdfTextPages(fixture.buffer, options))
|
||||
const offMain = await settle(() => extractPdfTextPagesOffMain(fixture.buffer, options, client))
|
||||
if (inline.ok && offMain.ok) expectIdentical(offMain.value, inline.value)
|
||||
else expect(describeError(offMain.ok ? undefined : offMain.error)).toEqual(describeError(inline.ok ? undefined : inline.error))
|
||||
}
|
||||
}
|
||||
for (const buffer of [createPptx(['a', 'b'], true), createImagePptx('native', 3), createPptx(['plain'])]) {
|
||||
const inline = extractPptxPages(buffer)
|
||||
const offMain = await extractPptxPagesOffMain(buffer, client)
|
||||
expectIdentical(offMain, inline)
|
||||
for (const page of offMain) {
|
||||
for (const image of page.images) expect(image.data).toBeInstanceOf(Uint8Array)
|
||||
}
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it('hashes identically', async () => {
|
||||
for (const buffer of [Buffer.alloc(0), Buffer.from('abc'), createLargePdf(40, 40), Buffer.alloc(3 * 1024 * 1024, 7)]) {
|
||||
expect(await sha256OffMain(buffer, client)).toBe(createHash('sha256').update(buffer).digest('hex'))
|
||||
}
|
||||
})
|
||||
|
||||
it('maps errors to the same types and messages as the inline parser', async () => {
|
||||
for (const fixture of failureFixtures) {
|
||||
const inline = await settle(() => parseDocument(fixture.name, fixture.buffer))
|
||||
const offMain = await settle(() => parseDocumentOffMain(fixture.name, fixture.buffer, undefined, client))
|
||||
expect(inline.ok, fixture.name).toBe(false)
|
||||
expect(offMain.ok, fixture.name).toBe(false)
|
||||
expect(describeError(offMain.ok ? undefined : offMain.error), fixture.name)
|
||||
.toEqual(describeError(inline.ok ? undefined : inline.error))
|
||||
}
|
||||
const tooManySections = {
|
||||
title: 'Too many', sourceFormat: '.txt', content: '', warnings: [],
|
||||
sections: Array.from({ length: 10_001 }, (_, index) => ({ locator: `s-${index}`, content: 'content' }))
|
||||
}
|
||||
const inline = await settle(async () => chunkDocumentAdvanced(tooManySections, chunkingModes[0]!))
|
||||
const offMain = await settle(() => chunkDocumentOffMain(tooManySections, chunkingModes[0]!, client))
|
||||
expect(describeError(offMain.ok ? undefined : offMain.error)).toEqual(describeError(inline.ok ? undefined : inline.error))
|
||||
const range = await settle(() => extractPdfTextPagesOffMain(createTextPdf([['x']]), { maximumPages: 0 }, client))
|
||||
expect(range.ok ? undefined : range.error).toBeInstanceOf(RangeError)
|
||||
})
|
||||
})
|
||||
|
||||
describe('document parse worker lifecycle', () => {
|
||||
it('cancels a running parse by terminating the worker and keeps serving later tasks', async () => {
|
||||
const local = new DocumentParseWorkerClient(workerPath)
|
||||
try {
|
||||
const big = createLargePdf(400, 50)
|
||||
const controller = new AbortController()
|
||||
const reason = new Error('文档解析已取消')
|
||||
const running = parseDocumentOffMain('big.pdf', big, controller.signal, local)
|
||||
const queued = parseDocumentOffMain('notes.md', Buffer.from(markdown), undefined, local)
|
||||
const queuedCancelled = new AbortController()
|
||||
const queuedAbort = expect(parseDocumentOffMain('plain.txt', Buffer.from('x'), queuedCancelled.signal, local)).rejects.toBe(reason)
|
||||
queuedCancelled.abort(reason)
|
||||
await new Promise((resolve) => setTimeout(resolve, 30))
|
||||
controller.abort(reason)
|
||||
await expect(running).rejects.toBe(reason)
|
||||
await queuedAbort
|
||||
expectIdentical(await queued, await parseDocument('notes.md', Buffer.from(markdown)))
|
||||
expect(local.pendingCount).toBe(0)
|
||||
} finally {
|
||||
local.close()
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it('rejects with the abort reason when already aborted, without starting the worker', async () => {
|
||||
const local = new DocumentParseWorkerClient(workerPath)
|
||||
try {
|
||||
const controller = new AbortController()
|
||||
const reason = new Error('cancelled')
|
||||
controller.abort(reason)
|
||||
await expect(parseDocumentOffMain('a.txt', Buffer.from('a'), controller.signal, local)).rejects.toBe(reason)
|
||||
} finally {
|
||||
local.close()
|
||||
}
|
||||
})
|
||||
|
||||
it('falls back to the inline path when the worker crashes or cannot start', async () => {
|
||||
const local = new DocumentParseWorkerClient(workerPath)
|
||||
try {
|
||||
const running = local.call('parseDocument', ['big.pdf', Uint8Array.from(createLargePdf(200, 50)).buffer])
|
||||
await new Promise((resolve) => setTimeout(resolve, 20))
|
||||
await local.terminateWorkerForTest()
|
||||
await expect(running).rejects.toBeInstanceOf(DocumentParseWorkerUnavailableError)
|
||||
expect(local.available).toBe(false)
|
||||
// While backing off, calls run inline with identical output.
|
||||
const buffer = Buffer.from(markdown)
|
||||
expectIdentical(await parseDocumentOffMain('notes.md', buffer, undefined, local), await parseDocument('notes.md', buffer))
|
||||
local.resetBackoffForTest()
|
||||
expect(local.available).toBe(true)
|
||||
expectIdentical(await parseDocumentOffMain('notes.md', buffer, undefined, local), await parseDocument('notes.md', buffer))
|
||||
} finally {
|
||||
local.close()
|
||||
}
|
||||
|
||||
const missing = new DocumentParseWorkerClient(`${workerPath}.missing.cjs`)
|
||||
try {
|
||||
const buffer = createDocx(['fallback'])
|
||||
expectIdentical(await parseDocumentOffMain('a.docx', buffer, undefined, missing), await parseDocument('a.docx', buffer))
|
||||
expect(missing.available).toBe(false)
|
||||
} finally {
|
||||
missing.close()
|
||||
}
|
||||
}, 60_000)
|
||||
|
||||
it('runs tasks one at a time in FIFO order', async () => {
|
||||
const local = new DocumentParseWorkerClient(workerPath)
|
||||
try {
|
||||
const names = fixtures.slice(0, 6)
|
||||
const results = await Promise.all(names.map((fixture) => parseDocumentOffMain(fixture.name, fixture.buffer, undefined, local)))
|
||||
for (const [index, fixture] of names.entries()) {
|
||||
expectIdentical(results[index], await parseDocument(fixture.name, fixture.buffer))
|
||||
}
|
||||
} finally {
|
||||
local.close()
|
||||
}
|
||||
}, 60_000)
|
||||
})
|
||||
@@ -0,0 +1,101 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { parentPort } from 'node:worker_threads'
|
||||
import type { KnowledgeChunkingSettings } from '../shared/knowledge-contracts'
|
||||
import {
|
||||
chunkDocumentAdvanced,
|
||||
extractPdfTextPages,
|
||||
parseDocument,
|
||||
type ParsedDocument
|
||||
} from './knowledge/document-parser'
|
||||
import { extractPptxPages } from './knowledge/pptx-parser'
|
||||
import {
|
||||
documentParseErrorExtraKeys,
|
||||
type DocumentParseRequest,
|
||||
type DocumentParseResponse,
|
||||
type SerializedDocumentParseError
|
||||
} from './document-parse-errors'
|
||||
|
||||
// PERF-15/16: CPU-heavy, pure document work (PDF / Office / text parsing,
|
||||
// chunking, hashing) runs here so Main only orchestrates. Every handler calls
|
||||
// the same production function as the inline path, so results are identical.
|
||||
// No database access and no file system access happen in this worker.
|
||||
|
||||
type Handler = (args: unknown[]) => Promise<unknown> | unknown
|
||||
|
||||
const asBuffer = (value: unknown): Buffer => {
|
||||
const bytes = value as ArrayBuffer
|
||||
return Buffer.from(bytes, 0, bytes.byteLength)
|
||||
}
|
||||
|
||||
const handlers: Record<string, Handler> = {
|
||||
parseDocument: ([name, bytes]) => parseDocument(name as string, asBuffer(bytes)),
|
||||
extractPdfTextPages: ([bytes, options]) => extractPdfTextPages(
|
||||
asBuffer(bytes),
|
||||
options as { maximumPages?: number; maximumCharacters?: number }
|
||||
),
|
||||
extractPptxPages: ([bytes]) => extractPptxPages(asBuffer(bytes)),
|
||||
chunkDocument: ([document, settings]) => chunkDocumentAdvanced(
|
||||
document as ParsedDocument,
|
||||
settings as KnowledgeChunkingSettings
|
||||
),
|
||||
sha256: ([bytes]) => createHash('sha256').update(asBuffer(bytes)).digest('hex')
|
||||
}
|
||||
|
||||
/** ArrayBuffers backing typed arrays in the result (PPTX images); transferred, not copied. */
|
||||
function transferables(value: unknown): ArrayBuffer[] {
|
||||
const buffers = new Set<ArrayBuffer>()
|
||||
if (Array.isArray(value)) {
|
||||
for (const page of value as Array<{ images?: Array<{ data?: unknown }> }>) {
|
||||
for (const image of page?.images ?? []) {
|
||||
const data = image?.data
|
||||
if (
|
||||
data instanceof Uint8Array &&
|
||||
data.buffer instanceof ArrayBuffer &&
|
||||
data.byteOffset === 0 &&
|
||||
data.byteLength === data.buffer.byteLength
|
||||
) {
|
||||
buffers.add(data.buffer)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return [...buffers]
|
||||
}
|
||||
|
||||
function serializeError(error: unknown, depth = 0): SerializedDocumentParseError {
|
||||
if (!(error instanceof Error)) return { name: 'Error', message: String(error) }
|
||||
const extra: Record<string, unknown> = {}
|
||||
for (const key of documentParseErrorExtraKeys) {
|
||||
const value = (error as unknown as Record<string, unknown>)[key]
|
||||
if (value !== undefined && (typeof value !== 'object' || value === null)) extra[key] = value
|
||||
}
|
||||
return {
|
||||
name: error.name,
|
||||
message: error.message,
|
||||
...(Object.keys(extra).length > 0 ? { extra } : {}),
|
||||
...(error.cause !== undefined && depth < 3 ? { cause: serializeError(error.cause, depth + 1) } : {})
|
||||
}
|
||||
}
|
||||
|
||||
// PDF.js can leave a stray rejection ("Controller is already closed") after a
|
||||
// text stream is cancelled at the character limit. Unhandled, it would kill
|
||||
// this thread and push the next tasks onto the inline fallback. Task errors are
|
||||
// awaited by the handler and still reported through the response.
|
||||
process.on('unhandledRejection', () => undefined)
|
||||
|
||||
parentPort!.on('message', (message: DocumentParseRequest) => {
|
||||
void (async () => {
|
||||
let response: DocumentParseResponse
|
||||
let transfer: ArrayBuffer[] = []
|
||||
try {
|
||||
const handler = Object.hasOwn(handlers, message.op) ? handlers[message.op] : undefined
|
||||
if (!handler) throw new Error(`Unknown document parse operation: ${message.op}`)
|
||||
const result = await handler(message.args)
|
||||
transfer = message.op === 'extractPptxPages' ? transferables(result) : []
|
||||
response = { id: message.id, result }
|
||||
} catch (error) {
|
||||
response = { id: message.id, error: serializeError(error) }
|
||||
}
|
||||
parentPort!.postMessage(response, transfer)
|
||||
})()
|
||||
})
|
||||
@@ -18,12 +18,14 @@ import { HttpDocumentOcr } from './http-document-ocr'
|
||||
import { hasExtractedDocumentText } from './document-extracted-text'
|
||||
import { renderOcrPdf, renderSelectedOcrPdf } from './render-ocr-pdf'
|
||||
import type { DocumentResultStorage } from './document-result-storage'
|
||||
import { extractPptxPages } from './knowledge/pptx-parser'
|
||||
import {
|
||||
extractPdfTextPagesOffMain,
|
||||
extractPptxPagesOffMain,
|
||||
parseDocumentOffMain
|
||||
} from './document-parse-client'
|
||||
import {
|
||||
assertDocumentBuffer,
|
||||
DocumentTextUnavailableError,
|
||||
extractPdfTextPages,
|
||||
parseDocument,
|
||||
type ParsedDocument,
|
||||
type ParsedSection,
|
||||
type PdfTextPage
|
||||
@@ -279,10 +281,10 @@ export class DocumentParsingService {
|
||||
return this.parsePptx(name, buffer, purpose, operation, signal)
|
||||
}
|
||||
if (extname(name).toLowerCase() !== '.pdf') {
|
||||
return parseDocument(name, buffer, signal)
|
||||
return parseDocumentOffMain(name, buffer, signal)
|
||||
}
|
||||
|
||||
const extracted = await extractPdfTextPages(buffer, { signal })
|
||||
const extracted = await extractPdfTextPagesOffMain(buffer, { signal })
|
||||
const { pages } = extracted
|
||||
ensureNotAborted(signal)
|
||||
const mode = effectiveOcrMode(settings, purpose)
|
||||
@@ -517,7 +519,7 @@ export class DocumentParsingService {
|
||||
const mode = effectiveOcrMode(settings, purpose)
|
||||
if (mode === 'disabled') {
|
||||
try {
|
||||
return await parseDocument(name, buffer, signal)
|
||||
return await parseDocumentOffMain(name, buffer, signal)
|
||||
} catch (error) {
|
||||
if (error instanceof DocumentTextUnavailableError) {
|
||||
throw new DocumentTextUnavailableError(
|
||||
@@ -527,7 +529,7 @@ export class DocumentParsingService {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const pages = extractPptxPages(buffer)
|
||||
const pages = await extractPptxPagesOffMain(buffer)
|
||||
ensureNotAborted(signal)
|
||||
const native: ParsedSection[] = pages
|
||||
.filter((page) => page.content)
|
||||
|
||||
+31
-15
@@ -1,3 +1,4 @@
|
||||
import { startupMark, startupSpan } from './startup-marks'
|
||||
import { localInferenceService } from './local-inference-service'
|
||||
import { NativeClientCoordinator } from './agent/native-client-coordinator'
|
||||
import { createEmbeddingUtilityTransport } from './knowledge/embedding-utility-transport'
|
||||
@@ -98,6 +99,7 @@ import { DocumentParsingSettingsStore } from './document-parsing-settings-store'
|
||||
import { DocumentOcrModelManager } from './document-ocr-model-manager'
|
||||
import { DocumentOcrBroker } from './document-ocr-broker'
|
||||
import { DocumentParsingService } from './document-parsing-service'
|
||||
import { configureDocumentParseWorker } from './document-parse-client'
|
||||
import { DocumentResultStorage } from './document-result-storage'
|
||||
import { ConversationAttachmentStorage } from './conversation-attachment-storage'
|
||||
import { ReleaseNotesService } from './release-notes-service'
|
||||
@@ -553,6 +555,7 @@ function buildTray(): Tray {
|
||||
const storageUpgradeController = new AbortController()
|
||||
let storageUpgrade: Promise<void> | undefined
|
||||
|
||||
startupMark('main:module-evaluated')
|
||||
if (hasSingleInstanceLock) {
|
||||
app.on('second-instance', () => {
|
||||
if (mainWindow) {
|
||||
@@ -561,6 +564,7 @@ if (hasSingleInstanceLock) {
|
||||
})
|
||||
|
||||
void app.whenReady().then(async () => {
|
||||
startupMark('main:when-ready')
|
||||
session.defaultSession.setPermissionRequestHandler(
|
||||
(webContents, permission, callback, details) => {
|
||||
const mediaTypes =
|
||||
@@ -582,15 +586,18 @@ if (hasSingleInstanceLock) {
|
||||
details.mediaType === 'audio'
|
||||
)
|
||||
|
||||
startupMark('main:create-window:start')
|
||||
mainWindow = createMainWindow(() => isQuitting, observeDesktopFailure)
|
||||
startupMark('main:create-window:end')
|
||||
registerDesktopNotificationActivation(mainWindow)
|
||||
tray = buildTray()
|
||||
startupMark('main:tray-built')
|
||||
storageUpgrade = prepareAssistantStorage(
|
||||
mainWindow,
|
||||
join(app.getPath('userData'), 'assistant.sqlite'),
|
||||
storageUpgradeController.signal
|
||||
)
|
||||
await storageUpgrade
|
||||
await startupSpan('main:storage-upgrade-check', () => storageUpgrade!)
|
||||
if (storageUpgradeController.signal.aborted) return
|
||||
const defaultWorkspace = process.env.GOODBUDDY_WORKSPACE ?? homedir()
|
||||
const secureCipher = {
|
||||
@@ -670,7 +677,7 @@ if (hasSingleInstanceLock) {
|
||||
},
|
||||
baseEnvironment: buildCredentialFilteredUserEnvironment()
|
||||
})
|
||||
await startupLocalToolEnvironmentService.initialize()
|
||||
await startupSpan('main:local-tool-environment', () => startupLocalToolEnvironmentService.initialize())
|
||||
localToolEnvironmentService =
|
||||
startupLocalToolEnvironmentService
|
||||
const sshHostStore = new SshHostStore(
|
||||
@@ -682,11 +689,11 @@ if (hasSingleInstanceLock) {
|
||||
resourcesPath: process.resourcesPath,
|
||||
packaged: app.isPackaged
|
||||
})
|
||||
const controlPlanePackageInstaller = await readFile(
|
||||
const controlPlanePackageInstaller = await startupSpan('main:read-control-plane-installer', () => readFile(
|
||||
resolveControlPlanePackageInstallerPath({
|
||||
appPath: app.getAppPath()
|
||||
})
|
||||
)
|
||||
))
|
||||
const agentPackageManager = new AgentPackageManager({
|
||||
userDataPath: app.getPath('userData'),
|
||||
desktopVersion: app.getVersion(),
|
||||
@@ -728,7 +735,7 @@ if (hasSingleInstanceLock) {
|
||||
return selection.provider === 'continue' ? resolved.continueModelProfile : resolved.opencodeModelProfile
|
||||
}
|
||||
})
|
||||
await startupManagedRemoteExecutionServices.initialize()
|
||||
await startupSpan('main:managed-remote-init', () => startupManagedRemoteExecutionServices.initialize())
|
||||
managedRemoteExecutionServices =
|
||||
startupManagedRemoteExecutionServices
|
||||
const beginRemoteHostInvalidation = (hostId: string): void => {
|
||||
@@ -777,10 +784,10 @@ if (hasSingleInstanceLock) {
|
||||
)
|
||||
})
|
||||
const [initialRuntimeSettings, initialResolvedSettings] =
|
||||
await Promise.all([
|
||||
await startupSpan('main:runtime-settings-load', () => Promise.all([
|
||||
settingsStore.getPublicSettings(),
|
||||
settingsStore.getResolvedSettings()
|
||||
])
|
||||
]))
|
||||
globalTlsPolicy = new GlobalTlsPolicy(app)
|
||||
globalTlsPolicy.install()
|
||||
const capabilityService = new CapabilityService(
|
||||
@@ -836,6 +843,7 @@ if (hasSingleInstanceLock) {
|
||||
(await applicationSettingsStore.get()).modelDownloadSource
|
||||
})
|
||||
documentOcrBroker = new DocumentOcrBroker(mainWindow)
|
||||
configureDocumentParseWorker(join(app.getAppPath(), 'out/main/document-parse-worker.js'))
|
||||
documentParsingService = new DocumentParsingService(
|
||||
documentParsingSettingsStore,
|
||||
documentOcrModelManager,
|
||||
@@ -1208,7 +1216,8 @@ if (hasSingleInstanceLock) {
|
||||
resolved.target
|
||||
)
|
||||
}
|
||||
const configuredRuntime = await runStartupPrerequisites({
|
||||
startupMark('main:services-constructed')
|
||||
const configuredRuntime = await startupSpan('main:prerequisites', () => runStartupPrerequisites({
|
||||
prepareDeepSeekHome: async () => {
|
||||
await mkdir(deepSeekHarnessHome, {
|
||||
recursive: true,
|
||||
@@ -1216,7 +1225,7 @@ if (hasSingleInstanceLock) {
|
||||
})
|
||||
},
|
||||
initializeKnowledgeAndGateway: async () => {
|
||||
await startupKnowledgeService.initialize()
|
||||
await startupSpan('main:knowledge-init', () => startupKnowledgeService.initialize())
|
||||
const embeddingProvider = await createEmbeddingProvider(
|
||||
initialResolvedSettings,
|
||||
startupEmbeddingModelManager
|
||||
@@ -1242,11 +1251,12 @@ if (hasSingleInstanceLock) {
|
||||
() => undefined
|
||||
)
|
||||
])
|
||||
await startupKnowledgeGateway.start()
|
||||
await startupSpan('main:knowledge-gateway-start', () => startupKnowledgeGateway.start())
|
||||
},
|
||||
hydrateConfiguredRuntime: () =>
|
||||
createConfiguredRuntime(initialResolvedSettings),
|
||||
startupSpan('main:runtime-hydrate', () => createConfiguredRuntime(initialResolvedSettings)),
|
||||
initializeAssistant: () => {
|
||||
startupMark('main:assistant-db-init:start')
|
||||
startupAssistantDatabase.initialize(defaultWorkspace)
|
||||
startupAssistantDatabase.enableReadonlyWorker(readonlyQueryWorkerPath)
|
||||
imageGenerationService!.initialize()
|
||||
@@ -1259,13 +1269,14 @@ if (hasSingleInstanceLock) {
|
||||
initialRuntimeSettings
|
||||
)
|
||||
)
|
||||
startupMark('main:assistant-db-init:end')
|
||||
}
|
||||
})
|
||||
}))
|
||||
const [initialSubagentRuntime, initialSubagentProfileRuntimes] =
|
||||
await Promise.all([
|
||||
await startupSpan('main:subagent-runtimes', () => Promise.all([
|
||||
createSubagentRuntime(initialResolvedSettings),
|
||||
createSubagentProfileRuntimes(initialResolvedSettings)
|
||||
])
|
||||
]))
|
||||
const subagentService = new SubagentService(
|
||||
initialSubagentRuntime,
|
||||
startupAssistantDatabase,
|
||||
@@ -1285,8 +1296,10 @@ if (hasSingleInstanceLock) {
|
||||
createSelectedStatusRuntime,
|
||||
(conversationId) => localRuntimeRegistry.releaseConversation(conversationId)
|
||||
)
|
||||
startupMark('main:attachments-reconcile:start')
|
||||
conversationAttachmentStorage = new ConversationAttachmentStorage(app.getPath('userData'), documentParsingService.results!)
|
||||
conversationAttachmentStorage.reconcile((conversationId, kind, ownerId) => startupAssistantDatabase.hasAttachmentOwner(conversationId, kind, ownerId), true)
|
||||
startupMark('main:attachments-reconcile:end')
|
||||
const contextManager = new ContextManager({
|
||||
parseDocument: documentParsingService.parse,
|
||||
assets: conversationAttachmentStorage,
|
||||
@@ -1308,7 +1321,7 @@ if (hasSingleInstanceLock) {
|
||||
},
|
||||
process.platform
|
||||
)
|
||||
await shortcutSettingsService.initialize()
|
||||
await startupSpan('main:shortcuts-init', () => shortcutSettingsService.initialize())
|
||||
|
||||
let runtimeReconfigurationQueue: Promise<void> = Promise.resolve()
|
||||
let runtimeReconfigurationClosing = false
|
||||
@@ -1513,6 +1526,7 @@ if (hasSingleInstanceLock) {
|
||||
obsidianService, launchEnvironmentProvider: startupLocalToolEnvironmentService.launchEnvironmentProvider
|
||||
})
|
||||
})
|
||||
startupMark('main:ipc-register:start')
|
||||
removeIpcHandlers = registerIpcHandlers(
|
||||
mainWindow,
|
||||
runtime,
|
||||
@@ -1581,7 +1595,9 @@ if (hasSingleInstanceLock) {
|
||||
mainWindow,
|
||||
startupFeedbackService
|
||||
)
|
||||
startupMark('main:ipc-register:end')
|
||||
loadMainWindow(mainWindow)
|
||||
startupMark('main:load-main-window')
|
||||
removeDeviceSharingIpc = registerDeviceSharingIpc(mainWindow,
|
||||
new DeviceSharingService(join(app.getPath('userData'), 'device-sharing-settings.json'), app.getVersion()))
|
||||
setImmediate(() => {
|
||||
|
||||
@@ -21,11 +21,14 @@ import {
|
||||
} from 'node:path'
|
||||
import {
|
||||
buildChunkContextPrefix,
|
||||
chunkDocumentAdvanced,
|
||||
parseDocument,
|
||||
supportedDocumentExtensions,
|
||||
type ParsedDocument
|
||||
} from './document-parser'
|
||||
import {
|
||||
chunkDocumentOffMain,
|
||||
parseDocumentOffMain,
|
||||
sha256OffMain
|
||||
} from '../document-parse-client'
|
||||
import type { DocumentResultStorage } from '../document-result-storage'
|
||||
import { parsedCompleteness } from '../document-result-storage'
|
||||
import {
|
||||
@@ -282,7 +285,7 @@ export class KnowledgeService {
|
||||
this.urlImporter = options.urlImporter ?? new UrlImporter()
|
||||
this.documentParser =
|
||||
options.parseDocument ??
|
||||
((name, buffer) => parseDocument(name, buffer))
|
||||
((name, buffer) => parseDocumentOffMain(name, buffer))
|
||||
this.embeddingProvider = options.embeddingProvider
|
||||
this.rerankProvider = options.rerankProvider
|
||||
const embeddingBatchSize = options.embeddingBatchSize ?? 16
|
||||
@@ -769,7 +772,7 @@ export class KnowledgeService {
|
||||
const documentId = input.id ?? randomUUID()
|
||||
const imagesOnly = parsedCompleteness(parsed) === 'images-only'
|
||||
if (imagesOnly && this.database.getDocument(documentId)?.metadata.status === 'ready') throw new Error('未提取到可索引文字，保留上次正文与索引')
|
||||
const chunks = imagesOnly ? [] : this.createDocumentChunks(parsed, library)
|
||||
const chunks = imagesOnly ? [] : await this.createDocumentChunks(parsed, library)
|
||||
let embeddingReplacement: PreparedEmbeddingReplacement | undefined
|
||||
let embeddingFailure: PreparedDocumentPublication['embeddingFailure']
|
||||
const embeddingProvider = this.embeddingProvider
|
||||
@@ -2484,7 +2487,7 @@ export class KnowledgeService {
|
||||
const prepared = await this.prepareDocumentPublication(
|
||||
{
|
||||
...document,
|
||||
checksum: createHash('sha256').update(buffer).digest('hex'),
|
||||
checksum: await sha256OffMain(buffer),
|
||||
metadata: {
|
||||
...document.metadata,
|
||||
status: 'ready',
|
||||
@@ -3137,7 +3140,7 @@ export class KnowledgeService {
|
||||
progress: 35,
|
||||
message: '正在解析文档内容'
|
||||
})
|
||||
const checksum = createHash('sha256').update(buffer).digest('hex')
|
||||
const checksum = await sha256OffMain(buffer)
|
||||
const previous = existing.find(
|
||||
(document) => document.externalId === file.relativePath
|
||||
)
|
||||
@@ -3835,11 +3838,12 @@ export class KnowledgeService {
|
||||
}
|
||||
}
|
||||
|
||||
private createDocumentChunks(
|
||||
private async createDocumentChunks(
|
||||
parsed: ParsedDocument,
|
||||
library: KnowledgeBase
|
||||
): ReplaceChunkInput[] {
|
||||
const chunks = chunkDocumentAdvanced(parsed, library.chunkingSettings)
|
||||
): Promise<ReplaceChunkInput[]> {
|
||||
// Pure CPU work runs off Main; chunk ids are still assigned here.
|
||||
const chunks = await chunkDocumentOffMain(parsed, library.chunkingSettings)
|
||||
const ids = chunks.map(() => randomUUID())
|
||||
return chunks.map((chunk) => ({
|
||||
id: ids[chunk.position],
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { lookup as dnsLookup } from 'node:dns/promises'
|
||||
import { request as httpRequest } from 'node:http'
|
||||
import { request as httpsRequest } from 'node:https'
|
||||
import { parseDocument, type ParsedDocument } from './document-parser'
|
||||
import type { ParsedDocument } from './document-parser'
|
||||
import { parseDocumentOffMain } from '../document-parse-client'
|
||||
|
||||
type ResolvedAddress = {
|
||||
address: string
|
||||
@@ -224,7 +225,7 @@ export class UrlImporter {
|
||||
.trim() || url.hostname
|
||||
).slice(0, 240)
|
||||
: url.pathname.split('/').filter(Boolean).at(-1) ?? url.hostname
|
||||
const document = await parseDocument(
|
||||
const document = await parseDocumentOffMain(
|
||||
isHtml ? `${title}.html` : `${title}.txt`,
|
||||
response.body
|
||||
)
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
import { performance } from 'node:perf_hooks'
|
||||
|
||||
// PERF-01 startup phase marks. Inactive unless GB_PERF_STARTUP_MARKS=1 (set by
|
||||
// build/run-startup-perf.cjs); when inactive each call is one boolean check.
|
||||
// Marks hold only fixed phase names and process-relative times, never user
|
||||
// content. The startup harness reads them from the global below.
|
||||
const enabled = process.env.GB_PERF_STARTUP_MARKS === '1'
|
||||
type StartupMark = { name: string; at: number }
|
||||
const marks: StartupMark[] | undefined = enabled ? [] : undefined
|
||||
if (marks) {
|
||||
;(globalThis as { __goodbuddyStartupMarks?: StartupMark[] }).__goodbuddyStartupMarks = marks
|
||||
}
|
||||
|
||||
export function startupMark(name: string): void {
|
||||
if (marks) marks.push({ name, at: performance.now() })
|
||||
}
|
||||
|
||||
// Takes a thunk so the span also covers the synchronous part of the operation
|
||||
// (everything before its first await).
|
||||
export function startupSpan<T>(name: string, operation: () => Promise<T>): Promise<T> {
|
||||
if (!marks) return operation()
|
||||
startupMark(`${name}:start`)
|
||||
let started: Promise<T>
|
||||
try {
|
||||
started = operation()
|
||||
} catch (error) {
|
||||
startupMark(`${name}:end`)
|
||||
throw error
|
||||
}
|
||||
startupMark(`${name}:sync-end`)
|
||||
return started.finally(() => startupMark(`${name}:end`))
|
||||
}
|
||||
@@ -0,0 +1,130 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { mkdir, rm } from 'node:fs/promises'
|
||||
import { join, resolve } from 'node:path'
|
||||
import { build } from 'esbuild'
|
||||
import { strToU8, zipSync } from 'fflate'
|
||||
|
||||
/**
|
||||
* Bundles the document parse worker for tests. The bundle lives under
|
||||
* node_modules/.cache so the external pdfjs-dist (ESM, loaded lazily by the
|
||||
* parser) resolves from the repository's node_modules.
|
||||
*/
|
||||
export async function buildDocumentParseWorker(): Promise<{ workerPath: string; dispose: () => Promise<void> }> {
|
||||
const directory = resolve('node_modules/.cache', `goodbuddy-document-parse-worker-${randomUUID()}`)
|
||||
await mkdir(directory, { recursive: true })
|
||||
const workerPath = join(directory, 'document-parse-worker.cjs')
|
||||
await build({
|
||||
entryPoints: [resolve('src/main/document-parse-worker.ts')],
|
||||
outfile: workerPath,
|
||||
bundle: true,
|
||||
platform: 'node',
|
||||
format: 'cjs',
|
||||
target: 'node22',
|
||||
logLevel: 'silent',
|
||||
external: ['pdfjs-dist', 'electron']
|
||||
})
|
||||
return {
|
||||
workerPath,
|
||||
dispose: () => rm(directory, { recursive: true, force: true, maxRetries: 10, retryDelay: 100 })
|
||||
}
|
||||
}
|
||||
|
||||
function escapePdfText(value: string): string {
|
||||
return value.replaceAll('\\', '\\\\').replaceAll('(', '\\(').replaceAll(')', '\\)')
|
||||
}
|
||||
|
||||
/** A text-layer PDF with one content stream per page; each page is a list of lines. */
|
||||
export function createTextPdf(pages: readonly (readonly string[])[]): Buffer {
|
||||
const pageCount = pages.length
|
||||
const objects: string[] = [
|
||||
'<< /Type /Catalog /Pages 2 0 R >>',
|
||||
`<< /Type /Pages /Kids [${pages.map((_, index) => `${4 + index * 2} 0 R`).join(' ')}] /Count ${pageCount} >>`,
|
||||
'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>'
|
||||
]
|
||||
for (const lines of pages) {
|
||||
const pageObject = objects.length + 1
|
||||
const stream = [
|
||||
'BT /F1 10 Tf 12 TL 40 760 Td',
|
||||
...lines.map((line, index) => `${index === 0 ? '' : 'T* '}(${escapePdfText(line)}) Tj`),
|
||||
'ET'
|
||||
].join('\n')
|
||||
objects.push(
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> >> /Contents ${pageObject + 1} 0 R >>`,
|
||||
`<< /Length ${Buffer.byteLength(stream, 'latin1')} >>\nstream\n${stream}\nendstream`
|
||||
)
|
||||
}
|
||||
let content = '%PDF-1.4\n'
|
||||
const offsets: number[] = []
|
||||
for (const [index, object] of objects.entries()) {
|
||||
offsets.push(Buffer.byteLength(content, 'latin1'))
|
||||
content += `${index + 1} 0 obj\n${object}\nendobj\n`
|
||||
}
|
||||
const xrefOffset = Buffer.byteLength(content, 'latin1')
|
||||
content += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`
|
||||
content += offsets.map((offset) => `${String(offset).padStart(10, '0')} 00000 n \n`).join('')
|
||||
content += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xrefOffset}\n%%EOF\n`
|
||||
return Buffer.from(content, 'latin1')
|
||||
}
|
||||
|
||||
const words = 'lighthouse harbor policy engine renderer stream vector index budget latency schedule archive network storage contract invoice release window memory review'.split(' ')
|
||||
|
||||
/** Deterministic pseudo-random sentence generator. */
|
||||
export function sentenceGenerator(seed = 42): (count: number) => string {
|
||||
let state = seed
|
||||
const random = (): number => ((state = (state * 1664525 + 1013904223) >>> 0) / 4294967296)
|
||||
return (count) => Array.from({ length: count }, () => words[Math.floor(random() * words.length)]).join(' ')
|
||||
}
|
||||
|
||||
export function createLargePdf(pageCount: number, linesPerPage: number): Buffer {
|
||||
const sentence = sentenceGenerator(7)
|
||||
return createTextPdf(Array.from({ length: pageCount }, (_, page) =>
|
||||
Array.from({ length: linesPerPage }, (_, line) => `Page ${page + 1} line ${line + 1}: ${sentence(10)}.`)))
|
||||
}
|
||||
|
||||
export function createDocx(paragraphs: readonly string[], tables: readonly (readonly (readonly string[])[])[] = []): Buffer {
|
||||
const body = [
|
||||
...paragraphs.map((text) => `<w:p><w:r><w:t>${text}</w:t></w:r><w:r><w:tab/><w:t>tab</w:t></w:r></w:p>`),
|
||||
...tables.map((rows) => `<w:tbl>${rows.map((row) => `<w:tr>${row.map((cell) => `<w:tc><w:p><w:t>${cell}</w:t></w:p></w:tc>`).join('')}</w:tr>`).join('')}</w:tbl>`),
|
||||
'<w:p><w:t>tail & <end> 中文</w:t></w:p>'
|
||||
].join('')
|
||||
return Buffer.from(zipSync({
|
||||
'[Content_Types].xml': strToU8('<Types/>'),
|
||||
'word/document.xml': strToU8(`<w:document><w:body>${body}</w:body></w:document>`)
|
||||
}))
|
||||
}
|
||||
|
||||
export function createXlsx(sheets: readonly (readonly (readonly string[])[])[]): Buffer {
|
||||
const shared: string[] = []
|
||||
const files: Record<string, Uint8Array> = {}
|
||||
sheets.forEach((rows, sheetIndex) => {
|
||||
const xmlRows = rows.map((row, rowIndex) => `<row r="${rowIndex + 1}">${row.map((cell, cellIndex) => {
|
||||
if (/^\d+$/u.test(cell)) return `<c r="${String.fromCharCode(65 + cellIndex)}${rowIndex + 1}"><v>${cell}</v></c>`
|
||||
if (cellIndex === 2) return `<c t="inlineStr"><is><t>${cell}</t></is></c>`
|
||||
shared.push(cell)
|
||||
return `<c t="s"><v>${shared.length - 1}</v></c>`
|
||||
}).join('')}</row>`).join('')
|
||||
files[`xl/worksheets/sheet${sheetIndex + 1}.xml`] = strToU8(`<worksheet><sheetData>${xmlRows}</sheetData></worksheet>`)
|
||||
})
|
||||
files['xl/sharedStrings.xml'] = strToU8(`<sst>${shared.map((value) => `<si><t>${value}</t></si>`).join('')}</sst>`)
|
||||
return Buffer.from(zipSync(files))
|
||||
}
|
||||
|
||||
export function createPptx(slides: readonly string[], withImage = false): Buffer {
|
||||
const files: Record<string, Uint8Array> = {}
|
||||
if (withImage) {
|
||||
files['ppt/media/image1.png'] = new Uint8Array(Array.from({ length: 4096 }, (_, index) => index % 251))
|
||||
files['ppt/media/image2.jpg'] = new Uint8Array([255, 216, 255, 224, 1, 2, 3])
|
||||
}
|
||||
slides.forEach((text, index) => {
|
||||
const page = index + 1
|
||||
files[`ppt/slides/slide${page}.xml`] = strToU8(
|
||||
`<p:sld><a:p><a:t>${text}</a:t></a:p>${withImage ? `<p:pic><a:blip r:embed="rId${(index % 2) + 1}"/></p:pic>` : ''}</p:sld>`
|
||||
)
|
||||
if (withImage) {
|
||||
files[`ppt/slides/_rels/slide${page}.xml.rels`] = strToU8(
|
||||
'<Relationships><Relationship Id="rId1" Target="../media/image1.png"/><Relationship Id="rId2" Target="../media/image2.jpg"/></Relationships>'
|
||||
)
|
||||
}
|
||||
})
|
||||
return Buffer.from(zipSync(files))
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
// Cold-start measurement driver (PERF-01). Launched by
|
||||
// build/run-startup-perf.cjs, once per measured launch. Runs inside the
|
||||
// Electron main process: isolates the profile, blocks every non-loopback
|
||||
// request, imports the current production Main and records when startup
|
||||
// milestones happen. All times are epoch milliseconds (timeOrigin + now) so the
|
||||
// supervisor can relate them to its spawn time. The report holds only phase
|
||||
// names, times and counts; never conversation text or file paths.
|
||||
//
|
||||
// GB_STARTUP_MODE=prepare builds a reusable profile instead of measuring:
|
||||
// acknowledges the release notes, disables the startup update check, seeds
|
||||
// GB_STARTUP_SEED synthetic conversations, then quits through the normal
|
||||
// shutdown path so the databases are checkpointed and closed.
|
||||
import { app, BrowserWindow, session } from 'electron'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { readFileSync, writeFileSync } from 'node:fs'
|
||||
import { join } from 'node:path'
|
||||
import { monitorEventLoopDelay, performance } from 'node:perf_hooks'
|
||||
import process from 'node:process'
|
||||
import { clearTimeout, setTimeout } from 'node:timers'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { pathToFileURL, URL } from 'node:url'
|
||||
|
||||
const now = () => performance.timeOrigin + performance.now()
|
||||
const driverStart = now()
|
||||
const directory = process.env.GB_STARTUP_DIR
|
||||
const resultPath = process.env.GB_STARTUP_RESULT
|
||||
const root = process.env.GB_PERF_ROOT
|
||||
if (!directory || !resultPath || !root) throw new Error('Run through build/run-startup-perf.cjs')
|
||||
const mode = process.env.GB_STARTUP_MODE === 'prepare' ? 'prepare' : 'measure'
|
||||
const profilePath = process.env.GB_STARTUP_CPU_PROFILE
|
||||
const packageJson = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8'))
|
||||
|
||||
// Same isolation as tests/support/app-perf-driver.mjs; it must happen before
|
||||
// the production Main module loads (single-instance lock, storage paths).
|
||||
app.setAppPath(root)
|
||||
app.setName(packageJson.name)
|
||||
app.setVersion(packageJson.version)
|
||||
app.setPath('userData', join(directory, 'profile'))
|
||||
app.setPath('sessionData', join(directory, 'session'))
|
||||
app.setAppLogsPath(join(directory, 'logs'))
|
||||
app.commandLine.appendSwitch('force-device-scale-factor', '1')
|
||||
app.commandLine.appendSwitch('disable-features', 'CalculateNativeWinOcclusion')
|
||||
|
||||
const report = { mode, status: 'running', times: { driverStart }, blockedRequests: { fetch: [], renderer: [] }, renderer: {}, flags: {} }
|
||||
const times = report.times
|
||||
const mark = name => { if (times[name] === undefined) times[name] = now() }
|
||||
const writeReport = () => writeFileSync(resultPath, JSON.stringify(report, null, 2))
|
||||
const mainDelay = monitorEventLoopDelay({ resolution: 5 })
|
||||
mainDelay.enable()
|
||||
|
||||
const deadline = setTimeout(() => {
|
||||
report.status = 'failed'
|
||||
report.error = 'Driver timeout'
|
||||
writeReport()
|
||||
app.exit(1)
|
||||
}, Number(process.env.GB_STARTUP_DRIVER_TIMEOUT_MS || 120_000))
|
||||
|
||||
// No external requests: the fake model and everything else must be loopback.
|
||||
const isLoopback = url => {
|
||||
try {
|
||||
const { hostname, protocol } = new URL(url)
|
||||
if (!['http:', 'https:', 'ws:', 'wss:'].includes(protocol)) return true
|
||||
return hostname === '127.0.0.1' || hostname === 'localhost' || hostname === '[::1]'
|
||||
} catch { return true }
|
||||
}
|
||||
const hostOf = url => { try { return new URL(url).hostname } catch { return 'invalid' } }
|
||||
const originalFetch = globalThis.fetch
|
||||
globalThis.fetch = (input, init) => {
|
||||
const url = typeof input === 'string' ? input : input instanceof URL ? input.href : input?.url
|
||||
if (url && !isLoopback(url)) {
|
||||
report.blockedRequests.fetch.push(hostOf(url))
|
||||
return Promise.reject(new TypeError('blocked by startup benchmark'))
|
||||
}
|
||||
return originalFetch(input, init)
|
||||
}
|
||||
|
||||
let profiler
|
||||
if (profilePath) {
|
||||
const inspector = await import('node:inspector')
|
||||
profiler = new inspector.Session()
|
||||
profiler.connect()
|
||||
const post = (method, params) => new Promise((resolve, reject) => profiler.post(method, params, (error, result) => error ? reject(error) : resolve(result)))
|
||||
profiler.postAsync = post
|
||||
await post('Profiler.enable')
|
||||
await post('Profiler.setSamplingInterval', { interval: 250 })
|
||||
await post('Profiler.start')
|
||||
}
|
||||
|
||||
app.once('ready', () => mark('appReady'))
|
||||
// Registered before Main imports, so it runs before Main's whenReady handler.
|
||||
void app.whenReady().then(() => {
|
||||
session.defaultSession.webRequest.onBeforeRequest({ urls: ['http://*/*', 'https://*/*', 'ws://*/*', 'wss://*/*'] }, (details, callback) => {
|
||||
if (isLoopback(details.url)) { callback({}); return }
|
||||
report.blockedRequests.renderer.push(hostOf(details.url))
|
||||
callback({ cancel: true })
|
||||
})
|
||||
})
|
||||
|
||||
let win
|
||||
app.on('browser-window-created', (_event, created) => {
|
||||
if (win) return
|
||||
win = created
|
||||
mark('windowCreated')
|
||||
created.once('ready-to-show', () => mark('readyToShow'))
|
||||
created.once('show', () => mark('windowShown'))
|
||||
created.webContents.once('did-start-loading', () => mark('didStartLoading'))
|
||||
created.webContents.once('dom-ready', () => { mark('domReady'); void installRendererProbe() })
|
||||
created.webContents.once('did-finish-load', () => mark('didFinishLoad'))
|
||||
created.webContents.on('render-process-gone', (_e, details) => { report.error = `render-process-gone ${details.reason}` })
|
||||
})
|
||||
|
||||
mark('mainImportStart')
|
||||
await import(pathToFileURL(join(root, 'out', 'main', 'index.js')).href)
|
||||
mark('mainImportEnd')
|
||||
|
||||
const js = code => win.webContents.executeJavaScript(code, true)
|
||||
|
||||
// Composer is interactive when it exists, is enabled, laid out, and not inside
|
||||
// an inert or hidden subtree (a modal dialog makes the app shell inert).
|
||||
const composerSelector = '.composer__input textarea'
|
||||
const expectedConversations = mode === 'measure' ? Math.max(0, Number(process.env.GB_STARTUP_EXPECT_CONVERSATIONS || 0)) : 0
|
||||
const rendererProbe = `new Promise(resolve => {
|
||||
const origin = performance.timeOrigin;
|
||||
const at = () => origin + performance.now();
|
||||
const r = { installedAt: at(), visibleAtInstall: document.visibilityState === 'visible' };
|
||||
// Chromium does not report first-contentful-paint when the first paint
|
||||
// happens while the page is hidden, and the main window is created hidden
|
||||
// until ready-to-show. "First visible frame" is the second animation frame
|
||||
// after the page becomes visible, i.e. a frame actually presented on screen.
|
||||
let frameDone = false;
|
||||
const firstVisibleFrame = () => {
|
||||
r.visibleAt = at();
|
||||
requestAnimationFrame(() => requestAnimationFrame(() => { r.firstVisibleFrame = at(); frameDone = true; if (check()) finish(); }));
|
||||
};
|
||||
if (document.visibilityState === 'visible') firstVisibleFrame();
|
||||
else document.addEventListener('visibilitychange', function onVisible() { if (document.visibilityState !== 'visible') return; document.removeEventListener('visibilitychange', onVisible); firstVisibleFrame(); });
|
||||
const check = () => {
|
||||
const c = document.querySelector(${JSON.stringify(composerSelector)});
|
||||
if (c && r.composerPresent === undefined) r.composerPresent = at();
|
||||
if (!r.firstConversationRow && document.querySelector('button.conversation-item')) r.firstConversationRow = at();
|
||||
if (!r.startupDialog && document.querySelector('section.release-notes-dialog')) r.startupDialog = at();
|
||||
if (c && r.composerInteractive === undefined && !c.disabled && c.getClientRects().length > 0 && !c.closest('[inert], [hidden], [aria-hidden="true"]')) r.composerInteractive = at();
|
||||
// Seeded profiles: the sidebar is populated once the virtualized list
|
||||
// reports the seeded total (or mounts enough rows to fill the viewport).
|
||||
if (${expectedConversations} > 0 && r.sidebarPopulated === undefined) {
|
||||
const rows = document.querySelectorAll('button.conversation-item').length;
|
||||
const entry = document.querySelector('.conversation-list [aria-setsize]');
|
||||
const total = entry ? Number(entry.getAttribute('aria-setsize')) : rows;
|
||||
if (total >= ${expectedConversations} && rows >= ${Math.min(10, expectedConversations)}) r.sidebarPopulated = at();
|
||||
}
|
||||
const sidebarDone = ${expectedConversations} === 0 || r.sidebarPopulated !== undefined;
|
||||
return frameDone && sidebarDone && (r.composerInteractive !== undefined || (r.composerPresent !== undefined && r.startupDialog !== undefined));
|
||||
};
|
||||
let observer;
|
||||
let settled = false;
|
||||
const finish = () => { if (settled) return; settled = true; observer?.disconnect(); resolve(r); };
|
||||
if (check()) { finish(); return; }
|
||||
observer = new MutationObserver(() => { if (check()) finish(); });
|
||||
observer.observe(document.documentElement, { subtree: true, childList: true, attributes: true, attributeFilter: ['inert', 'hidden', 'disabled', 'aria-hidden', 'class'] });
|
||||
setTimeout(() => { r.timedOut = true; finish(); }, 60000);
|
||||
})`
|
||||
let probeResult
|
||||
async function installRendererProbe() {
|
||||
try { probeResult = await js(rendererProbe) } catch (error) { report.error = `renderer probe: ${error instanceof Error ? error.message : error}` }
|
||||
}
|
||||
|
||||
async function collectRenderer() {
|
||||
const data = await js(`(() => {
|
||||
const origin = performance.timeOrigin;
|
||||
const nav = performance.getEntriesByType('navigation')[0];
|
||||
const paint = Object.fromEntries(performance.getEntriesByType('paint').map(e => [e.name, origin + e.startTime]));
|
||||
const scripts = performance.getEntriesByType('resource').filter(e => e.initiatorType === 'script' || /\\.js$/.test(e.name));
|
||||
return {
|
||||
timeOrigin: origin,
|
||||
navigationStart: nav ? origin + nav.startTime : null,
|
||||
responseEnd: nav ? origin + nav.responseEnd : null,
|
||||
domInteractive: nav ? origin + nav.domInteractive : null,
|
||||
domContentLoaded: nav ? origin + nav.domContentLoadedEventEnd : null,
|
||||
loadEventEnd: nav ? origin + nav.loadEventEnd : null,
|
||||
firstPaint: paint['first-paint'] ?? null,
|
||||
firstContentfulPaint: paint['first-contentful-paint'] ?? null,
|
||||
scriptResources: scripts.length,
|
||||
scriptBytes: scripts.reduce((a, e) => a + (e.decodedBodySize || 0), 0),
|
||||
jsHeapMB: performance.memory ? performance.memory.usedJSHeapSize / 1048576 : null,
|
||||
domNodes: document.getElementsByTagName('*').length,
|
||||
conversationRows: document.querySelectorAll('button.conversation-item').length
|
||||
};
|
||||
})()`)
|
||||
return data
|
||||
}
|
||||
|
||||
// Long animation frames are buffered by Chromium, so the ones during startup
|
||||
// are visible even though the observer is installed after dom-ready.
|
||||
const loafProbe = `new Promise(resolve => {
|
||||
const items = [];
|
||||
try {
|
||||
new PerformanceObserver(list => { for (const e of list.getEntries()) items.push(e); }).observe({ type: 'long-animation-frame', buffered: true });
|
||||
} catch { resolve(null); return; }
|
||||
setTimeout(() => {
|
||||
const origin = performance.timeOrigin;
|
||||
resolve({ count: items.length, totalMs: items.reduce((a, e) => a + e.duration, 0), blockingMs: items.reduce((a, e) => a + e.blockingDuration, 0),
|
||||
maxMs: items.reduce((a, e) => Math.max(a, e.duration), 0),
|
||||
scriptMs: items.reduce((a, e) => a + e.scripts.reduce((s, x) => s + x.duration, 0), 0),
|
||||
frames: items.slice(0, 20).map(e => ({ at: origin + e.startTime, duration: e.duration, scriptMs: e.scripts.reduce((s, x) => s + x.duration, 0) })) });
|
||||
}, 50);
|
||||
})`
|
||||
|
||||
function profileHotspots(profile) {
|
||||
const byId = new Map(profile.nodes.map(node => [node.id, node]))
|
||||
const parent = new Map()
|
||||
for (const node of profile.nodes) for (const child of node.children ?? []) parent.set(child, node.id)
|
||||
const self = new Map()
|
||||
for (let index = 0; index < profile.samples.length; index += 1) {
|
||||
const id = profile.samples[index]
|
||||
self.set(id, (self.get(id) ?? 0) + (profile.timeDeltas[index] ?? 0))
|
||||
}
|
||||
const keyOf = frame => {
|
||||
const file = frame.url ? frame.url.replace(/^file:\/\/\/?/, '').split(/[\\/]/).slice(-3).join('/') : `(${frame.functionName || 'native'})`
|
||||
return `${frame.functionName || '(anonymous)'} ${file}:${frame.lineNumber + 1}`
|
||||
}
|
||||
const selfByKey = new Map()
|
||||
const inclusiveByKey = new Map()
|
||||
let total = 0
|
||||
for (const [id, micros] of self) {
|
||||
total += micros
|
||||
const key = keyOf(byId.get(id).callFrame)
|
||||
selfByKey.set(key, (selfByKey.get(key) ?? 0) + micros)
|
||||
const seen = new Set()
|
||||
for (let cursor = id; cursor !== undefined; cursor = parent.get(cursor)) {
|
||||
const k = keyOf(byId.get(cursor).callFrame)
|
||||
if (seen.has(k)) continue
|
||||
seen.add(k)
|
||||
inclusiveByKey.set(k, (inclusiveByKey.get(k) ?? 0) + micros)
|
||||
}
|
||||
}
|
||||
const top = (map, count, filter = () => true) => [...map].filter(([key]) => filter(key)).sort((a, b) => b[1] - a[1]).slice(0, count)
|
||||
.map(([key, micros]) => ({ key, ms: Math.round(micros / 100) / 10 }))
|
||||
const own = key => /out\/main\//.test(key)
|
||||
return {
|
||||
sampledMs: Math.round(total / 1000),
|
||||
selfTop: top(selfByKey, 30),
|
||||
inclusiveOutMain: top(inclusiveByKey, 40, own)
|
||||
}
|
||||
}
|
||||
|
||||
async function prepare() {
|
||||
await js(`window.goodbuddy.updates.updateSettings({ checkUpdatesOnStartup: false }).then(() => true)`)
|
||||
await js(`window.goodbuddy.releaseNotes.getPending().then(s => s.releases.some(r => r.version === s.currentVersion) ? window.goodbuddy.releaseNotes.acknowledge(s.currentVersion) : undefined).then(() => true)`)
|
||||
const count = Number(process.env.GB_STARTUP_SEED || 0)
|
||||
const perConversation = Number(process.env.GB_STARTUP_SEED_MESSAGES || 20)
|
||||
if (count > 0) {
|
||||
const projectId = await js(`window.goodbuddy.projects.list(false).then(list => list.find(p => p.kind === 'user' && p.executionSpace.kind === 'local')?.id)`)
|
||||
if (!projectId) throw new Error('No local project to seed')
|
||||
const base = Date.now()
|
||||
const items = []
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
const updatedAt = base - (index + 1) * 60_000
|
||||
const messages = []
|
||||
for (let m = 0; m < perConversation; m += 1) {
|
||||
const role = m % 2 === 0 ? 'user' : 'assistant'
|
||||
const content = role === 'user'
|
||||
? `Question ${m}: how should the benchmark handle item ${m}?`
|
||||
: `## Answer ${m}\n\nThe benchmark answer for item ${m} contains **formatting**, \`inline code\`, and a list:\n\n- point one\n- point two\n\n\`\`\`ts\nconst value${m} = ${m}\n\`\`\``
|
||||
messages.push({ id: randomUUID(), role, content, createdAt: updatedAt - (perConversation - m) * 1000, state: 'complete' })
|
||||
}
|
||||
items.push({ header: { id: randomUUID(), projectId, title: `Perf conversation ${String(index + 1).padStart(4, '0')}`, updatedAt, knowledgeLibraryIds: [], knowledgeRetrievalMode: 'auto' }, messages })
|
||||
}
|
||||
for (let offset = 0; offset < items.length; offset += 100) {
|
||||
await js(`window.goodbuddy.conversations.saveLocal(${JSON.stringify(items.slice(offset, offset + 100))}).then(() => true)`)
|
||||
}
|
||||
report.seeded = { conversations: count, messagesPerConversation: perConversation }
|
||||
}
|
||||
}
|
||||
|
||||
async function run() {
|
||||
try {
|
||||
for (let i = 0; i < 600 && !win; i += 1) await sleep(50)
|
||||
if (!win) throw new Error('Main window was not created')
|
||||
for (let i = 0; i < 1200 && probeResult === undefined; i += 1) await sleep(25)
|
||||
if (!probeResult) throw new Error(report.error ?? 'Renderer probe did not finish')
|
||||
mark('probeResolvedInMain')
|
||||
if (profiler) {
|
||||
const { profile } = await profiler.postAsync('Profiler.stop')
|
||||
writeFileSync(profilePath, JSON.stringify(profile))
|
||||
report.cpuProfile = profileHotspots(profile)
|
||||
}
|
||||
Object.assign(report.renderer, probeResult)
|
||||
Object.assign(report.renderer, await collectRenderer())
|
||||
report.renderer.longAnimationFrames = await js(loafProbe).catch(() => null)
|
||||
report.startupMarks = (globalThis.__goodbuddyStartupMarks ?? []).map(({ name, at }) => ({ name, at: performance.timeOrigin + at }))
|
||||
report.mainEventLoopDelay = { maxMs: mainDelay.max / 1e6, p99Ms: mainDelay.percentile(99) / 1e6, meanMs: mainDelay.mean / 1e6 }
|
||||
report.flags.windowCount = BrowserWindow.getAllWindows().length
|
||||
if (mode === 'prepare') await prepare()
|
||||
report.status = 'passed'
|
||||
writeReport()
|
||||
clearTimeout(deadline)
|
||||
if (mode === 'prepare') app.quit()
|
||||
else app.exit(0)
|
||||
} catch (error) {
|
||||
report.status = 'failed'
|
||||
report.error = error instanceof Error ? error.message : String(error)
|
||||
writeReport()
|
||||
clearTimeout(deadline)
|
||||
app.exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Electron emits `ready` only after this ESM entry finishes evaluating.
|
||||
void app.whenReady().then(run)
|
||||
Reference in New Issue
Block a user