goodbuddy/build/run-app-perf.cjs

// Full-App interaction benchmark supervisor (PERF-11).
// Starts a loopback fake OpenAI-compatible model on 127.0.0.1:11434 (the
// fresh-profile default model endpoint), launches the current production
// build in an isolated profile through tests/support/app-perf-driver.mjs,
// enforces a total deadline and prints a summary of the driver report.
const { spawn } = require('node:child_process')
const { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } = require('node:fs')
const { createServer } = require('node:http')
const { join, resolve } = require('node:path')

const runnerRoot = resolve(__dirname, '..')
const root = resolve(process.env.GB_PERF_ROOT || runnerRoot)
const concurrentMode = process.env.GB_PERF_MODE?.startsWith('C')
// 11434 is the fresh-profile default endpoint. Another GoodBuddy (or Ollama)
// may already use it; GB_PERF_MODEL_PORT moves the fake model and points the
// isolated profile at it through the GOODBUDDY_MODEL_* environment variables.
const defaultModelPort = 11434
const modelPort = Number(process.env.GB_PERF_MODEL_PORT || defaultModelPort)
const totalTimeoutMs = Number(process.env.GB_PERF_TIMEOUT_MS || (process.env.GB_PERF_MODE === 'C06' ? 900_000 : 600_000))
const keepProfile = process.env.GB_PERF_KEEP_PROFILE === '1'
const streamBytes = Number(process.env.GB_PERF_STREAM_BYTES || 40_000)
const chunkChars = Number(process.env.GB_PERF_CHUNK_CHARS || 12)
const streamCharsPerSecond = Number(process.env.GB_PERF_STREAM_CPS || 2_000)
const tickIntervalMs = 10

function buildAnswer(targetBytes) {
  const blocks = [
    '## Section {n}\n\nThis paragraph explains the {n}th step of the benchmark answer with **bold**, `inline code`, and a [link](https://example.com/{n}).\n\n',
    '- First item for block {n}\n- Second item with *emphasis*\n  - Nested item {n}\n- Third item\n\n',
    '```ts\nexport function step{n}(input: number): number {\n  const value = input * {n}\n  return value + 1\n}\n```\n\n',
    '| Column | Value {n} | Notes |\n| --- | --- | --- |\n| alpha | {n} | first |\n| beta | {n}1 | second |\n\n',
    // Single-line display math is common in model output. GB_PERF_CORPUS=multiline-math
    // puts the delimiters on their own lines instead.
    process.env.GB_PERF_CORPUS === 'multiline-math'
      ? 'Inline math $a_{n} = b^2 + c$ and display math:\n\n$$\n\\sum_{i=1}^{{n}} i = \\frac{{n}({n}+1)}{2}\n$$\n\n'
      : 'Inline math $a_{n} = b^2 + c$ and display math:\n\n$$\\sum_{i=1}^{{n}} i = \\frac{{n}({n}+1)}{2}$$\n\n',
    '> Quoted note {n}: keep the streaming renderer honest with mixed blocks.\n\n'
  ]
  let text = ''
  let n = 1
  while (Buffer.byteLength(text) < targetBytes) {
    text += blocks[n % blocks.length].replaceAll('{n}', String(n))
    n += 1
  }
  return text
}

function startModelServer(ledger) {
  const answer = buildAnswer(streamBytes)
  const server = createServer((request, response) => {
    const started = Date.now()
    const entry = { method: request.method, path: request.url, startedAt: started }
    ledger.push(entry)
    if (request.method === 'GET' && request.url?.endsWith('/models')) {
      response.writeHead(200, { 'content-type': 'application/json' })
      response.end(JSON.stringify({ object: 'list', data: [{ id: 'qwen3', object: 'model' }] }))
      return
    }
    if (request.method === 'GET' && request.url === '/c03-knowledge.txt') {
      response.writeHead(200, { 'content-type': 'text/plain; charset=utf-8' })
      response.end('C03 deterministic knowledge source\n'.repeat(80))
      return
    }
    if (request.method !== 'POST' || !request.url?.endsWith('/chat/completions')) {
      response.writeHead(404).end()
      return
    }
    let body = ''
    request.on('data', chunk => { body += chunk })
    request.on('end', () => {
      let parsed = {}
      try { parsed = JSON.parse(body) } catch { /* counted below */ }
      const messages = Array.isArray(parsed.messages) ? parsed.messages : []
      const last = messages.at(-1)
      const lastText = typeof last?.content === 'string' ? last.content : JSON.stringify(last?.content ?? '')
      if (concurrentMode && lastText.includes('gb-fault:hang')) {
        entry.kind = 'fault-hung'
        entry.identity = 'fault-hung'
        entry.stream = true
        entry.aborted = false
        // Keep the production request in flight until the UI cancels it.
        response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' })
        const timer = setTimeout(() => {
          if (!response.destroyed) response.end('data: [DONE]\n\n')
        }, 120_000)
        response.on('close', () => { clearTimeout(timer); entry.aborted = !response.writableFinished; entry.finishedAt = Date.now() })
        return
      }
      if (concurrentMode && messages.some(message => typeof message.content === 'string' && message.content.includes('gb-concurrent:'))) {
        const marker = messages.map(message => typeof message.content === 'string' ? message.content : '').join('\n').match(/gb-concurrent:(parent|child):([a-zA-Z0-9-]+)/)
        const child = marker?.[1] === 'child'
        const toolsDone = messages.some(message => message.role === 'tool')
        entry.kind = child ? 'concurrent-child' : 'concurrent-parent'
        entry.identity = marker?.[2]
        entry.stream = parsed.stream === true
        const durationMs = child ? 12_000 : toolsDone ? 1_000 : 12_000
        const count = Math.ceil(durationMs / 16)
        const calls = !child && !toolsDone ? [0, 1].map(index => ({ index, id: `child-${marker?.[2]}-${index}`, type: 'function', function: { name: 'subagent_delegate', arguments: JSON.stringify({ task: `gb-concurrent:child:${marker?.[2]}-${index} deterministic output` }) } })) : []
        entry.offeredFragments = count
        entry.offeredToolCalls = calls.length
        entry.chunks = 0
        entry.bytes = 0
        entry.latenessMs = []
        response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache' })
        const send = (delta, finish_reason = null) => response.write(`data: ${JSON.stringify({ id: entry.identity, object: 'chat.completion.chunk', model: 'qwen3', choices: [{ index: 0, delta, finish_reason }] })}\n\n`)
        const started = performance.now()
        const tick = () => {
          if (response.destroyed) { entry.aborted = true; entry.settledAt = Date.now(); return }
          while (entry.chunks < count && performance.now() >= started + entry.chunks * 16) {
            entry.latenessMs.push(performance.now() - started - entry.chunks * 16)
            const text = `[${entry.identity}:${String(entry.chunks).padStart(5, '0')}] sample\n`
            send({ content: text })
            entry.bytes += Buffer.byteLength(text)
            entry.chunks++
          }
          if (entry.chunks < count) { setTimeout(tick, 4); return }
          if (calls.length) send({ tool_calls: calls })
          send({}, calls.length ? 'tool_calls' : 'stop')
          response.end('data: [DONE]\n\n')
          entry.finishedAt = Date.now()
        }
        tick()
        return
      }
      const benchmark = lastText.includes('perf-stream')
      entry.kind = benchmark ? 'benchmark-stream' : 'auxiliary'
      entry.stream = parsed.stream === true
      const content = benchmark ? answer : 'Benchmark title'
      if (!entry.stream) {
        response.writeHead(200, { 'content-type': 'application/json' })
        response.end(JSON.stringify({
          id: 'perf', object: 'chat.completion', model: 'qwen3',
          choices: [{ index: 0, finish_reason: 'stop', message: { role: 'assistant', content } }],
          usage: { prompt_tokens: 10, completion_tokens: 10, total_tokens: 20 }
        }))
        entry.finishedAt = Date.now()
        return
      }
      response.writeHead(200, { 'content-type': 'text/event-stream', 'cache-control': 'no-cache', connection: 'keep-alive' })
      const send = payload => response.write(`data: ${JSON.stringify(payload)}\n\n`)
      const delta = text => send({ id: 'perf', object: 'chat.completion.chunk', model: 'qwen3', choices: [{ index: 0, delta: { content: text }, finish_reason: null }] })
      send({ id: 'perf', object: 'chat.completion.chunk', model: 'qwen3', choices: [{ index: 0, delta: { role: 'assistant' }, finish_reason: null }] })
      let offset = 0
      let chunks = 0
      const finish = () => {
        send({ id: 'perf', object: 'chat.completion.chunk', model: 'qwen3', choices: [{ index: 0, delta: {}, finish_reason: 'stop' }] })
        send({ id: 'perf', object: 'chat.completion.chunk', model: 'qwen3', choices: [], usage: { prompt_tokens: 10, completion_tokens: Math.ceil(content.length / 4), total_tokens: 10 + Math.ceil(content.length / 4) } })
        response.write('data: [DONE]\n\n')
        response.end()
        entry.finishedAt = Date.now()
        entry.chunks = chunks
        entry.bytes = Buffer.byteLength(content)
      }
      if (!benchmark) { delta(content); finish(); return }
      // Emit at a fixed character rate, split into small SSE events like a
      // fast token stream. Pacing is time-based so coarse timers do not
      // change throughput.
      const streamStart = Date.now()
      const tick = () => {
        if (response.destroyed) { entry.aborted = true; return }
        const due = Math.min(content.length, Math.ceil(((Date.now() - streamStart) / 1000) * streamCharsPerSecond))
        while (offset < due) {
          delta(content.slice(offset, offset + chunkChars))
          offset += chunkChars
          chunks += 1
        }
        if (offset >= content.length) { finish(); return }
        setTimeout(tick, tickIntervalMs)
      }
      tick()
    })
  })
  return new Promise((resolvePromise, reject) => {
    server.once('error', reject)
    server.listen(modelPort, '127.0.0.1', () => resolvePromise(server))
  })
}

function format(value, digits = 1) {
  return value === null || value === undefined || Number.isNaN(value) ? '-' : Number(value).toFixed(digits)
}

function printSummary(report) {
  console.log(`\nGoodBuddy full-App benchmark (${report.status})`)
  console.log(`source ${report.source?.commit ?? '?'}${report.source?.dirty ? ' (dirty)' : ''}, Electron ${report.environment?.electron}, DPR ${report.environment?.devicePixelRatio}, ${report.environment?.cpu}`)
  if (report.startup) {
    console.log(`startup: window ${format(report.startup.windowCreatedMs, 0)} ms, ready-to-show ${format(report.startup.readyToShowMs, 0)} ms, composer ready ${format(report.startup.composerReadyMs, 0)} ms`)
  }
  const rows = (report.scenarios ?? []).map(s => ({
    scenario: s.id,
    ms: format(s.durationMs, 0),
    longTasks: `${s.renderer?.longTasks?.count ?? '-'} / ${format(s.renderer?.longTasks?.totalMs, 0)} / ${format(s.renderer?.longTasks?.maxMs, 0)}`,
    'frame p50/p95/max': `${format(s.renderer?.frames?.p50)} / ${format(s.renderer?.frames?.p95)} / ${format(s.renderer?.frames?.max, 0)}`,
    'jank%': format(s.renderer?.frames?.jankRatio * 100),
    'latency p50/p95/max': s.latency ? `${format(s.latency.p50)} / ${format(s.latency.p95)} / ${format(s.latency.max)}` : '-',
    'mainLag p99/max': `${format(s.main?.eventLoopDelay?.p99)} / ${format(s.main?.eventLoopDelay?.max)}`,
    'cpu% main/renderer/gpu': `${format(s.processes?.cpu?.Browser, 0)} / ${format(s.processes?.cpu?.Tab, 0)} / ${format(s.processes?.cpu?.GPU, 0)}`,
    domNodes: s.renderer?.domNodes ?? '-',
    heapMB: format(s.renderer?.heapUsedMB)
  }))
  if (rows.length) console.table(rows)
  if (report.memoryGrowth) {
    console.log(`memory growth (${report.memoryGrowth.description}):`)
    console.table(report.memoryGrowth.samples)
  }
  if (report.environment?.rendererCpuThrottle > 1) console.log(`renderer CPU throttled ${report.environment.rendererCpuThrottle}x`)
  if (report.error) console.log(`error: ${report.error}`)
}

async function main() {
  const taskDirectory = join(runnerRoot, 'temp', 'goodbuddy-concurrent-upgrade')
  mkdirSync(taskDirectory, { recursive: true })
  if (process.argv.includes('--launch')) {
    const { openSync, closeSync } = require('node:fs')
    const output = resolve(process.env.GB_PERF_OUTPUT || join(taskDirectory, `run-${Date.now()}`))
    mkdirSync(output, { recursive: true })
    const log = openSync(join(output, 'execution.log'), 'w')
    const child = spawn(process.execPath, [__filename], {
      detached: true, stdio: ['ignore', log, log],
      env: { ...process.env, GB_PERF_OUTPUT: output }
    })
    closeSync(log)
    writeFileSync(join(output, 'supervisor.json'), JSON.stringify({ pid: child.pid, startedAt: new Date().toISOString(), root, mode: process.env.GB_PERF_MODE, timeoutMs: totalTimeoutMs }))
    child.unref()
    console.log(`Supervisor ${child.pid}; logs ${output}`)
    return
  }
  const required = ['out/main/index.js', 'out/preload/index.cjs', 'out/renderer/index.html']
  const missing = required.filter(file => !existsSync(join(root, file)))
  if (missing.length) throw new Error(`Missing production build (${missing.join(', ')}); run npm run build:bundle first`)
  const runDirectory = mkdtempSync(join(taskDirectory, 'profile-'))
  const outputDirectory = resolve(process.env.GB_PERF_OUTPUT || join(runDirectory, 'artifacts'))
  mkdirSync(outputDirectory, { recursive: true })
  const ledger = []
  let server
  try {
    server = await startModelServer(ledger)
  } catch (error) {
    throw new Error(`Cannot listen on 127.0.0.1:${modelPort} for the fake model (${error.code ?? error.message}); stop the local service using that port, or set GB_PERF_MODEL_PORT to a free port`, { cause: error })
  }
  const environment = { ...process.env, GB_PERF_DIRECTORY: runDirectory, GB_PERF_ARTIFACTS: outputDirectory, GB_PERF_ROOT: root, TEMP: runDirectory, TMP: runDirectory, TMPDIR: runDirectory }
  if (concurrentMode) environment.GB_PERF_DRIVER_TIMEOUT_MS = String(totalTimeoutMs - 30_000)
  delete environment.ELECTRON_RUN_AS_NODE
  delete environment.ELECTRON_RENDERER_URL
  for (const key of ['GOODBUDDY_MODEL_API_KEY', 'GOODBUDDY_MODEL_BASE_URL', 'GOODBUDDY_MODEL_NAME', 'GOODBUDDY_BIGTOKEN_API_KEY', 'GOODBUDDY_BIGTOKEN_BASE_URL', 'GOODBUDDY_BIGTOKEN_MODEL']) delete environment[key]
  if (modelPort !== defaultModelPort) {
    // The fake model ignores the key; it only makes the default profile use the URL.
    environment.GOODBUDDY_MODEL_API_KEY = 'perf-loopback'
    environment.GOODBUDDY_MODEL_BASE_URL = `http://127.0.0.1:${modelPort}/v1`
    environment.GOODBUDDY_MODEL_NAME = 'qwen3'
  }
  environment.GOODBUDDY_WORKSPACE = join(runDirectory, 'workspace')
  mkdirSync(environment.GOODBUDDY_WORKSPACE, { recursive: true })
  const started = Date.now()
  const child = spawn(require('electron'), [join(runnerRoot, 'tests', 'support', 'app-perf-driver.mjs')], { cwd: root, env: environment, stdio: 'inherit', windowsHide: false })
  writeFileSync(join(outputDirectory, 'owned-processes.json'), JSON.stringify({ supervisorPid: process.pid, electronPid: child.pid, root, runDirectory, startedAt: new Date().toISOString() }))
  let timedOut = false
  const timer = setTimeout(() => {
    timedOut = true
    if (process.platform === 'win32') spawn('taskkill', ['/pid', String(child.pid), '/T', '/F'], { stdio: 'ignore' })
    else child.kill('SIGKILL')
  }, totalTimeoutMs)
  const exit = await new Promise(resolvePromise => child.once('exit', (code, signal) => resolvePromise({ code, signal })))
  clearTimeout(timer)
  await new Promise(resolvePromise => server.close(() => resolvePromise()))
  const resultPath = join(outputDirectory, 'result.json')
  let report
  if (existsSync(resultPath)) report = JSON.parse(readFileSync(resultPath, 'utf8'))
  else report = { status: 'failed', error: timedOut ? 'Supervisor timeout before report' : 'Driver exited without a report' }
  report.supervisor = { elapsedMs: Date.now() - started, timeoutMs: totalTimeoutMs, timedOut, exitCode: exit.code, signal: exit.signal }
  report.corpus = { kind: process.env.GB_PERF_CORPUS || 'single-line-math', bytes: streamBytes, charsPerSecond: streamCharsPerSecond, charsPerEvent: chunkChars }
  report.modelRequests = {
    boundary: 'loopback fake model only; no external requests',
    total: ledger.length,
    benchmarkStreams: ledger.filter(entry => entry.kind === 'benchmark-stream').length,
    auxiliary: ledger.filter(entry => entry.kind === 'auxiliary').length,
    ledger: ledger.map(({ method, path, kind, stream, chunks, bytes, aborted, startedAt, finishedAt, identity, offeredFragments, offeredToolCalls, latenessMs }) => ({ method, path, kind, stream, chunks, bytes, aborted, startedAt, finishedAt, identity, offeredFragments, offeredToolCalls, latenessMs, durationMs: finishedAt ? finishedAt - startedAt : null }))
  }
  if (timedOut || exit.code !== 0) report.status = 'failed'
  writeFileSync(resultPath, JSON.stringify(report, null, 2))
  printSummary(report)
  console.log(`\nreport: ${resultPath}`)
   if (!keepProfile) rmSync(runDirectory, { recursive: true, force: true, maxRetries: 5, retryDelay: 200 })
  process.exitCode = String(report.status).startsWith('passed') ? 0 : 1
  // PERF-17: GB_PERF_CHECK=1 gates the run on build/perf-thresholds.json
  // (exit 1 on a regression, 2 when the report cannot be judged, e.g. throttled).
  if (process.env.GB_PERF_CHECK === '1') process.exitCode = require('./check-perf-thresholds.cjs').run([resultPath])
}

main().catch(error => {
  console.error(error instanceof Error ? error.message : error)
  process.exitCode = 1
})