feat: enhance local knowledge retrieval
This commit is contained in:
lofyer
@@ -0,0 +1,443 @@
|
||||
{
|
||||
"version": 1,
|
||||
"id": "synthetic-bilingual-v1",
|
||||
"provenance": {
|
||||
"kind": "synthetic",
|
||||
"license": "CC0-1.0"
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"id": "doc-secrets",
|
||||
"title": "Credential storage",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-secrets-main",
|
||||
"content": "API tokens and service keys stay in the main process encrypted settings store. The renderer never receives the credential value."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-secrets-secondary",
|
||||
"title": "凭据轮换",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-secrets-secondary",
|
||||
"content": "轮换访问凭据后，应立即撤销旧令牌，并确认新密钥只保存在加密存储中。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-offline",
|
||||
"title": "Offline operation",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-offline-main",
|
||||
"content": "Offline search uses the local SQLite index and does not require a network connection. Local files remain on the device."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-cancel",
|
||||
"title": "任务取消",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-cancel-main",
|
||||
"content": "用户取消索引任务时，系统会中止当前批次，保留旧索引，并将任务标记为已取消。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-backup",
|
||||
"title": "Backup and restore",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-backup-main",
|
||||
"content": "Create a backup before migration. To restore, close the application and replace the database with the verified backup copy."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-accessibility",
|
||||
"title": "键盘导航",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-accessibility-main",
|
||||
"content": "分段控件支持键盘导航：左右方向键移动焦点，Tab 键离开控件，当前选项使用清晰的焦点样式。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-image",
|
||||
"title": "Image handling",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-image-main",
|
||||
"content": "Generated images are accepted only as bounded inline image data. Provider-returned image URLs are not fetched."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-sqlite",
|
||||
"title": "数据库更新",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-sqlite-main",
|
||||
"content": "SQLite 数据迁移在事务中执行；任一步骤失败都会回滚，以保护现有用户数据。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-runtime",
|
||||
"title": "Runtime modes",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-runtime-main",
|
||||
"content": "Ask mode is read-only at the runtime boundary. Execute mode can use tools only after the configured approval check."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-fonts",
|
||||
"title": "界面字体",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-fonts-main",
|
||||
"content": "界面使用随应用打包的 Inter Variable 和 Noto Sans SC Variable 字体，不发起远程字体请求。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-release",
|
||||
"title": "Release verification",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-release-main",
|
||||
"content": "Each release package includes a release manifest containing SHA-256 hashes for the packaged artifacts."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-chunking",
|
||||
"title": "上下文分块",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-chunking-main",
|
||||
"content": "检索结果可加入相邻分块来补充上下文，但合并后的内容必须遵守字符预算。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-language",
|
||||
"title": "Interface language",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-language-main",
|
||||
"content": "The interface supports Simplified Chinese and English. Release notes are shown in the current interface language."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-approval-secondary",
|
||||
"title": "审批记录",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-approval-secondary",
|
||||
"content": "执行敏感工具前应显示审批提示，并记录本次允许的操作范围。"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "doc-manifest-secondary",
|
||||
"title": "包清单检查",
|
||||
"chunks": [
|
||||
{
|
||||
"id": "chunk-manifest-secondary",
|
||||
"content": "发布检查会核对包清单，但测试构建可以跳过签名步骤。"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"queries": [
|
||||
{
|
||||
"id": "query-secrets-en",
|
||||
"language": "en",
|
||||
"query": "Where should API tokens be stored?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-secrets-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "API tokens and service keys stay in the main process encrypted settings store."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"chunkId": "chunk-secrets-secondary",
|
||||
"relevance": 1,
|
||||
"spans": [
|
||||
{
|
||||
"text": "新密钥只保存在加密存储中"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-secrets-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "怎样安全保存访问凭据",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-secrets-main",
|
||||
"relevance": 2,
|
||||
"spans": [
|
||||
{
|
||||
"text": "main process encrypted settings store"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"chunkId": "chunk-secrets-secondary",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "新密钥只保存在加密存储中"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-offline-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "没有网络时能否检索本地资料",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-offline-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "Offline search uses the local SQLite index and does not require a network connection."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-cancel-en",
|
||||
"language": "en",
|
||||
"query": "What happens when indexing is aborted?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-cancel-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "系统会中止当前批次，保留旧索引"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-backup-en",
|
||||
"language": "en",
|
||||
"query": "How do I restore the database backup?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-backup-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "To restore, close the application and replace the database with the verified backup copy."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-keyboard-en",
|
||||
"language": "en",
|
||||
"query": "How does focus move inside a segmented control?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-accessibility-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "左右方向键移动焦点"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-image-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "模型生成的图片数据如何接收",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-image-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "accepted only as bounded inline image data"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-sqlite-en",
|
||||
"language": "en",
|
||||
"query": "How are failed database migrations protected?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-sqlite-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "任一步骤失败都会回滚"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-runtime-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "询问模式是否允许写入，工具何时需要审批",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-runtime-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "Ask mode is read-only at the runtime boundary."
|
||||
},
|
||||
{
|
||||
"text": "Execute mode can use tools only after the configured approval check."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"chunkId": "chunk-approval-secondary",
|
||||
"relevance": 1,
|
||||
"spans": [
|
||||
{
|
||||
"text": "执行敏感工具前应显示审批提示"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-font-en",
|
||||
"language": "en",
|
||||
"query": "Are interface typefaces downloaded remotely?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-fonts-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "不发起远程字体请求"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-release-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "发布产物怎样校验完整性",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-release-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "release manifest containing SHA-256 hashes"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"chunkId": "chunk-manifest-secondary",
|
||||
"relevance": 1,
|
||||
"spans": [
|
||||
{
|
||||
"text": "发布检查会核对包清单"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-chunk-en",
|
||||
"language": "en",
|
||||
"query": "Can neighboring sections be added to retrieved context?",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-chunking-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "检索结果可加入相邻分块来补充上下文"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-language-zh",
|
||||
"language": "zh-CN",
|
||||
"query": "界面提供哪些语言",
|
||||
"noAnswer": false,
|
||||
"judgments": [
|
||||
{
|
||||
"chunkId": "chunk-language-main",
|
||||
"relevance": 3,
|
||||
"spans": [
|
||||
{
|
||||
"text": "supports Simplified Chinese and English"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "query-noanswer-weather",
|
||||
"language": "zh-CN",
|
||||
"query": "明天火星天气预报",
|
||||
"noAnswer": true,
|
||||
"judgments": []
|
||||
},
|
||||
{
|
||||
"id": "query-noanswer-payroll",
|
||||
"language": "en",
|
||||
"query": "employee payroll tax withholding schedule",
|
||||
"noAnswer": true,
|
||||
"judgments": []
|
||||
},
|
||||
{
|
||||
"id": "query-noanswer-recipe",
|
||||
"language": "en",
|
||||
"query": "sourdough cinnamon recipe temperature",
|
||||
"noAnswer": true,
|
||||
"judgments": []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { mkdtemp, mkdir, readFile, rm, symlink } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
createDeterministicTokenHashEmbeddingProvider,
|
||||
deterministicReportProjection,
|
||||
loadRetrievalFixture,
|
||||
runRetrievalEvaluation
|
||||
} from './support/knowledge-retrieval-evaluation'
|
||||
|
||||
describe('knowledge retrieval evaluation', () => {
|
||||
it('evaluates production retrieval ablations offline, privately, and deterministically', async () => {
|
||||
const fixture = await loadRetrievalFixture()
|
||||
expect(fixture.provenance.kind).toBe('synthetic')
|
||||
expect(fixture.queries.filter((query) => !query.noAnswer).length).toBeGreaterThanOrEqual(12)
|
||||
expect(new Set(fixture.queries.map((query) => query.language))).toEqual(
|
||||
new Set(['en', 'zh-CN'])
|
||||
)
|
||||
|
||||
const first = await runRetrievalEvaluation()
|
||||
const second = await runRetrievalEvaluation()
|
||||
expect(deterministicReportProjection(second)).toEqual(
|
||||
deterministicReportProjection(first)
|
||||
)
|
||||
|
||||
const byId = new Map(first.ablations.map((ablation) => [ablation.id, ablation]))
|
||||
const lexical = byId.get('lexical')!
|
||||
const tokenHash = byId.get('token-hash-vector')!
|
||||
const regressionVector = byId.get('regression-alias-vector')!
|
||||
const hybrid = byId.get('hybrid')!
|
||||
const rerank = byId.get('hybrid-rerank')!
|
||||
expect(first.corpusHash).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(first.evaluationDefinitionHash).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(first.providerFingerprintHash).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(first.evaluationDefinitionHash).not.toBe(first.corpusHash)
|
||||
expect(lexical.metrics.recallAt10).toBeGreaterThanOrEqual(0.2)
|
||||
expect(lexical.metrics.contextRecall).toBeGreaterThanOrEqual(0.2)
|
||||
expect(tokenHash.metrics.recallAt10).toBeGreaterThanOrEqual(0.15)
|
||||
expect(regressionVector.metrics.recallAt10).toBeGreaterThanOrEqual(0.9)
|
||||
expect(hybrid.metrics.recallAt5).toBeGreaterThanOrEqual(0.9)
|
||||
expect(rerank.metrics.mrrAt10).toBeGreaterThanOrEqual(0.78)
|
||||
expect(rerank.metrics.ndcgAt10).toBeGreaterThanOrEqual(0.75)
|
||||
expect(rerank.metrics.contextPrecision).toBeGreaterThanOrEqual(0.05)
|
||||
expect(rerank.metrics.contextRecall).toBeGreaterThanOrEqual(0.9)
|
||||
expect(rerank.metrics.noAnswerFalsePositiveRate).toBeLessThanOrEqual(0.34)
|
||||
for (const language of ['en', 'zh-CN'] as const) {
|
||||
expect(rerank.metricsByLanguage[language].recallAt10).toBeGreaterThanOrEqual(0.9)
|
||||
expect(rerank.metricsByLanguage[language].contextRecall).toBeGreaterThanOrEqual(0.85)
|
||||
}
|
||||
expect(
|
||||
rerank.metrics.ndcgAt10,
|
||||
'local rerank should not materially regress hybrid ranking quality'
|
||||
).toBeGreaterThanOrEqual(hybrid.metrics.ndcgAt10 - 0.05)
|
||||
expect(rerank.failures, JSON.stringify(rerank.failures)).toEqual([])
|
||||
|
||||
const serialized = JSON.stringify(first)
|
||||
expect(Object.keys(first).sort()).toEqual([
|
||||
'ablations',
|
||||
'corpusHash',
|
||||
'evaluationDefinitionHash',
|
||||
'fixtureId',
|
||||
'providerFingerprintHash',
|
||||
'queryIds',
|
||||
'schemaVersion'
|
||||
])
|
||||
for (const ablation of first.ablations) {
|
||||
expect(Object.keys(ablation).sort()).toEqual([
|
||||
'failures',
|
||||
'id',
|
||||
'latencyMs',
|
||||
'metrics',
|
||||
'metricsByLanguage'
|
||||
])
|
||||
expect(Object.keys(ablation.metrics).sort()).toEqual([
|
||||
'contextPrecision',
|
||||
'contextRecall',
|
||||
'mrrAt10',
|
||||
'ndcgAt10',
|
||||
'noAnswerFalsePositiveRate',
|
||||
'recallAt10',
|
||||
'recallAt5'
|
||||
])
|
||||
}
|
||||
const forbiddenReportValues = [
|
||||
...fixture.documents.map((document) => document.title),
|
||||
...fixture.documents.flatMap((document) =>
|
||||
document.chunks.map((chunk) => chunk.content)
|
||||
),
|
||||
...fixture.queries.map((query) => query.query),
|
||||
'fixture://',
|
||||
'goodbuddy:retrieval-eval',
|
||||
'handcrafted-alias-hash-v1',
|
||||
'topic-agnostic-token-hash-v1'
|
||||
]
|
||||
for (const value of forbiddenReportValues) {
|
||||
expect(serialized).not.toContain(value)
|
||||
}
|
||||
for (const document of fixture.documents) {
|
||||
expect(serialized).not.toContain(document.id.replace(/^doc-/u, 'fixture://'))
|
||||
}
|
||||
expect(serialized).not.toMatch(/[a-z]:[\\/]/iu)
|
||||
}, 30_000)
|
||||
|
||||
it('provides a deterministic topic-agnostic token-hash vectorizer', async () => {
|
||||
const provider = createDeterministicTokenHashEmbeddingProvider()
|
||||
const [first, second, unrelated] = await provider.embed([
|
||||
'alpha beta alpha',
|
||||
'beta alpha',
|
||||
'凭据'
|
||||
])
|
||||
|
||||
expect(first).toEqual(second)
|
||||
expect(first).not.toEqual(unrelated)
|
||||
expect(provider.model).toBe('topic-agnostic-token-hash-v1')
|
||||
expect(provider.fingerprint).not.toContain('synthetic-bilingual')
|
||||
})
|
||||
|
||||
it('writes only to non-symlink workspace-relative output paths', async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), 'goodbuddy-retrieval-output-'))
|
||||
const workspace = join(root, 'workspace')
|
||||
const outside = join(root, 'outside')
|
||||
await mkdir(workspace)
|
||||
await mkdir(outside)
|
||||
try {
|
||||
const outputPath = join('reports', 'retrieval.json')
|
||||
const report = await runRetrievalEvaluation({
|
||||
workingDirectory: workspace,
|
||||
outputPath
|
||||
})
|
||||
const persisted = JSON.parse(
|
||||
await readFile(join(workspace, outputPath), 'utf8')
|
||||
) as { evaluationDefinitionHash: string }
|
||||
expect(persisted.evaluationDefinitionHash).toBe(
|
||||
report.evaluationDefinitionHash
|
||||
)
|
||||
|
||||
await expect(runRetrievalEvaluation({
|
||||
workingDirectory: workspace,
|
||||
outputPath: join('..', 'outside.json')
|
||||
})).rejects.toThrow(/workspace-relative/u)
|
||||
|
||||
const link = join(workspace, 'linked')
|
||||
await symlink(outside, link, process.platform === 'win32' ? 'junction' : 'dir')
|
||||
await expect(runRetrievalEvaluation({
|
||||
workingDirectory: workspace,
|
||||
outputPath: join('linked', 'escaped.json')
|
||||
})).rejects.toThrow(/symlink/u)
|
||||
} finally {
|
||||
await rm(root, { recursive: true, force: true })
|
||||
}
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -0,0 +1,211 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import {
|
||||
computeEvaluationDefinitionHash,
|
||||
computeRetrievalMetrics,
|
||||
retrievalFixtureSchema,
|
||||
summarizeLatencies,
|
||||
type RetrievalFixture
|
||||
} from './support/knowledge-retrieval-evaluation'
|
||||
|
||||
const metricFixture: RetrievalFixture = retrievalFixtureSchema.parse({
|
||||
version: 1,
|
||||
id: 'metric-fixture',
|
||||
provenance: { kind: 'synthetic', license: 'CC0-1.0' },
|
||||
documents: Array.from({ length: 10 }, (_, index) => ({
|
||||
id: `document-${index}`,
|
||||
title: `Document ${index}`,
|
||||
chunks: [{
|
||||
id: `chunk-${index}`,
|
||||
content: `Synthetic content number ${index} with exact span ${index}.`
|
||||
}]
|
||||
})),
|
||||
queries: [
|
||||
{
|
||||
id: 'query-answer-one',
|
||||
language: 'en',
|
||||
query: 'first synthetic question',
|
||||
noAnswer: false,
|
||||
judgments: [
|
||||
{ chunkId: 'chunk-0', relevance: 3, spans: [{ text: 'exact span 0' }] },
|
||||
{ chunkId: 'chunk-1', relevance: 1, spans: [{ text: 'exact span 1' }] }
|
||||
]
|
||||
},
|
||||
{
|
||||
id: 'query-answer-two',
|
||||
language: 'zh-CN',
|
||||
query: '第二个合成测试问题',
|
||||
noAnswer: false,
|
||||
judgments: [
|
||||
{ chunkId: 'chunk-2', relevance: 2, spans: [{ text: 'exact span 2' }] }
|
||||
]
|
||||
},
|
||||
...Array.from({ length: 9 }, (_, index) => ({
|
||||
id: `query-padding-${index}`,
|
||||
language: (index === 0 ? 'zh-CN' : 'en') as 'en' | 'zh-CN',
|
||||
query: `padding synthetic query ${index}`,
|
||||
noAnswer: true,
|
||||
judgments: []
|
||||
})),
|
||||
{
|
||||
id: 'query-no-answer',
|
||||
language: 'en',
|
||||
query: 'unanswerable synthetic question',
|
||||
noAnswer: true,
|
||||
judgments: []
|
||||
}
|
||||
]
|
||||
})
|
||||
|
||||
describe('retrieval evaluation metrics', () => {
|
||||
it('computes cutoffs, reciprocal rank, graded nDCG, exact-span context, and no-answer rate', () => {
|
||||
const rankings = new Map([
|
||||
[
|
||||
'query-answer-one',
|
||||
[
|
||||
{ chunkId: 'chunk-9', context: 'irrelevant' },
|
||||
{ chunkId: 'chunk-1', context: 'exact span 1' },
|
||||
{ chunkId: 'chunk-0', context: 'prefix exact span 0 suffix' }
|
||||
]
|
||||
],
|
||||
[
|
||||
'query-answer-two',
|
||||
[
|
||||
{ chunkId: 'chunk-9', context: 'noise' },
|
||||
{ chunkId: 'chunk-8', context: 'noise' },
|
||||
{ chunkId: 'chunk-7', context: 'noise' },
|
||||
{ chunkId: 'chunk-6', context: 'noise' },
|
||||
{ chunkId: 'chunk-5', context: 'noise' },
|
||||
{ chunkId: 'chunk-2', context: 'exact span 2' }
|
||||
]
|
||||
],
|
||||
['query-no-answer', [{ chunkId: 'chunk-4', context: 'false positive' }]]
|
||||
])
|
||||
|
||||
const metrics = computeRetrievalMetrics(metricFixture, rankings)
|
||||
|
||||
expect(metrics.recallAt5).toBeCloseTo(0.5)
|
||||
expect(metrics.recallAt10).toBe(1)
|
||||
expect(metrics.mrrAt10).toBeCloseTo((1 / 2 + 1 / 6) / 2)
|
||||
const firstQueryDcg =
|
||||
(2 ** 1 - 1) / Math.log2(3) +
|
||||
(2 ** 3 - 1) / Math.log2(4)
|
||||
const firstQueryIdeal =
|
||||
(2 ** 3 - 1) / Math.log2(2) +
|
||||
(2 ** 1 - 1) / Math.log2(3)
|
||||
const secondQueryNdcg =
|
||||
((2 ** 2 - 1) / Math.log2(7)) /
|
||||
((2 ** 2 - 1) / Math.log2(2))
|
||||
expect(metrics.ndcgAt10).toBeCloseTo(
|
||||
(firstQueryDcg / firstQueryIdeal + secondQueryNdcg) / 2,
|
||||
12
|
||||
)
|
||||
expect(metrics.contextRecall).toBe(1)
|
||||
expect(metrics.contextPrecision).toBeCloseTo(36 / 85)
|
||||
expect(metrics.noAnswerFalsePositiveRate).toBeCloseTo(0.1)
|
||||
})
|
||||
|
||||
it('deduplicates rankings and unions overlapping evidence spans', () => {
|
||||
const fixture = structuredClone(metricFixture)
|
||||
fixture.queries[0]!.judgments[0]!.spans = [
|
||||
{ text: 'exact span' },
|
||||
{ text: 'span 0' },
|
||||
{ text: 'exact span 0' }
|
||||
]
|
||||
const rankings = new Map([
|
||||
[
|
||||
'query-answer-one',
|
||||
[
|
||||
{ chunkId: 'chunk-0', context: 'exact span 0' },
|
||||
{ chunkId: 'chunk-0', context: 'exact span 0' }
|
||||
]
|
||||
],
|
||||
['query-answer-two', [{ chunkId: 'chunk-2', context: 'exact span 2' }]]
|
||||
])
|
||||
|
||||
const metrics = computeRetrievalMetrics(fixture, rankings)
|
||||
|
||||
expect(metrics.recallAt5).toBeCloseTo(0.75)
|
||||
expect(metrics.mrrAt10).toBe(1)
|
||||
expect(metrics.ndcgAt10).toBeLessThanOrEqual(1)
|
||||
expect(metrics.contextRecall).toBeCloseTo(2 / 3)
|
||||
expect(metrics.contextPrecision).toBe(1)
|
||||
})
|
||||
|
||||
it('uses deterministic nearest-rank latency summaries', () => {
|
||||
expect(summarizeLatencies([9, 1, 5, 3, 7])).toEqual({
|
||||
count: 5,
|
||||
min: 1,
|
||||
median: 5,
|
||||
p95: 9,
|
||||
max: 9,
|
||||
mean: 5
|
||||
})
|
||||
expect(summarizeLatencies([])).toEqual({
|
||||
count: 0,
|
||||
min: 0,
|
||||
median: 0,
|
||||
p95: 0,
|
||||
max: 0,
|
||||
mean: 0
|
||||
})
|
||||
})
|
||||
|
||||
it('hashes fixture queries, judgments, settings, providers, and metric version', () => {
|
||||
const baseline = computeEvaluationDefinitionHash(metricFixture)
|
||||
const changedQuery = structuredClone(metricFixture)
|
||||
changedQuery.queries[0]!.query = 'changed synthetic question'
|
||||
const changedJudgment = structuredClone(metricFixture)
|
||||
changedJudgment.queries[0]!.judgments[0]!.relevance = 2
|
||||
|
||||
expect(baseline).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(computeEvaluationDefinitionHash(changedQuery)).not.toBe(baseline)
|
||||
expect(computeEvaluationDefinitionHash(changedJudgment)).not.toBe(baseline)
|
||||
})
|
||||
|
||||
it('rejects unknown fields, unsafe provenance, and inexact spans', () => {
|
||||
const unsafe = structuredClone(metricFixture) as unknown as {
|
||||
documents: Array<{ chunks: Array<{ content: string }> }>
|
||||
queries: Array<{ judgments: Array<{ spans: Array<{ text: string }> }> }>
|
||||
endpoint?: string
|
||||
}
|
||||
unsafe.endpoint = 'https://example.invalid'
|
||||
unsafe.queries[0]!.judgments[0]!.spans[0]!.text = 'not in corpus'
|
||||
expect(() => retrievalFixtureSchema.parse(unsafe)).toThrow()
|
||||
})
|
||||
|
||||
it('rejects all-answer, all-no-answer, and missing bilingual class coverage', () => {
|
||||
const allAnswer = structuredClone(metricFixture)
|
||||
allAnswer.queries = allAnswer.queries.map((query) => ({
|
||||
...query,
|
||||
noAnswer: false,
|
||||
judgments: [{
|
||||
chunkId: 'chunk-0',
|
||||
relevance: 1,
|
||||
spans: [{ text: 'exact span 0' }]
|
||||
}]
|
||||
}))
|
||||
expect(() => retrievalFixtureSchema.parse(allAnswer)).toThrow(
|
||||
/must include a no-answer query/u
|
||||
)
|
||||
|
||||
const allNoAnswer = structuredClone(metricFixture)
|
||||
allNoAnswer.queries = allNoAnswer.queries.map((query) => ({
|
||||
...query,
|
||||
noAnswer: true,
|
||||
judgments: []
|
||||
}))
|
||||
expect(() => retrievalFixtureSchema.parse(allNoAnswer)).toThrow(
|
||||
/must include an answerable query/u
|
||||
)
|
||||
|
||||
const noChineseNoAnswer = structuredClone(metricFixture)
|
||||
noChineseNoAnswer.queries = noChineseNoAnswer.queries.map((query) =>
|
||||
query.noAnswer ? { ...query, language: 'en' as const } : query
|
||||
)
|
||||
expect(() => retrievalFixtureSchema.parse(noChineseNoAnswer)).toThrow(
|
||||
/zh-CN: fixture must include a no-answer query/u
|
||||
)
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,780 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import {
|
||||
lstat,
|
||||
mkdtemp,
|
||||
mkdir,
|
||||
readFile,
|
||||
realpath,
|
||||
rename,
|
||||
rm,
|
||||
writeFile
|
||||
} from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { defineConfig } from 'vitest/config'
|
||||
import { z } from 'zod'
|
||||
import { embeddingStorageProvider } from '../../src/main/knowledge/embedding-provider-key'
|
||||
import { KnowledgeService } from '../../src/main/knowledge/knowledge-service'
|
||||
import { knowledgeRetrievalTerms } from '../../src/main/knowledge/retrieval-text'
|
||||
import { isPathInside } from '../../src/main/workspace-file-access'
|
||||
import type { EmbeddingProvider } from '../../src/main/knowledge/types'
|
||||
|
||||
const stableIdSchema = z
|
||||
.string()
|
||||
.regex(/^[a-z][a-z0-9-]{2,63}$/u, 'must be a stable lowercase ID')
|
||||
const spanSchema = z
|
||||
.object({ text: z.string().min(2).max(500) })
|
||||
.strict()
|
||||
const judgmentSchema = z
|
||||
.object({
|
||||
chunkId: stableIdSchema,
|
||||
relevance: z.number().int().min(1).max(3),
|
||||
spans: z.array(spanSchema).min(1).max(8)
|
||||
})
|
||||
.strict()
|
||||
const querySchema = z
|
||||
.object({
|
||||
id: stableIdSchema,
|
||||
language: z.enum(['en', 'zh-CN']),
|
||||
query: z.string().min(4).max(300),
|
||||
noAnswer: z.boolean(),
|
||||
judgments: z.array(judgmentSchema).max(10)
|
||||
})
|
||||
.strict()
|
||||
const chunkSchema = z
|
||||
.object({
|
||||
id: stableIdSchema,
|
||||
content: z.string().min(20).max(2_000)
|
||||
})
|
||||
.strict()
|
||||
const documentSchema = z
|
||||
.object({
|
||||
id: stableIdSchema,
|
||||
title: z.string().min(2).max(100),
|
||||
chunks: z.array(chunkSchema).min(1).max(12)
|
||||
})
|
||||
.strict()
|
||||
|
||||
export const retrievalFixtureSchema = z
|
||||
.object({
|
||||
version: z.literal(1),
|
||||
id: stableIdSchema,
|
||||
provenance: z
|
||||
.object({
|
||||
kind: z.enum(['synthetic', 'public']),
|
||||
license: z.string().min(2).max(64)
|
||||
})
|
||||
.strict(),
|
||||
documents: z.array(documentSchema).min(10).max(100),
|
||||
queries: z.array(querySchema).min(12).max(100)
|
||||
})
|
||||
.strict()
|
||||
.superRefine((fixture, context) => {
|
||||
const ids = new Set<string>()
|
||||
const chunks = new Map<string, string>()
|
||||
for (const document of fixture.documents) {
|
||||
if (ids.has(document.id)) {
|
||||
context.addIssue({ code: 'custom', message: `duplicate ID: ${document.id}` })
|
||||
}
|
||||
ids.add(document.id)
|
||||
for (const chunk of document.chunks) {
|
||||
if (ids.has(chunk.id)) {
|
||||
context.addIssue({ code: 'custom', message: `duplicate ID: ${chunk.id}` })
|
||||
}
|
||||
ids.add(chunk.id)
|
||||
chunks.set(chunk.id, chunk.content)
|
||||
}
|
||||
}
|
||||
for (const query of fixture.queries) {
|
||||
if (ids.has(query.id)) {
|
||||
context.addIssue({ code: 'custom', message: `duplicate ID: ${query.id}` })
|
||||
}
|
||||
ids.add(query.id)
|
||||
if (query.noAnswer !== (query.judgments.length === 0)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${query.id}: noAnswer must exactly match empty judgments`
|
||||
})
|
||||
}
|
||||
const judged = new Set<string>()
|
||||
for (const judgment of query.judgments) {
|
||||
const content = chunks.get(judgment.chunkId)
|
||||
if (!content) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${query.id}: unknown chunk ${judgment.chunkId}`
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (judged.has(judgment.chunkId)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${query.id}: duplicate judgment ${judgment.chunkId}`
|
||||
})
|
||||
}
|
||||
judged.add(judgment.chunkId)
|
||||
for (const span of judgment.spans) {
|
||||
if (!content.includes(span.text)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${query.id}: annotated span is not exact`
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const language of ['en', 'zh-CN'] as const) {
|
||||
const languageQueries = fixture.queries.filter(
|
||||
(query) => query.language === language
|
||||
)
|
||||
if (!languageQueries.some((query) => !query.noAnswer)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${language}: fixture must include an answerable query`
|
||||
})
|
||||
}
|
||||
if (!languageQueries.some((query) => query.noAnswer)) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: `${language}: fixture must include a no-answer query`
|
||||
})
|
||||
}
|
||||
}
|
||||
const forbidden = /(?:[a-z]:[\\/]|\/(?:users|home|var|etc)\/|https?:\/\/|api[_-]?key|bearer\s+[a-z0-9]|sk-[a-z0-9]{8})/iu
|
||||
if (forbidden.test(JSON.stringify(fixture))) {
|
||||
context.addIssue({
|
||||
code: 'custom',
|
||||
message: 'fixture contains a path, endpoint, or secret-like value'
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
export type RetrievalFixture = z.infer<typeof retrievalFixtureSchema>
|
||||
export type RetrievalAblation =
|
||||
| 'lexical'
|
||||
| 'token-hash-vector'
|
||||
| 'regression-alias-vector'
|
||||
| 'hybrid'
|
||||
| 'hybrid-rerank'
|
||||
|
||||
export interface RankedEvaluationItem {
|
||||
chunkId: string
|
||||
context?: string
|
||||
}
|
||||
|
||||
export interface RetrievalMetrics {
|
||||
recallAt5: number
|
||||
recallAt10: number
|
||||
mrrAt10: number
|
||||
ndcgAt10: number
|
||||
contextPrecision: number
|
||||
contextRecall: number
|
||||
noAnswerFalsePositiveRate: number
|
||||
}
|
||||
|
||||
export interface RetrievalEvaluationReport {
|
||||
schemaVersion: 1
|
||||
fixtureId: string
|
||||
corpusHash: string
|
||||
evaluationDefinitionHash: string
|
||||
providerFingerprintHash: string
|
||||
queryIds: string[]
|
||||
ablations: Array<{
|
||||
id: RetrievalAblation
|
||||
metrics: RetrievalMetrics
|
||||
metricsByLanguage: Record<'en' | 'zh-CN', RetrievalMetrics>
|
||||
latencyMs: {
|
||||
count: number
|
||||
min: number
|
||||
median: number
|
||||
p95: number
|
||||
max: number
|
||||
mean: number
|
||||
}
|
||||
failures: Array<{
|
||||
queryId: string
|
||||
reason: 'no-relevant-result-at-10' | 'no-answer-false-positive'
|
||||
}>
|
||||
}>
|
||||
}
|
||||
|
||||
const fixturePath = fileURLToPath(
|
||||
new URL('../fixtures/knowledge-retrieval/synthetic-bilingual-v1.json', import.meta.url)
|
||||
)
|
||||
const dimensions = 256
|
||||
const metricVersion = 2
|
||||
const retrievalSettings = {
|
||||
version: 1 as const,
|
||||
topK: 10,
|
||||
minimumVectorSimilarity: 0.18,
|
||||
graphWeight: 0,
|
||||
candidateMultiplier: 4,
|
||||
contextMaxCharacters: 16_000,
|
||||
adjacentChunkCount: 0
|
||||
}
|
||||
const providerDefinitions = {
|
||||
'regression-alias': {
|
||||
fingerprint: 'goodbuddy:retrieval-eval:regression-alias:v1',
|
||||
model: 'handcrafted-alias-hash-v1'
|
||||
},
|
||||
'token-hash': {
|
||||
fingerprint: 'goodbuddy:retrieval-eval:topic-agnostic-token-hash:v1',
|
||||
model: 'topic-agnostic-token-hash-v1'
|
||||
}
|
||||
} as const
|
||||
type EvaluationProviderId = keyof typeof providerDefinitions
|
||||
|
||||
// The repository-wide setup targets jsdom. This owned config keeps the
|
||||
// evaluation in Node and prevents that renderer setup from crossing boundaries.
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: 'node',
|
||||
setupFiles: []
|
||||
}
|
||||
})
|
||||
|
||||
function sha256(value: string): string {
|
||||
return createHash('sha256').update(value).digest('hex')
|
||||
}
|
||||
|
||||
function canonicalFixtureCorpus(fixture: RetrievalFixture): string {
|
||||
return fixture.documents
|
||||
.flatMap((document) =>
|
||||
document.chunks.map((chunk) => `${document.id}\0${chunk.id}\0${chunk.content}`)
|
||||
)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
function normalizedTerms(text: string): string[] {
|
||||
return knowledgeRetrievalTerms(text)
|
||||
}
|
||||
|
||||
function tokenHashVector(text: string): number[] {
|
||||
const vector: number[] = Array.from({ length: dimensions }, () => 0)
|
||||
for (const term of normalizedTerms(text)) {
|
||||
const digest = createHash('sha256').update(`token:${term}`).digest()
|
||||
const bucket = digest.readUInt16LE(0) % dimensions
|
||||
vector[bucket]! += digest[2]! % 2 === 0 ? 1 : -1
|
||||
}
|
||||
if (vector.every((value) => value === 0)) {
|
||||
return [1, ...vector.slice(1)]
|
||||
}
|
||||
return vector
|
||||
}
|
||||
|
||||
/*
|
||||
* Regression plumbing only: these fixture-specific bilingual aliases make the
|
||||
* production vector/hybrid path deterministic. They do not model embedding
|
||||
* quality and must not be presented as a provider-quality benchmark.
|
||||
*/
|
||||
function regressionAliasHashVector(text: string): number[] {
|
||||
const vector = tokenHashVector(text)
|
||||
const aliases: Record<string, string[]> = {
|
||||
credential: ['credential', 'credentials', 'token', 'tokens', 'key', 'keys', '凭据', '密钥', '令牌'],
|
||||
offline: ['offline', 'network', 'connection', '本地', '网络', '离线'],
|
||||
cancel: ['cancel', 'cancelled', 'aborted', 'indexing', '取消', '中止', '索引'],
|
||||
backup: ['backup', 'restore', 'database', '备份', '恢复', '数据库'],
|
||||
keyboard: ['keyboard', 'focus', 'segmented', 'arrow', '键盘', '焦点', '方向键', '分段控件'],
|
||||
image: ['image', 'images', 'generated', 'inline', '图片', '生成', '内联'],
|
||||
migration: ['migration', 'migrations', 'failed', 'rollback', '迁移', '失败', '回滚'],
|
||||
approval: ['ask', 'execute', 'approval', 'read-only', '询问', '执行', '审批', '写入'],
|
||||
font: ['font', 'fonts', 'typefaces', 'remote', '字体', '远程'],
|
||||
integrity: ['release', 'artifact', 'manifest', 'hash', 'integrity', '发布', '产物', '完整性', '校验'],
|
||||
context: ['context', 'neighboring', 'sections', 'chunks', '上下文', '相邻', '分块'],
|
||||
language: ['language', 'languages', 'interface', 'english', 'chinese', '语言', '界面', '英文', '中文']
|
||||
}
|
||||
for (const [concept, variants] of Object.entries(aliases)) {
|
||||
if (variants.some((variant) => text.toLowerCase().includes(variant))) {
|
||||
const digest = createHash('sha256').update(`concept:${concept}`).digest()
|
||||
const bucket = digest.readUInt16LE(0) % dimensions
|
||||
vector[bucket]! += digest[2]! % 2 === 0 ? 12 : -12
|
||||
}
|
||||
}
|
||||
return vector
|
||||
}
|
||||
|
||||
function createEvaluationEmbeddingProvider(
|
||||
providerId: EvaluationProviderId
|
||||
): EmbeddingProvider {
|
||||
const definition = providerDefinitions[providerId]
|
||||
const vectorize =
|
||||
providerId === 'token-hash' ? tokenHashVector : regressionAliasHashVector
|
||||
return {
|
||||
provider: 'retrieval-eval-memory',
|
||||
model: definition.model,
|
||||
fingerprint: definition.fingerprint,
|
||||
embed: async (input, signal) => {
|
||||
signal?.throwIfAborted()
|
||||
return input.map(vectorize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Topic-agnostic deterministic plumbing for lexical-overlap vector ablations. */
|
||||
export function createDeterministicTokenHashEmbeddingProvider(): EmbeddingProvider {
|
||||
return createEvaluationEmbeddingProvider('token-hash')
|
||||
}
|
||||
|
||||
/** Fixture-aware regression plumbing; not an embedding-quality model. */
|
||||
export function createRegressionAliasEmbeddingProvider(): EmbeddingProvider {
|
||||
return createEvaluationEmbeddingProvider('regression-alias')
|
||||
}
|
||||
|
||||
export async function loadRetrievalFixture(
|
||||
path = fixturePath
|
||||
): Promise<RetrievalFixture> {
|
||||
const parsed: unknown = JSON.parse(await readFile(path, 'utf8'))
|
||||
return retrievalFixtureSchema.parse(parsed)
|
||||
}
|
||||
|
||||
function relevantAt(
|
||||
query: RetrievalFixture['queries'][number],
|
||||
ranked: readonly RankedEvaluationItem[],
|
||||
limit: number
|
||||
): number {
|
||||
const relevant = new Set(query.judgments.map((judgment) => judgment.chunkId))
|
||||
return ranked.slice(0, limit).filter((item) => relevant.has(item.chunkId)).length /
|
||||
Math.max(1, relevant.size)
|
||||
}
|
||||
|
||||
function dedupeRanking(
|
||||
ranked: readonly RankedEvaluationItem[]
|
||||
): RankedEvaluationItem[] {
|
||||
const seen = new Set<string>()
|
||||
return ranked.filter((item) => {
|
||||
if (seen.has(item.chunkId)) {
|
||||
return false
|
||||
}
|
||||
seen.add(item.chunkId)
|
||||
return true
|
||||
})
|
||||
}
|
||||
|
||||
function unionRangeLength(ranges: Array<readonly [number, number]>): number {
|
||||
const sorted = ranges
|
||||
.filter(([start, end]) => end > start)
|
||||
.sort(([leftStart, leftEnd], [rightStart, rightEnd]) =>
|
||||
leftStart - rightStart || leftEnd - rightEnd
|
||||
)
|
||||
let total = 0
|
||||
let currentStart = -1
|
||||
let currentEnd = -1
|
||||
for (const [start, end] of sorted) {
|
||||
if (currentStart < 0) {
|
||||
currentStart = start
|
||||
currentEnd = end
|
||||
} else if (start <= currentEnd) {
|
||||
currentEnd = Math.max(currentEnd, end)
|
||||
} else {
|
||||
total += currentEnd - currentStart
|
||||
currentStart = start
|
||||
currentEnd = end
|
||||
}
|
||||
}
|
||||
return currentStart < 0 ? 0 : total + currentEnd - currentStart
|
||||
}
|
||||
|
||||
function spanRanges(
|
||||
content: string,
|
||||
spans: readonly { text: string }[]
|
||||
): Array<readonly [number, number]> {
|
||||
return spans.flatMap((span) => {
|
||||
const start = content.indexOf(span.text)
|
||||
return start < 0 ? [] : [[start, start + span.text.length] as const]
|
||||
})
|
||||
}
|
||||
|
||||
export function computeRetrievalMetrics(
|
||||
fixture: RetrievalFixture,
|
||||
rankings: ReadonlyMap<string, readonly RankedEvaluationItem[]>
|
||||
): RetrievalMetrics {
|
||||
const answerable = fixture.queries.filter((query) => !query.noAnswer)
|
||||
const noAnswer = fixture.queries.filter((query) => query.noAnswer)
|
||||
let recall5 = 0
|
||||
let recall10 = 0
|
||||
let reciprocalRank = 0
|
||||
let ndcg = 0
|
||||
let matchedSpanCharacters = 0
|
||||
let returnedContextCharacters = 0
|
||||
let annotatedSpanCharacters = 0
|
||||
const chunksById = new Map(
|
||||
fixture.documents.flatMap((document) =>
|
||||
document.chunks.map((chunk) => [chunk.id, chunk.content] as const)
|
||||
)
|
||||
)
|
||||
|
||||
for (const query of answerable) {
|
||||
const ranked = dedupeRanking(rankings.get(query.id) ?? [])
|
||||
const grades = new Map(
|
||||
query.judgments.map((judgment) => [judgment.chunkId, judgment.relevance])
|
||||
)
|
||||
recall5 += relevantAt(query, ranked, 5)
|
||||
recall10 += relevantAt(query, ranked, 10)
|
||||
const firstRelevant = ranked
|
||||
.slice(0, 10)
|
||||
.findIndex((item) => (grades.get(item.chunkId) ?? 0) > 0)
|
||||
reciprocalRank += firstRelevant < 0 ? 0 : 1 / (firstRelevant + 1)
|
||||
const dcg = ranked.slice(0, 10).reduce((total, item, index) => {
|
||||
const grade = grades.get(item.chunkId) ?? 0
|
||||
return total + (2 ** grade - 1) / Math.log2(index + 2)
|
||||
}, 0)
|
||||
const ideal = [...grades.values()]
|
||||
.sort((left, right) => right - left)
|
||||
.slice(0, 10)
|
||||
.reduce((total, grade, index) => total + (2 ** grade - 1) / Math.log2(index + 2), 0)
|
||||
ndcg += ideal === 0 ? 0 : dcg / ideal
|
||||
|
||||
for (const judgment of query.judgments) {
|
||||
annotatedSpanCharacters += unionRangeLength(
|
||||
spanRanges(chunksById.get(judgment.chunkId) ?? '', judgment.spans)
|
||||
)
|
||||
}
|
||||
for (const item of ranked.slice(0, 10)) {
|
||||
const context = item.context ?? ''
|
||||
returnedContextCharacters += context.length
|
||||
const judgment = query.judgments.find((candidate) => candidate.chunkId === item.chunkId)
|
||||
matchedSpanCharacters += unionRangeLength(
|
||||
spanRanges(context, judgment?.spans ?? [])
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const falsePositives = noAnswer.filter(
|
||||
(query) => dedupeRanking(rankings.get(query.id) ?? []).length > 0
|
||||
).length
|
||||
return {
|
||||
recallAt5: recall5 / answerable.length,
|
||||
recallAt10: recall10 / answerable.length,
|
||||
mrrAt10: reciprocalRank / answerable.length,
|
||||
ndcgAt10: ndcg / answerable.length,
|
||||
contextPrecision:
|
||||
returnedContextCharacters === 0 ? 0 : matchedSpanCharacters / returnedContextCharacters,
|
||||
contextRecall:
|
||||
annotatedSpanCharacters === 0 ? 0 : matchedSpanCharacters / annotatedSpanCharacters,
|
||||
noAnswerFalsePositiveRate:
|
||||
noAnswer.length === 0 ? 0 : falsePositives / noAnswer.length
|
||||
}
|
||||
}
|
||||
|
||||
export function summarizeLatencies(values: readonly number[]) {
|
||||
if (values.length === 0) {
|
||||
return { count: 0, min: 0, median: 0, p95: 0, max: 0, mean: 0 }
|
||||
}
|
||||
const sorted = [...values].sort((left, right) => left - right)
|
||||
const percentile = (fraction: number): number =>
|
||||
sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * fraction) - 1)]!
|
||||
return {
|
||||
count: sorted.length,
|
||||
min: sorted[0]!,
|
||||
median: percentile(0.5),
|
||||
p95: percentile(0.95),
|
||||
max: sorted.at(-1)!,
|
||||
mean: sorted.reduce((sum, value) => sum + value, 0) / sorted.length
|
||||
}
|
||||
}
|
||||
|
||||
async function createSeededService(
|
||||
fixture: RetrievalFixture,
|
||||
providerId: EvaluationProviderId
|
||||
) {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-retrieval-eval-'))
|
||||
const provider = createEvaluationEmbeddingProvider(providerId)
|
||||
const vectorize =
|
||||
providerId === 'token-hash' ? tokenHashVector : regressionAliasHashVector
|
||||
const service = new KnowledgeService({
|
||||
databasePath: join(directory, 'knowledge.sqlite'),
|
||||
managedRoot: join(directory, 'managed'),
|
||||
embeddingProvider: provider
|
||||
})
|
||||
await service.initialize()
|
||||
const library = service.createLibrary({
|
||||
id: 'library-retrieval-eval',
|
||||
name: 'Synthetic retrieval evaluation',
|
||||
storageMode: 'reference',
|
||||
graphEnabled: false
|
||||
})
|
||||
const source = service.database.upsertSource({
|
||||
id: 'source-retrieval-eval',
|
||||
knowledgeBaseId: library.id,
|
||||
type: 'file',
|
||||
location: 'fixture://synthetic-bilingual-v1',
|
||||
displayName: 'Synthetic fixture',
|
||||
status: 'ready'
|
||||
})
|
||||
for (const document of fixture.documents) {
|
||||
service.database.upsertDocument(
|
||||
{
|
||||
id: document.id,
|
||||
knowledgeBaseId: library.id,
|
||||
sourceId: source.id,
|
||||
externalId: document.id,
|
||||
title: document.title,
|
||||
mimeType: 'text/plain',
|
||||
metadata: { status: 'ready', fixtureId: fixture.id }
|
||||
},
|
||||
document.chunks.map((chunk, ordinal) => ({
|
||||
id: chunk.id,
|
||||
ordinal,
|
||||
content: chunk.content,
|
||||
role: 'standalone' as const
|
||||
}))
|
||||
)
|
||||
service.database.replaceDocumentEmbeddings(
|
||||
document.id,
|
||||
embeddingStorageProvider(provider),
|
||||
provider.model,
|
||||
document.chunks.map((chunk) => ({
|
||||
chunkId: chunk.id,
|
||||
contentChecksum: sha256(chunk.content),
|
||||
vector: vectorize(chunk.content)
|
||||
}))
|
||||
)
|
||||
}
|
||||
return { directory, service, libraryId: library.id }
|
||||
}
|
||||
|
||||
const ablationSettings: Record<RetrievalAblation, {
|
||||
providerId: EvaluationProviderId
|
||||
ftsWeight: number
|
||||
vectorWeight: number
|
||||
localRerankEnabled: boolean
|
||||
}> = {
|
||||
lexical: {
|
||||
providerId: 'token-hash',
|
||||
ftsWeight: 1,
|
||||
vectorWeight: 0,
|
||||
localRerankEnabled: false
|
||||
},
|
||||
'token-hash-vector': {
|
||||
providerId: 'token-hash',
|
||||
ftsWeight: 0,
|
||||
vectorWeight: 1,
|
||||
localRerankEnabled: false
|
||||
},
|
||||
'regression-alias-vector': {
|
||||
providerId: 'regression-alias',
|
||||
ftsWeight: 0,
|
||||
vectorWeight: 1,
|
||||
localRerankEnabled: false
|
||||
},
|
||||
hybrid: {
|
||||
providerId: 'regression-alias',
|
||||
ftsWeight: 1,
|
||||
vectorWeight: 1,
|
||||
localRerankEnabled: false
|
||||
},
|
||||
'hybrid-rerank': {
|
||||
providerId: 'regression-alias',
|
||||
ftsWeight: 1,
|
||||
vectorWeight: 1,
|
||||
localRerankEnabled: true
|
||||
}
|
||||
}
|
||||
|
||||
async function safeOutputPath(rawPath: string, workingDirectory: string): Promise<string> {
|
||||
if (rawPath.includes('\0')) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT contains a null byte')
|
||||
}
|
||||
if (isAbsolute(rawPath)) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT must be a workspace-relative path')
|
||||
}
|
||||
const workspace = await realpath(workingDirectory)
|
||||
const output = resolve(workspace, rawPath)
|
||||
if (!isPathInside(workspace, output) || output === workspace) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT must be a workspace-relative file')
|
||||
}
|
||||
|
||||
const relativeParent = relative(workspace, dirname(output))
|
||||
let current = workspace
|
||||
for (const component of relativeParent.split(sep).filter(Boolean)) {
|
||||
current = join(current, component)
|
||||
try {
|
||||
const status = await lstat(current)
|
||||
if (status.isSymbolicLink()) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT may not traverse a symlink')
|
||||
}
|
||||
if (!status.isDirectory()) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT parent must be a directory')
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
await mkdir(current)
|
||||
}
|
||||
const canonicalCurrent = await realpath(current)
|
||||
if (!isPathInside(workspace, canonicalCurrent)) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT escapes the workspace')
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if ((await lstat(output)).isSymbolicLink()) {
|
||||
throw new Error('GOODBUDDY_RETRIEVAL_EVAL_OUTPUT may not be a symlink')
|
||||
}
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
async function writeWorkspaceReport(
|
||||
destination: string,
|
||||
report: RetrievalEvaluationReport
|
||||
): Promise<void> {
|
||||
const temporaryDirectory = await mkdtemp(join(dirname(destination), '.retrieval-eval-'))
|
||||
const temporaryPath = join(temporaryDirectory, 'report.json')
|
||||
try {
|
||||
await writeFile(temporaryPath, `${JSON.stringify(report, null, 2)}\n`, {
|
||||
encoding: 'utf8',
|
||||
flag: 'wx'
|
||||
})
|
||||
await rename(temporaryPath, destination)
|
||||
} finally {
|
||||
await rm(temporaryDirectory, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
function metricsForLanguage(
|
||||
fixture: RetrievalFixture,
|
||||
rankings: ReadonlyMap<string, readonly RankedEvaluationItem[]>,
|
||||
language: 'en' | 'zh-CN'
|
||||
): RetrievalMetrics {
|
||||
return computeRetrievalMetrics(
|
||||
{
|
||||
...fixture,
|
||||
queries: fixture.queries.filter((query) => query.language === language)
|
||||
},
|
||||
rankings
|
||||
)
|
||||
}
|
||||
|
||||
export function computeEvaluationDefinitionHash(
|
||||
fixture: RetrievalFixture
|
||||
): string {
|
||||
return sha256(JSON.stringify({
|
||||
fixtureVersion: fixture.version,
|
||||
fixtureId: fixture.id,
|
||||
queries: fixture.queries.map((query) => ({
|
||||
id: query.id,
|
||||
language: query.language,
|
||||
query: query.query,
|
||||
noAnswer: query.noAnswer,
|
||||
judgments: query.judgments
|
||||
})),
|
||||
retrievalSettings,
|
||||
ablations: ablationSettings,
|
||||
providers: providerDefinitions,
|
||||
metricVersion
|
||||
}))
|
||||
}
|
||||
|
||||
export async function runRetrievalEvaluation(options: {
|
||||
outputPath?: string
|
||||
workingDirectory?: string
|
||||
} = {}): Promise<RetrievalEvaluationReport> {
|
||||
const fixture = await loadRetrievalFixture()
|
||||
const ablations: RetrievalEvaluationReport['ablations'] = []
|
||||
for (const id of Object.keys(ablationSettings) as RetrievalAblation[]) {
|
||||
const settings = ablationSettings[id]
|
||||
const seeded = await createSeededService(fixture, settings.providerId)
|
||||
try {
|
||||
const rankings = new Map<string, RankedEvaluationItem[]>()
|
||||
const latencies: number[] = []
|
||||
for (const query of fixture.queries) {
|
||||
const response = await seeded.service.retrieve({
|
||||
knowledgeBaseId: seeded.libraryId,
|
||||
query: query.query,
|
||||
settings: {
|
||||
...retrievalSettings,
|
||||
ftsWeight: settings.ftsWeight,
|
||||
vectorWeight: settings.vectorWeight,
|
||||
localRerankEnabled: settings.localRerankEnabled
|
||||
}
|
||||
})
|
||||
latencies.push(response.durationMs)
|
||||
const contexts = new Map(
|
||||
response.context.groups.map((group) => [group.resultChunkId, group.content])
|
||||
)
|
||||
rankings.set(
|
||||
query.id,
|
||||
response.results.map((result) => ({
|
||||
chunkId: result.chunkId,
|
||||
context: contexts.get(result.chunkId)
|
||||
}))
|
||||
)
|
||||
}
|
||||
const failures: RetrievalEvaluationReport['ablations'][number]['failures'] = []
|
||||
for (const query of fixture.queries) {
|
||||
const ranking = rankings.get(query.id) ?? []
|
||||
if (query.noAnswer) {
|
||||
if (ranking.length > 0) {
|
||||
failures.push({
|
||||
queryId: query.id,
|
||||
reason: 'no-answer-false-positive'
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
const relevant = new Set(query.judgments.map((judgment) => judgment.chunkId))
|
||||
if (!ranking.slice(0, 10).some((item) => relevant.has(item.chunkId))) {
|
||||
failures.push({
|
||||
queryId: query.id,
|
||||
reason: 'no-relevant-result-at-10'
|
||||
})
|
||||
}
|
||||
}
|
||||
ablations.push({
|
||||
id,
|
||||
metrics: computeRetrievalMetrics(fixture, rankings),
|
||||
metricsByLanguage: {
|
||||
en: metricsForLanguage(fixture, rankings, 'en'),
|
||||
'zh-CN': metricsForLanguage(fixture, rankings, 'zh-CN')
|
||||
},
|
||||
latencyMs: summarizeLatencies(latencies),
|
||||
failures
|
||||
})
|
||||
} finally {
|
||||
await seeded.service.dispose()
|
||||
await rm(seeded.directory, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
const report: RetrievalEvaluationReport = {
|
||||
schemaVersion: 1,
|
||||
fixtureId: fixture.id,
|
||||
corpusHash: sha256(canonicalFixtureCorpus(fixture)),
|
||||
evaluationDefinitionHash: computeEvaluationDefinitionHash(fixture),
|
||||
providerFingerprintHash: sha256(JSON.stringify(providerDefinitions)),
|
||||
queryIds: fixture.queries.map((query) => query.id),
|
||||
ablations
|
||||
}
|
||||
const outputPath = options.outputPath ?? process.env.GOODBUDDY_RETRIEVAL_EVAL_OUTPUT
|
||||
if (outputPath) {
|
||||
const destination = await safeOutputPath(
|
||||
outputPath,
|
||||
options.workingDirectory ?? process.cwd()
|
||||
)
|
||||
await writeWorkspaceReport(destination, report)
|
||||
}
|
||||
return report
|
||||
}
|
||||
|
||||
export function deterministicReportProjection(report: RetrievalEvaluationReport) {
|
||||
return {
|
||||
schemaVersion: report.schemaVersion,
|
||||
fixtureId: report.fixtureId,
|
||||
corpusHash: report.corpusHash,
|
||||
evaluationDefinitionHash: report.evaluationDefinitionHash,
|
||||
providerFingerprintHash: report.providerFingerprintHash,
|
||||
queryIds: report.queryIds,
|
||||
ablations: report.ablations.map((ablation) => ({
|
||||
id: ablation.id,
|
||||
metrics: ablation.metrics,
|
||||
metricsByLanguage: ablation.metricsByLanguage,
|
||||
failures: ablation.failures
|
||||
}))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user