fix: record an experience application only when it touches the same knowledge

This commit is contained in:
mesalogo
2026-10-02 19:08:05 +08:00
parent c3fa262832
commit 4d1ce38dad
3 changed files with 30 additions and 6 deletions
@@ -18,7 +18,9 @@
| 经验跨运行一致性 | run2 的 38 条里 23 条能在 run1 中找到形成依据重合(Jaccard ≥ 0.3)的对应经验,表述基本一致 |
| 经验依据抽查(10 条) | 形成依据基本都对得上经验内容;后续“应用”较弱,抽到的 5 条应用里约 2 条和经验关系不大(例如“需求有歧义先确认”被记到“询问微信附件导入能力”) |
结论与后续:功能边界在明确“提交归领域”后明显稳定;仍有约四分之一的事件对在两次运行间归属不同,主要发生在界面、笔记、帮助提示等相邻功能之间,可以通过手动合并修正。经验的形成依据可信,应用记录需要更严格的约束,下一步可要求应用事件与经验共享实体或在描述中体现经验内容。
结论:功能边界在明确“提交归领域”后明显稳定;仍有约四分之一的事件对在两次运行间归属不同,主要发生在界面、笔记、帮助提示等相邻功能之间,可以通过手动合并修正。经验的形成依据可信,应用记录原先过宽。
随后收紧经验应用:模型给出的应用事件必须与该经验的某个形成事件共享实体,否则丢弃(同批其他内容保留)。在 off3 的故事上重新抽取(6 次调用,约 3.8 分钟):45 条经验,应用记录从此前的 53 条降到 6 条,抽查 6 条全部与经验内容相符;依据无来源 0、应用早于形成 0、再次抽取 0 次调用。代价是跨故事的经验连线变少(3 条),时间螺旋和“可用经验”建议能显示的跨故事复用也随之减少。`supervision-experiences.test.ts` 新增“无关知识的事件不记为应用”。
## 2026-10-02 时间螺旋显示经验节点
@@ -23,13 +23,18 @@ async function fixture() {
cleanups.push(async () => { sql.close(); db.close(); await rm(directory, { recursive: true, force: true }) })
const project = db.listProjects()[0]!
// Publishes events of one project in time order, then places them all in one feature story.
const publish = async (items: Array<[string, Kind]>, day: number, story = 'Review scheduling') => {
// Every event mentions the entity `topic`; an application must share an entity with the formation.
const publish = async (items: Array<[string, Kind]>, day: number, story = 'Review scheduling', topic = 'Long jobs') => {
const request = { trigger: 'manual' as const, scope: { kind: 'projects' as const, projectIds: [project.id] }, timeRange: { from: '2026-09-01T00:00:00Z', to: '2026-10-01T00:00:00Z' } }
const candidates = db.listSupervisionCandidates(request)
const known = candidates.find(entity => entity.label === topic)
db.saveSupervisionResult({
request: { trigger: 'manual', scope: { kind: 'projects', projectIds: [project.id] }, timeRange: { from: '2026-09-01T00:00:00Z', to: '2026-10-01T00:00:00Z' } },
request, candidates,
evidence: items.map(([title], index) => ({ id: `s${index}`, sourceType: 'conversation' as const, sourceId: 'c', title, content: title,
occurredAt: `2026-09-${day}T0${index}:00:00.000Z`, locator: { source: `message:${randomUUID()}`, revision: 'r', projectId: project.id, start: 0, end: title.length } })),
output: { summary: 'x', changeDigest: '', openItems: [], entities: [], entityChanges: [], relations: [],
events: items.map(([title, eventType], index) => ({ title, description: title, occurredAt: `2026-09-${day}T0${index}:00:00.000Z`, eventType, entityIds: [], sourceReferenceIds: [`s${index}`] })) }
output: { summary: 'x', changeDigest: '', openItems: [], entityChanges: [], relations: [],
entities: [{ id: 'topic', label: topic, description: topic, sourceReferenceIds: ['s0'], ...(known ? { persistedId: known.id } : {}) }],
events: items.map(([title, eventType], index) => ({ title, description: title, occurredAt: `2026-09-${day}T0${index}:00:00.000Z`, eventType, entityIds: ['topic'], sourceReferenceIds: [`s${index}`] })) }
})
const existing = db.supervisionStories().candidates(project.id, false).find(row => row.name === story)
await assignStories(db.supervisionStories(), async prompt => {
@@ -98,6 +103,12 @@ it('rejects invented evidence and saves nothing from a rejected answer; drops ap
await f.publish([['D', 'decision']], 19)
await extractExperiences(f.store, async () => JSON.stringify({ applications: [{ experience: 'x_1', event: 'e_1' }] }), options)
expect(f.store.list()[0]!.events.map(event => [event.role, event.title])).toEqual([['formed', 'B'], ['applied', 'C']])
// A later event about unrelated knowledge is not recorded as an application.
await f.publish([['Unrelated decision', 'decision']], 25, 'Review scheduling', 'Billing')
await expect(extractExperiences(f.store, async prompt => {
const ref = prompt.split('\n').find(line => line.includes('Unrelated decision'))!.match(/"ref":"(e_\d+)"/)![1]
return JSON.stringify({ applications: [{ experience: 'x_1', event: ref, note: 'loosely related' }] })
}, options)).resolves.toMatchObject({ applied: 0 })
})
it('protects edits, keeps removed experiences removed and undoes adjustments', async () => {
+12 -1
View File
@@ -139,9 +139,20 @@ export class SupervisionExperienceStore {
}
// An application must come after the experience formed; the real model sometimes cites earlier events. Those
// citations are dropped, the rest of the answer is kept.
// An application must also touch the same knowledge as the formation: share an entity with one of its
// formation events. The real model otherwise records loosely related events as applications.
const entitiesOf = this.db.prepare('SELECT entity_id FROM supervision_event_entities WHERE event_id = ?')
const formedEvents = new Map<string, string[]>()
for (const item of output.experiences) formedEvents.set(item.key, item.formed.map(ref => refs.get(ref)!.id))
const formedOf = this.db.prepare("SELECT event_id FROM supervision_experience_events WHERE experience_id = ? AND role = 'formed'")
for (const [ref, row] of xRefs) formedEvents.set(ref, formedOf.all(String(row.id)).map(item => String(item.event_id)))
const related = (experience: string, eventId: string) => {
const formed = new Set((formedEvents.get(experience) ?? []).flatMap(id => entitiesOf.all(id).map(item => String(item.entity_id))))
return entitiesOf.all(eventId).some(item => formed.has(String(item.entity_id)))
}
const timely = output.applications.filter(application => {
const since = formedAt.get(application.experience)
return !since || refs.get(application.event)!.at > since
return (!since || refs.get(application.event)!.at > since) && related(application.experience, refs.get(application.event)!.id)
})
const removed = this.db.prepare("SELECT id, statement FROM supervision_experiences WHERE status = 'removed'").all()
const removedStatements = new Set(removed.map(row => statementKey(String(row.statement))))