feat: enhance knowledge workflows and refresh interface
This commit is contained in:
lofyer
@@ -520,6 +520,20 @@ function getKnowledgeSnapshot(
|
||||
documentsById.get(item.documentId)?.title ?? '未知文档',
|
||||
excerpt: item.quote ?? '',
|
||||
location: item.location
|
||||
})),
|
||||
tasks: snapshot.tasks.map((task) => ({
|
||||
id: task.id,
|
||||
libraryId: task.libraryId,
|
||||
sourceId: task.sourceId,
|
||||
documentId: task.documentId,
|
||||
documentName: task.documentName,
|
||||
kind: task.kind,
|
||||
status: task.status,
|
||||
progress: task.progress,
|
||||
message: task.message,
|
||||
createdAt: task.createdAt,
|
||||
startedAt: task.startedAt,
|
||||
completedAt: task.completedAt
|
||||
}))
|
||||
}
|
||||
}
|
||||
@@ -3515,12 +3529,22 @@ export function registerIpcHandlers(
|
||||
assertTrustedSender(event, window)
|
||||
const value = knowledgeUpdateLibrarySchema.parse(input)
|
||||
knowledgeService.database.updateKnowledgeBase(value.libraryId, {
|
||||
name: value.name,
|
||||
description: value.description,
|
||||
graphEnabled: value.graphEnabled,
|
||||
graphStrategy: value.graphStrategy
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
ipcMain.handle(
|
||||
ipcChannels.knowledgeReextractGraph,
|
||||
async (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
return knowledgeService.reextractGraph(knowledgeIdSchema.parse(input))
|
||||
}
|
||||
)
|
||||
|
||||
ipcMain.handle(
|
||||
ipcChannels.knowledgeSelectFiles,
|
||||
async (event, input: unknown) => {
|
||||
|
||||
@@ -348,6 +348,28 @@ describe('extraction strategies', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('propagates model extraction failures for hybrid and model strategies', async () => {
|
||||
const chunks = [{ id: 'fallback', content: '# Local Entity' }]
|
||||
for (const strategy of ['hybrid', 'model'] as const) {
|
||||
await expect(
|
||||
extractKnowledgeGraph(chunks, {
|
||||
strategy,
|
||||
extractStructured: async () => {
|
||||
throw new Error('模型未返回图谱内容')
|
||||
}
|
||||
})
|
||||
).rejects.toThrow('模型未返回图谱内容')
|
||||
}
|
||||
await expect(
|
||||
extractKnowledgeGraph(chunks, {
|
||||
strategy: 'hybrid',
|
||||
extractStructured: async () => {
|
||||
return { invalid: true }
|
||||
}
|
||||
})
|
||||
).rejects.toThrow()
|
||||
})
|
||||
|
||||
it('supports rules, model, and ask behavior without an implicit model call', async () => {
|
||||
const chunks = [{ id: 'strategy', content: '# Local Entity' }]
|
||||
const callback = vi.fn()
|
||||
@@ -359,14 +381,12 @@ describe('extraction strategies', () => {
|
||||
strategy: 'ask',
|
||||
extractStructured: callback
|
||||
})
|
||||
const unavailable = await extractKnowledgeGraph(chunks, {
|
||||
strategy: 'model'
|
||||
})
|
||||
|
||||
expect(callback).not.toHaveBeenCalled()
|
||||
expect(rules.requiresModelApproval).toBe(false)
|
||||
expect(ask.requiresModelApproval).toBe(true)
|
||||
expect(unavailable.warnings).toEqual(['Model extraction is unavailable'])
|
||||
await expect(
|
||||
extractKnowledgeGraph(chunks, { strategy: 'model' })
|
||||
).rejects.toThrow('Model extraction is unavailable')
|
||||
})
|
||||
|
||||
it('honors cancellation before and after the injected model callback', async () => {
|
||||
|
||||
@@ -679,12 +679,7 @@ export async function extractKnowledgeGraph(
|
||||
}
|
||||
}
|
||||
if (!options.extractStructured) {
|
||||
return {
|
||||
...rules,
|
||||
strategy,
|
||||
requiresModelApproval: false,
|
||||
warnings: ['Model extraction is unavailable']
|
||||
}
|
||||
throw new Error('Model extraction is unavailable')
|
||||
}
|
||||
|
||||
const output = await options.extractStructured(
|
||||
@@ -692,7 +687,11 @@ export async function extractKnowledgeGraph(
|
||||
options.signal
|
||||
)
|
||||
throwIfAborted(options.signal)
|
||||
const model = validateModelGraph(output, prepared)
|
||||
const parsedOutput = parseModelOutput(output)
|
||||
if (!modelEnvelopeSchema.safeParse(parsedOutput).success) {
|
||||
throw new Error('模型返回的图谱结构无效')
|
||||
}
|
||||
const model = validateModelGraph(parsedOutput, prepared)
|
||||
const graph =
|
||||
strategy === 'hybrid' ? mergeKnowledgeGraphs(rules, model) : model
|
||||
return {
|
||||
|
||||
@@ -905,6 +905,54 @@ export class KnowledgeDatabase {
|
||||
)
|
||||
}
|
||||
|
||||
pruneUnreferencedGeneratedGraph(knowledgeBaseId: string): {
|
||||
entities: number
|
||||
relations: number
|
||||
} {
|
||||
const normalizedId = requiredString(
|
||||
knowledgeBaseId,
|
||||
'knowledgeBaseId',
|
||||
MAX_ID_LENGTH
|
||||
)
|
||||
const database = this.requireDatabase()
|
||||
let entities = 0
|
||||
let relations = 0
|
||||
this.transaction(database, () => {
|
||||
relations = Number(
|
||||
database
|
||||
.prepare(
|
||||
`DELETE FROM graph_relations
|
||||
WHERE knowledge_base_id = ?
|
||||
AND locked = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM graph_evidence
|
||||
WHERE relation_id = graph_relations.id
|
||||
)`
|
||||
)
|
||||
.run(normalizedId).changes
|
||||
)
|
||||
entities = Number(
|
||||
database
|
||||
.prepare(
|
||||
`DELETE FROM graph_entities
|
||||
WHERE knowledge_base_id = ?
|
||||
AND locked = 0
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM graph_evidence
|
||||
WHERE entity_id = graph_entities.id
|
||||
)
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM graph_relations
|
||||
WHERE source_entity_id = graph_entities.id
|
||||
OR target_entity_id = graph_entities.id
|
||||
)`
|
||||
)
|
||||
.run(normalizedId).changes
|
||||
)
|
||||
})
|
||||
return { entities, relations }
|
||||
}
|
||||
|
||||
listChunks(documentId: string, limit = MAX_LIST_LIMIT): Chunk[] {
|
||||
const normalizedId = requiredString(
|
||||
documentId,
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
import type { ExtractStructured } from './graph-extractor'
|
||||
import { KnowledgeService } from './knowledge-service'
|
||||
import type { EmbeddingProvider } from './types'
|
||||
import { UrlImporter } from './url-importer'
|
||||
@@ -17,7 +18,8 @@ const services: KnowledgeService[] = []
|
||||
|
||||
async function createService(
|
||||
urlImporter?: UrlImporter,
|
||||
embeddingProvider?: EmbeddingProvider
|
||||
embeddingProvider?: EmbeddingProvider,
|
||||
extractStructured?: ExtractStructured
|
||||
): Promise<{ directory: string; service: KnowledgeService }> {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-knowledge-service-'))
|
||||
temporaryDirectories.push(directory)
|
||||
@@ -25,7 +27,8 @@ async function createService(
|
||||
databasePath: join(directory, 'knowledge.sqlite'),
|
||||
managedRoot: join(directory, 'managed'),
|
||||
urlImporter,
|
||||
embeddingProvider
|
||||
embeddingProvider,
|
||||
extractStructured
|
||||
})
|
||||
await service.initialize()
|
||||
services.push(service)
|
||||
@@ -145,9 +148,101 @@ describe('KnowledgeService', () => {
|
||||
|
||||
expect(snapshot.entities.length).toBeGreaterThan(0)
|
||||
expect(snapshot.evidence.length).toBeGreaterThan(0)
|
||||
expect(snapshot.tasks).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
kind: 'parsing',
|
||||
status: 'succeeded',
|
||||
progress: 100
|
||||
}),
|
||||
expect.objectContaining({
|
||||
kind: 'embedding',
|
||||
status: 'skipped',
|
||||
progress: 100
|
||||
}),
|
||||
expect.objectContaining({
|
||||
kind: 'graph',
|
||||
status: 'succeeded',
|
||||
progress: 100
|
||||
})
|
||||
])
|
||||
)
|
||||
await service.dispose()
|
||||
})
|
||||
|
||||
it('reextracts graph evidence and removes only stale generated entities', async () => {
|
||||
const { directory, service } = await createService()
|
||||
const sourcePath = join(directory, 'reextract.md')
|
||||
await writeFile(
|
||||
sourcePath,
|
||||
'GoodBuddy（产品）依赖 Electron（框架）。',
|
||||
'utf8'
|
||||
)
|
||||
const library = service.createLibrary({
|
||||
name: '重新抽取',
|
||||
storageMode: 'reference',
|
||||
graphEnabled: true,
|
||||
graphStrategy: 'rules'
|
||||
})
|
||||
await service.importPaths(library.id, [sourcePath])
|
||||
const stale = service.database.createEntity({
|
||||
knowledgeBaseId: library.id,
|
||||
name: '过期实体',
|
||||
type: '概念',
|
||||
locked: false
|
||||
})
|
||||
const manual = service.database.createEntity({
|
||||
knowledgeBaseId: library.id,
|
||||
name: '人工实体',
|
||||
type: '概念',
|
||||
locked: true
|
||||
})
|
||||
|
||||
await service.reextractGraph(library.id)
|
||||
|
||||
const snapshot = service.snapshot(library.id)
|
||||
expect(snapshot.evidence.length).toBeGreaterThan(0)
|
||||
expect(service.database.getEntity(stale.id)).toBeUndefined()
|
||||
expect(service.database.getEntity(manual.id)).toBeDefined()
|
||||
})
|
||||
|
||||
it('fails hybrid reextraction when model extraction fails', async () => {
|
||||
const extractStructured = vi.fn(async () => {
|
||||
throw new Error('模型未返回图谱内容')
|
||||
})
|
||||
const { directory, service } = await createService(
|
||||
undefined,
|
||||
undefined,
|
||||
extractStructured
|
||||
)
|
||||
const sourcePath = join(directory, 'hybrid-fallback.md')
|
||||
await writeFile(sourcePath, '# 本地实体', 'utf8')
|
||||
const library = service.createLibrary({
|
||||
name: '混合抽取',
|
||||
storageMode: 'reference',
|
||||
graphEnabled: false,
|
||||
graphStrategy: 'hybrid'
|
||||
})
|
||||
await service.importPaths(library.id, [sourcePath])
|
||||
service.database.updateKnowledgeBase(library.id, {
|
||||
graphEnabled: true
|
||||
})
|
||||
|
||||
await expect(service.reextractGraph(library.id)).rejects.toThrow(
|
||||
'模型未返回图谱内容'
|
||||
)
|
||||
expect(service.snapshot(library.id).entities).toHaveLength(0)
|
||||
expect(service.snapshot(library.id).tasks).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
kind: 'graph',
|
||||
status: 'failed',
|
||||
message: '模型未返回图谱内容'
|
||||
})
|
||||
])
|
||||
)
|
||||
})
|
||||
|
||||
it('indexes optional embeddings and performs vector-backed hybrid search', async () => {
|
||||
const provider: EmbeddingProvider = {
|
||||
provider: 'test-provider',
|
||||
|
||||
@@ -23,7 +23,8 @@ import { classifyEmbeddingError } from './embedding-errors'
|
||||
import {
|
||||
extractKnowledgeGraph,
|
||||
normalizeEntityAlias,
|
||||
type ExtractStructured
|
||||
type ExtractStructured,
|
||||
type GraphExtractionResult
|
||||
} from './graph-extractor'
|
||||
import { KnowledgeDatabase } from './knowledge-database'
|
||||
import type {
|
||||
@@ -66,6 +67,21 @@ export type KnowledgeDocumentSnapshot = Document & {
|
||||
error?: string
|
||||
}
|
||||
|
||||
export type KnowledgeTaskSnapshot = {
|
||||
id: string
|
||||
libraryId: string
|
||||
sourceId?: string
|
||||
documentId?: string
|
||||
documentName: string
|
||||
kind: 'parsing' | 'embedding' | 'graph'
|
||||
status: 'queued' | 'running' | 'succeeded' | 'failed' | 'skipped'
|
||||
progress: number
|
||||
message?: string
|
||||
createdAt: string
|
||||
startedAt?: string
|
||||
completedAt?: string
|
||||
}
|
||||
|
||||
export type KnowledgeSnapshot = {
|
||||
libraries: KnowledgeLibrarySnapshot[]
|
||||
sources: KnowledgeSourceSnapshot[]
|
||||
@@ -73,6 +89,7 @@ export type KnowledgeSnapshot = {
|
||||
entities: GraphEntity[]
|
||||
relations: GraphRelation[]
|
||||
evidence: ReturnType<KnowledgeDatabase['listEvidence']>
|
||||
tasks: KnowledgeTaskSnapshot[]
|
||||
}
|
||||
|
||||
export type KnowledgeServiceOptions = {
|
||||
@@ -89,6 +106,7 @@ const maximumFileBytes = 20 * 1024 * 1024
|
||||
const maximumSourceBytes = 500 * 1024 * 1024
|
||||
const maximumFilesPerSource = 2_000
|
||||
const maximumEmbeddingChunksPerBatch = 32
|
||||
const maximumKnowledgeTasks = 500
|
||||
|
||||
function isInside(root: string, candidate: string): boolean {
|
||||
const path = relative(resolve(root), resolve(candidate))
|
||||
@@ -105,6 +123,7 @@ export class KnowledgeService {
|
||||
private readonly watchers = new Map<string, FSWatcher>()
|
||||
private readonly syncTimers = new Map<string, ReturnType<typeof setTimeout>>()
|
||||
private readonly activeSyncs = new Map<string, Promise<void>>()
|
||||
private readonly tasks = new Map<string, KnowledgeTaskSnapshot>()
|
||||
private readonly lifecycleController = new AbortController()
|
||||
|
||||
constructor(options: KnowledgeServiceOptions) {
|
||||
@@ -163,6 +182,109 @@ export class KnowledgeService {
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
private createKnowledgeTask(input: {
|
||||
libraryId: string
|
||||
sourceId?: string
|
||||
documentId?: string
|
||||
documentName: string
|
||||
kind: KnowledgeTaskSnapshot['kind']
|
||||
status?: KnowledgeTaskSnapshot['status']
|
||||
message?: string
|
||||
}): KnowledgeTaskSnapshot {
|
||||
while (this.tasks.size >= maximumKnowledgeTasks) {
|
||||
const oldestTaskId = this.tasks.keys().next().value as
|
||||
| string
|
||||
| undefined
|
||||
if (!oldestTaskId) {
|
||||
break
|
||||
}
|
||||
this.tasks.delete(oldestTaskId)
|
||||
}
|
||||
const now = new Date().toISOString()
|
||||
const status = input.status ?? 'queued'
|
||||
const task: KnowledgeTaskSnapshot = {
|
||||
id: randomUUID(),
|
||||
libraryId: input.libraryId,
|
||||
sourceId: input.sourceId,
|
||||
documentId: input.documentId,
|
||||
documentName: input.documentName.slice(0, 512),
|
||||
kind: input.kind,
|
||||
status,
|
||||
progress: status === 'succeeded' || status === 'skipped' ? 100 : 0,
|
||||
message: input.message?.slice(0, 1_000),
|
||||
createdAt: now,
|
||||
startedAt: status === 'running' ? now : undefined,
|
||||
completedAt:
|
||||
status === 'succeeded' ||
|
||||
status === 'failed' ||
|
||||
status === 'skipped'
|
||||
? now
|
||||
: undefined
|
||||
}
|
||||
this.tasks.set(task.id, task)
|
||||
return task
|
||||
}
|
||||
|
||||
private updateKnowledgeTask(
|
||||
taskId: string,
|
||||
update: {
|
||||
status?: KnowledgeTaskSnapshot['status']
|
||||
progress?: number
|
||||
message?: string
|
||||
documentId?: string
|
||||
documentName?: string
|
||||
}
|
||||
): void {
|
||||
const current = this.tasks.get(taskId)
|
||||
if (!current) {
|
||||
return
|
||||
}
|
||||
const status = update.status ?? current.status
|
||||
const terminal =
|
||||
status === 'succeeded' ||
|
||||
status === 'failed' ||
|
||||
status === 'skipped'
|
||||
this.tasks.set(taskId, {
|
||||
...current,
|
||||
status,
|
||||
documentId: update.documentId ?? current.documentId,
|
||||
documentName:
|
||||
update.documentName?.slice(0, 512) ?? current.documentName,
|
||||
progress:
|
||||
status === 'succeeded' || status === 'skipped'
|
||||
? 100
|
||||
: update.progress === undefined
|
||||
? current.progress
|
||||
: Math.max(0, Math.min(100, Math.round(update.progress))),
|
||||
message:
|
||||
update.message === undefined
|
||||
? current.message
|
||||
: update.message.slice(0, 1_000),
|
||||
startedAt:
|
||||
status === 'running' && !current.startedAt
|
||||
? new Date().toISOString()
|
||||
: current.startedAt,
|
||||
completedAt:
|
||||
terminal && !current.completedAt
|
||||
? new Date().toISOString()
|
||||
: current.completedAt
|
||||
})
|
||||
}
|
||||
|
||||
private failKnowledgeTask(taskId: string, error: unknown): void {
|
||||
const current = this.tasks.get(taskId)
|
||||
if (
|
||||
current?.status === 'succeeded' ||
|
||||
current?.status === 'skipped'
|
||||
) {
|
||||
return
|
||||
}
|
||||
this.updateKnowledgeTask(taskId, {
|
||||
status: 'failed',
|
||||
message: error instanceof Error ? error.message : '任务失败'
|
||||
})
|
||||
}
|
||||
|
||||
createLibrary(input: CreateKnowledgeBaseInput): KnowledgeBase {
|
||||
return this.database.createKnowledgeBase(input)
|
||||
}
|
||||
@@ -175,6 +297,11 @@ export class KnowledgeService {
|
||||
for (const source of this.database.listSources(id)) {
|
||||
this.stopWatcher(source.id)
|
||||
}
|
||||
for (const task of this.tasks.values()) {
|
||||
if (task.libraryId === id) {
|
||||
this.tasks.delete(task.id)
|
||||
}
|
||||
}
|
||||
const deleted = this.database.deleteKnowledgeBase(id)
|
||||
if (deleted && library.storageMode === 'managed') {
|
||||
const path = join(this.managedRoot, id)
|
||||
@@ -206,7 +333,8 @@ export class KnowledgeService {
|
||||
documents: [],
|
||||
entities: [],
|
||||
relations: [],
|
||||
evidence: []
|
||||
evidence: [],
|
||||
tasks: []
|
||||
}
|
||||
}
|
||||
const sources = this.database.listSources(libraryId).map((source) => ({
|
||||
@@ -253,7 +381,12 @@ export class KnowledgeService {
|
||||
documents,
|
||||
entities: this.database.listEntities(libraryId),
|
||||
relations: this.database.listRelations(libraryId),
|
||||
evidence: this.database.listEvidence(libraryId)
|
||||
evidence: this.database.listEvidence(libraryId),
|
||||
tasks: [...this.tasks.values()]
|
||||
.filter((task) => task.libraryId === libraryId)
|
||||
.sort((left, right) =>
|
||||
right.createdAt.localeCompare(left.createdAt)
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -427,7 +560,28 @@ export class KnowledgeService {
|
||||
this.lifecycleController.signal,
|
||||
AbortSignal.timeout(60_000)
|
||||
])
|
||||
const result = await this.urlImporter.import(input, effectiveSignal)
|
||||
const parsingTask = this.createKnowledgeTask({
|
||||
libraryId: library.id,
|
||||
sourceId,
|
||||
documentName: new URL(input).hostname,
|
||||
kind: 'parsing'
|
||||
})
|
||||
let result: Awaited<ReturnType<UrlImporter['import']>>
|
||||
try {
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
status: 'running',
|
||||
progress: 10,
|
||||
message: '正在抓取并解析网页'
|
||||
})
|
||||
result = await this.urlImporter.import(input, effectiveSignal)
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
progress: 70,
|
||||
message: '正在保存网页内容'
|
||||
})
|
||||
} catch (error) {
|
||||
this.failKnowledgeTask(parsingTask.id, error)
|
||||
throw error
|
||||
}
|
||||
let source = this.database.upsertSource({
|
||||
id: sourceId,
|
||||
knowledgeBaseId,
|
||||
@@ -465,6 +619,12 @@ export class KnowledgeService {
|
||||
location: chunk.locator
|
||||
}))
|
||||
)
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
status: 'succeeded',
|
||||
documentId: document.id,
|
||||
documentName: document.title,
|
||||
message: '网页解析完成'
|
||||
})
|
||||
await this.indexDocumentEmbeddings(document)
|
||||
await this.extractGraph(effectiveLibrary, document)
|
||||
source = this.database.upsertSource({
|
||||
@@ -477,6 +637,7 @@ export class KnowledgeService {
|
||||
}
|
||||
})
|
||||
} catch (error) {
|
||||
this.failKnowledgeTask(parsingTask.id, error)
|
||||
this.database.upsertSource({
|
||||
...source,
|
||||
status: 'error',
|
||||
@@ -511,6 +672,62 @@ export class KnowledgeService {
|
||||
return this.syncSource(sourceId)
|
||||
}
|
||||
|
||||
async reextractGraph(knowledgeBaseId: string): Promise<void> {
|
||||
const library = this.requireLibrary(knowledgeBaseId)
|
||||
if (!library.graphEnabled) {
|
||||
throw new Error('请先启用知识图谱')
|
||||
}
|
||||
if (library.graphStrategy === 'ask') {
|
||||
throw new Error('按需询问策略不会自动抽取，请在设置中选择其他策略')
|
||||
}
|
||||
const documents = this.database.listDocuments(library.id)
|
||||
const tasks = documents.map((document) =>
|
||||
this.createKnowledgeTask({
|
||||
libraryId: library.id,
|
||||
sourceId: document.sourceId,
|
||||
documentId: document.id,
|
||||
documentName: document.title,
|
||||
kind: 'graph',
|
||||
message: '等待重新抽取'
|
||||
})
|
||||
)
|
||||
for (let index = 0; index < documents.length; index += 1) {
|
||||
const document = documents[index]
|
||||
const task = tasks[index]
|
||||
if (!document || !task) {
|
||||
continue
|
||||
}
|
||||
try {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'running',
|
||||
progress: 10,
|
||||
message: '正在重新抽取知识图谱'
|
||||
})
|
||||
const result = await this.extractGraphResult(library, document)
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
progress: 85,
|
||||
message: '正在保存实体和关系'
|
||||
})
|
||||
this.database.removeEvidenceForDocument(document.id)
|
||||
this.storeExtractedGraph(library, document, result)
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'succeeded',
|
||||
message: `已抽取 ${result.entities.length} 个实体、${result.relations.length} 条关系`
|
||||
})
|
||||
} catch (error) {
|
||||
this.failKnowledgeTask(task.id, error)
|
||||
for (const pendingTask of tasks.slice(index + 1)) {
|
||||
this.updateKnowledgeTask(pendingTask.id, {
|
||||
status: 'skipped',
|
||||
message: '因前序图谱任务失败而未执行'
|
||||
})
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||
}
|
||||
|
||||
async removeSource(sourceId: string): Promise<boolean> {
|
||||
const source = this.requireSource(sourceId)
|
||||
const library = this.requireLibrary(source.knowledgeBaseId)
|
||||
@@ -594,19 +811,42 @@ export class KnowledgeService {
|
||||
if (!file) {
|
||||
continue
|
||||
}
|
||||
const parsingTask = this.createKnowledgeTask({
|
||||
libraryId: library.id,
|
||||
sourceId: source.id,
|
||||
documentName: file.relativePath,
|
||||
kind: 'parsing'
|
||||
})
|
||||
try {
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
status: 'running',
|
||||
progress: 10,
|
||||
message: '正在读取文档'
|
||||
})
|
||||
const buffer = await this.readBoundedFile(file.absolutePath)
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
progress: 35,
|
||||
message: '正在解析文档内容'
|
||||
})
|
||||
const checksum = createHash('sha256').update(buffer).digest('hex')
|
||||
const previous = existing.find(
|
||||
(document) => document.externalId === file.relativePath
|
||||
)
|
||||
if (previous?.checksum === checksum) {
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
status: 'skipped',
|
||||
message: '文档内容未发生变化'
|
||||
})
|
||||
continue
|
||||
}
|
||||
const parsed = await parseDocument(
|
||||
basename(file.absolutePath),
|
||||
buffer
|
||||
)
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
progress: 75,
|
||||
message: '正在保存解析结果'
|
||||
})
|
||||
const document = this.database.upsertDocument(
|
||||
{
|
||||
knowledgeBaseId: library.id,
|
||||
@@ -630,10 +870,17 @@ export class KnowledgeService {
|
||||
location: chunk.locator
|
||||
}))
|
||||
)
|
||||
this.updateKnowledgeTask(parsingTask.id, {
|
||||
status: 'succeeded',
|
||||
documentId: document.id,
|
||||
documentName: document.title,
|
||||
message: '文档解析完成'
|
||||
})
|
||||
this.database.removeEvidenceForDocument(document.id)
|
||||
await this.indexDocumentEmbeddings(document)
|
||||
await this.extractGraph(library, document)
|
||||
} catch (error) {
|
||||
this.failKnowledgeTask(parsingTask.id, error)
|
||||
failures.push(
|
||||
`${file.relativePath}: ${
|
||||
error instanceof Error ? error.message : '解析失败'
|
||||
@@ -661,10 +908,26 @@ export class KnowledgeService {
|
||||
requestedProvider?: EmbeddingProvider
|
||||
): Promise<void> {
|
||||
const provider = requestedProvider ?? this.embeddingProvider
|
||||
const task = this.createKnowledgeTask({
|
||||
libraryId: document.knowledgeBaseId,
|
||||
sourceId: document.sourceId,
|
||||
documentId: document.id,
|
||||
documentName: document.title,
|
||||
kind: 'embedding'
|
||||
})
|
||||
if (!provider) {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'skipped',
|
||||
message: '未启用向量化'
|
||||
})
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'running',
|
||||
progress: 5,
|
||||
message: '正在准备文档分块'
|
||||
})
|
||||
const chunks = this.database.listChunks(document.id, 10_000)
|
||||
const embeddings: Array<{
|
||||
chunkId: string
|
||||
@@ -704,8 +967,21 @@ export class KnowledgeService {
|
||||
vector
|
||||
})
|
||||
}
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
progress:
|
||||
5 +
|
||||
((offset + batch.length) / Math.max(chunks.length, 1)) * 85,
|
||||
message: `正在向量化 ${Math.min(
|
||||
offset + batch.length,
|
||||
chunks.length
|
||||
)}/${chunks.length} 个分块`
|
||||
})
|
||||
}
|
||||
if (this.embeddingProvider !== provider) {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'skipped',
|
||||
message: '向量模型配置已变化'
|
||||
})
|
||||
return
|
||||
}
|
||||
this.database.replaceDocumentEmbeddings(
|
||||
@@ -714,11 +990,17 @@ export class KnowledgeService {
|
||||
provider.model,
|
||||
embeddings
|
||||
)
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'succeeded',
|
||||
message: `已向量化 ${chunks.length} 个分块`
|
||||
})
|
||||
} catch (error) {
|
||||
if (this.lifecycleController.signal.aborted) {
|
||||
this.failKnowledgeTask(task.id, new Error('向量化已取消'))
|
||||
return
|
||||
}
|
||||
const safeError = classifyEmbeddingError(error)
|
||||
this.failKnowledgeTask(task.id, safeError)
|
||||
try {
|
||||
this.database.recordEmbeddingIndexError(
|
||||
document.id,
|
||||
@@ -736,11 +1018,55 @@ export class KnowledgeService {
|
||||
library: KnowledgeBase,
|
||||
document: Document
|
||||
): Promise<void> {
|
||||
if (!library.graphEnabled || library.graphStrategy === 'ask') {
|
||||
const task = this.createKnowledgeTask({
|
||||
libraryId: library.id,
|
||||
sourceId: document.sourceId,
|
||||
documentId: document.id,
|
||||
documentName: document.title,
|
||||
kind: 'graph'
|
||||
})
|
||||
if (!library.graphEnabled) {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'skipped',
|
||||
message: '知识图谱未启用'
|
||||
})
|
||||
return
|
||||
}
|
||||
if (library.graphStrategy === 'ask') {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'skipped',
|
||||
message: '按需询问策略不自动抽取'
|
||||
})
|
||||
return
|
||||
}
|
||||
try {
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'running',
|
||||
progress: 10,
|
||||
message: '正在准备图谱抽取'
|
||||
})
|
||||
const result = await this.extractGraphResult(library, document)
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
progress: 85,
|
||||
message: '正在保存实体和关系'
|
||||
})
|
||||
this.storeExtractedGraph(library, document, result)
|
||||
this.updateKnowledgeTask(task.id, {
|
||||
status: 'succeeded',
|
||||
message: `已抽取 ${result.entities.length} 个实体、${result.relations.length} 条关系`
|
||||
})
|
||||
} catch (error) {
|
||||
this.failKnowledgeTask(task.id, error)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async extractGraphResult(
|
||||
library: KnowledgeBase,
|
||||
document: Document
|
||||
): Promise<GraphExtractionResult> {
|
||||
const chunks = this.database.listChunks(document.id)
|
||||
const result = await extractKnowledgeGraph(
|
||||
return extractKnowledgeGraph(
|
||||
chunks.map((chunk) => ({
|
||||
id: chunk.id,
|
||||
content: chunk.content
|
||||
@@ -750,6 +1076,13 @@ export class KnowledgeService {
|
||||
extractStructured: this.extractStructured
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
private storeExtractedGraph(
|
||||
library: KnowledgeBase,
|
||||
document: Document,
|
||||
result: GraphExtractionResult
|
||||
): void {
|
||||
const existingEntities = this.database.listEntities(library.id)
|
||||
const entityIds = new Map<string, string>()
|
||||
for (const entity of result.entities) {
|
||||
|
||||
@@ -65,7 +65,12 @@ describe('createModelGraphExtractor', () => {
|
||||
choices: [
|
||||
{
|
||||
message: {
|
||||
content: '```json\n{"relations":[]}\n```'
|
||||
content: [
|
||||
{
|
||||
type: 'text',
|
||||
text: '```json\n{"relations":[]}\n```'
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -136,6 +141,22 @@ describe('createModelGraphExtractor', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('accepts top-level output text from compatible Responses providers', async () => {
|
||||
const extract = createModelGraphExtractor(
|
||||
store({ modelProtocol: 'openai-responses' }),
|
||||
vi.fn(async () =>
|
||||
jsonResponse({
|
||||
output_text: '{"entities":[],"relations":[]}'
|
||||
})
|
||||
)
|
||||
)
|
||||
|
||||
await expect(extract('extract this')).resolves.toEqual({
|
||||
entities: [],
|
||||
relations: []
|
||||
})
|
||||
})
|
||||
|
||||
it('requires a key only for API-key authentication', async () => {
|
||||
const extract = createModelGraphExtractor(
|
||||
store({
|
||||
|
||||
@@ -95,12 +95,28 @@ function openAIChatText(payload: unknown): string {
|
||||
if (!Array.isArray(choices)) {
|
||||
return ''
|
||||
}
|
||||
const message = record(record(choices[0])?.message)
|
||||
return typeof message?.content === 'string' ? message.content : ''
|
||||
const choice = record(choices[0])
|
||||
const message = record(choice?.message)
|
||||
if (typeof message?.content === 'string') {
|
||||
return message.content
|
||||
}
|
||||
if (Array.isArray(message?.content)) {
|
||||
return message.content
|
||||
.flatMap((part) => {
|
||||
const value = record(part)
|
||||
return typeof value?.text === 'string' ? [value.text] : []
|
||||
})
|
||||
.join('')
|
||||
}
|
||||
return typeof choice?.text === 'string' ? choice.text : ''
|
||||
}
|
||||
|
||||
function openAIResponsesText(payload: unknown): string {
|
||||
const output = record(payload)?.output
|
||||
const response = record(payload)
|
||||
if (typeof response?.output_text === 'string') {
|
||||
return response.output_text
|
||||
}
|
||||
const output = response?.output
|
||||
if (!Array.isArray(output)) {
|
||||
return ''
|
||||
}
|
||||
@@ -111,7 +127,7 @@ function openAIResponsesText(payload: unknown): string {
|
||||
})
|
||||
.flatMap((part) => {
|
||||
const value = record(part)
|
||||
return value?.type === 'output_text' &&
|
||||
return (value?.type === 'output_text' || value?.type === 'text') &&
|
||||
typeof value.text === 'string'
|
||||
? [value.text]
|
||||
: []
|
||||
@@ -211,7 +227,9 @@ export function createModelGraphExtractor(
|
||||
? openAIResponsesText(payload)
|
||||
: openAIChatText(payload)
|
||||
if (!text) {
|
||||
throw new Error('模型未返回图谱内容')
|
||||
throw new Error(
|
||||
'模型未返回图谱内容，请重试或在知识库设置中切换到规则抽取'
|
||||
)
|
||||
}
|
||||
return extractJsonText(text)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user