fix: preserve knowledge indexing state
This commit is contained in:
@@ -86,7 +86,7 @@ describe('KnowledgeDatabase', () => {
|
|||||||
const inspection = new DatabaseSync(path)
|
const inspection = new DatabaseSync(path)
|
||||||
expect(
|
expect(
|
||||||
inspection.prepare('PRAGMA user_version').get()
|
inspection.prepare('PRAGMA user_version').get()
|
||||||
).toEqual({ user_version: 9 })
|
).toEqual({ user_version: 10 })
|
||||||
expect(
|
expect(
|
||||||
inspection
|
inspection
|
||||||
.prepare('SELECT version FROM schema_migrations ORDER BY version')
|
.prepare('SELECT version FROM schema_migrations ORDER BY version')
|
||||||
@@ -100,7 +100,8 @@ describe('KnowledgeDatabase', () => {
|
|||||||
{ version: 6 },
|
{ version: 6 },
|
||||||
{ version: 7 },
|
{ version: 7 },
|
||||||
{ version: 8 },
|
{ version: 8 },
|
||||||
{ version: 9 }
|
{ version: 9 },
|
||||||
|
{ version: 10 }
|
||||||
])
|
])
|
||||||
inspection.close()
|
inspection.close()
|
||||||
|
|
||||||
@@ -192,7 +193,7 @@ describe('KnowledgeDatabase', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
it('upgrades an existing v1 database through knowledge schema v9', async () => {
|
it('upgrades an existing v1 database through knowledge schema v10', async () => {
|
||||||
const { database, path } = await createDatabase()
|
const { database, path } = await createDatabase()
|
||||||
const knowledgeBase = database.createKnowledgeBase({
|
const knowledgeBase = database.createKnowledgeBase({
|
||||||
name: 'Version one data',
|
name: 'Version one data',
|
||||||
@@ -208,7 +209,8 @@ describe('KnowledgeDatabase', () => {
|
|||||||
DROP TABLE embedding_index_state;
|
DROP TABLE embedding_index_state;
|
||||||
DROP TABLE chunk_embeddings;
|
DROP TABLE chunk_embeddings;
|
||||||
DROP TABLE knowledge_tasks;
|
DROP TABLE knowledge_tasks;
|
||||||
DELETE FROM schema_migrations WHERE version IN (2, 3, 4, 5, 6, 7, 8, 9);
|
DELETE FROM schema_migrations
|
||||||
|
WHERE version IN (2, 3, 4, 5, 6, 7, 8, 9, 10);
|
||||||
PRAGMA user_version = 1;
|
PRAGMA user_version = 1;
|
||||||
`)
|
`)
|
||||||
downgrade.close()
|
downgrade.close()
|
||||||
@@ -218,7 +220,7 @@ describe('KnowledgeDatabase', () => {
|
|||||||
upgraded.initialize()
|
upgraded.initialize()
|
||||||
const inspection = new DatabaseSync(path)
|
const inspection = new DatabaseSync(path)
|
||||||
expect(inspection.prepare('PRAGMA user_version').get()).toEqual({
|
expect(inspection.prepare('PRAGMA user_version').get()).toEqual({
|
||||||
user_version: 9
|
user_version: 10
|
||||||
})
|
})
|
||||||
expect(
|
expect(
|
||||||
inspection
|
inspection
|
||||||
@@ -325,7 +327,7 @@ describe('KnowledgeDatabase', () => {
|
|||||||
WHERE new.enabled = 1 AND new.role <> 'parent';
|
WHERE new.enabled = 1 AND new.role <> 'parent';
|
||||||
END;
|
END;
|
||||||
INSERT INTO chunks_fts(chunks_fts) VALUES ('rebuild');
|
INSERT INTO chunks_fts(chunks_fts) VALUES ('rebuild');
|
||||||
DELETE FROM schema_migrations WHERE version = 9;
|
DELETE FROM schema_migrations WHERE version IN (9, 10);
|
||||||
PRAGMA user_version = 8;
|
PRAGMA user_version = 8;
|
||||||
`)
|
`)
|
||||||
downgrade
|
downgrade
|
||||||
@@ -478,6 +480,38 @@ describe('KnowledgeDatabase', () => {
|
|||||||
).toMatchObject({ status: 'interrupted', progress: 30 })
|
).toMatchObject({ status: 'interrupted', progress: 30 })
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('bounds task status text from runtime failures', async () => {
|
||||||
|
const { database } = await createDatabase()
|
||||||
|
const library = database.createKnowledgeBase({
|
||||||
|
name: 'Bounded task status',
|
||||||
|
storageMode: 'reference'
|
||||||
|
})
|
||||||
|
const task = database.createKnowledgeTask({
|
||||||
|
libraryId: library.id,
|
||||||
|
documentName: library.name,
|
||||||
|
scope: 'library',
|
||||||
|
kind: 'library-rebuild'
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(
|
||||||
|
database.updateKnowledgeTask(task.id, {
|
||||||
|
status: 'failed',
|
||||||
|
message: 'm'.repeat(2_000),
|
||||||
|
error: {
|
||||||
|
message: 'e'.repeat(2_000),
|
||||||
|
remedy: 'r'.repeat(2_000)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
).toMatchObject({
|
||||||
|
status: 'failed',
|
||||||
|
message: 'm'.repeat(1_000),
|
||||||
|
error: {
|
||||||
|
message: 'e'.repeat(1_000),
|
||||||
|
remedy: 'r'.repeat(1_000)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
it('lists every active task in addition to the terminal history limit', async () => {
|
it('lists every active task in addition to the terminal history limit', async () => {
|
||||||
const { database } = await createDatabase()
|
const { database } = await createDatabase()
|
||||||
const library = database.createKnowledgeBase({
|
const library = database.createKnowledgeBase({
|
||||||
@@ -779,6 +813,63 @@ describe('KnowledgeDatabase', () => {
|
|||||||
expect(database.deleteEntity(other.id)).toBe(true)
|
expect(database.deleteEntity(other.id)).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('finds graph identities beyond the ordinary list limit', async () => {
|
||||||
|
const { database } = await createDatabase()
|
||||||
|
const knowledgeBase = database.createKnowledgeBase({
|
||||||
|
name: 'Large graph identity',
|
||||||
|
storageMode: 'reference'
|
||||||
|
})
|
||||||
|
let expectedEntityId = ''
|
||||||
|
const targetEntityIds: string[] = []
|
||||||
|
for (let index = 0; index < 501; index += 1) {
|
||||||
|
const entity = database.createEntity({
|
||||||
|
knowledgeBaseId: knowledgeBase.id,
|
||||||
|
name: `Entity ${String(index).padStart(3, '0')}`,
|
||||||
|
type: 'CONCEPT',
|
||||||
|
locked: false
|
||||||
|
})
|
||||||
|
targetEntityIds.push(entity.id)
|
||||||
|
if (index === 500) {
|
||||||
|
expectedEntityId = entity.id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const source = database.createEntity({
|
||||||
|
knowledgeBaseId: knowledgeBase.id,
|
||||||
|
name: 'Relation Source',
|
||||||
|
type: 'CONCEPT',
|
||||||
|
locked: false
|
||||||
|
})
|
||||||
|
let expectedRelationId = ''
|
||||||
|
for (let index = 0; index < 501; index += 1) {
|
||||||
|
const relation = database.createRelation({
|
||||||
|
knowledgeBaseId: knowledgeBase.id,
|
||||||
|
sourceEntityId: source.id,
|
||||||
|
targetEntityId: targetEntityIds[index]!,
|
||||||
|
type: 'RELATED_TO',
|
||||||
|
locked: false
|
||||||
|
})
|
||||||
|
if (index === 500) {
|
||||||
|
expectedRelationId = relation.id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
expect(
|
||||||
|
database.findEntityByCanonicalName(
|
||||||
|
knowledgeBase.id,
|
||||||
|
'CONCEPT',
|
||||||
|
'Entity 500'
|
||||||
|
)?.id
|
||||||
|
).toBe(expectedEntityId)
|
||||||
|
expect(
|
||||||
|
database.findRelationByIdentity(
|
||||||
|
knowledgeBase.id,
|
||||||
|
source.id,
|
||||||
|
targetEntityIds[500]!,
|
||||||
|
'RELATED_TO'
|
||||||
|
)?.id
|
||||||
|
).toBe(expectedRelationId)
|
||||||
|
})
|
||||||
|
|
||||||
it('persists controlled ontology updates and marks graph rebuild state', async () => {
|
it('persists controlled ontology updates and marks graph rebuild state', async () => {
|
||||||
const { database } = await createDatabase()
|
const { database } = await createDatabase()
|
||||||
const library = database.createKnowledgeBase({
|
const library = database.createKnowledgeBase({
|
||||||
@@ -1229,6 +1320,12 @@ describe('KnowledgeDatabase', () => {
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
const concurrentReplacementId =
|
||||||
|
database.beginDocumentEmbeddingReplacement(
|
||||||
|
document.id,
|
||||||
|
'openai-compatible',
|
||||||
|
'embed-v1'
|
||||||
|
)
|
||||||
|
|
||||||
expect(
|
expect(
|
||||||
database.vectorSearch({
|
database.vectorSearch({
|
||||||
@@ -1244,6 +1341,9 @@ describe('KnowledgeDatabase', () => {
|
|||||||
'openai-compatible',
|
'openai-compatible',
|
||||||
'embed-v1'
|
'embed-v1'
|
||||||
)
|
)
|
||||||
|
database.discardDocumentEmbeddingReplacement(
|
||||||
|
concurrentReplacementId
|
||||||
|
)
|
||||||
expect(
|
expect(
|
||||||
database.vectorSearch({
|
database.vectorSearch({
|
||||||
knowledgeBaseId: knowledgeBase.id,
|
knowledgeBaseId: knowledgeBase.id,
|
||||||
@@ -1254,6 +1354,55 @@ describe('KnowledgeDatabase', () => {
|
|||||||
).toBe(item.id)
|
).toBe(item.id)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('rejects a staged embedding replacement after chunk content changes', async () => {
|
||||||
|
const { database } = await createDatabase()
|
||||||
|
const knowledgeBase = database.createKnowledgeBase({
|
||||||
|
name: 'Stale replacement',
|
||||||
|
storageMode: 'reference'
|
||||||
|
})
|
||||||
|
const seeded = seedDocument(
|
||||||
|
database,
|
||||||
|
knowledgeBase.id,
|
||||||
|
'stale-replacement'
|
||||||
|
)
|
||||||
|
const document =
|
||||||
|
database.getEmbeddingIndexDocument(seeded.documentId)!
|
||||||
|
const item = document.items[0]!
|
||||||
|
const replacementId =
|
||||||
|
database.beginDocumentEmbeddingReplacement(
|
||||||
|
document.id,
|
||||||
|
'openai-compatible',
|
||||||
|
'embed-v1'
|
||||||
|
)
|
||||||
|
database.appendDocumentEmbeddingBatch(
|
||||||
|
replacementId,
|
||||||
|
document.id,
|
||||||
|
'openai-compatible',
|
||||||
|
'embed-v1',
|
||||||
|
[{
|
||||||
|
chunkId: item.id,
|
||||||
|
contentChecksum: item.contentChecksum!,
|
||||||
|
vector: [1, 0]
|
||||||
|
}]
|
||||||
|
)
|
||||||
|
database.updateChunk({
|
||||||
|
knowledgeBaseId: knowledgeBase.id,
|
||||||
|
documentId: document.id,
|
||||||
|
chunkId: item.id,
|
||||||
|
content: 'newer content'
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(() =>
|
||||||
|
database.finishDocumentEmbeddingReplacement(
|
||||||
|
replacementId,
|
||||||
|
document.id,
|
||||||
|
'openai-compatible',
|
||||||
|
'embed-v1'
|
||||||
|
)
|
||||||
|
).toThrow('content changed')
|
||||||
|
database.discardDocumentEmbeddingReplacement(replacementId)
|
||||||
|
})
|
||||||
|
|
||||||
it('persists embedding index jobs independently by knowledge base', async () => {
|
it('persists embedding index jobs independently by knowledge base', async () => {
|
||||||
const created = await createDatabase()
|
const created = await createDatabase()
|
||||||
let database = created.database
|
let database = created.database
|
||||||
@@ -1442,6 +1591,20 @@ describe('KnowledgeDatabase', () => {
|
|||||||
0
|
0
|
||||||
).map((chunk) => chunk.id)
|
).map((chunk) => chunk.id)
|
||||||
).toEqual(['parent-chunk'])
|
).toEqual(['parent-chunk'])
|
||||||
|
const graphEntity = database.createEntity({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
name: 'Chunk evidence',
|
||||||
|
type: 'CONCEPT',
|
||||||
|
locked: false
|
||||||
|
})
|
||||||
|
const evidence = database.createEvidence({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
entityId: graphEntity.id,
|
||||||
|
documentId: document.id,
|
||||||
|
chunkId: 'child-chunk',
|
||||||
|
quote: 'recallable child text',
|
||||||
|
source: 'rules'
|
||||||
|
})
|
||||||
|
|
||||||
database.updateChunk({
|
database.updateChunk({
|
||||||
knowledgeBaseId: library.id,
|
knowledgeBaseId: library.id,
|
||||||
@@ -1452,6 +1615,12 @@ describe('KnowledgeDatabase', () => {
|
|||||||
expect(
|
expect(
|
||||||
database.search({ knowledgeBaseId: library.id, query: 'recallable' })
|
database.search({ knowledgeBaseId: library.id, query: 'recallable' })
|
||||||
).toEqual([])
|
).toEqual([])
|
||||||
|
expect(database.listEvidence(library.id)).toEqual([
|
||||||
|
expect.objectContaining({ id: evidence.id })
|
||||||
|
])
|
||||||
|
expect(
|
||||||
|
database.graphSearch(library.id, 'Chunk evidence')
|
||||||
|
).toEqual([])
|
||||||
database.updateChunk({
|
database.updateChunk({
|
||||||
knowledgeBaseId: library.id,
|
knowledgeBaseId: library.id,
|
||||||
documentId: document.id,
|
documentId: document.id,
|
||||||
@@ -1462,6 +1631,7 @@ describe('KnowledgeDatabase', () => {
|
|||||||
expect(
|
expect(
|
||||||
database.search({ knowledgeBaseId: library.id, query: '人工修正' })
|
database.search({ knowledgeBaseId: library.id, query: '人工修正' })
|
||||||
).toHaveLength(1)
|
).toHaveLength(1)
|
||||||
|
expect(database.listEvidence(library.id)).toEqual([])
|
||||||
expect(
|
expect(
|
||||||
database.listChunksPage({
|
database.listChunksPage({
|
||||||
knowledgeBaseId: library.id,
|
knowledgeBaseId: library.id,
|
||||||
|
|||||||
@@ -45,6 +45,7 @@ import {
|
|||||||
createCjkSearchText,
|
createCjkSearchText,
|
||||||
knowledgeRetrievalTerms
|
knowledgeRetrievalTerms
|
||||||
} from './retrieval-text'
|
} from './retrieval-text'
|
||||||
|
import { normalizeEntityAlias } from './graph-extractor'
|
||||||
import type {
|
import type {
|
||||||
Chunk,
|
Chunk,
|
||||||
ChunkEmbeddingInput,
|
ChunkEmbeddingInput,
|
||||||
@@ -79,7 +80,7 @@ import type {
|
|||||||
VectorSearchOptions
|
VectorSearchOptions
|
||||||
} from './types'
|
} from './types'
|
||||||
|
|
||||||
const DATABASE_VERSION = 9
|
const DATABASE_VERSION = 10
|
||||||
const MAX_ID_LENGTH = 128
|
const MAX_ID_LENGTH = 128
|
||||||
const MAX_NAME_LENGTH = 512
|
const MAX_NAME_LENGTH = 512
|
||||||
const MAX_LOCATION_LENGTH = 8192
|
const MAX_LOCATION_LENGTH = 8192
|
||||||
@@ -202,6 +203,17 @@ function optionalString(
|
|||||||
return requiredString(value, field, maximum, trim)
|
return requiredString(value, field, maximum, trim)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function truncatedOptionalString(
|
||||||
|
value: string | null | undefined,
|
||||||
|
maximum: number
|
||||||
|
): string | undefined {
|
||||||
|
if (value === undefined || value === null || value === '') {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
const normalized = String(value).trim()
|
||||||
|
return normalized ? normalized.slice(0, maximum) : undefined
|
||||||
|
}
|
||||||
|
|
||||||
function boundedInteger(
|
function boundedInteger(
|
||||||
value: number,
|
value: number,
|
||||||
field: string,
|
field: string,
|
||||||
@@ -1340,25 +1352,20 @@ export class KnowledgeDatabase {
|
|||||||
? undefined
|
? undefined
|
||||||
: update.message === undefined
|
: update.message === undefined
|
||||||
? current.message
|
? current.message
|
||||||
: optionalString(
|
: truncatedOptionalString(update.message, MAX_TASK_TEXT_LENGTH)
|
||||||
update.message,
|
|
||||||
'message',
|
|
||||||
MAX_TASK_TEXT_LENGTH
|
|
||||||
)
|
|
||||||
const error =
|
const error =
|
||||||
update.error === null
|
update.error === null
|
||||||
? undefined
|
? undefined
|
||||||
: update.error === undefined
|
: update.error === undefined
|
||||||
? current.error
|
? current.error
|
||||||
: {
|
: {
|
||||||
message: requiredString(
|
message:
|
||||||
update.error.message,
|
truncatedOptionalString(
|
||||||
'error.message',
|
update.error.message,
|
||||||
MAX_TASK_TEXT_LENGTH
|
MAX_TASK_TEXT_LENGTH
|
||||||
),
|
) ?? '任务失败',
|
||||||
remedy: optionalString(
|
remedy: truncatedOptionalString(
|
||||||
update.error.remedy,
|
update.error.remedy,
|
||||||
'error.remedy',
|
|
||||||
MAX_TASK_TEXT_LENGTH
|
MAX_TASK_TEXT_LENGTH
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2089,6 +2096,11 @@ export class KnowledgeDatabase {
|
|||||||
const database = this.requireDatabase()
|
const database = this.requireDatabase()
|
||||||
const now = new Date().toISOString()
|
const now = new Date().toISOString()
|
||||||
this.transaction(database, () => {
|
this.transaction(database, () => {
|
||||||
|
if (content !== current.chunk.content) {
|
||||||
|
database
|
||||||
|
.prepare('DELETE FROM graph_evidence WHERE chunk_id = ?')
|
||||||
|
.run(current.chunk.id)
|
||||||
|
}
|
||||||
database
|
database
|
||||||
.prepare(
|
.prepare(
|
||||||
`UPDATE chunks SET content = ?, enabled = ?, manually_edited = 1,
|
`UPDATE chunks SET content = ?, enabled = ?, manually_edited = 1,
|
||||||
@@ -2110,9 +2122,6 @@ export class KnowledgeDatabase {
|
|||||||
database
|
database
|
||||||
.prepare('DELETE FROM embedding_index_state WHERE document_id = ?')
|
.prepare('DELETE FROM embedding_index_state WHERE document_id = ?')
|
||||||
.run(current.document.id)
|
.run(current.document.id)
|
||||||
database
|
|
||||||
.prepare('DELETE FROM graph_evidence WHERE chunk_id = ?')
|
|
||||||
.run(current.chunk.id)
|
|
||||||
})
|
})
|
||||||
return this.getChunkForReference(
|
return this.getChunkForReference(
|
||||||
input.knowledgeBaseId,
|
input.knowledgeBaseId,
|
||||||
@@ -2335,12 +2344,12 @@ export class KnowledgeDatabase {
|
|||||||
'documentId',
|
'documentId',
|
||||||
MAX_ID_LENGTH
|
MAX_ID_LENGTH
|
||||||
)
|
)
|
||||||
const normalizedProvider = requiredString(
|
requiredString(
|
||||||
provider,
|
provider,
|
||||||
'provider',
|
'provider',
|
||||||
MAX_EMBEDDING_PROVIDER_LENGTH
|
MAX_EMBEDDING_PROVIDER_LENGTH
|
||||||
)
|
)
|
||||||
const normalizedModel = requiredString(
|
requiredString(
|
||||||
model,
|
model,
|
||||||
'model',
|
'model',
|
||||||
MAX_EMBEDDING_MODEL_LENGTH
|
MAX_EMBEDDING_MODEL_LENGTH
|
||||||
@@ -2353,16 +2362,6 @@ export class KnowledgeDatabase {
|
|||||||
) {
|
) {
|
||||||
throw new Error(`Document not found: ${normalizedDocumentId}`)
|
throw new Error(`Document not found: ${normalizedDocumentId}`)
|
||||||
}
|
}
|
||||||
database
|
|
||||||
.prepare(
|
|
||||||
`DELETE FROM embedding_rebuild_staging
|
|
||||||
WHERE document_id = ? AND provider = ? AND model = ?`
|
|
||||||
)
|
|
||||||
.run(
|
|
||||||
normalizedDocumentId,
|
|
||||||
normalizedProvider,
|
|
||||||
normalizedModel
|
|
||||||
)
|
|
||||||
return randomUUID()
|
return randomUUID()
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2544,6 +2543,40 @@ export class KnowledgeDatabase {
|
|||||||
) {
|
) {
|
||||||
throw new Error('Embeddings must cover every current document chunk')
|
throw new Error('Embeddings must cover every current document chunk')
|
||||||
}
|
}
|
||||||
|
const stagedRows = database
|
||||||
|
.prepare(
|
||||||
|
`SELECT
|
||||||
|
staged.content_checksum AS staged_checksum,
|
||||||
|
current.index_content AS current_content
|
||||||
|
FROM embedding_rebuild_staging staged
|
||||||
|
LEFT JOIN chunks current
|
||||||
|
ON current.id = staged.chunk_id
|
||||||
|
AND current.document_id = staged.document_id
|
||||||
|
AND current.enabled = 1
|
||||||
|
AND current.role <> 'parent'
|
||||||
|
WHERE staged.replacement_id = ?
|
||||||
|
AND staged.document_id = ?
|
||||||
|
AND staged.provider = ?
|
||||||
|
AND staged.model = ?`
|
||||||
|
)
|
||||||
|
.all(
|
||||||
|
normalizedReplacementId,
|
||||||
|
normalizedDocumentId,
|
||||||
|
normalizedProvider,
|
||||||
|
normalizedModel
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
stagedRows.some((row) => {
|
||||||
|
const currentContent = asOptionalString(row, 'current_content')
|
||||||
|
return (
|
||||||
|
currentContent === undefined ||
|
||||||
|
asString(row, 'staged_checksum') !==
|
||||||
|
contentChecksum(currentContent)
|
||||||
|
)
|
||||||
|
})
|
||||||
|
) {
|
||||||
|
throw new Error('Embedding content changed during replacement')
|
||||||
|
}
|
||||||
const indexHash = createHash('sha256')
|
const indexHash = createHash('sha256')
|
||||||
let dimensions: number | undefined
|
let dimensions: number | undefined
|
||||||
let firstChecksum = true
|
let firstChecksum = true
|
||||||
@@ -3270,6 +3303,45 @@ export class KnowledgeDatabase {
|
|||||||
.map(mapEntity)
|
.map(mapEntity)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
findEntityByCanonicalName(
|
||||||
|
knowledgeBaseId: string,
|
||||||
|
type: string,
|
||||||
|
name: string
|
||||||
|
): GraphEntity | undefined {
|
||||||
|
requiredString(knowledgeBaseId, 'knowledgeBaseId', MAX_ID_LENGTH)
|
||||||
|
const normalizedType = requiredString(type, 'type', MAX_NAME_LENGTH)
|
||||||
|
const normalizedName = normalizeEntityAlias(name)
|
||||||
|
if (!normalizedName) {
|
||||||
|
return undefined
|
||||||
|
}
|
||||||
|
return this.listEntitiesForIdentity(knowledgeBaseId)
|
||||||
|
.find(
|
||||||
|
(candidate) =>
|
||||||
|
candidate.type === normalizedType &&
|
||||||
|
(
|
||||||
|
normalizeEntityAlias(candidate.name) === normalizedName ||
|
||||||
|
candidate.aliases.some(
|
||||||
|
(alias) => normalizeEntityAlias(alias) === normalizedName
|
||||||
|
)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
listEntitiesForIdentity(knowledgeBaseId: string): GraphEntity[] {
|
||||||
|
const normalizedId = requiredString(
|
||||||
|
knowledgeBaseId,
|
||||||
|
'knowledgeBaseId',
|
||||||
|
MAX_ID_LENGTH
|
||||||
|
)
|
||||||
|
return this.requireDatabase()
|
||||||
|
.prepare(
|
||||||
|
`SELECT * FROM graph_entities WHERE knowledge_base_id = ?
|
||||||
|
ORDER BY name COLLATE NOCASE ASC, id ASC`
|
||||||
|
)
|
||||||
|
.all(normalizedId)
|
||||||
|
.map(mapEntity)
|
||||||
|
}
|
||||||
|
|
||||||
updateEntity(id: string, input: UpdateGraphEntityInput): GraphEntity {
|
updateEntity(id: string, input: UpdateGraphEntityInput): GraphEntity {
|
||||||
const current = this.requiredEntity(id)
|
const current = this.requiredEntity(id)
|
||||||
const ontology =
|
const ontology =
|
||||||
@@ -3430,6 +3502,32 @@ export class KnowledgeDatabase {
|
|||||||
.map(mapRelation)
|
.map(mapRelation)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
findRelationByIdentity(
|
||||||
|
knowledgeBaseId: string,
|
||||||
|
sourceEntityId: string,
|
||||||
|
targetEntityId: string,
|
||||||
|
type: string
|
||||||
|
): GraphRelation | undefined {
|
||||||
|
const row = this.requireDatabase()
|
||||||
|
.prepare(
|
||||||
|
`SELECT * FROM graph_relations
|
||||||
|
WHERE knowledge_base_id = ? AND source_entity_id = ?
|
||||||
|
AND target_entity_id = ? AND type = ?
|
||||||
|
ORDER BY created_at ASC, id ASC LIMIT 1`
|
||||||
|
)
|
||||||
|
.get(
|
||||||
|
requiredString(
|
||||||
|
knowledgeBaseId,
|
||||||
|
'knowledgeBaseId',
|
||||||
|
MAX_ID_LENGTH
|
||||||
|
),
|
||||||
|
requiredString(sourceEntityId, 'sourceEntityId', MAX_ID_LENGTH),
|
||||||
|
requiredString(targetEntityId, 'targetEntityId', MAX_ID_LENGTH),
|
||||||
|
requiredString(type, 'type', MAX_NAME_LENGTH)
|
||||||
|
)
|
||||||
|
return row ? mapRelation(row) : undefined
|
||||||
|
}
|
||||||
|
|
||||||
updateRelation(
|
updateRelation(
|
||||||
id: string,
|
id: string,
|
||||||
input: UpdateGraphRelationInput
|
input: UpdateGraphRelationInput
|
||||||
@@ -3626,6 +3724,42 @@ export class KnowledgeDatabase {
|
|||||||
.map(mapEvidence)
|
.map(mapEvidence)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
listGraphSnapshot(knowledgeBaseId: string): {
|
||||||
|
entities: GraphEntity[]
|
||||||
|
relations: GraphRelation[]
|
||||||
|
evidence: Evidence[]
|
||||||
|
} {
|
||||||
|
const normalizedId = requiredString(
|
||||||
|
knowledgeBaseId,
|
||||||
|
'knowledgeBaseId',
|
||||||
|
MAX_ID_LENGTH
|
||||||
|
)
|
||||||
|
const database = this.requireDatabase()
|
||||||
|
return {
|
||||||
|
entities: database
|
||||||
|
.prepare(
|
||||||
|
`SELECT * FROM graph_entities WHERE knowledge_base_id = ?
|
||||||
|
ORDER BY name COLLATE NOCASE ASC, id ASC`
|
||||||
|
)
|
||||||
|
.all(normalizedId)
|
||||||
|
.map(mapEntity),
|
||||||
|
relations: database
|
||||||
|
.prepare(
|
||||||
|
`SELECT * FROM graph_relations WHERE knowledge_base_id = ?
|
||||||
|
ORDER BY created_at ASC, id ASC`
|
||||||
|
)
|
||||||
|
.all(normalizedId)
|
||||||
|
.map(mapRelation),
|
||||||
|
evidence: database
|
||||||
|
.prepare(
|
||||||
|
`SELECT * FROM graph_evidence WHERE knowledge_base_id = ?
|
||||||
|
ORDER BY created_at ASC, id ASC`
|
||||||
|
)
|
||||||
|
.all(normalizedId)
|
||||||
|
.map(mapEvidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
updateEvidence(id: string, input: UpdateEvidenceInput): Evidence {
|
updateEvidence(id: string, input: UpdateEvidenceInput): Evidence {
|
||||||
const current = this.requiredEvidence(id)
|
const current = this.requiredEvidence(id)
|
||||||
const next = {
|
const next = {
|
||||||
@@ -3961,6 +4095,8 @@ export class KnowledgeDatabase {
|
|||||||
AND ev.knowledge_base_id = ge.knowledge_base_id
|
AND ev.knowledge_base_id = ge.knowledge_base_id
|
||||||
AND ec.knowledge_base_id = ge.knowledge_base_id
|
AND ec.knowledge_base_id = ge.knowledge_base_id
|
||||||
AND ec.document_id = ev.document_id
|
AND ec.document_id = ev.document_id
|
||||||
|
AND ec.enabled = 1
|
||||||
|
AND ec.role <> 'parent'
|
||||||
)
|
)
|
||||||
ORDER BY ge.name COLLATE NOCASE ASC, ge.id ASC
|
ORDER BY ge.name COLLATE NOCASE ASC, ge.id ASC
|
||||||
LIMIT 24
|
LIMIT 24
|
||||||
@@ -3988,6 +4124,8 @@ export class KnowledgeDatabase {
|
|||||||
AND rev.knowledge_base_id = gr.knowledge_base_id
|
AND rev.knowledge_base_id = gr.knowledge_base_id
|
||||||
AND rc.knowledge_base_id = gr.knowledge_base_id
|
AND rc.knowledge_base_id = gr.knowledge_base_id
|
||||||
AND rc.document_id = rev.document_id
|
AND rc.document_id = rev.document_id
|
||||||
|
AND rc.enabled = 1
|
||||||
|
AND rc.role <> 'parent'
|
||||||
)
|
)
|
||||||
AND EXISTS (
|
AND EXISTS (
|
||||||
SELECT 1 FROM graph_evidence nev
|
SELECT 1 FROM graph_evidence nev
|
||||||
@@ -4000,6 +4138,8 @@ export class KnowledgeDatabase {
|
|||||||
AND nev.knowledge_base_id = gr.knowledge_base_id
|
AND nev.knowledge_base_id = gr.knowledge_base_id
|
||||||
AND nc.knowledge_base_id = gr.knowledge_base_id
|
AND nc.knowledge_base_id = gr.knowledge_base_id
|
||||||
AND nc.document_id = nev.document_id
|
AND nc.document_id = nev.document_id
|
||||||
|
AND nc.enabled = 1
|
||||||
|
AND nc.role <> 'parent'
|
||||||
)
|
)
|
||||||
),
|
),
|
||||||
reached(id, depth) AS (
|
reached(id, depth) AS (
|
||||||
@@ -4042,6 +4182,7 @@ export class KnowledgeDatabase {
|
|||||||
JOIN knowledge_sources s ON s.id = d.source_id
|
JOIN knowledge_sources s ON s.id = d.source_id
|
||||||
WHERE c.knowledge_base_id = ? AND d.knowledge_base_id = ?
|
WHERE c.knowledge_base_id = ? AND d.knowledge_base_id = ?
|
||||||
AND s.knowledge_base_id = ?
|
AND s.knowledge_base_id = ?
|
||||||
|
AND c.enabled = 1 AND c.role <> 'parent'
|
||||||
GROUP BY c.id
|
GROUP BY c.id
|
||||||
ORDER BY MIN(backed_evidence.graph_depth) ASC, c.id ASC
|
ORDER BY MIN(backed_evidence.graph_depth) ASC, c.id ASC
|
||||||
LIMIT ?`
|
LIMIT ?`
|
||||||
@@ -4175,6 +4316,14 @@ export class KnowledgeDatabase {
|
|||||||
)
|
)
|
||||||
.run(9, new Date().toISOString())
|
.run(9, new Date().toISOString())
|
||||||
}
|
}
|
||||||
|
if (currentVersion < 10) {
|
||||||
|
this.migrateToVersion10(database)
|
||||||
|
database
|
||||||
|
.prepare(
|
||||||
|
'INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)'
|
||||||
|
)
|
||||||
|
.run(10, new Date().toISOString())
|
||||||
|
}
|
||||||
database.exec(`PRAGMA user_version = ${DATABASE_VERSION}`)
|
database.exec(`PRAGMA user_version = ${DATABASE_VERSION}`)
|
||||||
database.exec('COMMIT')
|
database.exec('COMMIT')
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -4783,6 +4932,21 @@ export class KnowledgeDatabase {
|
|||||||
`)
|
`)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private migrateToVersion10(database: DatabaseSync): void {
|
||||||
|
database.exec(`
|
||||||
|
CREATE INDEX IF NOT EXISTS graph_evidence_chunk_idx
|
||||||
|
ON graph_evidence(knowledge_base_id, chunk_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS graph_evidence_entity_idx
|
||||||
|
ON graph_evidence(knowledge_base_id, entity_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS graph_evidence_relation_idx
|
||||||
|
ON graph_evidence(knowledge_base_id, relation_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS graph_relations_source_idx
|
||||||
|
ON graph_relations(knowledge_base_id, source_entity_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS graph_relations_target_idx
|
||||||
|
ON graph_relations(knowledge_base_id, target_entity_id);
|
||||||
|
`)
|
||||||
|
}
|
||||||
|
|
||||||
private normalizeChunks(chunks: ReplaceChunkInput[]): Array<{
|
private normalizeChunks(chunks: ReplaceChunkInput[]): Array<{
|
||||||
id: string
|
id: string
|
||||||
ordinal: number
|
ordinal: number
|
||||||
|
|||||||
@@ -433,6 +433,37 @@ describe('KnowledgeService', () => {
|
|||||||
expect(service.database.getEntity(manual.id)).toBeDefined()
|
expect(service.database.getEntity(manual.id)).toBeDefined()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('prunes generated graph records after chunk evidence is removed', async () => {
|
||||||
|
const { directory, service } = await createService()
|
||||||
|
const sourcePath = join(directory, 'chunk-graph.md')
|
||||||
|
await writeFile(sourcePath, '# Disposable Entity', 'utf8')
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Chunk graph cleanup',
|
||||||
|
storageMode: 'reference',
|
||||||
|
graphEnabled: true,
|
||||||
|
graphStrategy: 'rules'
|
||||||
|
})
|
||||||
|
await service.importPaths(library.id, [sourcePath])
|
||||||
|
const snapshot = service.snapshot(library.id)
|
||||||
|
const document = snapshot.documents[0]!
|
||||||
|
const generatedEntity = snapshot.entities.find(
|
||||||
|
(entity) => entity.name === 'Disposable Entity'
|
||||||
|
)!
|
||||||
|
const evidence = snapshot.evidence.find(
|
||||||
|
(item) => item.entityId === generatedEntity.id
|
||||||
|
)!
|
||||||
|
|
||||||
|
await service.updateChunk({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
documentId: document.id,
|
||||||
|
chunkId: evidence.chunkId!,
|
||||||
|
content: 'Replacement text without graph evidence'
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(service.database.getEntity(generatedEntity.id)).toBeUndefined()
|
||||||
|
expect(service.snapshot(library.id).evidence).toEqual([])
|
||||||
|
})
|
||||||
|
|
||||||
it('fails hybrid reextraction when model extraction fails', async () => {
|
it('fails hybrid reextraction when model extraction fails', async () => {
|
||||||
const extractStructured = vi.fn(async () => {
|
const extractStructured = vi.fn(async () => {
|
||||||
throw new Error('模型未返回图谱内容')
|
throw new Error('模型未返回图谱内容')
|
||||||
@@ -633,6 +664,44 @@ describe('KnowledgeService', () => {
|
|||||||
expect(results[0]?.retrieval.channels).toContain('fts')
|
expect(results[0]?.retrieval.channels).toContain('fts')
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('bounds long indexing failures so task persistence does not mask them', async () => {
|
||||||
|
const directory = await mkdtemp(
|
||||||
|
join(tmpdir(), 'goodbuddy-knowledge-service-')
|
||||||
|
)
|
||||||
|
temporaryDirectories.push(directory)
|
||||||
|
const service = new KnowledgeService({
|
||||||
|
databasePath: join(directory, 'knowledge.sqlite'),
|
||||||
|
managedRoot: join(directory, 'managed'),
|
||||||
|
parseDocument: async () => {
|
||||||
|
throw new Error('x'.repeat(2_000))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
await service.initialize()
|
||||||
|
services.push(service)
|
||||||
|
const sourcePath = join(directory, 'failure.txt')
|
||||||
|
await writeFile(sourcePath, 'failure source', 'utf8')
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Bounded task errors',
|
||||||
|
storageMode: 'reference',
|
||||||
|
graphEnabled: false
|
||||||
|
})
|
||||||
|
|
||||||
|
await expect(
|
||||||
|
service.importPaths(library.id, [sourcePath])
|
||||||
|
).rejects.toThrow()
|
||||||
|
const failedTasks = service
|
||||||
|
.snapshot(library.id)
|
||||||
|
.tasks.filter((task) => task.status === 'failed')
|
||||||
|
expect(failedTasks.length).toBeGreaterThan(0)
|
||||||
|
expect(
|
||||||
|
failedTasks.every(
|
||||||
|
(task) =>
|
||||||
|
(task.message?.length ?? 0) <= 1_000 &&
|
||||||
|
(task.error?.message.length ?? 0) <= 1_000
|
||||||
|
)
|
||||||
|
).toBe(true)
|
||||||
|
})
|
||||||
|
|
||||||
it('defers existing-document rebuilds when an embedding provider is enabled', async () => {
|
it('defers existing-document rebuilds when an embedding provider is enabled', async () => {
|
||||||
const { directory, service } = await createService()
|
const { directory, service } = await createService()
|
||||||
const sourcePath = join(directory, 'existing.txt')
|
const sourcePath = join(directory, 'existing.txt')
|
||||||
@@ -1202,6 +1271,149 @@ describe('KnowledgeService', () => {
|
|||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('marks an existing URL source failed when refresh fails before parsing', async () => {
|
||||||
|
const importer = {
|
||||||
|
import: vi.fn(async () => {
|
||||||
|
throw new Error('URL refresh unavailable')
|
||||||
|
})
|
||||||
|
} as unknown as UrlImporter
|
||||||
|
const { service } = await createService(importer)
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Failed URL refresh',
|
||||||
|
storageMode: 'managed',
|
||||||
|
graphEnabled: false
|
||||||
|
})
|
||||||
|
const source = service.database.upsertSource({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
type: 'url',
|
||||||
|
location: 'https://example.com/failure',
|
||||||
|
displayName: 'Existing URL',
|
||||||
|
status: 'ready'
|
||||||
|
})
|
||||||
|
|
||||||
|
await expect(service.syncSource(source.id)).rejects.toThrow(
|
||||||
|
'URL refresh unavailable'
|
||||||
|
)
|
||||||
|
expect(service.database.getSource(source.id)).toMatchObject({
|
||||||
|
status: 'error',
|
||||||
|
lastError: 'URL refresh unavailable'
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('preserves refreshed URL metadata when later indexing fails', async () => {
|
||||||
|
const importer = {
|
||||||
|
import: vi.fn(async () => ({
|
||||||
|
url: 'https://example.com/redirected',
|
||||||
|
title: 'Redirected title',
|
||||||
|
contentType: 'text/html',
|
||||||
|
etag: 'fresh-etag',
|
||||||
|
lastModified: undefined,
|
||||||
|
discoveredUrls: [],
|
||||||
|
document: {
|
||||||
|
title: 'Redirected title',
|
||||||
|
sourceFormat: '.html',
|
||||||
|
content: 'refreshed content',
|
||||||
|
sections: [{
|
||||||
|
locator: '网页正文',
|
||||||
|
content: 'refreshed content'
|
||||||
|
}],
|
||||||
|
warnings: []
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
} as unknown as UrlImporter
|
||||||
|
const { service } = await createService(importer)
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Failed refreshed URL',
|
||||||
|
storageMode: 'managed',
|
||||||
|
graphEnabled: false
|
||||||
|
})
|
||||||
|
const source = service.database.upsertSource({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
type: 'url',
|
||||||
|
location: 'https://example.com/original',
|
||||||
|
displayName: 'Original title',
|
||||||
|
status: 'ready'
|
||||||
|
})
|
||||||
|
vi.spyOn(service.database, 'upsertDocument').mockImplementationOnce(
|
||||||
|
() => {
|
||||||
|
throw new Error('synthetic indexing failure')
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
await expect(service.syncSource(source.id)).rejects.toThrow(
|
||||||
|
'synthetic indexing failure'
|
||||||
|
)
|
||||||
|
expect(service.database.getSource(source.id)).toMatchObject({
|
||||||
|
location: 'https://example.com/redirected',
|
||||||
|
displayName: 'Redirected title',
|
||||||
|
status: 'error',
|
||||||
|
metadata: expect.objectContaining({ etag: 'fresh-etag' })
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
|
it('serializes background embedding reindexes and awaits the rerun on disposal', async () => {
|
||||||
|
const resolvers: Array<() => void> = []
|
||||||
|
let activeEmbeddings = 0
|
||||||
|
let maximumActiveEmbeddings = 0
|
||||||
|
const provider: EmbeddingProvider = {
|
||||||
|
provider: 'serial-background-provider',
|
||||||
|
model: 'serial-background-model',
|
||||||
|
embed: vi.fn(async () => {
|
||||||
|
activeEmbeddings += 1
|
||||||
|
maximumActiveEmbeddings = Math.max(
|
||||||
|
maximumActiveEmbeddings,
|
||||||
|
activeEmbeddings
|
||||||
|
)
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
resolvers.push(resolve)
|
||||||
|
})
|
||||||
|
activeEmbeddings -= 1
|
||||||
|
return [[1, 0]]
|
||||||
|
})
|
||||||
|
}
|
||||||
|
const { directory, service } = await createService()
|
||||||
|
const sourcePath = join(directory, 'serial-background.txt')
|
||||||
|
await writeFile(sourcePath, 'initial content', 'utf8')
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Serialized background embeddings',
|
||||||
|
storageMode: 'reference',
|
||||||
|
graphEnabled: false
|
||||||
|
})
|
||||||
|
await service.importPaths(library.id, [sourcePath])
|
||||||
|
await service.setEmbeddingProvider(provider)
|
||||||
|
const document = service.snapshot(library.id).documents[0]!
|
||||||
|
const chunk = service.database.listChunks(document.id, 1)[0]!
|
||||||
|
|
||||||
|
await service.updateChunk({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
documentId: document.id,
|
||||||
|
chunkId: chunk.id,
|
||||||
|
content: 'first edit'
|
||||||
|
})
|
||||||
|
await vi.waitFor(() => expect(resolvers).toHaveLength(1))
|
||||||
|
await service.updateChunk({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
documentId: document.id,
|
||||||
|
chunkId: chunk.id,
|
||||||
|
content: 'second edit'
|
||||||
|
})
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 300))
|
||||||
|
expect(maximumActiveEmbeddings).toBe(1)
|
||||||
|
|
||||||
|
resolvers.shift()?.()
|
||||||
|
await vi.waitFor(() => expect(resolvers).toHaveLength(1))
|
||||||
|
const disposing = service.dispose()
|
||||||
|
let disposed = false
|
||||||
|
void disposing.then(() => {
|
||||||
|
disposed = true
|
||||||
|
})
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||||
|
expect(disposed).toBe(false)
|
||||||
|
resolvers.shift()?.()
|
||||||
|
await disposing
|
||||||
|
expect(maximumActiveEmbeddings).toBe(1)
|
||||||
|
})
|
||||||
|
|
||||||
it('cancels and awaits active graph work before deleting a library', async () => {
|
it('cancels and awaits active graph work before deleting a library', async () => {
|
||||||
let extractionSignal: AbortSignal | undefined
|
let extractionSignal: AbortSignal | undefined
|
||||||
const extractStructured: ExtractStructured = (_prompt, signal) => {
|
const extractStructured: ExtractStructured = (_prompt, signal) => {
|
||||||
@@ -1280,6 +1492,54 @@ describe('KnowledgeService', () => {
|
|||||||
).toBe(true)
|
).toBe(true)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it('deduplicates a source retry against an active manual sync', async () => {
|
||||||
|
let releaseImport: (() => void) | undefined
|
||||||
|
const importer = {
|
||||||
|
import: vi.fn(
|
||||||
|
async () => {
|
||||||
|
await new Promise<void>((resolve) => {
|
||||||
|
releaseImport = resolve
|
||||||
|
})
|
||||||
|
throw new Error('synthetic sync failure')
|
||||||
|
}
|
||||||
|
)
|
||||||
|
} as unknown as UrlImporter
|
||||||
|
const { service } = await createService(importer)
|
||||||
|
const library = service.createLibrary({
|
||||||
|
name: 'Dedupe source retry',
|
||||||
|
storageMode: 'managed',
|
||||||
|
graphEnabled: false
|
||||||
|
})
|
||||||
|
const source = service.database.upsertSource({
|
||||||
|
knowledgeBaseId: library.id,
|
||||||
|
type: 'url',
|
||||||
|
location: 'https://example.com/dedupe',
|
||||||
|
displayName: 'Dedupe URL',
|
||||||
|
status: 'ready'
|
||||||
|
})
|
||||||
|
const failed = service.database.createKnowledgeTask({
|
||||||
|
libraryId: library.id,
|
||||||
|
sourceId: source.id,
|
||||||
|
documentName: source.displayName,
|
||||||
|
scope: 'source',
|
||||||
|
kind: 'source-sync',
|
||||||
|
status: 'failed',
|
||||||
|
error: { message: 'retry me' }
|
||||||
|
})
|
||||||
|
|
||||||
|
const syncing = service.syncSource(source.id)
|
||||||
|
void syncing.catch(() => undefined)
|
||||||
|
await vi.waitFor(() => expect(importer.import).toHaveBeenCalledOnce())
|
||||||
|
const retrying = service.retryTask(failed.id)
|
||||||
|
void retrying.catch(() => undefined)
|
||||||
|
await new Promise((resolve) => setTimeout(resolve, 0))
|
||||||
|
expect(importer.import).toHaveBeenCalledOnce()
|
||||||
|
|
||||||
|
releaseImport?.()
|
||||||
|
await expect(syncing).rejects.toThrow('synthetic sync failure')
|
||||||
|
await expect(retrying).rejects.toThrow('synthetic sync failure')
|
||||||
|
})
|
||||||
|
|
||||||
it('reconciles embedding task status during ordinary snapshots', async () => {
|
it('reconciles embedding task status during ordinary snapshots', async () => {
|
||||||
let resolveEmbedding: ((value: number[][]) => void) | undefined
|
let resolveEmbedding: ((value: number[][]) => void) | undefined
|
||||||
const provider: EmbeddingProvider = {
|
const provider: EmbeddingProvider = {
|
||||||
|
|||||||
@@ -2,7 +2,6 @@ import {
|
|||||||
cp,
|
cp,
|
||||||
lstat,
|
lstat,
|
||||||
mkdir,
|
mkdir,
|
||||||
open,
|
|
||||||
readdir,
|
readdir,
|
||||||
realpath,
|
realpath,
|
||||||
rm,
|
rm,
|
||||||
@@ -91,7 +90,10 @@ import type {
|
|||||||
} from './types'
|
} from './types'
|
||||||
import { UrlImporter } from './url-importer'
|
import { UrlImporter } from './url-importer'
|
||||||
import { mimeTypeFromFileName } from '../file-media-type'
|
import { mimeTypeFromFileName } from '../file-media-type'
|
||||||
import { isPathInside } from '../workspace-file-access'
|
import {
|
||||||
|
isPathInside,
|
||||||
|
readBoundedFile
|
||||||
|
} from '../workspace-file-access'
|
||||||
|
|
||||||
type ScannedFile = {
|
type ScannedFile = {
|
||||||
absolutePath: string
|
absolutePath: string
|
||||||
@@ -182,6 +184,11 @@ export class KnowledgeService {
|
|||||||
string,
|
string,
|
||||||
Promise<Document>
|
Promise<Document>
|
||||||
>()
|
>()
|
||||||
|
private readonly backgroundEmbeddingReindexes = new Map<
|
||||||
|
string,
|
||||||
|
Promise<void>
|
||||||
|
>()
|
||||||
|
private readonly pendingEmbeddingReindexes = new Set<string>()
|
||||||
private readonly taskControllers = new Map<string, AbortController>()
|
private readonly taskControllers = new Map<string, AbortController>()
|
||||||
private readonly taskOperations = new Map<string, Promise<unknown>>()
|
private readonly taskOperations = new Map<string, Promise<unknown>>()
|
||||||
private readonly sourceSyncTaskIds = new Map<string, string>()
|
private readonly sourceSyncTaskIds = new Map<string, string>()
|
||||||
@@ -253,6 +260,7 @@ export class KnowledgeService {
|
|||||||
clearTimeout(timer)
|
clearTimeout(timer)
|
||||||
}
|
}
|
||||||
this.embeddingEditTimers.clear()
|
this.embeddingEditTimers.clear()
|
||||||
|
this.pendingEmbeddingReindexes.clear()
|
||||||
for (const watcher of this.watchers.values()) {
|
for (const watcher of this.watchers.values()) {
|
||||||
watcher.close()
|
watcher.close()
|
||||||
}
|
}
|
||||||
@@ -266,6 +274,7 @@ export class KnowledgeService {
|
|||||||
await Promise.allSettled([
|
await Promise.allSettled([
|
||||||
...this.activeSyncs.values(),
|
...this.activeSyncs.values(),
|
||||||
...this.activeDocumentRebuilds.values(),
|
...this.activeDocumentRebuilds.values(),
|
||||||
|
...this.backgroundEmbeddingReindexes.values(),
|
||||||
...this.taskOperations.values(),
|
...this.taskOperations.values(),
|
||||||
...embeddingCompletions
|
...embeddingCompletions
|
||||||
])
|
])
|
||||||
@@ -451,6 +460,11 @@ export class KnowledgeService {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private static taskErrorMessage(error: unknown): string {
|
||||||
|
const message = error instanceof Error ? error.message : '任务失败'
|
||||||
|
return message.slice(0, 1_000) || '任务失败'
|
||||||
|
}
|
||||||
|
|
||||||
private failKnowledgeTask(taskId: string, error: unknown): void {
|
private failKnowledgeTask(taskId: string, error: unknown): void {
|
||||||
const current = this.database.getKnowledgeTask(taskId)
|
const current = this.database.getKnowledgeTask(taskId)
|
||||||
if (
|
if (
|
||||||
@@ -459,7 +473,7 @@ export class KnowledgeService {
|
|||||||
) {
|
) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
const message = error instanceof Error ? error.message : '任务失败'
|
const message = KnowledgeService.taskErrorMessage(error)
|
||||||
this.database.updateKnowledgeTask(taskId, {
|
this.database.updateKnowledgeTask(taskId, {
|
||||||
status: 'failed',
|
status: 'failed',
|
||||||
message,
|
message,
|
||||||
@@ -712,13 +726,14 @@ export class KnowledgeService {
|
|||||||
: undefined
|
: undefined
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
const graph = this.database.listGraphSnapshot(libraryId)
|
||||||
return {
|
return {
|
||||||
libraries,
|
libraries,
|
||||||
sources,
|
sources,
|
||||||
documents,
|
documents,
|
||||||
entities: this.database.listEntities(libraryId),
|
entities: graph.entities,
|
||||||
relations: this.database.listRelations(libraryId),
|
relations: graph.relations,
|
||||||
evidence: this.database.listEvidence(libraryId),
|
evidence: graph.evidence,
|
||||||
tasks: this.database.listKnowledgeTasks(libraryId)
|
tasks: this.database.listKnowledgeTasks(libraryId)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1223,6 +1238,15 @@ export class KnowledgeService {
|
|||||||
) {
|
) {
|
||||||
this.scheduleEmbeddingReindex(input.documentId)
|
this.scheduleEmbeddingReindex(input.documentId)
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
current &&
|
||||||
|
input.content !== undefined &&
|
||||||
|
input.content !== current.content
|
||||||
|
) {
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(
|
||||||
|
input.knowledgeBaseId
|
||||||
|
)
|
||||||
|
}
|
||||||
return chunk
|
return chunk
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1231,6 +1255,9 @@ export class KnowledgeService {
|
|||||||
const deleted = this.database.deleteChunk(input)
|
const deleted = this.database.deleteChunk(input)
|
||||||
if (deleted) {
|
if (deleted) {
|
||||||
this.scheduleEmbeddingReindex(input.documentId)
|
this.scheduleEmbeddingReindex(input.documentId)
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(
|
||||||
|
input.knowledgeBaseId
|
||||||
|
)
|
||||||
}
|
}
|
||||||
return deleted
|
return deleted
|
||||||
}
|
}
|
||||||
@@ -1618,6 +1645,7 @@ export class KnowledgeService {
|
|||||||
effectiveLibrary
|
effectiveLibrary
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||||
this.updateKnowledgeTask(parsingTask.id, {
|
this.updateKnowledgeTask(parsingTask.id, {
|
||||||
documentId: document.id,
|
documentId: document.id,
|
||||||
documentName: document.title,
|
documentName: document.title,
|
||||||
@@ -1702,11 +1730,21 @@ export class KnowledgeService {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async syncSource(sourceId: string): Promise<void> {
|
async syncSource(sourceId: string): Promise<void> {
|
||||||
|
return this.syncSourceOperation(sourceId)
|
||||||
|
}
|
||||||
|
|
||||||
|
private async syncSourceOperation(
|
||||||
|
sourceId: string,
|
||||||
|
retryOfTaskId?: string
|
||||||
|
): Promise<void> {
|
||||||
const existing = this.activeSyncs.get(sourceId)
|
const existing = this.activeSyncs.get(sourceId)
|
||||||
if (existing) {
|
if (existing) {
|
||||||
return existing
|
return existing
|
||||||
}
|
}
|
||||||
const operation = this.performSyncSource(sourceId).finally(() => {
|
const operation = this.performSyncSource(
|
||||||
|
sourceId,
|
||||||
|
retryOfTaskId
|
||||||
|
).finally(() => {
|
||||||
if (this.activeSyncs.get(sourceId) === operation) {
|
if (this.activeSyncs.get(sourceId) === operation) {
|
||||||
this.activeSyncs.delete(sourceId)
|
this.activeSyncs.delete(sourceId)
|
||||||
}
|
}
|
||||||
@@ -1836,7 +1874,12 @@ export class KnowledgeService {
|
|||||||
progress: 5,
|
progress: 5,
|
||||||
message: '正在读取文档'
|
message: '正在读取文档'
|
||||||
})
|
})
|
||||||
const buffer = await this.readBoundedFile(location)
|
const buffer = await readBoundedFile(
|
||||||
|
location,
|
||||||
|
maximumFileBytes,
|
||||||
|
'文件超过 20MB',
|
||||||
|
'文件不是普通文件'
|
||||||
|
)
|
||||||
effectiveSignal.throwIfAborted()
|
effectiveSignal.throwIfAborted()
|
||||||
this.updateKnowledgeTask(task.id, {
|
this.updateKnowledgeTask(task.id, {
|
||||||
stage: 'parsing',
|
stage: 'parsing',
|
||||||
@@ -1868,6 +1911,7 @@ export class KnowledgeService {
|
|||||||
},
|
},
|
||||||
this.createDocumentChunks(parsed, library)
|
this.createDocumentChunks(parsed, library)
|
||||||
)
|
)
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||||
this.updateKnowledgeTask(task.id, {
|
this.updateKnowledgeTask(task.id, {
|
||||||
stage: 'embedding',
|
stage: 'embedding',
|
||||||
progress: 65,
|
progress: 65,
|
||||||
@@ -2073,7 +2117,7 @@ export class KnowledgeService {
|
|||||||
if (!task.sourceId) {
|
if (!task.sourceId) {
|
||||||
throw new Error('任务缺少知识来源')
|
throw new Error('任务缺少知识来源')
|
||||||
}
|
}
|
||||||
await this.performSyncSource(task.sourceId, task.id)
|
await this.syncSourceOperation(task.sourceId, task.id)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
case 'document-rebuild': {
|
case 'document-rebuild': {
|
||||||
@@ -2151,7 +2195,8 @@ export class KnowledgeService {
|
|||||||
if (library.graphStrategy === 'ask') {
|
if (library.graphStrategy === 'ask') {
|
||||||
throw new Error('按需询问策略不会自动抽取,请在设置中选择其他策略')
|
throw new Error('按需询问策略不会自动抽取,请在设置中选择其他策略')
|
||||||
}
|
}
|
||||||
const documents = this.database.listDocuments(library.id)
|
const documents =
|
||||||
|
this.database.listDocumentsForLibraryRebuild(library.id)
|
||||||
const parentTask = this.createKnowledgeTask({
|
const parentTask = this.createKnowledgeTask({
|
||||||
libraryId: library.id,
|
libraryId: library.id,
|
||||||
retryOfTaskId,
|
retryOfTaskId,
|
||||||
@@ -2293,6 +2338,9 @@ export class KnowledgeService {
|
|||||||
'Knowledge source deleted'
|
'Knowledge source deleted'
|
||||||
)
|
)
|
||||||
const removed = this.database.removeSource(sourceId)
|
const removed = this.database.removeSource(sourceId)
|
||||||
|
if (removed) {
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||||
|
}
|
||||||
if (
|
if (
|
||||||
removed &&
|
removed &&
|
||||||
library.storageMode === 'managed' &&
|
library.storageMode === 'managed' &&
|
||||||
@@ -2395,13 +2443,18 @@ export class KnowledgeService {
|
|||||||
} else {
|
} else {
|
||||||
this.failKnowledgeTask(task.id, error)
|
this.failKnowledgeTask(task.id, error)
|
||||||
}
|
}
|
||||||
if (!effectiveSignal.aborted && source.type !== 'url') {
|
if (!effectiveSignal.aborted) {
|
||||||
this.database.upsertSource({
|
const currentSource = this.database.getSource(source.id)
|
||||||
...source,
|
if (currentSource) {
|
||||||
status: 'error',
|
this.database.upsertSource({
|
||||||
lastError:
|
...currentSource,
|
||||||
error instanceof Error ? error.message.slice(0, 1_000) : '同步失败'
|
status: 'error',
|
||||||
})
|
lastError:
|
||||||
|
error instanceof Error
|
||||||
|
? error.message.slice(0, 1_000)
|
||||||
|
: '同步失败'
|
||||||
|
})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
throw error
|
throw error
|
||||||
} finally {
|
} finally {
|
||||||
@@ -2431,6 +2484,11 @@ export class KnowledgeService {
|
|||||||
this.database.removeDocument(document.id)
|
this.database.removeDocument(document.id)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (existing.some(
|
||||||
|
(document) => !currentExternalIds.has(document.externalId)
|
||||||
|
)) {
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||||
|
}
|
||||||
|
|
||||||
const failures: string[] = []
|
const failures: string[] = []
|
||||||
for (let index = 0; index < files.length; index += 1) {
|
for (let index = 0; index < files.length; index += 1) {
|
||||||
@@ -2454,7 +2512,12 @@ export class KnowledgeService {
|
|||||||
progress: 10,
|
progress: 10,
|
||||||
message: '正在读取文档'
|
message: '正在读取文档'
|
||||||
})
|
})
|
||||||
const buffer = await this.readBoundedFile(file.absolutePath)
|
const buffer = await readBoundedFile(
|
||||||
|
file.absolutePath,
|
||||||
|
maximumFileBytes,
|
||||||
|
'文件超过 20MB',
|
||||||
|
'文件不是普通文件'
|
||||||
|
)
|
||||||
signal.throwIfAborted()
|
signal.throwIfAborted()
|
||||||
this.updateKnowledgeTask(parsingTask.id, {
|
this.updateKnowledgeTask(parsingTask.id, {
|
||||||
stage: 'parsing',
|
stage: 'parsing',
|
||||||
@@ -2507,6 +2570,7 @@ export class KnowledgeService {
|
|||||||
},
|
},
|
||||||
this.createDocumentChunks(parsed, effectiveLibrary)
|
this.createDocumentChunks(parsed, effectiveLibrary)
|
||||||
)
|
)
|
||||||
|
this.database.pruneUnreferencedGeneratedGraph(library.id)
|
||||||
this.updateKnowledgeTask(parsingTask.id, {
|
this.updateKnowledgeTask(parsingTask.id, {
|
||||||
documentId: document.id,
|
documentId: document.id,
|
||||||
documentName: document.title,
|
documentName: document.title,
|
||||||
@@ -2550,7 +2614,7 @@ export class KnowledgeService {
|
|||||||
this.failKnowledgeTask(parsingTask.id, error)
|
this.failKnowledgeTask(parsingTask.id, error)
|
||||||
failures.push(
|
failures.push(
|
||||||
`${file.relativePath}: ${
|
`${file.relativePath}: ${
|
||||||
error instanceof Error ? error.message : '解析失败'
|
KnowledgeService.taskErrorMessage(error)
|
||||||
}`
|
}`
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -2564,8 +2628,9 @@ export class KnowledgeService {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
if (failures.length > 0) {
|
if (failures.length > 0) {
|
||||||
|
const detail = failures.slice(0, 5).join(';')
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`${failures.length} 个文件处理失败:${failures.slice(0, 5).join(';')}`
|
`${failures.length} 个文件处理失败:${detail}`.slice(0, 1_000)
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2577,15 +2642,49 @@ export class KnowledgeService {
|
|||||||
}
|
}
|
||||||
const timer = setTimeout(() => {
|
const timer = setTimeout(() => {
|
||||||
this.embeddingEditTimers.delete(documentId)
|
this.embeddingEditTimers.delete(documentId)
|
||||||
const document = this.database.getDocument(documentId)
|
if (this.backgroundEmbeddingReindexes.has(documentId)) {
|
||||||
if (document && this.embeddingProvider) {
|
this.pendingEmbeddingReindexes.add(documentId)
|
||||||
void this.indexDocumentEmbeddings(document)
|
return
|
||||||
}
|
}
|
||||||
|
this.startBackgroundEmbeddingReindex(documentId)
|
||||||
}, 250)
|
}, 250)
|
||||||
timer.unref?.()
|
timer.unref?.()
|
||||||
this.embeddingEditTimers.set(documentId, timer)
|
this.embeddingEditTimers.set(documentId, timer)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private startBackgroundEmbeddingReindex(documentId: string): void {
|
||||||
|
if (
|
||||||
|
this.backgroundEmbeddingReindexes.has(documentId) ||
|
||||||
|
this.lifecycleController.signal.aborted
|
||||||
|
) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const operation = (async () => {
|
||||||
|
do {
|
||||||
|
this.pendingEmbeddingReindexes.delete(documentId)
|
||||||
|
const document = this.database.getDocument(documentId)
|
||||||
|
if (
|
||||||
|
!document ||
|
||||||
|
!this.embeddingProvider ||
|
||||||
|
this.lifecycleController.signal.aborted
|
||||||
|
) {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
await this.indexDocumentEmbeddings(document)
|
||||||
|
} while (this.pendingEmbeddingReindexes.delete(documentId))
|
||||||
|
})()
|
||||||
|
.catch(() => undefined)
|
||||||
|
.finally(() => {
|
||||||
|
this.pendingEmbeddingReindexes.delete(documentId)
|
||||||
|
if (
|
||||||
|
this.backgroundEmbeddingReindexes.get(documentId) === operation
|
||||||
|
) {
|
||||||
|
this.backgroundEmbeddingReindexes.delete(documentId)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
this.backgroundEmbeddingReindexes.set(documentId, operation)
|
||||||
|
}
|
||||||
|
|
||||||
private cancelScheduledEmbeddingReindex(documentId: string): void {
|
private cancelScheduledEmbeddingReindex(documentId: string): void {
|
||||||
const timer = this.embeddingEditTimers.get(documentId)
|
const timer = this.embeddingEditTimers.get(documentId)
|
||||||
if (timer) {
|
if (timer) {
|
||||||
@@ -2850,23 +2949,24 @@ export class KnowledgeService {
|
|||||||
document: Document,
|
document: Document,
|
||||||
result: GraphExtractionResult
|
result: GraphExtractionResult
|
||||||
): void {
|
): void {
|
||||||
const existingEntities = this.database.listEntities(library.id)
|
|
||||||
const chunksById = new Map(
|
const chunksById = new Map(
|
||||||
this.database
|
this.database
|
||||||
.listChunks(document.id)
|
.listChunks(document.id)
|
||||||
.map((chunk) => [chunk.id, chunk])
|
.map((chunk) => [chunk.id, chunk])
|
||||||
)
|
)
|
||||||
const entityIds = new Map<string, string>()
|
const entityIds = new Map<string, string>()
|
||||||
|
const existingEntitiesByIdentity = new Map<string, GraphEntity>()
|
||||||
|
for (const entity of this.database.listEntitiesForIdentity(library.id)) {
|
||||||
|
for (const name of [entity.name, ...entity.aliases]) {
|
||||||
|
existingEntitiesByIdentity.set(
|
||||||
|
`${entity.type}\0${normalizeEntityAlias(name)}`,
|
||||||
|
entity
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
for (const entity of result.entities) {
|
for (const entity of result.entities) {
|
||||||
const normalized = normalizeEntityAlias(entity.name)
|
const identity = `${entity.type}\0${normalizeEntityAlias(entity.name)}`
|
||||||
const existing = existingEntities.find(
|
const existing = existingEntitiesByIdentity.get(identity)
|
||||||
(candidate) =>
|
|
||||||
candidate.type === entity.type &&
|
|
||||||
(normalizeEntityAlias(candidate.name) === normalized ||
|
|
||||||
candidate.aliases.some(
|
|
||||||
(alias) => normalizeEntityAlias(alias) === normalized
|
|
||||||
))
|
|
||||||
)
|
|
||||||
const stored = existing
|
const stored = existing
|
||||||
? existing.locked
|
? existing.locked
|
||||||
? existing
|
? existing
|
||||||
@@ -2880,6 +2980,12 @@ export class KnowledgeService {
|
|||||||
aliases: entity.aliases,
|
aliases: entity.aliases,
|
||||||
locked: false
|
locked: false
|
||||||
})
|
})
|
||||||
|
for (const name of [stored.name, ...stored.aliases]) {
|
||||||
|
existingEntitiesByIdentity.set(
|
||||||
|
`${stored.type}\0${normalizeEntityAlias(name)}`,
|
||||||
|
stored
|
||||||
|
)
|
||||||
|
}
|
||||||
entityIds.set(entity.id, stored.id)
|
entityIds.set(entity.id, stored.id)
|
||||||
for (const evidence of entity.evidence) {
|
for (const evidence of entity.evidence) {
|
||||||
this.database.createEvidence({
|
this.database.createEvidence({
|
||||||
@@ -2900,18 +3006,17 @@ export class KnowledgeService {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
const existingRelations = this.database.listRelations(library.id)
|
|
||||||
for (const relation of result.relations) {
|
for (const relation of result.relations) {
|
||||||
const sourceEntityId = entityIds.get(relation.sourceId)
|
const sourceEntityId = entityIds.get(relation.sourceId)
|
||||||
const targetEntityId = entityIds.get(relation.targetId)
|
const targetEntityId = entityIds.get(relation.targetId)
|
||||||
if (!sourceEntityId || !targetEntityId) {
|
if (!sourceEntityId || !targetEntityId) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const existing = existingRelations.find(
|
const existing = this.database.findRelationByIdentity(
|
||||||
(candidate) =>
|
library.id,
|
||||||
candidate.sourceEntityId === sourceEntityId &&
|
sourceEntityId,
|
||||||
candidate.targetEntityId === targetEntityId &&
|
targetEntityId,
|
||||||
candidate.type === relation.type
|
relation.type
|
||||||
)
|
)
|
||||||
const stored =
|
const stored =
|
||||||
existing ??
|
existing ??
|
||||||
@@ -3027,24 +3132,6 @@ export class KnowledgeService {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private async readBoundedFile(path: string): Promise<Buffer> {
|
|
||||||
const handle = await open(path, 'r')
|
|
||||||
try {
|
|
||||||
const fileStat = await handle.stat()
|
|
||||||
if (!fileStat.isFile() || fileStat.size > maximumFileBytes) {
|
|
||||||
throw new Error('文件超过 20MB 或不是普通文件')
|
|
||||||
}
|
|
||||||
const buffer = Buffer.alloc(fileStat.size + 1)
|
|
||||||
const result = await handle.read(buffer, 0, buffer.length, 0)
|
|
||||||
if (result.bytesRead > maximumFileBytes) {
|
|
||||||
throw new Error('文件超过 20MB')
|
|
||||||
}
|
|
||||||
return buffer.subarray(0, result.bytesRead)
|
|
||||||
} finally {
|
|
||||||
await handle.close()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private startWatcher(source: KnowledgeSource): void {
|
private startWatcher(source: KnowledgeSource): void {
|
||||||
this.stopWatcher(source.id)
|
this.stopWatcher(source.id)
|
||||||
try {
|
try {
|
||||||
|
|||||||
Reference in New Issue
Block a user