diff --git a/src/main/knowledge/external/external-knowledge-service.test.ts b/src/main/knowledge/external/external-knowledge-service.test.ts index 3a1759e4..3b663bd7 100644 --- a/src/main/knowledge/external/external-knowledge-service.test.ts +++ b/src/main/knowledge/external/external-knowledge-service.test.ts @@ -119,8 +119,9 @@ it('cancels pending probes on disable and prevents shutdown writeback', async () const probe = service.external.testInstance(saved.id) service.external.setEnabled(saved.id, false) release(Response.json({ data: [], has_more: false })) - await expect(probe).rejects.toThrow('EXTERNAL_KB_CONFIG_CHANGED') - expect(service.external.listInstances()[0]?.enabled).toBe(false) + await expect(probe).rejects.toMatchObject({ code: 'EXTERNAL_KB_CANCELLED' }) + expect(service.external.listInstances()[0]).toMatchObject({ enabled: false, probeStatus: 'untested' }) + expect(service.database.externalStore.getInstance(saved.id)?.lastTestedAt).toBeUndefined() service.external.setEnabled(saved.id, true) const pending = service.external.saveBinding(bindingInput(saved.id)) const rejected = expect(pending).rejects.toThrow() diff --git a/src/main/knowledge/external/external-knowledge-service.ts b/src/main/knowledge/external/external-knowledge-service.ts index c4c0c113..7a5293e9 100644 --- a/src/main/knowledge/external/external-knowledge-service.ts +++ b/src/main/knowledge/external/external-knowledge-service.ts @@ -1,154 +1,416 @@ import { randomUUID } from 'node:crypto' -import { externalKnowledgeBindingSaveInputSchema, externalKnowledgeBindingTestInputSchema, externalKnowledgeBindingUpdateInputSchema, externalKnowledgeInstanceSaveInputSchema, type ExternalKnowledgeBinding, type ExternalKnowledgeBindingSaveInput, type ExternalKnowledgeBindingTestInput, type ExternalKnowledgeInstanceSaveInput, type ExternalKnowledgeInstanceSummary, type ExternalKnowledgeTestResult } from '../../../shared/external-knowledge-contracts' -import { decryptSettingsCredential, encryptSettingsCredential, type SettingsCredentialCipher } from '../../settings-credential-cipher' +import { + externalKnowledgeBindingSaveInputSchema, + externalKnowledgeBindingTestInputSchema, + externalKnowledgeBindingUpdateInputSchema, + externalKnowledgeInstanceSaveInputSchema, + externalKnowledgeRetrievalInputSchema, + type ExternalKnowledgeBinding, + type ExternalKnowledgeBindingSaveInput, + type ExternalKnowledgeBindingTestInput, + type ExternalKnowledgeCommonConfig, + type ExternalKnowledgeInstanceSaveInput, + type ExternalKnowledgeInstanceSummary, + type ExternalKnowledgeRemoteResult, + type ExternalKnowledgeRetrievalInput, + type ExternalKnowledgeTestResult +} from '../../../shared/external-knowledge-contracts' +import { + decryptSettingsCredential, + encryptSettingsCredential, + type SettingsCredentialCipher +} from '../../settings-credential-cipher' import type { KnowledgeDatabase } from '../knowledge-database' import { ExternalKnowledgeClient, ExternalKnowledgeError } from './external-knowledge-client' import type { StoredExternalInstance } from './external-knowledge-store' -// Probe results are persisted metadata, not a change to an in-flight request's configuration. -function instanceConfiguration(value: StoredExternalInstance): string { - return JSON.stringify([value.id, value.name, value.provider, value.baseUrl, value.enabled, value.credential]) -} +const SHARED_SNIPPET_BUDGET = 48_000 export class ExternalKnowledgeService { private disposed = false - private readonly controllers = new Map() - constructor(private readonly database: KnowledgeDatabase, private readonly cipher?: SettingsCredentialCipher, private readonly fetcher?: typeof fetch) {} - dispose(): void { this.disposed = true; for (const controller of this.controllers.keys()) controller.abort(); this.controllers.clear() } - private checkActive(): void { if (this.disposed) throw new Error('EXTERNAL_KB_CANCELLED') } - private cancelInstance(id: string): void { - for (const [controller, instanceId] of this.controllers) if (instanceId === id) controller.abort() + private readonly controllers = new Map() + + constructor( + private readonly database: KnowledgeDatabase, + private readonly cipher?: SettingsCredentialCipher, + private readonly fetcher?: typeof fetch + ) {} + + dispose(): void { + this.disposed = true + for (const controller of this.controllers.keys()) { + controller.abort() + } + this.controllers.clear() } - private instance(id:string): StoredExternalInstance { + + private checkActive(): void { + if (this.disposed) { + throw new Error('EXTERNAL_KB_CANCELLED') + } + } + + private cancelInstance(id: string): void { + for (const [controller, instanceId] of this.controllers) { + if (instanceId === id) { + controller.abort() + } + } + } + + private instance(id: string): StoredExternalInstance { this.checkActive() - const value=this.database.externalStore.getInstance(id) - if (!value) throw new Error('EXTERNAL_KB_NOT_FOUND') + const value = this.database.externalStore.getInstance(id) + if (!value) { + throw new Error('EXTERNAL_KB_NOT_FOUND') + } return value } - listInstances(): ExternalKnowledgeInstanceSummary[] { - this.checkActive() - const bindings=this.database.externalStore.listBindings() - return this.database.externalStore.listInstances().map(({credential,...value})=>{ - let credentialStatus: ExternalKnowledgeInstanceSummary['credentialStatus'] = 'missing' - if (credential) { - try { this.credential({ ...value, credential }); credentialStatus = 'configured' } - catch { credentialStatus = 'unavailable' } - } - return {...value,credentialStatus,bindingCount:bindings.filter(binding=>binding.instanceId===value.id).length} - }) - } + private credential(instance: StoredExternalInstance): string { try { - if (!instance.credential || !this.cipher?.isAvailable()) throw new Error() + if (!instance.credential || !this.cipher?.isAvailable()) { + throw new Error() + } const key = decryptSettingsCredential(this.cipher, instance.credential) - if (typeof key !== 'string' || !key.trim()) throw new Error() + if (typeof key !== 'string' || !key.trim()) { + throw new Error() + } return key - } catch { throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') } + } catch { + throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') + } } - saveInstance(raw:ExternalKnowledgeInstanceSaveInput):ExternalKnowledgeInstanceSummary { + + listInstances(): ExternalKnowledgeInstanceSummary[] { this.checkActive() - const input=externalKnowledgeInstanceSaveInputSchema.parse(raw) - const previous=input.id ? this.instance(input.id):undefined - if (previous && previous.provider!==input.provider && this.database.externalStore.listBindings().some(binding=>binding.instanceId===previous.id)) throw new Error('EXTERNAL_KB_INSTANCE_IN_USE') - let credential=previous?.credential - if (input.credential.action==='clear') credential=undefined - if (input.credential.action==='replace') { - if (!this.cipher?.isAvailable()) throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') - try { credential=encryptSettingsCredential(this.cipher,input.credential.value) } - catch { throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') } + const bindingCounts = new Map() + for (const binding of this.database.externalStore.listBindings()) { + bindingCounts.set(binding.instanceId, (bindingCounts.get(binding.instanceId) ?? 0) + 1) + } + return this.database.externalStore.listInstances().map(({ credential, ...value }) => { + let credentialStatus: ExternalKnowledgeInstanceSummary['credentialStatus'] = 'missing' + if (credential) { + try { + this.credential({ ...value, credential }) + credentialStatus = 'configured' + } catch { + credentialStatus = 'unavailable' + } + } + return { ...value, credentialStatus, bindingCount: bindingCounts.get(value.id) ?? 0 } + }) + } + + private summary(id: string): ExternalKnowledgeInstanceSummary { + const summary = this.listInstances().find((item) => item.id === id) + if (!summary) { + throw new Error('EXTERNAL_KB_NOT_FOUND') + } + return summary + } + + saveInstance(raw: ExternalKnowledgeInstanceSaveInput): ExternalKnowledgeInstanceSummary { + this.checkActive() + const input = externalKnowledgeInstanceSaveInputSchema.parse(raw) + const previous = input.id ? this.instance(input.id) : undefined + if ( + previous && + previous.provider !== input.provider && + this.database.externalStore.getBindingsForInstance(previous.id).length > 0 + ) { + throw new Error('EXTERNAL_KB_INSTANCE_IN_USE') + } + let credential = previous?.credential + if (input.credential.action === 'clear') { + credential = undefined + } + if (input.credential.action === 'replace') { + if (!this.cipher?.isAvailable()) { + throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') + } + try { + credential = encryptSettingsCredential(this.cipher, input.credential.value) + } catch { + throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') + } + } + const value: StoredExternalInstance = { + id: previous?.id ?? randomUUID(), + name: input.name, + provider: input.provider, + baseUrl: input.baseUrl.replace(/\/+$/, ''), + enabled: input.enabled, + credential, + probeStatus: 'untested' } - const value:StoredExternalInstance={id:previous?.id??randomUUID(),name:input.name,provider:input.provider,baseUrl:input.baseUrl.replace(/\/+$/,''),enabled:input.enabled,credential,probeStatus:'untested'} this.database.externalStore.saveInstance(value) + // In-flight requests were authorized against the replaced configuration. this.cancelInstance(value.id) - return this.listInstances().find(item=>item.id===value.id)! + return this.summary(value.id) } - setEnabled(id:string,enabled:boolean):ExternalKnowledgeInstanceSummary { - const instance=this.instance(id) - this.database.externalStore.saveInstance({...instance,enabled}) - if (!enabled) for(const [controller,instanceId] of this.controllers) if(instanceId===id) controller.abort() - return this.listInstances().find(item=>item.id===id)! + + setEnabled(id: string, enabled: boolean): ExternalKnowledgeInstanceSummary { + const instance = this.instance(id) + this.database.externalStore.saveInstance({ ...instance, enabled }) + if (!enabled) { + this.cancelInstance(id) + } + return this.summary(id) } - deleteInstance(id:string):void { + + deleteInstance(id: string): void { this.instance(id) - if(this.database.externalStore.listBindings().some(binding=>binding.instanceId===id)) throw new Error('EXTERNAL_KB_INSTANCE_IN_USE') - for(const [controller,instanceId] of this.controllers) if(instanceId===id) controller.abort() + if (this.database.externalStore.getBindingsForInstance(id).length > 0) { + throw new Error('EXTERNAL_KB_INSTANCE_IN_USE') + } + this.cancelInstance(id) this.database.externalStore.deleteInstance(id) } - private async request(id:string,timeout:number,signal:AbortSignal|undefined,operation:(client:ExternalKnowledgeClient,signal:AbortSignal)=>Promise):Promise { - const instance=this.instance(id) - if(!instance.enabled) throw new Error('EXTERNAL_KB_DISABLED') + + private async request( + id: string, + timeout: number, + signal: AbortSignal | undefined, + operation: (client: ExternalKnowledgeClient, signal: AbortSignal) => Promise + ): Promise { + const instance = this.instance(id) + if (!instance.enabled) { + throw new Error('EXTERNAL_KB_DISABLED') + } const key = this.credential(instance) - const controller=new AbortController() - this.controllers.set(controller,id) + const controller = new AbortController() + this.controllers.set(controller, id) const timeoutSignal = AbortSignal.timeout(timeout) - const combined = AbortSignal.any([controller.signal, timeoutSignal, ...(signal ? [signal] : [])]) + const combined = AbortSignal.any([ + controller.signal, + timeoutSignal, + ...(signal ? [signal] : []) + ]) try { combined.throwIfAborted() - const result = await operation(new ExternalKnowledgeClient({provider:instance.provider,baseUrl:instance.baseUrl,apiKey:key,timeoutMs:timeout,fetcher:this.fetcher}),combined) - combined.throwIfAborted() - return result + return await operation( + new ExternalKnowledgeClient({ + provider: instance.provider, + baseUrl: instance.baseUrl, + apiKey: key, + timeoutMs: timeout, + fetcher: this.fetcher + }), + combined + ) } catch (error) { if (combined.aborted) { const timedOut = timeoutSignal.aborted && combined.reason === timeoutSignal.reason - throw new ExternalKnowledgeError(timedOut ? 'EXTERNAL_KB_TIMEOUT' : 'EXTERNAL_KB_CANCELLED', - timedOut ? 'External knowledge request timed out' : 'External knowledge request was cancelled') + throw new ExternalKnowledgeError( + timedOut ? 'EXTERNAL_KB_TIMEOUT' : 'EXTERNAL_KB_CANCELLED', + timedOut + ? 'External knowledge request timed out' + : 'External knowledge request was cancelled' + ) } throw error - } finally {this.controllers.delete(controller)} - } - async testInstance(id:string):Promise { - const instance=this.instance(id) - let probeStatus:StoredExternalInstance['probeStatus']='catalog-ready' - let lastErrorCode:string|undefined - try {await this.listCatalog({instanceId:id})} catch(error) { - lastErrorCode=error instanceof ExternalKnowledgeError ? error.code : error instanceof Error && /^EXTERNAL_KB_/.test(error.message) ? error.message : 'EXTERNAL_KB_NETWORK' - probeStatus=lastErrorCode==='EXTERNAL_KB_FORBIDDEN'?'list-restricted':lastErrorCode==='EXTERNAL_KB_AUTH'?'auth-failed':lastErrorCode==='EXTERNAL_KB_NETWORK'||lastErrorCode==='EXTERNAL_KB_TIMEOUT'?'unreachable':'failed' + } finally { + this.controllers.delete(controller) } - const current = this.instance(id) - if (instanceConfiguration(current) !== instanceConfiguration(instance)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED') - this.database.externalStore.saveInstance({...current,probeStatus,lastErrorCode,lastTestedAt:new Date().toISOString()}) - return this.listInstances().find(item=>item.id===id)! } - listCatalog(input:{instanceId:string;page?:number;pageSize?:number;parentId?:string|null;search?:string},signal?:AbortSignal) { - return this.request(input.instanceId,15000,signal,(client,current)=>client.listKnowledgeBases(current,input)) - } - getCatalog(input:{instanceId:string;remoteKnowledgeBaseId:string},signal?:AbortSignal) { - return this.request(input.instanceId,15000,signal,(client,current)=>client.getKnowledgeBase(input.remoteKnowledgeBaseId,current)) - } - async testRetrieval(raw:ExternalKnowledgeBindingTestInput,signal?:AbortSignal):Promise { - const input=externalKnowledgeBindingTestInputSchema.parse(raw) - const instance=this.instance(input.instanceId) - if(instance.provider!==input.providerConfig.provider) throw new Error('EXTERNAL_KB_CONFIG_INVALID') - const started=Date.now() - const results=await this.request(input.instanceId,input.commonConfig.requestTimeoutMs,signal,async (client,current)=>{ - if(input.providerConfig.provider==='ragflow' && (input.providerConfig.useKg||input.providerConfig.includeKnowledgeCompilation)) { - const detail=await client.getKnowledgeBase(input.remoteKnowledgeBaseId,current) - if((input.providerConfig.useKg&&!detail.graphEnabled)||(input.providerConfig.includeKnowledgeCompilation&&!detail.knowledgeCompilationEnabled)) throw new Error('EXTERNAL_KB_CAPABILITY_UNAVAILABLE') + + async testInstance(id: string): Promise { + this.instance(id) + let probeStatus: StoredExternalInstance['probeStatus'] = 'catalog-ready' + let lastErrorCode: string | undefined + try { + await this.listCatalog({ instanceId: id }) + } catch (error) { + lastErrorCode = + error instanceof ExternalKnowledgeError + ? error.code + : error instanceof Error && /^EXTERNAL_KB_/.test(error.message) + ? error.message + : 'EXTERNAL_KB_NETWORK' + // Cancellation means the configuration changed under the probe, so it + // learned nothing about reachability and must not overwrite the record. + if (lastErrorCode === 'EXTERNAL_KB_CANCELLED') { + throw error } - return client.retrieve(input.remoteKnowledgeBaseId,input.testQuery,input.providerConfig,current) + probeStatus = + lastErrorCode === 'EXTERNAL_KB_FORBIDDEN' + ? 'list-restricted' + : lastErrorCode === 'EXTERNAL_KB_AUTH' + ? 'auth-failed' + : lastErrorCode === 'EXTERNAL_KB_NETWORK' || lastErrorCode === 'EXTERNAL_KB_TIMEOUT' + ? 'unreachable' + : 'failed' + } + const current = this.database.externalStore.getInstance(id) + if (!current) { + throw new Error('EXTERNAL_KB_NOT_FOUND') + } + this.database.externalStore.saveInstance({ + ...current, + probeStatus, + lastErrorCode, + lastTestedAt: new Date().toISOString() }) - if (instanceConfiguration(instance) !== instanceConfiguration(this.instance(input.instanceId))) throw new Error('EXTERNAL_KB_CONFIG_CHANGED') - let remaining = 48_000 - const bounded = results.slice(0, input.commonConfig.resultLimit).map(item => { - const snippet = item.snippet.slice(0, Math.min(remaining, input.commonConfig.maxSnippetCharacters)) - remaining -= snippet.length - return { ...item, snippet } - }).filter(item => item.snippet.length > 0) - return {results:bounded,durationMs:Date.now()-started} + return this.summary(id) } - async saveBinding(raw:ExternalKnowledgeBindingSaveInput & {knowledgeBaseId?:string}):Promise { + + listCatalog( + input: { + instanceId: string + page?: number + pageSize?: number + parentId?: string | null + search?: string + }, + signal?: AbortSignal + ) { + return this.request(input.instanceId, 15_000, signal, (client, current) => + client.listKnowledgeBases(current, input) + ) + } + + getCatalog(input: { instanceId: string; remoteKnowledgeBaseId: string }, signal?: AbortSignal) { + return this.request(input.instanceId, 15_000, signal, (client, current) => + client.getKnowledgeBase(input.remoteKnowledgeBaseId, current) + ) + } + + private boundResults( + results: readonly ExternalKnowledgeRemoteResult[], + common: ExternalKnowledgeCommonConfig + ): ExternalKnowledgeRemoteResult[] { + let remaining = SHARED_SNIPPET_BUDGET + return results + .slice(0, common.resultLimit) + .map((item) => { + const snippet = item.snippet.slice( + 0, + Math.min(remaining, common.maxSnippetCharacters) + ) + remaining -= snippet.length + return { ...item, snippet } + }) + .filter((item) => item.snippet.length > 0) + } + + /** Retrieves from the bound remote knowledge base. This is the production path. */ + async retrieve( + raw: ExternalKnowledgeRetrievalInput, + signal?: AbortSignal + ): Promise { + const input = externalKnowledgeRetrievalInputSchema.parse(raw) + if (this.instance(input.instanceId).provider !== input.providerConfig.provider) { + throw new Error('EXTERNAL_KB_CONFIG_INVALID') + } + const started = Date.now() + const results = await this.request( + input.instanceId, + input.commonConfig.requestTimeoutMs, + signal, + (client, current) => + client.retrieve(input.remoteKnowledgeBaseId, input.query, input.providerConfig, current) + ) + return { + results: this.boundResults(results, input.commonConfig), + durationMs: Date.now() - started + } + } + + /** + * Verifies a binding before it is saved. Unlike retrieve, this also confirms + * that the remote deployment provides the requested capabilities. The check + * shares the configured request timeout so the whole verification stays + * within the budget the user chose. + */ + async testRetrieval( + raw: ExternalKnowledgeBindingTestInput, + signal?: AbortSignal + ): Promise { + const input = externalKnowledgeBindingTestInputSchema.parse(raw) + const config = input.providerConfig + if (this.instance(input.instanceId).provider !== config.provider) { + throw new Error('EXTERNAL_KB_CONFIG_INVALID') + } + const started = Date.now() + const results = await this.request( + input.instanceId, + input.commonConfig.requestTimeoutMs, + signal, + async (client, current) => { + if (config.provider === 'ragflow' && (config.useKg || config.includeKnowledgeCompilation)) { + const detail = await client.getKnowledgeBase(input.remoteKnowledgeBaseId, current) + if ( + (config.useKg && !detail.graphEnabled) || + (config.includeKnowledgeCompilation && !detail.knowledgeCompilationEnabled) + ) { + throw new Error('EXTERNAL_KB_CAPABILITY_UNAVAILABLE') + } + } + return client.retrieve(input.remoteKnowledgeBaseId, input.testQuery, config, current) + } + ) + return { + results: this.boundResults(results, input.commonConfig), + durationMs: Date.now() - started + } + } + + async saveBinding( + raw: ExternalKnowledgeBindingSaveInput & { knowledgeBaseId?: string } + ): Promise { this.checkActive() - const input=raw.knowledgeBaseId !== undefined ? externalKnowledgeBindingUpdateInputSchema.parse(raw):externalKnowledgeBindingSaveInputSchema.parse(raw) - const knowledgeBaseId = 'knowledgeBaseId' in input ? input.knowledgeBaseId : undefined - const existing=knowledgeBaseId ? this.database.externalStore.listBindings().find(item=>item.knowledgeBaseId===knowledgeBaseId):undefined - if(knowledgeBaseId&&!existing) throw new Error('EXTERNAL_KB_NOT_FOUND') - if(this.database.externalStore.listBindings().some(item=>item.instanceId===input.instanceId&&item.remoteKnowledgeBaseId===input.remoteKnowledgeBaseId&&item.knowledgeBaseId!==existing?.knowledgeBaseId)) throw new Error('EXTERNAL_KB_DUPLICATE_BINDING') - const instance = this.instance(input.instanceId) - await this.testRetrieval({instanceId:input.instanceId,remoteKnowledgeBaseId:input.remoteKnowledgeBaseId,commonConfig:input.commonConfig,providerConfig:input.providerConfig,testQuery:input.testQuery}) - if (instanceConfiguration(instance) !== instanceConfiguration(this.instance(input.instanceId))) throw new Error('EXTERNAL_KB_CONFIG_CHANGED') - const bindings = this.database.externalStore.listBindings() - if (existing && JSON.stringify(bindings.find(item => item.knowledgeBaseId === existing.knowledgeBaseId)) !== JSON.stringify(existing)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED') - if (bindings.some(item => item.instanceId === input.instanceId && item.remoteKnowledgeBaseId === input.remoteKnowledgeBaseId && item.knowledgeBaseId !== existing?.knowledgeBaseId)) throw new Error('EXTERNAL_KB_DUPLICATE_BINDING') - return this.database.saveExternalBinding({instanceId:input.instanceId,provider:input.providerConfig.provider,remoteKnowledgeBaseId:input.remoteKnowledgeBaseId,remoteName:input.remoteName,commonConfig:input.commonConfig,providerConfig:input.providerConfig,lastVerifiedAt:new Date().toISOString()}, {name:input.name,description:input.description,knowledgeBaseId:existing?.knowledgeBaseId}) + const update = + raw.knowledgeBaseId !== undefined + ? externalKnowledgeBindingUpdateInputSchema.parse(raw) + : undefined + const input = update ?? externalKnowledgeBindingSaveInputSchema.parse(raw) + const existing = update + ? this.database.externalStore.getBinding(update.knowledgeBaseId) + : undefined + if (update && !existing) { + throw new Error('EXTERNAL_KB_NOT_FOUND') + } + this.instance(input.instanceId) + const duplicate = (): boolean => + this.database.externalStore + .listBindings() + .some( + (item) => + item.instanceId === input.instanceId && + item.remoteKnowledgeBaseId === input.remoteKnowledgeBaseId && + item.knowledgeBaseId !== existing?.knowledgeBaseId + ) + if (duplicate()) { + throw new Error('EXTERNAL_KB_DUPLICATE_BINDING') + } + await this.testRetrieval({ + instanceId: input.instanceId, + remoteKnowledgeBaseId: input.remoteKnowledgeBaseId, + commonConfig: input.commonConfig, + providerConfig: input.providerConfig, + testQuery: input.testQuery + }) + // Another binding for the same remote target may have landed while the + // verification request was in flight; the unique index is the final guard. + if (duplicate()) { + throw new Error('EXTERNAL_KB_DUPLICATE_BINDING') + } + return this.database.saveExternalBinding( + { + instanceId: input.instanceId, + provider: input.providerConfig.provider, + remoteKnowledgeBaseId: input.remoteKnowledgeBaseId, + remoteName: input.remoteName, + commonConfig: input.commonConfig, + providerConfig: input.providerConfig, + lastVerifiedAt: new Date().toISOString() + }, + { + name: input.name, + description: input.description, + knowledgeBaseId: existing?.knowledgeBaseId + } + ) } } diff --git a/src/main/knowledge/knowledge-service.ts b/src/main/knowledge/knowledge-service.ts index 96731b35..b63445d3 100644 --- a/src/main/knowledge/knowledge-service.ts +++ b/src/main/knowledge/knowledge-service.ts @@ -1247,15 +1247,14 @@ export class KnowledgeService { const input = knowledgeRetrieveInputSchema.parse(rawInput) const library = this.requireLibrary(input.knowledgeBaseId, false) const settings = input.settings ?? library.retrievalSettings - const binding = this.database.externalStore.listBindings().find(item => item.knowledgeBaseId === library.id) + const binding = this.database.externalStore.getBinding(library.id) if (binding) { - const response = await this.external.testRetrieval({instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,commonConfig:binding.commonConfig,providerConfig:binding.providerConfig,testQuery:input.query}, signal) + const response = await this.external.retrieve({instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,commonConfig:binding.commonConfig,providerConfig:binding.providerConfig,query:input.query}, signal) signal.throwIfAborted() - if (JSON.stringify(this.database.externalStore.listBindings().find(item => item.knowledgeBaseId === library.id)) !== JSON.stringify(binding)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED') return { query:input.query,durationMs:response.durationMs,settings, diagnostics:{external:{provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId},requestedChannels:[],usedChannels:[],degradedChannels:[],candidateCounts:{},channelDurationMs:{},vectorScannedCount:0,filteredByThresholdCount:0,filteredByBudgetCount:0,rerank:{requested:'none',used:'none',status:'skipped',candidateCount:0,durationMs:0}}, - results:response.results.map((item,index)=>({knowledgeBaseId:library.id,documentTitle:item.documentTitle,sourceDisplayName:item.sourceDisplayName,snippet:item.snippet,rank:index+1,channels:[],scores:{fusedScore:0},external:{kind:'external' as const,provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,remoteDocumentId:item.remoteDocumentId,remoteChunkId:item.remoteChunkId,providerScore:item.providerScore,providerScores:item.providerScores,location:item.location,sourceUrl:item.sourceUrl}})), + results:response.results.map((item,index)=>({knowledgeBaseId:library.id,documentTitle:item.documentTitle,sourceDisplayName:item.sourceDisplayName,snippet:item.snippet,rank:index+1,channels:[],scores:{fusedScore:0},external:{kind:'external' as const,provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,remoteDocumentId:item.remoteDocumentId,remoteChunkId:item.remoteChunkId,providerScore:item.providerScore,providerScores:item.providerScores,location:item.location}})), context:{characterCount:response.results.reduce((sum,item)=>sum+item.snippet.length,0),truncated:false,groups:[]} } } @@ -1693,7 +1692,7 @@ export class KnowledgeService { this.requireLibrary(id, false) ) const preparedEmbedding = libraries.some( - (library) => !this.database.externalStore.listBindings().some(item=>item.knowledgeBaseId===library.id) && library.retrievalSettings.vectorWeight > 0 + (library) => !this.database.externalStore.hasBinding(library.id) && library.retrievalSettings.vectorWeight > 0 ) ? await this.prepareQueryEmbedding(query, signal) : undefined @@ -1869,45 +1868,6 @@ export class KnowledgeService { }) } - async searchHybridMany( - knowledgeBaseIds: readonly string[], - query: string, - limitPerLibrary = 6, - signal?: AbortSignal - ): Promise< - Array<{ knowledgeBaseId: string; result: HybridSearchResult }> - > { - const vector = await this.embedQuery(query, signal) - return knowledgeBaseIds.flatMap((knowledgeBaseId) => { - const library = this.requireLibrary(knowledgeBaseId) - const settings = library.retrievalSettings - return this.database - .hybridSearch({ - knowledgeBaseId, - query, - limit: limitPerLibrary, - provider: - vector && this.embeddingProvider - ? embeddingStorageProvider(this.embeddingProvider) - : undefined, - model: vector ? this.embeddingProvider?.model : undefined, - vector, - graphEnabled: library.graphEnabled, - minimumVectorSimilarity: settings.minimumVectorSimilarity, - candidateMultiplier: settings.candidateMultiplier, - ftsWeight: settings.ftsWeight, - vectorWeight: settings.vectorWeight, - graphWeight: settings.graphWeight, - signal - }) - .map((result) => ({ knowledgeBaseId, result })) - }).sort( - (left, right) => - right.result.retrieval.score - left.result.retrieval.score || - left.result.chunk.id.localeCompare(right.result.chunk.id) - ).slice(0, limitPerLibrary) - } - private async embedQuery( query: string, signal?: AbortSignal @@ -3863,7 +3823,7 @@ export class KnowledgeService { } private requireLibrary(id: string, localOnly = true): KnowledgeBase { - if (localOnly && this.database.externalStore.listBindings().some(item => item.knowledgeBaseId === id)) { + if (localOnly && this.database.externalStore.hasBinding(id)) { throw new Error('EXTERNAL_KB_READ_ONLY') } const library = this.database.getKnowledgeBase(id) diff --git a/src/shared/external-knowledge-contracts.ts b/src/shared/external-knowledge-contracts.ts index f56b337e..ce5445c6 100644 --- a/src/shared/external-knowledge-contracts.ts +++ b/src/shared/external-knowledge-contracts.ts @@ -37,15 +37,17 @@ export const externalKnowledgeInstanceInputSchema = z.object({instanceId:id}).st export const externalKnowledgeInstanceEnabledInputSchema = externalKnowledgeInstanceInputSchema.extend({enabled:z.boolean()}) export const externalKnowledgeCatalogListInputSchema = externalKnowledgeInstanceInputSchema.extend({page:z.number().int().min(1).max(500).optional(),pageSize:z.number().int().min(1).max(100).optional(),parentId:id.nullable().optional(),search:z.string().trim().max(512).optional()}) export const externalKnowledgeCatalogGetInputSchema = externalKnowledgeInstanceInputSchema.extend({remoteKnowledgeBaseId:id}) +export const externalKnowledgeRetrievalInputSchema = externalKnowledgeCatalogGetInputSchema.extend({commonConfig:externalKnowledgeCommonConfigSchema,providerConfig:externalKnowledgeProviderConfigSchema,query:z.string().trim().min(1).max(4000)}) +export type ExternalKnowledgeRetrievalInput = z.infer export const externalKnowledgeBindingTestInputSchema = externalKnowledgeCatalogGetInputSchema.extend({commonConfig:externalKnowledgeCommonConfigSchema,providerConfig:externalKnowledgeProviderConfigSchema,testQuery:z.string().trim().min(1).max(4000)}) export const externalKnowledgeBindingSaveInputSchema = externalKnowledgeBindingTestInputSchema.extend({name:z.string().trim().min(1).max(512),description:z.string().max(4000).optional(),remoteName:z.string().trim().min(1).max(512)}) export const externalKnowledgeBindingUpdateInputSchema = externalKnowledgeBindingSaveInputSchema.extend({knowledgeBaseId:id}) export type ExternalKnowledgeBindingTestInput = z.infer export type ExternalKnowledgeBindingSaveInput = z.infer -export type ExternalKnowledgeInstanceSummary = {id:string;name:string;provider:ExternalKnowledgeProvider;baseUrl:string;enabled:boolean;credentialStatus:'configured'|'missing'|'unavailable';probeStatus:'untested'|'catalog-ready'|'healthy'|'list-restricted'|'auth-failed'|'unreachable'|'failed';bindingCount:number;lastTestedAt?:string;lastErrorCode?:string} +export type ExternalKnowledgeInstanceSummary = {id:string;name:string;provider:ExternalKnowledgeProvider;baseUrl:string;enabled:boolean;credentialStatus:'configured'|'missing'|'unavailable';probeStatus:'untested'|'catalog-ready'|'list-restricted'|'auth-failed'|'unreachable'|'failed';bindingCount:number;lastTestedAt?:string;lastErrorCode?:string} export type ExternalKnowledgeCatalogItem = {id:string;name:string;description?:string;kind?:'dataset'|'folder';graphEnabled?:boolean;knowledgeCompilationEnabled?:boolean} export type ExternalKnowledgeCatalogPage = {items:ExternalKnowledgeCatalogItem[];total?:number;hasMore:boolean} -export type ExternalKnowledgeRemoteResult = {documentTitle:string;sourceDisplayName:string;snippet:string;providerScore?:number;providerScores?:{type:string;value:number;index?:number}[];remoteDocumentId?:string;remoteChunkId?:string;location?:string;sourceUrl?:string} +export type ExternalKnowledgeRemoteResult = {documentTitle:string;sourceDisplayName:string;snippet:string;providerScore?:number;providerScores?:{type:string;value:number;index?:number}[];remoteDocumentId?:string;remoteChunkId?:string;location?:string} export type ExternalKnowledgeTestResult = {results:ExternalKnowledgeRemoteResult[];durationMs:number} export type ExternalKnowledgeBinding = {knowledgeBaseId:string;instanceId:string;provider:ExternalKnowledgeProvider;remoteKnowledgeBaseId:string;remoteName:string;commonConfig:ExternalKnowledgeCommonConfig;providerConfig:ExternalKnowledgeProviderConfig;lastVerifiedAt:string} export const externalKnowledgeLocatorSchema = z.object({kind:z.literal('external'),provider:externalKnowledgeProviderSchema,instanceId:id,remoteKnowledgeBaseId:id,remoteDocumentId:id.optional(),remoteChunkId:id.optional(),providerScore:z.number().finite().optional(),providerScores:z.array(z.object({type:z.string().max(128),value:z.number().finite(),index:z.number().int().optional()}).strict()).max(20).optional(),location:z.string().max(8192).optional(),sourceUrl:z.string().max(8192).optional()}).strict()