fix: stop external knowledge chat retrieval from running binding checks
Conversation retrieval called the binding verification helper, so every question sent the user query as a test query and RAGFlow bindings with graph or knowledge compilation enabled spent an extra detail request before retrieving. A failed capability check could also surface EXTERNAL_KB_CAPABILITY_UNAVAILABLE during a normal conversation. Retrieval and verification are now separate service entry points sharing one snippet-bounding helper. The capability pre-check stays in the verification path and keeps using the binding's configured request timeout. Replace the five configuration snapshot comparisons and the EXTERNAL_KB_CONFIG_CHANGED code with the existing per-instance cancellation. A connection test cancelled by a configuration change now rejects without writing back a probe result, so disabling an instance no longer records it as a failed probe. Duplicate binding safety relies on a re-check plus the existing unique index. Release note: 外部知识库对话检索不再执行仅用于保存校验的能力预检，RAGFlow 图谱绑定每次 提问少一次请求；停用实例时不再把它错误标记为连接失败。
This commit is contained in:
mesalogo
@@ -119,8 +119,9 @@ it('cancels pending probes on disable and prevents shutdown writeback', async ()
|
||||
const probe = service.external.testInstance(saved.id)
|
||||
service.external.setEnabled(saved.id, false)
|
||||
release(Response.json({ data: [], has_more: false }))
|
||||
await expect(probe).rejects.toThrow('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
expect(service.external.listInstances()[0]?.enabled).toBe(false)
|
||||
await expect(probe).rejects.toMatchObject({ code: 'EXTERNAL_KB_CANCELLED' })
|
||||
expect(service.external.listInstances()[0]).toMatchObject({ enabled: false, probeStatus: 'untested' })
|
||||
expect(service.database.externalStore.getInstance(saved.id)?.lastTestedAt).toBeUndefined()
|
||||
service.external.setEnabled(saved.id, true)
|
||||
const pending = service.external.saveBinding(bindingInput(saved.id))
|
||||
const rejected = expect(pending).rejects.toThrow()
|
||||
|
||||
+373
-111
@@ -1,154 +1,416 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { externalKnowledgeBindingSaveInputSchema, externalKnowledgeBindingTestInputSchema, externalKnowledgeBindingUpdateInputSchema, externalKnowledgeInstanceSaveInputSchema, type ExternalKnowledgeBinding, type ExternalKnowledgeBindingSaveInput, type ExternalKnowledgeBindingTestInput, type ExternalKnowledgeInstanceSaveInput, type ExternalKnowledgeInstanceSummary, type ExternalKnowledgeTestResult } from '../../../shared/external-knowledge-contracts'
|
||||
import { decryptSettingsCredential, encryptSettingsCredential, type SettingsCredentialCipher } from '../../settings-credential-cipher'
|
||||
import {
|
||||
externalKnowledgeBindingSaveInputSchema,
|
||||
externalKnowledgeBindingTestInputSchema,
|
||||
externalKnowledgeBindingUpdateInputSchema,
|
||||
externalKnowledgeInstanceSaveInputSchema,
|
||||
externalKnowledgeRetrievalInputSchema,
|
||||
type ExternalKnowledgeBinding,
|
||||
type ExternalKnowledgeBindingSaveInput,
|
||||
type ExternalKnowledgeBindingTestInput,
|
||||
type ExternalKnowledgeCommonConfig,
|
||||
type ExternalKnowledgeInstanceSaveInput,
|
||||
type ExternalKnowledgeInstanceSummary,
|
||||
type ExternalKnowledgeRemoteResult,
|
||||
type ExternalKnowledgeRetrievalInput,
|
||||
type ExternalKnowledgeTestResult
|
||||
} from '../../../shared/external-knowledge-contracts'
|
||||
import {
|
||||
decryptSettingsCredential,
|
||||
encryptSettingsCredential,
|
||||
type SettingsCredentialCipher
|
||||
} from '../../settings-credential-cipher'
|
||||
import type { KnowledgeDatabase } from '../knowledge-database'
|
||||
import { ExternalKnowledgeClient, ExternalKnowledgeError } from './external-knowledge-client'
|
||||
import type { StoredExternalInstance } from './external-knowledge-store'
|
||||
|
||||
// Probe results are persisted metadata, not a change to an in-flight request's configuration.
|
||||
function instanceConfiguration(value: StoredExternalInstance): string {
|
||||
return JSON.stringify([value.id, value.name, value.provider, value.baseUrl, value.enabled, value.credential])
|
||||
}
|
||||
const SHARED_SNIPPET_BUDGET = 48_000
|
||||
|
||||
export class ExternalKnowledgeService {
|
||||
private disposed = false
|
||||
private readonly controllers = new Map<AbortController,string>()
|
||||
constructor(private readonly database: KnowledgeDatabase, private readonly cipher?: SettingsCredentialCipher, private readonly fetcher?: typeof fetch) {}
|
||||
dispose(): void { this.disposed = true; for (const controller of this.controllers.keys()) controller.abort(); this.controllers.clear() }
|
||||
private checkActive(): void { if (this.disposed) throw new Error('EXTERNAL_KB_CANCELLED') }
|
||||
private cancelInstance(id: string): void {
|
||||
for (const [controller, instanceId] of this.controllers) if (instanceId === id) controller.abort()
|
||||
private readonly controllers = new Map<AbortController, string>()
|
||||
|
||||
constructor(
|
||||
private readonly database: KnowledgeDatabase,
|
||||
private readonly cipher?: SettingsCredentialCipher,
|
||||
private readonly fetcher?: typeof fetch
|
||||
) {}
|
||||
|
||||
dispose(): void {
|
||||
this.disposed = true
|
||||
for (const controller of this.controllers.keys()) {
|
||||
controller.abort()
|
||||
}
|
||||
this.controllers.clear()
|
||||
}
|
||||
private instance(id:string): StoredExternalInstance {
|
||||
|
||||
private checkActive(): void {
|
||||
if (this.disposed) {
|
||||
throw new Error('EXTERNAL_KB_CANCELLED')
|
||||
}
|
||||
}
|
||||
|
||||
private cancelInstance(id: string): void {
|
||||
for (const [controller, instanceId] of this.controllers) {
|
||||
if (instanceId === id) {
|
||||
controller.abort()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private instance(id: string): StoredExternalInstance {
|
||||
this.checkActive()
|
||||
const value=this.database.externalStore.getInstance(id)
|
||||
if (!value) throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
const value = this.database.externalStore.getInstance(id)
|
||||
if (!value) {
|
||||
throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
}
|
||||
return value
|
||||
}
|
||||
listInstances(): ExternalKnowledgeInstanceSummary[] {
|
||||
this.checkActive()
|
||||
const bindings=this.database.externalStore.listBindings()
|
||||
return this.database.externalStore.listInstances().map(({credential,...value})=>{
|
||||
let credentialStatus: ExternalKnowledgeInstanceSummary['credentialStatus'] = 'missing'
|
||||
if (credential) {
|
||||
try { this.credential({ ...value, credential }); credentialStatus = 'configured' }
|
||||
catch { credentialStatus = 'unavailable' }
|
||||
}
|
||||
return {...value,credentialStatus,bindingCount:bindings.filter(binding=>binding.instanceId===value.id).length}
|
||||
})
|
||||
}
|
||||
|
||||
private credential(instance: StoredExternalInstance): string {
|
||||
try {
|
||||
if (!instance.credential || !this.cipher?.isAvailable()) throw new Error()
|
||||
if (!instance.credential || !this.cipher?.isAvailable()) {
|
||||
throw new Error()
|
||||
}
|
||||
const key = decryptSettingsCredential(this.cipher, instance.credential)
|
||||
if (typeof key !== 'string' || !key.trim()) throw new Error()
|
||||
if (typeof key !== 'string' || !key.trim()) {
|
||||
throw new Error()
|
||||
}
|
||||
return key
|
||||
} catch { throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') }
|
||||
} catch {
|
||||
throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE')
|
||||
}
|
||||
}
|
||||
saveInstance(raw:ExternalKnowledgeInstanceSaveInput):ExternalKnowledgeInstanceSummary {
|
||||
|
||||
listInstances(): ExternalKnowledgeInstanceSummary[] {
|
||||
this.checkActive()
|
||||
const input=externalKnowledgeInstanceSaveInputSchema.parse(raw)
|
||||
const previous=input.id ? this.instance(input.id):undefined
|
||||
if (previous && previous.provider!==input.provider && this.database.externalStore.listBindings().some(binding=>binding.instanceId===previous.id)) throw new Error('EXTERNAL_KB_INSTANCE_IN_USE')
|
||||
let credential=previous?.credential
|
||||
if (input.credential.action==='clear') credential=undefined
|
||||
if (input.credential.action==='replace') {
|
||||
if (!this.cipher?.isAvailable()) throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE')
|
||||
try { credential=encryptSettingsCredential(this.cipher,input.credential.value) }
|
||||
catch { throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE') }
|
||||
const bindingCounts = new Map<string, number>()
|
||||
for (const binding of this.database.externalStore.listBindings()) {
|
||||
bindingCounts.set(binding.instanceId, (bindingCounts.get(binding.instanceId) ?? 0) + 1)
|
||||
}
|
||||
return this.database.externalStore.listInstances().map(({ credential, ...value }) => {
|
||||
let credentialStatus: ExternalKnowledgeInstanceSummary['credentialStatus'] = 'missing'
|
||||
if (credential) {
|
||||
try {
|
||||
this.credential({ ...value, credential })
|
||||
credentialStatus = 'configured'
|
||||
} catch {
|
||||
credentialStatus = 'unavailable'
|
||||
}
|
||||
}
|
||||
return { ...value, credentialStatus, bindingCount: bindingCounts.get(value.id) ?? 0 }
|
||||
})
|
||||
}
|
||||
|
||||
private summary(id: string): ExternalKnowledgeInstanceSummary {
|
||||
const summary = this.listInstances().find((item) => item.id === id)
|
||||
if (!summary) {
|
||||
throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
}
|
||||
return summary
|
||||
}
|
||||
|
||||
saveInstance(raw: ExternalKnowledgeInstanceSaveInput): ExternalKnowledgeInstanceSummary {
|
||||
this.checkActive()
|
||||
const input = externalKnowledgeInstanceSaveInputSchema.parse(raw)
|
||||
const previous = input.id ? this.instance(input.id) : undefined
|
||||
if (
|
||||
previous &&
|
||||
previous.provider !== input.provider &&
|
||||
this.database.externalStore.getBindingsForInstance(previous.id).length > 0
|
||||
) {
|
||||
throw new Error('EXTERNAL_KB_INSTANCE_IN_USE')
|
||||
}
|
||||
let credential = previous?.credential
|
||||
if (input.credential.action === 'clear') {
|
||||
credential = undefined
|
||||
}
|
||||
if (input.credential.action === 'replace') {
|
||||
if (!this.cipher?.isAvailable()) {
|
||||
throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE')
|
||||
}
|
||||
try {
|
||||
credential = encryptSettingsCredential(this.cipher, input.credential.value)
|
||||
} catch {
|
||||
throw new Error('EXTERNAL_KB_CREDENTIAL_UNAVAILABLE')
|
||||
}
|
||||
}
|
||||
const value: StoredExternalInstance = {
|
||||
id: previous?.id ?? randomUUID(),
|
||||
name: input.name,
|
||||
provider: input.provider,
|
||||
baseUrl: input.baseUrl.replace(/\/+$/, ''),
|
||||
enabled: input.enabled,
|
||||
credential,
|
||||
probeStatus: 'untested'
|
||||
}
|
||||
const value:StoredExternalInstance={id:previous?.id??randomUUID(),name:input.name,provider:input.provider,baseUrl:input.baseUrl.replace(/\/+$/,''),enabled:input.enabled,credential,probeStatus:'untested'}
|
||||
this.database.externalStore.saveInstance(value)
|
||||
// In-flight requests were authorized against the replaced configuration.
|
||||
this.cancelInstance(value.id)
|
||||
return this.listInstances().find(item=>item.id===value.id)!
|
||||
return this.summary(value.id)
|
||||
}
|
||||
setEnabled(id:string,enabled:boolean):ExternalKnowledgeInstanceSummary {
|
||||
const instance=this.instance(id)
|
||||
this.database.externalStore.saveInstance({...instance,enabled})
|
||||
if (!enabled) for(const [controller,instanceId] of this.controllers) if(instanceId===id) controller.abort()
|
||||
return this.listInstances().find(item=>item.id===id)!
|
||||
|
||||
setEnabled(id: string, enabled: boolean): ExternalKnowledgeInstanceSummary {
|
||||
const instance = this.instance(id)
|
||||
this.database.externalStore.saveInstance({ ...instance, enabled })
|
||||
if (!enabled) {
|
||||
this.cancelInstance(id)
|
||||
}
|
||||
return this.summary(id)
|
||||
}
|
||||
deleteInstance(id:string):void {
|
||||
|
||||
deleteInstance(id: string): void {
|
||||
this.instance(id)
|
||||
if(this.database.externalStore.listBindings().some(binding=>binding.instanceId===id)) throw new Error('EXTERNAL_KB_INSTANCE_IN_USE')
|
||||
for(const [controller,instanceId] of this.controllers) if(instanceId===id) controller.abort()
|
||||
if (this.database.externalStore.getBindingsForInstance(id).length > 0) {
|
||||
throw new Error('EXTERNAL_KB_INSTANCE_IN_USE')
|
||||
}
|
||||
this.cancelInstance(id)
|
||||
this.database.externalStore.deleteInstance(id)
|
||||
}
|
||||
private async request<T>(id:string,timeout:number,signal:AbortSignal|undefined,operation:(client:ExternalKnowledgeClient,signal:AbortSignal)=>Promise<T>):Promise<T> {
|
||||
const instance=this.instance(id)
|
||||
if(!instance.enabled) throw new Error('EXTERNAL_KB_DISABLED')
|
||||
|
||||
private async request<T>(
|
||||
id: string,
|
||||
timeout: number,
|
||||
signal: AbortSignal | undefined,
|
||||
operation: (client: ExternalKnowledgeClient, signal: AbortSignal) => Promise<T>
|
||||
): Promise<T> {
|
||||
const instance = this.instance(id)
|
||||
if (!instance.enabled) {
|
||||
throw new Error('EXTERNAL_KB_DISABLED')
|
||||
}
|
||||
const key = this.credential(instance)
|
||||
const controller=new AbortController()
|
||||
this.controllers.set(controller,id)
|
||||
const controller = new AbortController()
|
||||
this.controllers.set(controller, id)
|
||||
const timeoutSignal = AbortSignal.timeout(timeout)
|
||||
const combined = AbortSignal.any([controller.signal, timeoutSignal, ...(signal ? [signal] : [])])
|
||||
const combined = AbortSignal.any([
|
||||
controller.signal,
|
||||
timeoutSignal,
|
||||
...(signal ? [signal] : [])
|
||||
])
|
||||
try {
|
||||
combined.throwIfAborted()
|
||||
const result = await operation(new ExternalKnowledgeClient({provider:instance.provider,baseUrl:instance.baseUrl,apiKey:key,timeoutMs:timeout,fetcher:this.fetcher}),combined)
|
||||
combined.throwIfAborted()
|
||||
return result
|
||||
return await operation(
|
||||
new ExternalKnowledgeClient({
|
||||
provider: instance.provider,
|
||||
baseUrl: instance.baseUrl,
|
||||
apiKey: key,
|
||||
timeoutMs: timeout,
|
||||
fetcher: this.fetcher
|
||||
}),
|
||||
combined
|
||||
)
|
||||
} catch (error) {
|
||||
if (combined.aborted) {
|
||||
const timedOut = timeoutSignal.aborted && combined.reason === timeoutSignal.reason
|
||||
throw new ExternalKnowledgeError(timedOut ? 'EXTERNAL_KB_TIMEOUT' : 'EXTERNAL_KB_CANCELLED',
|
||||
timedOut ? 'External knowledge request timed out' : 'External knowledge request was cancelled')
|
||||
throw new ExternalKnowledgeError(
|
||||
timedOut ? 'EXTERNAL_KB_TIMEOUT' : 'EXTERNAL_KB_CANCELLED',
|
||||
timedOut
|
||||
? 'External knowledge request timed out'
|
||||
: 'External knowledge request was cancelled'
|
||||
)
|
||||
}
|
||||
throw error
|
||||
} finally {this.controllers.delete(controller)}
|
||||
}
|
||||
async testInstance(id:string):Promise<ExternalKnowledgeInstanceSummary> {
|
||||
const instance=this.instance(id)
|
||||
let probeStatus:StoredExternalInstance['probeStatus']='catalog-ready'
|
||||
let lastErrorCode:string|undefined
|
||||
try {await this.listCatalog({instanceId:id})} catch(error) {
|
||||
lastErrorCode=error instanceof ExternalKnowledgeError ? error.code : error instanceof Error && /^EXTERNAL_KB_/.test(error.message) ? error.message : 'EXTERNAL_KB_NETWORK'
|
||||
probeStatus=lastErrorCode==='EXTERNAL_KB_FORBIDDEN'?'list-restricted':lastErrorCode==='EXTERNAL_KB_AUTH'?'auth-failed':lastErrorCode==='EXTERNAL_KB_NETWORK'||lastErrorCode==='EXTERNAL_KB_TIMEOUT'?'unreachable':'failed'
|
||||
} finally {
|
||||
this.controllers.delete(controller)
|
||||
}
|
||||
const current = this.instance(id)
|
||||
if (instanceConfiguration(current) !== instanceConfiguration(instance)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
this.database.externalStore.saveInstance({...current,probeStatus,lastErrorCode,lastTestedAt:new Date().toISOString()})
|
||||
return this.listInstances().find(item=>item.id===id)!
|
||||
}
|
||||
listCatalog(input:{instanceId:string;page?:number;pageSize?:number;parentId?:string|null;search?:string},signal?:AbortSignal) {
|
||||
return this.request(input.instanceId,15000,signal,(client,current)=>client.listKnowledgeBases(current,input))
|
||||
}
|
||||
getCatalog(input:{instanceId:string;remoteKnowledgeBaseId:string},signal?:AbortSignal) {
|
||||
return this.request(input.instanceId,15000,signal,(client,current)=>client.getKnowledgeBase(input.remoteKnowledgeBaseId,current))
|
||||
}
|
||||
async testRetrieval(raw:ExternalKnowledgeBindingTestInput,signal?:AbortSignal):Promise<ExternalKnowledgeTestResult> {
|
||||
const input=externalKnowledgeBindingTestInputSchema.parse(raw)
|
||||
const instance=this.instance(input.instanceId)
|
||||
if(instance.provider!==input.providerConfig.provider) throw new Error('EXTERNAL_KB_CONFIG_INVALID')
|
||||
const started=Date.now()
|
||||
const results=await this.request(input.instanceId,input.commonConfig.requestTimeoutMs,signal,async (client,current)=>{
|
||||
if(input.providerConfig.provider==='ragflow' && (input.providerConfig.useKg||input.providerConfig.includeKnowledgeCompilation)) {
|
||||
const detail=await client.getKnowledgeBase(input.remoteKnowledgeBaseId,current)
|
||||
if((input.providerConfig.useKg&&!detail.graphEnabled)||(input.providerConfig.includeKnowledgeCompilation&&!detail.knowledgeCompilationEnabled)) throw new Error('EXTERNAL_KB_CAPABILITY_UNAVAILABLE')
|
||||
|
||||
async testInstance(id: string): Promise<ExternalKnowledgeInstanceSummary> {
|
||||
this.instance(id)
|
||||
let probeStatus: StoredExternalInstance['probeStatus'] = 'catalog-ready'
|
||||
let lastErrorCode: string | undefined
|
||||
try {
|
||||
await this.listCatalog({ instanceId: id })
|
||||
} catch (error) {
|
||||
lastErrorCode =
|
||||
error instanceof ExternalKnowledgeError
|
||||
? error.code
|
||||
: error instanceof Error && /^EXTERNAL_KB_/.test(error.message)
|
||||
? error.message
|
||||
: 'EXTERNAL_KB_NETWORK'
|
||||
// Cancellation means the configuration changed under the probe, so it
|
||||
// learned nothing about reachability and must not overwrite the record.
|
||||
if (lastErrorCode === 'EXTERNAL_KB_CANCELLED') {
|
||||
throw error
|
||||
}
|
||||
return client.retrieve(input.remoteKnowledgeBaseId,input.testQuery,input.providerConfig,current)
|
||||
probeStatus =
|
||||
lastErrorCode === 'EXTERNAL_KB_FORBIDDEN'
|
||||
? 'list-restricted'
|
||||
: lastErrorCode === 'EXTERNAL_KB_AUTH'
|
||||
? 'auth-failed'
|
||||
: lastErrorCode === 'EXTERNAL_KB_NETWORK' || lastErrorCode === 'EXTERNAL_KB_TIMEOUT'
|
||||
? 'unreachable'
|
||||
: 'failed'
|
||||
}
|
||||
const current = this.database.externalStore.getInstance(id)
|
||||
if (!current) {
|
||||
throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
}
|
||||
this.database.externalStore.saveInstance({
|
||||
...current,
|
||||
probeStatus,
|
||||
lastErrorCode,
|
||||
lastTestedAt: new Date().toISOString()
|
||||
})
|
||||
if (instanceConfiguration(instance) !== instanceConfiguration(this.instance(input.instanceId))) throw new Error('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
let remaining = 48_000
|
||||
const bounded = results.slice(0, input.commonConfig.resultLimit).map(item => {
|
||||
const snippet = item.snippet.slice(0, Math.min(remaining, input.commonConfig.maxSnippetCharacters))
|
||||
remaining -= snippet.length
|
||||
return { ...item, snippet }
|
||||
}).filter(item => item.snippet.length > 0)
|
||||
return {results:bounded,durationMs:Date.now()-started}
|
||||
return this.summary(id)
|
||||
}
|
||||
async saveBinding(raw:ExternalKnowledgeBindingSaveInput & {knowledgeBaseId?:string}):Promise<ExternalKnowledgeBinding> {
|
||||
|
||||
listCatalog(
|
||||
input: {
|
||||
instanceId: string
|
||||
page?: number
|
||||
pageSize?: number
|
||||
parentId?: string | null
|
||||
search?: string
|
||||
},
|
||||
signal?: AbortSignal
|
||||
) {
|
||||
return this.request(input.instanceId, 15_000, signal, (client, current) =>
|
||||
client.listKnowledgeBases(current, input)
|
||||
)
|
||||
}
|
||||
|
||||
getCatalog(input: { instanceId: string; remoteKnowledgeBaseId: string }, signal?: AbortSignal) {
|
||||
return this.request(input.instanceId, 15_000, signal, (client, current) =>
|
||||
client.getKnowledgeBase(input.remoteKnowledgeBaseId, current)
|
||||
)
|
||||
}
|
||||
|
||||
private boundResults(
|
||||
results: readonly ExternalKnowledgeRemoteResult[],
|
||||
common: ExternalKnowledgeCommonConfig
|
||||
): ExternalKnowledgeRemoteResult[] {
|
||||
let remaining = SHARED_SNIPPET_BUDGET
|
||||
return results
|
||||
.slice(0, common.resultLimit)
|
||||
.map((item) => {
|
||||
const snippet = item.snippet.slice(
|
||||
0,
|
||||
Math.min(remaining, common.maxSnippetCharacters)
|
||||
)
|
||||
remaining -= snippet.length
|
||||
return { ...item, snippet }
|
||||
})
|
||||
.filter((item) => item.snippet.length > 0)
|
||||
}
|
||||
|
||||
/** Retrieves from the bound remote knowledge base. This is the production path. */
|
||||
async retrieve(
|
||||
raw: ExternalKnowledgeRetrievalInput,
|
||||
signal?: AbortSignal
|
||||
): Promise<ExternalKnowledgeTestResult> {
|
||||
const input = externalKnowledgeRetrievalInputSchema.parse(raw)
|
||||
if (this.instance(input.instanceId).provider !== input.providerConfig.provider) {
|
||||
throw new Error('EXTERNAL_KB_CONFIG_INVALID')
|
||||
}
|
||||
const started = Date.now()
|
||||
const results = await this.request(
|
||||
input.instanceId,
|
||||
input.commonConfig.requestTimeoutMs,
|
||||
signal,
|
||||
(client, current) =>
|
||||
client.retrieve(input.remoteKnowledgeBaseId, input.query, input.providerConfig, current)
|
||||
)
|
||||
return {
|
||||
results: this.boundResults(results, input.commonConfig),
|
||||
durationMs: Date.now() - started
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies a binding before it is saved. Unlike retrieve, this also confirms
|
||||
* that the remote deployment provides the requested capabilities. The check
|
||||
* shares the configured request timeout so the whole verification stays
|
||||
* within the budget the user chose.
|
||||
*/
|
||||
async testRetrieval(
|
||||
raw: ExternalKnowledgeBindingTestInput,
|
||||
signal?: AbortSignal
|
||||
): Promise<ExternalKnowledgeTestResult> {
|
||||
const input = externalKnowledgeBindingTestInputSchema.parse(raw)
|
||||
const config = input.providerConfig
|
||||
if (this.instance(input.instanceId).provider !== config.provider) {
|
||||
throw new Error('EXTERNAL_KB_CONFIG_INVALID')
|
||||
}
|
||||
const started = Date.now()
|
||||
const results = await this.request(
|
||||
input.instanceId,
|
||||
input.commonConfig.requestTimeoutMs,
|
||||
signal,
|
||||
async (client, current) => {
|
||||
if (config.provider === 'ragflow' && (config.useKg || config.includeKnowledgeCompilation)) {
|
||||
const detail = await client.getKnowledgeBase(input.remoteKnowledgeBaseId, current)
|
||||
if (
|
||||
(config.useKg && !detail.graphEnabled) ||
|
||||
(config.includeKnowledgeCompilation && !detail.knowledgeCompilationEnabled)
|
||||
) {
|
||||
throw new Error('EXTERNAL_KB_CAPABILITY_UNAVAILABLE')
|
||||
}
|
||||
}
|
||||
return client.retrieve(input.remoteKnowledgeBaseId, input.testQuery, config, current)
|
||||
}
|
||||
)
|
||||
return {
|
||||
results: this.boundResults(results, input.commonConfig),
|
||||
durationMs: Date.now() - started
|
||||
}
|
||||
}
|
||||
|
||||
async saveBinding(
|
||||
raw: ExternalKnowledgeBindingSaveInput & { knowledgeBaseId?: string }
|
||||
): Promise<ExternalKnowledgeBinding> {
|
||||
this.checkActive()
|
||||
const input=raw.knowledgeBaseId !== undefined ? externalKnowledgeBindingUpdateInputSchema.parse(raw):externalKnowledgeBindingSaveInputSchema.parse(raw)
|
||||
const knowledgeBaseId = 'knowledgeBaseId' in input ? input.knowledgeBaseId : undefined
|
||||
const existing=knowledgeBaseId ? this.database.externalStore.listBindings().find(item=>item.knowledgeBaseId===knowledgeBaseId):undefined
|
||||
if(knowledgeBaseId&&!existing) throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
if(this.database.externalStore.listBindings().some(item=>item.instanceId===input.instanceId&&item.remoteKnowledgeBaseId===input.remoteKnowledgeBaseId&&item.knowledgeBaseId!==existing?.knowledgeBaseId)) throw new Error('EXTERNAL_KB_DUPLICATE_BINDING')
|
||||
const instance = this.instance(input.instanceId)
|
||||
await this.testRetrieval({instanceId:input.instanceId,remoteKnowledgeBaseId:input.remoteKnowledgeBaseId,commonConfig:input.commonConfig,providerConfig:input.providerConfig,testQuery:input.testQuery})
|
||||
if (instanceConfiguration(instance) !== instanceConfiguration(this.instance(input.instanceId))) throw new Error('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
const bindings = this.database.externalStore.listBindings()
|
||||
if (existing && JSON.stringify(bindings.find(item => item.knowledgeBaseId === existing.knowledgeBaseId)) !== JSON.stringify(existing)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
if (bindings.some(item => item.instanceId === input.instanceId && item.remoteKnowledgeBaseId === input.remoteKnowledgeBaseId && item.knowledgeBaseId !== existing?.knowledgeBaseId)) throw new Error('EXTERNAL_KB_DUPLICATE_BINDING')
|
||||
return this.database.saveExternalBinding({instanceId:input.instanceId,provider:input.providerConfig.provider,remoteKnowledgeBaseId:input.remoteKnowledgeBaseId,remoteName:input.remoteName,commonConfig:input.commonConfig,providerConfig:input.providerConfig,lastVerifiedAt:new Date().toISOString()}, {name:input.name,description:input.description,knowledgeBaseId:existing?.knowledgeBaseId})
|
||||
const update =
|
||||
raw.knowledgeBaseId !== undefined
|
||||
? externalKnowledgeBindingUpdateInputSchema.parse(raw)
|
||||
: undefined
|
||||
const input = update ?? externalKnowledgeBindingSaveInputSchema.parse(raw)
|
||||
const existing = update
|
||||
? this.database.externalStore.getBinding(update.knowledgeBaseId)
|
||||
: undefined
|
||||
if (update && !existing) {
|
||||
throw new Error('EXTERNAL_KB_NOT_FOUND')
|
||||
}
|
||||
this.instance(input.instanceId)
|
||||
const duplicate = (): boolean =>
|
||||
this.database.externalStore
|
||||
.listBindings()
|
||||
.some(
|
||||
(item) =>
|
||||
item.instanceId === input.instanceId &&
|
||||
item.remoteKnowledgeBaseId === input.remoteKnowledgeBaseId &&
|
||||
item.knowledgeBaseId !== existing?.knowledgeBaseId
|
||||
)
|
||||
if (duplicate()) {
|
||||
throw new Error('EXTERNAL_KB_DUPLICATE_BINDING')
|
||||
}
|
||||
await this.testRetrieval({
|
||||
instanceId: input.instanceId,
|
||||
remoteKnowledgeBaseId: input.remoteKnowledgeBaseId,
|
||||
commonConfig: input.commonConfig,
|
||||
providerConfig: input.providerConfig,
|
||||
testQuery: input.testQuery
|
||||
})
|
||||
// Another binding for the same remote target may have landed while the
|
||||
// verification request was in flight; the unique index is the final guard.
|
||||
if (duplicate()) {
|
||||
throw new Error('EXTERNAL_KB_DUPLICATE_BINDING')
|
||||
}
|
||||
return this.database.saveExternalBinding(
|
||||
{
|
||||
instanceId: input.instanceId,
|
||||
provider: input.providerConfig.provider,
|
||||
remoteKnowledgeBaseId: input.remoteKnowledgeBaseId,
|
||||
remoteName: input.remoteName,
|
||||
commonConfig: input.commonConfig,
|
||||
providerConfig: input.providerConfig,
|
||||
lastVerifiedAt: new Date().toISOString()
|
||||
},
|
||||
{
|
||||
name: input.name,
|
||||
description: input.description,
|
||||
knowledgeBaseId: existing?.knowledgeBaseId
|
||||
}
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1247,15 +1247,14 @@ export class KnowledgeService {
|
||||
const input = knowledgeRetrieveInputSchema.parse(rawInput)
|
||||
const library = this.requireLibrary(input.knowledgeBaseId, false)
|
||||
const settings = input.settings ?? library.retrievalSettings
|
||||
const binding = this.database.externalStore.listBindings().find(item => item.knowledgeBaseId === library.id)
|
||||
const binding = this.database.externalStore.getBinding(library.id)
|
||||
if (binding) {
|
||||
const response = await this.external.testRetrieval({instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,commonConfig:binding.commonConfig,providerConfig:binding.providerConfig,testQuery:input.query}, signal)
|
||||
const response = await this.external.retrieve({instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,commonConfig:binding.commonConfig,providerConfig:binding.providerConfig,query:input.query}, signal)
|
||||
signal.throwIfAborted()
|
||||
if (JSON.stringify(this.database.externalStore.listBindings().find(item => item.knowledgeBaseId === library.id)) !== JSON.stringify(binding)) throw new Error('EXTERNAL_KB_CONFIG_CHANGED')
|
||||
return {
|
||||
query:input.query,durationMs:response.durationMs,settings,
|
||||
diagnostics:{external:{provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId},requestedChannels:[],usedChannels:[],degradedChannels:[],candidateCounts:{},channelDurationMs:{},vectorScannedCount:0,filteredByThresholdCount:0,filteredByBudgetCount:0,rerank:{requested:'none',used:'none',status:'skipped',candidateCount:0,durationMs:0}},
|
||||
results:response.results.map((item,index)=>({knowledgeBaseId:library.id,documentTitle:item.documentTitle,sourceDisplayName:item.sourceDisplayName,snippet:item.snippet,rank:index+1,channels:[],scores:{fusedScore:0},external:{kind:'external' as const,provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,remoteDocumentId:item.remoteDocumentId,remoteChunkId:item.remoteChunkId,providerScore:item.providerScore,providerScores:item.providerScores,location:item.location,sourceUrl:item.sourceUrl}})),
|
||||
results:response.results.map((item,index)=>({knowledgeBaseId:library.id,documentTitle:item.documentTitle,sourceDisplayName:item.sourceDisplayName,snippet:item.snippet,rank:index+1,channels:[],scores:{fusedScore:0},external:{kind:'external' as const,provider:binding.provider,instanceId:binding.instanceId,remoteKnowledgeBaseId:binding.remoteKnowledgeBaseId,remoteDocumentId:item.remoteDocumentId,remoteChunkId:item.remoteChunkId,providerScore:item.providerScore,providerScores:item.providerScores,location:item.location}})),
|
||||
context:{characterCount:response.results.reduce((sum,item)=>sum+item.snippet.length,0),truncated:false,groups:[]}
|
||||
}
|
||||
}
|
||||
@@ -1693,7 +1692,7 @@ export class KnowledgeService {
|
||||
this.requireLibrary(id, false)
|
||||
)
|
||||
const preparedEmbedding = libraries.some(
|
||||
(library) => !this.database.externalStore.listBindings().some(item=>item.knowledgeBaseId===library.id) && library.retrievalSettings.vectorWeight > 0
|
||||
(library) => !this.database.externalStore.hasBinding(library.id) && library.retrievalSettings.vectorWeight > 0
|
||||
)
|
||||
? await this.prepareQueryEmbedding(query, signal)
|
||||
: undefined
|
||||
@@ -1869,45 +1868,6 @@ export class KnowledgeService {
|
||||
})
|
||||
}
|
||||
|
||||
async searchHybridMany(
|
||||
knowledgeBaseIds: readonly string[],
|
||||
query: string,
|
||||
limitPerLibrary = 6,
|
||||
signal?: AbortSignal
|
||||
): Promise<
|
||||
Array<{ knowledgeBaseId: string; result: HybridSearchResult }>
|
||||
> {
|
||||
const vector = await this.embedQuery(query, signal)
|
||||
return knowledgeBaseIds.flatMap((knowledgeBaseId) => {
|
||||
const library = this.requireLibrary(knowledgeBaseId)
|
||||
const settings = library.retrievalSettings
|
||||
return this.database
|
||||
.hybridSearch({
|
||||
knowledgeBaseId,
|
||||
query,
|
||||
limit: limitPerLibrary,
|
||||
provider:
|
||||
vector && this.embeddingProvider
|
||||
? embeddingStorageProvider(this.embeddingProvider)
|
||||
: undefined,
|
||||
model: vector ? this.embeddingProvider?.model : undefined,
|
||||
vector,
|
||||
graphEnabled: library.graphEnabled,
|
||||
minimumVectorSimilarity: settings.minimumVectorSimilarity,
|
||||
candidateMultiplier: settings.candidateMultiplier,
|
||||
ftsWeight: settings.ftsWeight,
|
||||
vectorWeight: settings.vectorWeight,
|
||||
graphWeight: settings.graphWeight,
|
||||
signal
|
||||
})
|
||||
.map((result) => ({ knowledgeBaseId, result }))
|
||||
}).sort(
|
||||
(left, right) =>
|
||||
right.result.retrieval.score - left.result.retrieval.score ||
|
||||
left.result.chunk.id.localeCompare(right.result.chunk.id)
|
||||
).slice(0, limitPerLibrary)
|
||||
}
|
||||
|
||||
private async embedQuery(
|
||||
query: string,
|
||||
signal?: AbortSignal
|
||||
@@ -3863,7 +3823,7 @@ export class KnowledgeService {
|
||||
}
|
||||
|
||||
private requireLibrary(id: string, localOnly = true): KnowledgeBase {
|
||||
if (localOnly && this.database.externalStore.listBindings().some(item => item.knowledgeBaseId === id)) {
|
||||
if (localOnly && this.database.externalStore.hasBinding(id)) {
|
||||
throw new Error('EXTERNAL_KB_READ_ONLY')
|
||||
}
|
||||
const library = this.database.getKnowledgeBase(id)
|
||||
|
||||
@@ -37,15 +37,17 @@ export const externalKnowledgeInstanceInputSchema = z.object({instanceId:id}).st
|
||||
export const externalKnowledgeInstanceEnabledInputSchema = externalKnowledgeInstanceInputSchema.extend({enabled:z.boolean()})
|
||||
export const externalKnowledgeCatalogListInputSchema = externalKnowledgeInstanceInputSchema.extend({page:z.number().int().min(1).max(500).optional(),pageSize:z.number().int().min(1).max(100).optional(),parentId:id.nullable().optional(),search:z.string().trim().max(512).optional()})
|
||||
export const externalKnowledgeCatalogGetInputSchema = externalKnowledgeInstanceInputSchema.extend({remoteKnowledgeBaseId:id})
|
||||
export const externalKnowledgeRetrievalInputSchema = externalKnowledgeCatalogGetInputSchema.extend({commonConfig:externalKnowledgeCommonConfigSchema,providerConfig:externalKnowledgeProviderConfigSchema,query:z.string().trim().min(1).max(4000)})
|
||||
export type ExternalKnowledgeRetrievalInput = z.infer<typeof externalKnowledgeRetrievalInputSchema>
|
||||
export const externalKnowledgeBindingTestInputSchema = externalKnowledgeCatalogGetInputSchema.extend({commonConfig:externalKnowledgeCommonConfigSchema,providerConfig:externalKnowledgeProviderConfigSchema,testQuery:z.string().trim().min(1).max(4000)})
|
||||
export const externalKnowledgeBindingSaveInputSchema = externalKnowledgeBindingTestInputSchema.extend({name:z.string().trim().min(1).max(512),description:z.string().max(4000).optional(),remoteName:z.string().trim().min(1).max(512)})
|
||||
export const externalKnowledgeBindingUpdateInputSchema = externalKnowledgeBindingSaveInputSchema.extend({knowledgeBaseId:id})
|
||||
export type ExternalKnowledgeBindingTestInput = z.infer<typeof externalKnowledgeBindingTestInputSchema>
|
||||
export type ExternalKnowledgeBindingSaveInput = z.infer<typeof externalKnowledgeBindingSaveInputSchema>
|
||||
export type ExternalKnowledgeInstanceSummary = {id:string;name:string;provider:ExternalKnowledgeProvider;baseUrl:string;enabled:boolean;credentialStatus:'configured'|'missing'|'unavailable';probeStatus:'untested'|'catalog-ready'|'healthy'|'list-restricted'|'auth-failed'|'unreachable'|'failed';bindingCount:number;lastTestedAt?:string;lastErrorCode?:string}
|
||||
export type ExternalKnowledgeInstanceSummary = {id:string;name:string;provider:ExternalKnowledgeProvider;baseUrl:string;enabled:boolean;credentialStatus:'configured'|'missing'|'unavailable';probeStatus:'untested'|'catalog-ready'|'list-restricted'|'auth-failed'|'unreachable'|'failed';bindingCount:number;lastTestedAt?:string;lastErrorCode?:string}
|
||||
export type ExternalKnowledgeCatalogItem = {id:string;name:string;description?:string;kind?:'dataset'|'folder';graphEnabled?:boolean;knowledgeCompilationEnabled?:boolean}
|
||||
export type ExternalKnowledgeCatalogPage = {items:ExternalKnowledgeCatalogItem[];total?:number;hasMore:boolean}
|
||||
export type ExternalKnowledgeRemoteResult = {documentTitle:string;sourceDisplayName:string;snippet:string;providerScore?:number;providerScores?:{type:string;value:number;index?:number}[];remoteDocumentId?:string;remoteChunkId?:string;location?:string;sourceUrl?:string}
|
||||
export type ExternalKnowledgeRemoteResult = {documentTitle:string;sourceDisplayName:string;snippet:string;providerScore?:number;providerScores?:{type:string;value:number;index?:number}[];remoteDocumentId?:string;remoteChunkId?:string;location?:string}
|
||||
export type ExternalKnowledgeTestResult = {results:ExternalKnowledgeRemoteResult[];durationMs:number}
|
||||
export type ExternalKnowledgeBinding = {knowledgeBaseId:string;instanceId:string;provider:ExternalKnowledgeProvider;remoteKnowledgeBaseId:string;remoteName:string;commonConfig:ExternalKnowledgeCommonConfig;providerConfig:ExternalKnowledgeProviderConfig;lastVerifiedAt:string}
|
||||
export const externalKnowledgeLocatorSchema = z.object({kind:z.literal('external'),provider:externalKnowledgeProviderSchema,instanceId:id,remoteKnowledgeBaseId:id,remoteDocumentId:id.optional(),remoteChunkId:id.optional(),providerScore:z.number().finite().optional(),providerScores:z.array(z.object({type:z.string().max(128),value:z.number().finite(),index:z.number().int().optional()}).strict()).max(20).optional(),location:z.string().max(8192).optional(),sourceUrl:z.string().max(8192).optional()}).strict()
|
||||
|
||||
Reference in New Issue
Block a user