fix: distinguish external knowledge probe failures from empty results

Validate Dify and RAGFlow outcomes, normalize FastGPT URLs, and report response envelopes, score structures and actual rerank state. Align integration documentation with official APIs and archive 33 redacted read-only requests with strict HTTPS verification.

Validation: 43 focused tests, typecheck and lint passed. Full test runs exceeded 60s and 240s without a summary; this limitation is recorded in the feature progress document.

tests/external-knowledge-probe-fastgpt.test.ts

@@ -20,7 +20,7 @@ type ProviderReport = {
     count?: number
     contentPresent?: boolean
   }>
-  features: { searchModes: string[]; rerankProbed: boolean; rerankSucceeded: boolean }
+  features: { searchModes: string[]; rerankProbed: boolean; rerankRequestSucceeded: boolean }
   operations: Array<{ name: string; status: number | 'failed' }>
 }
 
@@ -40,8 +40,10 @@ async function probe(
     globalThis.fetch = async (url, init) => {
       const path = new URL(url).pathname;
       let fixture;
-      if (path.endsWith('/list')) fixture = catalog;
-      else if (path.endsWith('/detail')) fixture = {body: {code: 200, data: {}}};
+      if (path === '/prefix/api/core/dataset/list') {
+        if (JSON.parse(init.body).parentId !== null) throw new Error('Invalid root parent');
+        fixture = catalog;
+      } else if (path.endsWith('/detail')) fixture = {body: {code: 200, data: {}}};
       else if (path.endsWith('/searchTest')) {
         const input = JSON.parse(init.body);
         fixture = fixtures[input.usingReRank ? 'rerank' : input.searchMode];
@@ -60,7 +62,7 @@ async function probe(
     {
       env: {
         ...process.env,
-        GOODBUDDY_FASTGPT_BASE_URL: 'http://fixture.invalid',
+        GOODBUDDY_FASTGPT_BASE_URL: 'http://fixture.invalid/prefix/api/core/dataset/',
         GOODBUDDY_FASTGPT_API_KEY: 'fixture-only-key'
       },
       timeout: 10_000
@@ -88,7 +90,7 @@ describe('FastGPT probe success evidence', () => {
       embedding: fixture, fullTextRecall: fixture, mixedRecall: fixture, rerank: fixture
     })
     expect(report.features).toEqual({
-      searchModes: [], rerankProbed: true, rerankSucceeded: false,
+      searchModes: [], rerankProbed: true, rerankRequestSucceeded: false,
       extensionQueryProbed: false
     })
     expect(report.retrievals).toHaveLength(4)
@@ -108,7 +110,7 @@ describe('FastGPT probe success evidence', () => {
     })
     expect(report.features).toMatchObject({
       searchModes: ['embedding', 'fullTextRecall'],
-      rerankProbed: true, rerankSucceeded: true
+      rerankProbed: true, rerankRequestSucceeded: true
     })
     expect(report.retrievals[0]).toMatchObject({ success: true, count: 0, contentPresent: false })
     expect(report.retrievals[1]).toMatchObject({ success: true, count: 0 })
@@ -126,8 +128,31 @@ describe('FastGPT probe success evidence', () => {
     expect(report.catalog).not.toHaveProperty('count')
     expect(report.retrievals).toEqual([])
     expect(report.features).toMatchObject({
-      searchModes: [], rerankProbed: false, rerankSucceeded: false
+      searchModes: [], rerankProbed: false, rerankRequestSucceeded: false
     })
     expect(report.operations).toHaveLength(1)
   })
+
+  it('reports both retrieval envelopes, score structures and actual rerank response separately', async () => {
+    const report = await probe({
+      embedding: { body: { code: 200, data: [{ score: 0.6, q: 'private text' }] } },
+      fullTextRecall: { body: { code: 200, data: { list: [], usingReRank: true } } },
+      mixedRecall: success(),
+      rerank: { body: { code: 200, data: {
+        usingReRank: false,
+        list: [{ score: [{ type: 'private score type', value: 0.2, index: 1 }], q: 'private text' }]
+      } } }
+    })
+    expect(report.retrievals[0]).toMatchObject({
+      envelope: 'data-array', scoreTypes: ['number'], scoreRange: { minimum: 0.6, maximum: 0.6 }
+    })
+    expect(report.retrievals[3]).toMatchObject({
+      envelope: 'data-list', scoreTypes: ['array'], scoreItemKeys: ['index', 'type', 'value'],
+      scoreValueRange: { minimum: 0.2, maximum: 0.2 }, rerankEnabled: false
+    })
+    expect(report.retrievals[1]).toMatchObject({ rerankEnabled: true })
+    expect(report.retrievals[0]).not.toHaveProperty('rerankEnabled')
+    expect(report.features.rerankRequestSucceeded).toBe(true)
+    expect(JSON.stringify(report)).not.toContain('private')
+  })
 })

tests/external-knowledge-probe-outcomes.test.ts

@@ -0,0 +1,127 @@
+import { execFile } from 'node:child_process'
+import { resolve } from 'node:path'
+import { pathToFileURL } from 'node:url'
+import { promisify } from 'node:util'
+import { describe, expect, it } from 'vitest'
+
+type Fixture = { status?: number; body?: unknown; raw?: string; causeCode?: string }
+type Outcome = { success: boolean; failure?: string; count?: number }
+type Report = {
+  catalog: Outcome
+  detail: Outcome
+  retrievals: Outcome[]
+  features: Record<string, unknown>
+  operations: Array<{ status: number | 'failed'; businessCode?: number | string; causeCode?: string }>
+}
+
+async function probe(provider: 'dify' | 'ragflow', fixtures: Fixture[]): Promise<Report> {
+  const script = `
+    const fixtures = ${JSON.stringify(fixtures)};
+    globalThis.fetch = async () => {
+      const fixture = fixtures.shift();
+      if (!fixture) throw new Error('Unexpected request');
+      if (fixture.causeCode) throw new TypeError('private network message', {
+        cause: { code: fixture.causeCode, message: 'private cause' }
+      });
+      return new Response(fixture.raw ?? JSON.stringify(fixture.body), { status: fixture.status ?? 200 });
+    };
+    process.argv = [process.execPath, 'probe', '--provider=${provider}', '--extended'];
+    await import(${JSON.stringify(pathToFileURL(resolve('scripts/external-knowledge-probe.mjs')).href)});
+  `
+  const { stdout } = await promisify(execFile)(process.execPath, ['--input-type=module', '--eval', script], {
+    env: {
+      ...process.env,
+      [`GOODBUDDY_${provider.toUpperCase()}_BASE_URL`]: 'https://fixture.invalid',
+      [`GOODBUDDY_${provider.toUpperCase()}_API_KEY`]: 'fixture-only-key'
+    },
+    timeout: 10_000
+  })
+  expect(stdout).not.toMatch(/private|fixture.invalid|fixture-only-key|fixture-dataset/)
+  return JSON.parse(stdout).providers[0] as Report
+}
+
+describe.each(['dify', 'ragflow'] as const)('%s probe outcomes', (provider) => {
+  const catalog = { body: { code: 0, data: [{ id: 'fixture-dataset', retrieval_model_dict: {} }] } }
+  const detail = { body: provider === 'dify' ? { id: 'fixture-dataset' } : catalog.body }
+  const empty = { body: provider === 'dify' ? { records: [] } : { code: 0, data: { chunks: [] } } }
+
+  it('reports successful empty catalogs without claiming uncalled features', async () => {
+    const report = await probe(provider, [{ body: { code: 0, data: [] } }])
+    expect(report.catalog).toMatchObject({ success: true, count: 0 })
+    expect(report.detail).toMatchObject({ success: false, failure: 'not-probed' })
+    expect(report.retrievals).toEqual([])
+    expect(Object.values(report.features)).not.toContain(true)
+    expect(report.operations).toHaveLength(1)
+  })
+
+  it.each([
+    [{ status: 403, body: { code: 'forbidden', message: 'private message' } }, 'http-error'],
+    [{ status: 502, raw: '<html>private body</html>' }, 'http-error'],
+    [{ body: { code: 0, data: {} } }, 'invalid-shape'],
+    [{ raw: '{broken' }, 'request-failed']
+  ] as const)('does not turn catalog failure into an empty catalog: %j', async (fixture, failure) => {
+    const report = await probe(provider, [fixture])
+    expect(report.catalog).toMatchObject({ success: false, failure })
+    expect(report.catalog).not.toHaveProperty('count')
+    expect(report.detail).toMatchObject({ success: false, failure: 'not-probed' })
+    expect(report.retrievals).toEqual([])
+    expect(Object.values(report.features)).not.toContain(true)
+    expect(report.operations[0]).toMatchObject({ status: (fixture as Fixture).status ?? 200 })
+  })
+
+  it('distinguishes successful empty retrievals from failed detail and retrievals', async () => {
+    const failure = { status: 500, body: { code: 500, message: 'private error' } }
+    const report = await probe(provider, [catalog, failure, empty, failure, failure])
+    expect(report.catalog).toMatchObject({ success: true, count: 1 })
+    expect(report.detail).toMatchObject({ success: false, failure: 'http-error' })
+    expect(report.retrievals[0]).toMatchObject({ success: true, count: 0 })
+    for (const retrieval of report.retrievals.slice(1)) {
+      expect(retrieval).toMatchObject({ success: false, failure: 'http-error' })
+      expect(retrieval).not.toHaveProperty('count')
+    }
+    expect(report.operations[1]).toMatchObject({ status: 500, businessCode: 500 })
+  })
+
+  it('validates detail and retrieval shapes', async () => {
+    const invalid = { body: { code: 0, records: [null], data: { chunks: null } } }
+    const report = await probe(provider, [catalog, invalid, invalid, invalid, invalid])
+    expect(report.detail).toMatchObject({ success: false, failure: 'invalid-shape' })
+    expect(report.retrievals.every((item) => !item.success && item.failure === 'invalid-shape')).toBe(true)
+  })
+
+  it('recognizes valid detail responses', async () => {
+    const report = await probe(provider, [catalog, detail, empty, empty, empty])
+    expect(report.detail.success).toBe(true)
+  })
+
+  it.each(['UNABLE_TO_VERIFY_LEAF_SIGNATURE', 'ERR_TLS_CERT_ALTNAME_INVALID', 'private-code'])(
+    'only exposes allowlisted network cause codes: %s', async (causeCode) => {
+      const report = await probe(provider, [{ causeCode }])
+      expect(report.operations[0]).toMatchObject({ status: 'failed' })
+      expect(report.operations[0]?.causeCode).toBe(causeCode === 'private-code' ? undefined : causeCode)
+      expect(report.catalog).toMatchObject({ success: false, failure: 'request-failed' })
+    }
+  )
+
+  it.each(['forbidden', 'private-code'])('only exposes known string business codes: %s', async (code) => {
+    const report = await probe(provider, [{ status: 403, body: { code, message: 'private message' } }])
+    expect(report.operations[0]).toMatchObject({ status: 403 })
+    expect(report.operations[0]?.businessCode).toBe(code === 'forbidden' ? code : undefined)
+  })
+})
+
+it('rejects RAGFlow nonzero business codes in catalog, detail and retrieval', async () => {
+  const failure = { body: { code: 102, data: [{ id: 'fixture-dataset' }] } }
+  const catalogReport = await probe('ragflow', [failure])
+  expect(catalogReport.catalog).toMatchObject({ success: false, failure: 'business-error' })
+  expect(catalogReport.catalog).not.toHaveProperty('count')
+  const report = await probe('ragflow', [
+    { body: { code: 0, data: [{ id: 'fixture-dataset' }] } },
+    failure, failure, failure, failure
+  ])
+  expect(report.detail).toMatchObject({ success: false, failure: 'business-error' })
+  for (const retrieval of report.retrievals) {
+    expect(retrieval).toMatchObject({ success: false, failure: 'business-error' })
+    expect(retrieval).not.toHaveProperty('count')
+  }
+})

tests/external-knowledge-probe-report.test.ts

@@ -23,18 +23,22 @@ type ProbeReport = {
   providers: ProbeProvider[]
 }
 
-async function report(): Promise<ProbeReport> {
+async function report(filename = 'external-knowledge-probe-baseline.json'): Promise<ProbeReport> {
   return JSON.parse(
     await readFile(
-      resolve('docs/features/knowledge-base/external-knowledge-probe-baseline.json'),
+      resolve('docs/features/knowledge-base', filename),
       'utf8'
     )
   ) as ProbeReport
 }
 
 describe('external knowledge probe baseline', () => {
-  it('contains no endpoint, credential, remote identifier, or mutation operation', async () => {
-    const value = await report()
+  it.each([
+    'external-knowledge-probe-baseline.json',
+    'external-knowledge-probe-2026-09-10-strict.json',
+    'external-knowledge-probe-2026-09-10-extended.json'
+  ])('%s contains no endpoint, credential, remote identifier, or mutation operation', async (filename) => {
+    const value = await report(filename)
     const serialized = JSON.stringify(value)
 
     expect(serialized).not.toMatch(/https?:\/\//)