diff --git a/docs/features/knowledge-base/README.md b/docs/features/knowledge-base/README.md index 31d3ec6f..6919bec6 100644 --- a/docs/features/knowledge-base/README.md +++ b/docs/features/knowledge-base/README.md @@ -20,6 +20,8 @@ | [外部知识库 UI 设计](./external-knowledge-ui-design.md) | 知识库页面、外部实例管理和创建向导交互 | | [外部知识库技术设计](./external-knowledge-technical-design.md) | Provider Adapter、凭据、持久化、IPC 和检索标准化 | | [外部知识库实测基线](./external-knowledge-probe-baseline.json) | 脱敏记录真实实例的接口状态、字段结构、特色能力和耗时 | +| [2026-09-10 基础复测](./external-knowledge-probe-2026-09-10-strict.json) | 开启证书校验的目录、详情和基础检索结果 | +| [2026-09-10 扩展复测](./external-knowledge-probe-2026-09-10-extended.json) | 配置覆盖、重排、图谱与知识编译开关的请求结果及结构 | ## 术语 diff --git a/docs/features/knowledge-base/external-knowledge-logic-design.md b/docs/features/knowledge-base/external-knowledge-logic-design.md index 5dba110f..7b493465 100644 --- a/docs/features/knowledge-base/external-knowledge-logic-design.md +++ b/docs/features/knowledge-base/external-knowledge-logic-design.md @@ -253,7 +253,9 @@ GoodBuddy 固定安全上限 ``` - 固定安全上限不可被实例或用户扩大。 -- 远端知识库自身默认值只用于创建表单的建议值，不在每次请求时覆盖本地已保存值。 +- 显式保存 Provider 参数时，远端默认值只用于创建表单的建议值。Dify + `useDatasetDefaults=true` 是例外：每次请求省略 `retrieval_model`，沿用远端当时的配置； + 远端管理员修改配置后，后续检索随之变化。本地不保存一份默认参数快照。 - Provider 增加新字段不会自动启用。 - 当前实例版本离开 Adapter 支持矩阵，或 Adapter 新版本无法解析已保存配置时，绑定标记 配置失效；用户确认前不重写配置。 diff --git a/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-extended.json b/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-extended.json new file mode 100644 index 00000000..4400dbce --- /dev/null +++ b/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-extended.json @@ -0,0 +1,1043 @@ +{ + "schemaVersion": 1, + "probedAt": "2026-09-10T14:05:05.709Z", + "mode": "extended-read-only", + "queryCharacters": 3, + "privacy": "No credentials, origins, IDs, names, queries, or content are included.", + "providers": [ + { + "provider": "dify", + "profile": "0.x", + "detectedVersion": "0.15.8", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 3, + "total": 3, + "itemKeys": [ + "app_count", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "external_knowledge_info", + "external_retrieval_model", + "id", + "indexing_technique", + "name", + "permission", + "provider", + "retrieval_model_dict", + "tags", + "updated_at", + "updated_by", + "word_count" + ], + "retrievalModelKeys": [ + "reranking_enable", + "reranking_mode", + "reranking_model", + "score_threshold", + "score_threshold_enabled", + "search_method", + "top_k", + "weights" + ], + "indexingTechniques": [ + "high_quality" + ], + "metadataSchemaCount": 0, + "multimodalCount": 0, + "pipelineConfiguredCount": 0, + "summaryIndexConfiguredCount": 0 + }, + "detail": { + "success": false, + "failure": "http-error", + "keys": [ + "code", + "message", + "status" + ] + }, + "retrievals": [ + { + "mode": "dataset-defaults", + "success": true, + "count": 0, + "itemKeys": [], + "segmentKeys": [], + "documentKeys": [], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": false, + "childContentPresent": false, + "summaryPresent": false + }, + { + "mode": "explicit-current-config", + "success": true, + "count": 0, + "itemKeys": [], + "segmentKeys": [], + "documentKeys": [], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": false, + "childContentPresent": false, + "summaryPresent": false + } + ], + "features": { + "datasetDefaults": true, + "detailSupported": false, + "retrievalOverrideObserved": true, + "retrievalOverrideProbed": true, + "nonEmptyResultObserved": false + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/v1/datasets", + "status": 200, + "durationMs": 279, + "responseKeys": [ + "data", + "has_more", + "limit", + "page", + "total" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/v1/datasets/{id}", + "status": 405, + "durationMs": 105, + "responseKeys": [ + "code", + "message", + "status" + ] + }, + { + "name": "retrieve-defaults", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 296, + "responseKeys": [ + "query", + "records" + ] + }, + { + "name": "retrieve-explicit-config", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 158, + "responseKeys": [ + "query", + "records" + ] + } + ] + }, + { + "provider": "dify", + "profile": "1.x", + "detectedVersion": "1.17.0", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 1, + "total": 1, + "itemKeys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ], + "retrievalModelKeys": [ + "reranking_enable", + "reranking_mode", + "reranking_model", + "score_threshold", + "score_threshold_enabled", + "search_method", + "top_k", + "weights" + ], + "indexingTechniques": [ + "high_quality" + ], + "metadataSchemaCount": 0, + "multimodalCount": 0, + "pipelineConfiguredCount": 0, + "summaryIndexConfiguredCount": 1 + }, + "detail": { + "success": true, + "keys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ] + }, + "retrievals": [ + { + "mode": "dataset-defaults", + "success": true, + "count": 3, + "itemKeys": [ + "child_chunks", + "files", + "score", + "segment", + "summary", + "tsne_position" + ], + "segmentKeys": [ + "answer", + "completed_at", + "content", + "created_at", + "created_by", + "disabled_at", + "disabled_by", + "document", + "document_id", + "enabled", + "error", + "hit_count", + "id", + "index_node_hash", + "index_node_id", + "indexing_at", + "keywords", + "position", + "sign_content", + "status", + "stopped_at", + "tokens", + "word_count" + ], + "documentKeys": [ + "data_source_type", + "doc_metadata", + "doc_type", + "id", + "name" + ], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": true, + "childContentPresent": false, + "summaryPresent": false, + "scoreRange": { + "minimum": 0.5038624, + "maximum": 0.6006959 + } + }, + { + "mode": "explicit-current-config", + "success": true, + "count": 3, + "itemKeys": [ + "child_chunks", + "files", + "score", + "segment", + "summary", + "tsne_position" + ], + "segmentKeys": [ + "answer", + "completed_at", + "content", + "created_at", + "created_by", + "disabled_at", + "disabled_by", + "document", + "document_id", + "enabled", + "error", + "hit_count", + "id", + "index_node_hash", + "index_node_id", + "indexing_at", + "keywords", + "position", + "sign_content", + "status", + "stopped_at", + "tokens", + "word_count" + ], + "documentKeys": [ + "data_source_type", + "doc_metadata", + "doc_type", + "id", + "name" + ], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": true, + "childContentPresent": false, + "summaryPresent": false, + "scoreRange": { + "minimum": 0.5038624, + "maximum": 0.6006959 + } + } + ], + "features": { + "datasetDefaults": true, + "detailSupported": true, + "retrievalOverrideObserved": true, + "retrievalOverrideProbed": true, + "nonEmptyResultObserved": true + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/v1/datasets", + "status": 200, + "durationMs": 1348, + "responseKeys": [ + "data", + "has_more", + "limit", + "page", + "total" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/v1/datasets/{id}", + "status": 200, + "durationMs": 3297, + "responseKeys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ] + }, + { + "name": "retrieve-defaults", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 2387, + "responseKeys": [ + "query", + "records" + ] + }, + { + "name": "retrieve-explicit-config", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 1998, + "responseKeys": [ + "query", + "records" + ] + } + ] + }, + { + "provider": "fastgpt", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 1, + "envelope": "data-array", + "itemKeys": [ + "_id", + "avatar", + "createTime", + "inheritPermission", + "intro", + "name", + "permission", + "private", + "sourceMember", + "tmbId", + "type", + "updateTime", + "vectorModel" + ], + "itemTypes": [ + "dataset" + ] + }, + "detail": { + "keys": [ + "_id", + "agentModel", + "agentModelId", + "avatar", + "deleteTime", + "inheritPermission", + "intro", + "name", + "parentId", + "permission", + "status", + "teamId", + "tmbId", + "type", + "updateTime", + "vectorModel", + "vectorModelId", + "vlmModel", + "vlmModelId" + ], + "vectorModelKeys": [ + "avatar", + "charsPointsPrice", + "config", + "isActive", + "isCustom", + "model", + "modelId", + "name", + "provider", + "scope", + "testMode", + "type" + ] + }, + "retrievals": [ + { + "searchMode": "embedding", + "success": true, + "envelope": "data-list", + "count": 8, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.27005115151405334, + "maximum": 0.3160889148712158 + }, + "rerankEnabled": false + }, + { + "searchMode": "fullTextRecall", + "success": true, + "envelope": "data-list", + "count": 2, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.5047619047619047, + "maximum": 0.505 + }, + "rerankEnabled": false + }, + { + "searchMode": "mixedRecall", + "success": true, + "envelope": "data-list", + "count": 7, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.007575757575757576, + "maximum": 0.505 + }, + "rerankEnabled": false + }, + { + "searchMode": "mixedRecall", + "usingRerank": true, + "success": true, + "envelope": "data-list", + "count": 9, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.01334503950834065, + "maximum": 0.505 + }, + "rerankEnabled": true + } + ], + "features": { + "searchModes": [ + "embedding", + "fullTextRecall", + "mixedRecall" + ], + "rerankProbed": true, + "rerankRequestSucceeded": true, + "extensionQueryProbed": false + }, + "operations": [ + { + "name": "catalog", + "method": "POST", + "path": "/api/core/dataset/list", + "status": 200, + "businessCode": 200, + "durationMs": 2291, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/api/core/dataset/detail", + "status": 200, + "businessCode": 200, + "durationMs": 1000, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-embedding", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 3001, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-fullTextRecall", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 501, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-mixedRecall", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 1597, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-rerank", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 3634, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + } + ] + }, + { + "provider": "ragflow", + "transport": { + "protocol": "http:" + }, + "catalog": { + "success": true, + "count": 29, + "total": 29, + "itemKeys": [ + "artifact_task_finish_at", + "artifact_task_id", + "avatar", + "chunk_count", + "chunk_method", + "create_date", + "create_time", + "created_by", + "description", + "document_count", + "embedding_model", + "graphrag_task_finish_at", + "graphrag_task_id", + "id", + "language", + "mindmap_task_finish_at", + "mindmap_task_id", + "name", + "nickname", + "pagerank", + "parser_config", + "permission", + "pipeline_id", + "raptor_task_finish_at", + "raptor_task_id", + "similarity_threshold", + "skill_task_finish_at", + "skill_task_id", + "status", + "tenant_avatar", + "tenant_embd_id", + "tenant_id", + "token_num", + "update_date", + "update_time", + "vector_similarity_weight" + ], + "parserConfigKeys": [ + "auto_keywords", + "auto_questions", + "built_in_metadata", + "children_delimiter", + "chunk_token_num", + "compilation_template_group_id", + "delimiter", + "enable_children", + "enable_metadata", + "ext", + "filename_embd_weight", + "graphrag", + "html4excel", + "image_context_size", + "image_table_context_window", + "layout_recognize", + "llm_id", + "metadata", + "mineru_formula_enable", + "mineru_lang", + "mineru_parse_method", + "mineru_table_enable", + "overlapped_percent", + "pages", + "parent_child", + "raptor", + "table_column_mode", + "table_column_names", + "table_column_roles", + "table_context_size", + "tag_kb_ids", + "task_page_size", + "tenant_llm_id", + "toc_extraction", + "topn_tags" + ], + "graphConfiguredCount": 18, + "graphCompletedCount": 16, + "knowledgeCompilationConfiguredCount": 0 + }, + "detail": { + "success": true, + "keys": [ + "artifact_task_finish_at", + "artifact_task_id", + "avatar", + "chunk_count", + "chunk_method", + "create_date", + "create_time", + "created_by", + "description", + "document_count", + "embedding_model", + "graphrag_task_finish_at", + "graphrag_task_id", + "id", + "language", + "mindmap_task_finish_at", + "mindmap_task_id", + "name", + "nickname", + "pagerank", + "parser_config", + "permission", + "pipeline_id", + "raptor_task_finish_at", + "raptor_task_id", + "similarity_threshold", + "skill_task_finish_at", + "skill_task_id", + "status", + "tenant_avatar", + "tenant_embd_id", + "tenant_id", + "token_num", + "update_date", + "update_time", + "vector_similarity_weight" + ] + }, + "retrievals": [ + { + "feature": "baseline", + "success": true, + "count": 2, + "itemKeys": [ + "content", + "content_ltks", + "dataset_id", + "doc_type_kwd", + "document_id", + "document_keyword", + "id", + "image_id", + "important_keywords", + "mom_id", + "positions", + "row_id", + "similarity", + "tag_kwd", + "term_similarity", + "vector_similarity" + ], + "contentPresent": true, + "scoreRange": { + "minimum": 0.18431304078777125, + "maximum": 0.2623782504333604 + } + }, + { + "feature": "graph", + "success": true, + "count": 3, + "itemKeys": [ + "content", + "content_ltks", + "dataset_id", + "doc_type_kwd", + "document_id", + "document_keyword", + "id", + "image_id", + "important_keywords", + "mom_id", + "positions", + "row_id", + "similarity", + "tag_kwd", + "term_similarity", + "vector_similarity" + ], + "contentPresent": true, + "scoreRange": { + "minimum": 0.18431304078777125, + "maximum": 1 + } + }, + { + "feature": "knowledge-compilation", + "success": true, + "count": 2, + "itemKeys": [ + "content", + "content_ltks", + "dataset_id", + "doc_type_kwd", + "document_id", + "document_keyword", + "id", + "image_id", + "important_keywords", + "mom_id", + "positions", + "row_id", + "similarity", + "tag_kwd", + "term_similarity", + "vector_similarity" + ], + "contentPresent": true, + "scoreRange": { + "minimum": 0.18431304078777125, + "maximum": 0.2623782504333604 + } + } + ], + "features": { + "graphRetrievalProbed": true, + "knowledgeCompilationProbed": true, + "metadataConditionProbed": false + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/api/v1/datasets", + "status": 200, + "businessCode": 0, + "durationMs": 2804, + "responseKeys": [ + "code", + "data", + "total_datasets" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/api/v1/datasets", + "status": 200, + "businessCode": 0, + "durationMs": 153, + "responseKeys": [ + "code", + "data", + "total_datasets" + ] + }, + { + "name": "retrieve-baseline", + "method": "POST", + "path": "/api/v1/retrieval", + "status": 200, + "businessCode": 0, + "durationMs": 2561, + "responseKeys": [ + "code", + "data" + ] + }, + { + "name": "retrieve-graph", + "method": "POST", + "path": "/api/v1/retrieval", + "status": 200, + "businessCode": 0, + "durationMs": 5121, + "responseKeys": [ + "code", + "data" + ] + }, + { + "name": "retrieve-knowledge-compilation", + "method": "POST", + "path": "/api/v1/retrieval", + "status": 200, + "businessCode": 0, + "durationMs": 2320, + "responseKeys": [ + "code", + "data" + ] + } + ] + } + ] +} diff --git a/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-strict.json b/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-strict.json new file mode 100644 index 00000000..32cd0405 --- /dev/null +++ b/docs/features/knowledge-base/external-knowledge-probe-2026-09-10-strict.json @@ -0,0 +1,828 @@ +{ + "schemaVersion": 1, + "probedAt": "2026-09-10T14:02:51.600Z", + "mode": "basic-read-only", + "queryCharacters": 3, + "privacy": "No credentials, origins, IDs, names, queries, or content are included.", + "providers": [ + { + "provider": "dify", + "profile": "0.x", + "detectedVersion": "0.15.8", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 3, + "total": 3, + "itemKeys": [ + "app_count", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "external_knowledge_info", + "external_retrieval_model", + "id", + "indexing_technique", + "name", + "permission", + "provider", + "retrieval_model_dict", + "tags", + "updated_at", + "updated_by", + "word_count" + ], + "retrievalModelKeys": [ + "reranking_enable", + "reranking_mode", + "reranking_model", + "score_threshold", + "score_threshold_enabled", + "search_method", + "top_k", + "weights" + ], + "indexingTechniques": [ + "high_quality" + ], + "metadataSchemaCount": 0, + "multimodalCount": 0, + "pipelineConfiguredCount": 0, + "summaryIndexConfiguredCount": 0 + }, + "detail": { + "success": false, + "failure": "http-error", + "keys": [ + "code", + "message", + "status" + ] + }, + "retrievals": [ + { + "mode": "dataset-defaults", + "success": true, + "count": 0, + "itemKeys": [], + "segmentKeys": [], + "documentKeys": [], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": false, + "childContentPresent": false, + "summaryPresent": false + } + ], + "features": { + "datasetDefaults": true, + "detailSupported": false, + "retrievalOverrideObserved": true, + "retrievalOverrideProbed": false, + "nonEmptyResultObserved": false + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/v1/datasets", + "status": 200, + "durationMs": 1255, + "responseKeys": [ + "data", + "has_more", + "limit", + "page", + "total" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/v1/datasets/{id}", + "status": 405, + "durationMs": 105, + "responseKeys": [ + "code", + "message", + "status" + ] + }, + { + "name": "retrieve-defaults", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 582, + "responseKeys": [ + "query", + "records" + ] + } + ] + }, + { + "provider": "dify", + "profile": "1.x", + "detectedVersion": "1.17.0", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 1, + "total": 1, + "itemKeys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ], + "retrievalModelKeys": [ + "reranking_enable", + "reranking_mode", + "reranking_model", + "score_threshold", + "score_threshold_enabled", + "search_method", + "top_k", + "weights" + ], + "indexingTechniques": [ + "high_quality" + ], + "metadataSchemaCount": 0, + "multimodalCount": 0, + "pipelineConfiguredCount": 0, + "summaryIndexConfiguredCount": 1 + }, + "detail": { + "success": true, + "keys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ] + }, + "retrievals": [ + { + "mode": "dataset-defaults", + "success": true, + "count": 3, + "itemKeys": [ + "child_chunks", + "files", + "score", + "segment", + "summary", + "tsne_position" + ], + "segmentKeys": [ + "answer", + "completed_at", + "content", + "created_at", + "created_by", + "disabled_at", + "disabled_by", + "document", + "document_id", + "enabled", + "error", + "hit_count", + "id", + "index_node_hash", + "index_node_id", + "indexing_at", + "keywords", + "position", + "sign_content", + "status", + "stopped_at", + "tokens", + "word_count" + ], + "documentKeys": [ + "data_source_type", + "doc_metadata", + "doc_type", + "id", + "name" + ], + "childChunkKeys": [], + "fileKeys": [], + "contentPresent": true, + "childContentPresent": false, + "summaryPresent": false, + "scoreRange": { + "minimum": 0.5038624, + "maximum": 0.6006959 + } + } + ], + "features": { + "datasetDefaults": true, + "detailSupported": true, + "retrievalOverrideObserved": true, + "retrievalOverrideProbed": false, + "nonEmptyResultObserved": true + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/v1/datasets", + "status": 200, + "durationMs": 2261, + "responseKeys": [ + "data", + "has_more", + "limit", + "page", + "total" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/v1/datasets/{id}", + "status": 200, + "durationMs": 1680, + "responseKeys": [ + "app_count", + "author_name", + "built_in_field_enabled", + "chunk_structure", + "created_at", + "created_by", + "data_source_type", + "description", + "doc_form", + "doc_metadata", + "document_count", + "embedding_available", + "embedding_model", + "embedding_model_provider", + "enable_api", + "external_knowledge_info", + "external_retrieval_model", + "icon_info", + "id", + "indexing_technique", + "is_multimodal", + "is_published", + "maintainer", + "name", + "permission", + "pipeline_id", + "provider", + "retrieval_model_dict", + "runtime_mode", + "summary_index_setting", + "tags", + "total_available_documents", + "total_documents", + "updated_at", + "updated_by", + "word_count" + ] + }, + { + "name": "retrieve-defaults", + "method": "POST", + "path": "/v1/datasets/{id}/retrieve", + "status": 200, + "durationMs": 3814, + "responseKeys": [ + "query", + "records" + ] + } + ] + }, + { + "provider": "fastgpt", + "transport": { + "protocol": "https:", + "tlsVerificationDisabled": false + }, + "catalog": { + "success": true, + "count": 1, + "envelope": "data-array", + "itemKeys": [ + "_id", + "avatar", + "createTime", + "inheritPermission", + "intro", + "name", + "permission", + "private", + "sourceMember", + "tmbId", + "type", + "updateTime", + "vectorModel" + ], + "itemTypes": [ + "dataset" + ] + }, + "detail": { + "keys": [ + "_id", + "agentModel", + "agentModelId", + "avatar", + "deleteTime", + "inheritPermission", + "intro", + "name", + "parentId", + "permission", + "status", + "teamId", + "tmbId", + "type", + "updateTime", + "vectorModel", + "vectorModelId", + "vlmModel", + "vlmModelId" + ], + "vectorModelKeys": [ + "avatar", + "charsPointsPrice", + "config", + "isActive", + "isCustom", + "model", + "modelId", + "name", + "provider", + "scope", + "testMode", + "type" + ] + }, + "retrievals": [ + { + "searchMode": "embedding", + "success": true, + "envelope": "data-list", + "count": 8, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.27009665966033936, + "maximum": 0.3160914182662964 + }, + "rerankEnabled": false + }, + { + "searchMode": "fullTextRecall", + "success": true, + "envelope": "data-list", + "count": 2, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.5047619047619047, + "maximum": 0.505 + }, + "rerankEnabled": false + }, + { + "searchMode": "mixedRecall", + "success": true, + "envelope": "data-list", + "count": 7, + "itemKeys": [ + "chunkIndex", + "collectionId", + "datasetId", + "id", + "q", + "score", + "sourceId", + "sourceName", + "updateTime" + ], + "contentPresent": true, + "scoreTypes": [ + "array" + ], + "scoreItemKeys": [ + "index", + "type", + "value" + ], + "scoreValueRange": { + "minimum": 0.007575757575757576, + "maximum": 0.505 + }, + "rerankEnabled": false + } + ], + "features": { + "searchModes": [ + "embedding", + "fullTextRecall", + "mixedRecall" + ], + "rerankProbed": false, + "rerankRequestSucceeded": false, + "extensionQueryProbed": false + }, + "operations": [ + { + "name": "catalog", + "method": "POST", + "path": "/api/core/dataset/list", + "status": 200, + "businessCode": 200, + "durationMs": 1785, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/api/core/dataset/detail", + "status": 200, + "businessCode": 200, + "durationMs": 571, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-embedding", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 1255, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-fullTextRecall", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 777, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + }, + { + "name": "retrieve-mixedRecall", + "method": "POST", + "path": "/api/core/dataset/searchTest", + "status": 200, + "businessCode": 200, + "durationMs": 2326, + "responseKeys": [ + "code", + "data", + "message", + "statusText" + ] + } + ] + }, + { + "provider": "ragflow", + "transport": { + "protocol": "http:" + }, + "catalog": { + "success": true, + "count": 29, + "total": 29, + "itemKeys": [ + "artifact_task_finish_at", + "artifact_task_id", + "avatar", + "chunk_count", + "chunk_method", + "create_date", + "create_time", + "created_by", + "description", + "document_count", + "embedding_model", + "graphrag_task_finish_at", + "graphrag_task_id", + "id", + "language", + "mindmap_task_finish_at", + "mindmap_task_id", + "name", + "nickname", + "pagerank", + "parser_config", + "permission", + "pipeline_id", + "raptor_task_finish_at", + "raptor_task_id", + "similarity_threshold", + "skill_task_finish_at", + "skill_task_id", + "status", + "tenant_avatar", + "tenant_embd_id", + "tenant_id", + "token_num", + "update_date", + "update_time", + "vector_similarity_weight" + ], + "parserConfigKeys": [ + "auto_keywords", + "auto_questions", + "built_in_metadata", + "children_delimiter", + "chunk_token_num", + "compilation_template_group_id", + "delimiter", + "enable_children", + "enable_metadata", + "ext", + "filename_embd_weight", + "graphrag", + "html4excel", + "image_context_size", + "image_table_context_window", + "layout_recognize", + "llm_id", + "metadata", + "mineru_formula_enable", + "mineru_lang", + "mineru_parse_method", + "mineru_table_enable", + "overlapped_percent", + "pages", + "parent_child", + "raptor", + "table_column_mode", + "table_column_names", + "table_column_roles", + "table_context_size", + "tag_kb_ids", + "task_page_size", + "tenant_llm_id", + "toc_extraction", + "topn_tags" + ], + "graphConfiguredCount": 18, + "graphCompletedCount": 16, + "knowledgeCompilationConfiguredCount": 0 + }, + "detail": { + "success": true, + "keys": [ + "artifact_task_finish_at", + "artifact_task_id", + "avatar", + "chunk_count", + "chunk_method", + "create_date", + "create_time", + "created_by", + "description", + "document_count", + "embedding_model", + "graphrag_task_finish_at", + "graphrag_task_id", + "id", + "language", + "mindmap_task_finish_at", + "mindmap_task_id", + "name", + "nickname", + "pagerank", + "parser_config", + "permission", + "pipeline_id", + "raptor_task_finish_at", + "raptor_task_id", + "similarity_threshold", + "skill_task_finish_at", + "skill_task_id", + "status", + "tenant_avatar", + "tenant_embd_id", + "tenant_id", + "token_num", + "update_date", + "update_time", + "vector_similarity_weight" + ] + }, + "retrievals": [ + { + "feature": "baseline", + "success": true, + "count": 2, + "itemKeys": [ + "content", + "content_ltks", + "dataset_id", + "doc_type_kwd", + "document_id", + "document_keyword", + "id", + "image_id", + "important_keywords", + "mom_id", + "positions", + "row_id", + "similarity", + "tag_kwd", + "term_similarity", + "vector_similarity" + ], + "contentPresent": true, + "scoreRange": { + "minimum": 0.18431304078777125, + "maximum": 0.2623782504333604 + } + } + ], + "features": { + "graphRetrievalProbed": false, + "knowledgeCompilationProbed": false, + "metadataConditionProbed": false + }, + "operations": [ + { + "name": "catalog", + "method": "GET", + "path": "/api/v1/datasets", + "status": 200, + "businessCode": 0, + "durationMs": 2817, + "responseKeys": [ + "code", + "data", + "total_datasets" + ] + }, + { + "name": "detail", + "method": "GET", + "path": "/api/v1/datasets", + "status": 200, + "businessCode": 0, + "durationMs": 159, + "responseKeys": [ + "code", + "data", + "total_datasets" + ] + }, + { + "name": "retrieve-baseline", + "method": "POST", + "path": "/api/v1/retrieval", + "status": 200, + "businessCode": 0, + "durationMs": 776, + "responseKeys": [ + "code", + "data" + ] + } + ] + } + ] +} diff --git a/docs/features/knowledge-base/external-knowledge-technical-design.md b/docs/features/knowledge-base/external-knowledge-technical-design.md index fa81e3ec..010c6c89 100644 --- a/docs/features/knowledge-base/external-knowledge-technical-design.md +++ b/docs/features/knowledge-base/external-knowledge-technical-design.md @@ -355,9 +355,28 @@ knowledge:external-retrieval:test ## 9. Provider 映射 -本节记录 2026-09-03 核对的初始公开接口。实施时应把每个受支持版本的真实响应保存为 +本节依据 2026-09-10 核对的官方文档、官方源码和真实实例响应。实施时应把每个受支持版本的真实响应保存为 去敏 fixture，并以 fixture 和真实实例测试作为发布依据。 +官方来源与版本边界： + +- Dify：[Knowledge API 与密钥范围](https://docs.dify.ai/en/api-reference/guides/knowledge)、 + [列表](https://docs.dify.ai/en/api-reference/knowledge-bases/list-knowledge-bases)、 + [详情](https://docs.dify.ai/en/api-reference/knowledge-bases/get-knowledge-base)、 + [检索](https://docs.dify.ai/en/api-reference/knowledge-bases/retrieve-chunks-from-a-knowledge-base-test-retrieval)。 + 缺省配置顺序另对照[官方实现](https://github.com/langgenius/dify/blob/main/api/services/hit_testing_service.py)。 +- FastGPT：[API 说明](https://doc.fastgpt.io/zh-CN/openapi/intro)和 + [已停止维护的手工接口页](https://doc.fastgpt.io/zh-CN/openapi/dataset)。当前契约对照 + [生成文档 Schema](https://github.com/labring/FastGPT/blob/main/packages/global/openapi/core/dataset/api.ts)、 + [检索处理器](https://github.com/labring/FastGPT/blob/main/projects/app/src/pages/api/core/dataset/searchTest.ts)和 + [结果项 Schema](https://github.com/labring/FastGPT/blob/main/packages/global/core/dataset/type.ts)。 + 云端 `/apidoc/` 本次只取得应用壳，未读取渲染后的部署 Schema。 +- RAGFlow：[官网 HTTP API](https://ragflow.io/docs/dev/http_api_reference)正文未能有效提取， + 采用[官方当前文档原文](https://github.com/infiniflow/ragflow/blob/main/docs/references/http_api_reference.md)，并与 + [v0.24.0 固定版本](https://github.com/infiniflow/ragflow/blob/v0.24.0/docs/references/http_api_reference.md)比较。 + +`main` 分支文档与源码会变化，不是部署版本凭据。FastGPT、RAGFlow 的实际产品版本仍未确认。 + ### 9.0 真实实例验证基线 2026-09-09 使用本机未入库的测试地址和 API Key 完成第一轮只读验证。验证只记录状态、 @@ -370,10 +389,15 @@ knowledge:external-retrieval:test | FastGPT | 响应未提供版本，只能确认当前托管实例 | 基址包含 `/api`；列表和详情均返回 200，列表 `data` 直接为数组，本次 1 个 Dataset | embedding 8 条、fullTextRecall 2 条、mixedRecall 7 条、mixedRecall + Rerank 9 条 | 三种检索模式和 Rerank 均已实测；当前结果包含 `q`，未包含 `a` | | RAGFlow | 响应未提供产品版本，Server 为 nginx，不能据此推断版本 | `GET /api/v1/datasets` 返回 200，共 28 项；17 项配置 GraphRAG，15 项有完成证据 | 选定已完成图谱的库，基础检索 0 条，`use_kg=true` 返回 1 条，Knowledge Compilation 0 条 | 图谱检索已产生区别于基础检索的结果；当前没有库具备 Knowledge Compilation 配置证据 | -当前 Node 测试进程设置了 `NODE_TLS_REJECT_UNAUTHORIZED=0`，因此两个 HTTPS 实例的请求 +2026-09-09 的 Node 测试进程设置了 `NODE_TLS_REJECT_UNAUTHORIZED=0`，因此当时 HTTPS 实例的请求 使用了放宽的证书校验。这只能证明接口在当前测试环境可用，不能证明严格 TLS 校验通过。 产品实现不得读取该进程级设置作为隐式默认；实例测试和保存必须按 8.1 节处理传输风险。 +2026-09-10 复测显式设置 `NODE_TLS_REJECT_UNAUTHORIZED=1`，两个 Dify 实例和 FastGPT +均通过证书校验；RAGFlow 配置为 HTTP，不涉及 TLS。基础 14 次、扩展 19 次，共 33 次只读 +请求。每个实例从目录选择一个库检索，未遍历全部库；未调用聊天或生成端点，检索仍可能 +使用远端 Embedding、Rerank 并记录用量。两份报告与逐项结果见[实施进度](./progress.md)。 + ### 9.1 Dify | 操作 | 接口 | @@ -385,10 +409,15 @@ knowledge:external-retrieval:test Adapter 只接受 Knowledge Service API Key，不调用应用 `/info`、Chat 或 Workflow。默认不 发送 `retrieval_model`，沿用远端知识库配置。用户选择覆盖时，Adapter 必须一次收集并验证 -当前接口要求的搜索方式、Top K、阈值、`reranking_mode`，以及相应的 Rerank Provider/模型 -或关键词/向量权重与 Embedding Provider/模型；缺少任何必填标识时不构造请求。响应从 +当前接口要求的 `search_method`、`reranking_enable`、`top_k`、`score_threshold_enabled`。 +开启重排时再验证 `reranking_mode`，以及该模式所需的 Provider、模型或权重；缺少必填 +标识时不构造请求。显式对象不会与知识库配置逐字段合并。省略整个对象时，服务端优先使用 +知识库配置，未保存配置时才使用内置默认。响应从 `records[].segment`、`score`、文档名称和元数据提取标准化引用。 +当前官方文档说明限定知识库范围的 Key 不能调用列表，但可访问授权 ID 的详情和检索。 +列表 403 不足以认定密钥无效；本轮测试配置均能列目录，尚未验证 Scoped Key。 + Dify 当前文档限制查询不超过 250 字符。GoodBuddy 不静默截断；超出时检索测试和聊天 诊断返回 Provider 限制，并允许后续单独设计查询压缩策略。 @@ -421,19 +450,25 @@ Dify 1.x 官方 RetrievalModel 还声明 `metadata_filtering_conditions`，并 | 检索 | `POST /api/core/dataset/searchTest` | | 认证 | `Authorization: Bearer ` | -列表响应同时包含文件夹和知识库，只有 `type = dataset` 可以绑定。当前实测列表的 -`data` 直接为数组，不是 `data.list`；Adapter 可以仅对已经实测的这两种明确 envelope -做版本化解析，不递归猜测任意嵌套。检索映射 `datasetId`、 +列表响应同时包含文件夹和知识库，只有 `type = dataset` 可以绑定。根列表请求发送 +`parentId: null`；文件夹需按父 ID 继续读取。当前实测列表为 `data[]`，检索为 +`data.list[]`，这是两个不同端点的结构。旧手工文档还展示检索 `data[]`，探测脚本接受 +该旧结构并单独记录 envelope；当前客户端只接受检索 `data.list[]`，旧结构尚未经过真实 +部署验证。不递归猜测任意嵌套。检索映射 `datasetId`、 `text`、Token `limit`、`similarity`、`searchMode` 和 `usingReRank`。初始版本不发送查询优化 字段，避免调用 FastGPT 配置的 LLM。 当前实测响应的 `data` 包含 `list/duration/limit/searchMode/usingReRank/similarity`；结果项 包含 `q/id/datasetId/collectionId/sourceName/sourceId/chunkIndex/score`，未返回 `a`。 Adapter 以 `q` 为必需片段，`a` 仅在远端实际返回时追加，不能要求 `a` 存在。 +2026-09-10 实测 `score` 为 `{type, value, index}[]`，与当前官方结果 Schema 一致； +旧手工文档中的数值分数不能作为当前统一映射依据。应保留评分类型或按明确规则选分， +不能默认取第一项作为总分。当前客户端仅提取数值分数，数组评分的映射仍待实现。 当前实例的 embedding、fullTextRecall、mixedRecall 和 mixedRecall + Rerank 均返回 200 和 非空结果，因此这四种界面组合可以启用。查询扩展会调用模型，仍不在初始界面和探测脚本中 启用。 -FastGPT 4.15.0 起部署实例的 `/apidoc/devapi` 是接口事实来源，手工文档可能落后。支持矩阵 +FastGPT 4.15.0 起应优先查阅部署实例的自动生成文档，`/apidoc/devapi` 与 +`/apidoc/systemopenapi` 的认证范围不同，Dev API 中出现不等于允许 API Key 调用。支持矩阵 必须按真实版本测试；不能因为健康页可访问就假定列表和检索 Schema 相同。`searchTest` 可能记录 `SEARCH_TEST` 等远端审计并更新 API Key 用量；Embedding 和 Rerank 也可能产生 费用。GoodBuddy 将这些行为作为检索成本显示，不宣称该接口没有副作用。 @@ -448,7 +483,11 @@ FastGPT 4.15.0 起部署实例的 `/apidoc/devapi` 是接口事实来源，手 检索映射 `question`、`dataset_ids`、`page_size`、`similarity_threshold`、 `vector_similarity_weight`、`knn_top_k`、重排、`use_kg` 和 -`include_knowledge_compilation`。不发送已弃用的 `top_k`。响应从 `data.chunks` 和 +`include_knowledge_compilation`。当前文档以 `knn_top_k` 取代 `top_k`；v0.24.0 文档只声明 +`top_k`，因此当前客户端不发送 `top_k` 的行为不能作为旧版本兼容承诺。 +`knn_top_k` 控制向量候选数量，`page_size` 控制返回数量；请求成功不能单独证明未知参数 +已生效。本轮未验证候选数参数的效果，也未确定新字段首次发布版本。 +HTTP 成功且业务 `code === 0` 才能解析结果。响应从 `data.chunks` 和 `data.doc_aggs` 提取片段、文档、位置和原始相似度。当前实测非空 chunk 使用 `document_keyword` 表示文档名，并包含 `id/document_id/content/positions/similarity/` `term_similarity/vector_similarity`；初始 Adapter 应按该字段映射，不假定存在 @@ -467,10 +506,13 @@ FastGPT 4.15.0 起部署实例的 `/apidoc/devapi` 是接口事实来源，手 配置证据；仅仅请求返回 200 不能启用界面开关。 - Provider 没有返回图节点或路径证据时，GoodBuddy 不生成图谱引用。 -当前实例 28 个库中 17 个配置 GraphRAG、15 个存在完成证据。对一个已完成库使用同一查询， +2026-09-09 实例 28 个库中 17 个配置 GraphRAG、15 个存在完成证据。对一个已完成库使用同一查询， 基础检索返回 0 条，`use_kg=true` 返回 1 条且原始相似度为 1，说明图谱开关对实际结果有 -影响。当前 28 个库均没有 Knowledge Compilation 配置证据；该请求虽返回 200 和零结果， +影响。当时 28 个库均没有 Knowledge Compilation 配置证据；该请求虽返回 200 和零结果， 界面仍必须禁用开关并显示“当前知识库未配置 Knowledge Compilation”。 +2026-09-10 目录增至 29 项，18 项配置图谱、16 项有完成证据；所选库基础检索 2 条、 +图谱检索 3 条。Knowledge Compilation 开关请求返回 2 条，目录仍无配置证据，不能据此 +认定编译内容参与了检索。 ## 10. 检索编排 @@ -601,8 +643,8 @@ Adapter 可以保存少量受控事实，例如“目录接口可用”“详情 Mock 或公开文档通过不能代替真实实例验收。 2026-09-09 基线已完成 Dify `0.15.8`/`1.17.0` 目录、详情差异和 1.17 非空检索结构， -FastGPT/RAGFlow 产品版本、严格 TLS、401/403/404/429、超时和取消尚未完成，因此不满足 -本节完整验收条件。 +2026-09-10 补充验证了 HTTPS 实例的严格 TLS。FastGPT/RAGFlow 产品版本、真实实例 +401/403/404/429、超时和取消尚未完成，因此不满足本节完整验收条件。 ### 14.6 可重复探测脚本 @@ -621,14 +663,18 @@ node scripts/external-knowledge-probe.mjs --extended --output= Rerank、RAGFlow GraphRAG 和 Knowledge Compilation 检索。脚本只有固定 GET/POST 读取与 检索端点，不实现或调用远端创建、更新、上传和删除。 -FastGPT 报告将请求成功与命中数量分开：HTTP 必须成功、业务 `code` 必须为 `200`， -且 `data` 或 `data.list` 必须是数组，才记录 `success: true` 和 `count`。成功的空数组 +三家报告均将请求成功与命中数量分开：HTTP 必须成功，FastGPT 业务 `code` 必须为 `200`、 +RAGFlow 必须为 `0`，并分别校验 Dify `records`、FastGPT `data` 或 `data.list`、RAGFlow +`data.chunks` 数组结构，才记录检索 `success: true` 和 `count`。成功的空数组 记录 `count: 0`；HTTP、业务、响应结构、JSON 解析或网络失败记录 `success: false` 与 固定失败类别，不用零命中代替失败。目录失败时不继续调用详情和检索。 `features.searchModes` 只包含基础检索实际成功的模式；`rerankProbed` 表示实际尝试， -`rerankSucceeded` 单独表示成功。历史基线保留原始证据，不追填未观测的成功字段。 -`tests/external-knowledge-probe-fastgpt.test.ts` 使用内存 fetch fixture 运行真实 CLI， -覆盖错误、成功零命中和非空结果，不连接外部知识服务。 +`rerankRequestSucceeded` 表示请求成功，结果 `rerankEnabled` 单独记录远端实际返回的 +`usingReRank` 布尔值。FastGPT 另记录检索 envelope、分数类型和分数数组项结构； +operation 保留 HTTP 状态、安全业务码及白名单网络错误码，不记录远端错误消息。 +历史基线保留原始证据，不追填未观测的成功字段。 +`tests/external-knowledge-probe-fastgpt.test.ts` 和 `tests/external-knowledge-probe-outcomes.test.ts` +使用内存 fetch fixture 运行真实 CLI，覆盖错误、成功零命中和非空结果，不连接外部知识服务。 ### 14.7 CRUD 验收边界 diff --git a/docs/features/knowledge-base/progress.md b/docs/features/knowledge-base/progress.md index f1ad8d40..1328b071 100644 --- a/docs/features/knowledge-base/progress.md +++ b/docs/features/knowledge-base/progress.md @@ -15,6 +15,34 @@ ## 验证证据 +2026-09-10 官方 API 核对与真实实例复测： + +- 官方资料链接与版本差异见[技术设计第 9 节](./external-knowledge-technical-design.md#9-provider-映射)。 + 已修正 Dify 默认配置优先级、FastGPT 端点响应与数组评分说明、RAGFlow 新旧候选数参数边界。 +- 使用现有本地测试配置与脚本默认查询，执行基础 14 次、扩展 19 次，共 33 次只读请求， + 其中检索 17 次。未调用远端内容写入、聊天或生成端点；检索所用 Embedding、重排及审计 + 用量由远端服务处理，不据此推断底层模型调用次数。 +- 两轮均在独立 Node 进程显式设置 `NODE_TLS_REJECT_UNAUTHORIZED=1`。两个 Dify 实例与 + FastGPT 的 HTTPS 请求通过严格证书校验；RAGFlow 使用 HTTP。 +- [基础报告](./external-knowledge-probe-2026-09-10-strict.json)和 + [扩展报告](./external-knowledge-probe-2026-09-10-extended.json)保留脱敏状态、数量、字段结构 + 和耗时。每个实例只对目录选出的一个库执行检索，未遍历全部库。 + +| 实例 | 目录与详情 | 检索结果 | +| --- | --- | --- | +| Dify 0.15.8 | 目录 3 项；详情 HTTP 405 | 默认与显式当前配置请求均成功，均为 0 条；非空结果尚未验证 | +| Dify 1.17.0 | 目录 1 项；详情成功 | 默认与显式当前配置均为 3 条非空结果 | +| FastGPT，版本未确认 | 根目录 1 个 Dataset；详情 HTTP 200、业务码 200 | 向量 8 条、全文 2 条、混合 7 条、混合加重排 9 条；重排响应明确为启用；检索包装为 `data.list`，`score` 为数组 | +| RAGFlow，版本未确认 | 目录 29 项；详情成功；18 项配置图谱、16 项有完成证据 | 基础 2 条、图谱 3 条、知识编译开关 2 条；请求均 HTTP 200、业务码 0，但没有知识编译配置证据 | + +- 修正探测脚本将 Dify/RAGFlow 失败记成零命中的问题，增加实际调用与成功标记，统一 + FastGPT 地址规范化并记录分数结构、响应中的重排状态。未修改产品客户端；旧 FastGPT + 检索数组包装、数组评分映射及上一轮发现的取消分类问题仍待处理。 +- 三组探测测试共 43 项通过，包含两份新报告的脱敏与只读操作检查；`npm run typecheck`、`npm run lint` 通过。 + `npm test` 先后达到 60 秒和 240 秒执行时限，均未输出汇总，全量回归尚未完成。 +- 本轮未验证真实 401/403/404/429、超时、取消、Scoped Key、分页、元数据过滤或候选数 + 参数效果。探测脚本不打包到桌面或 GoodBuddy Agent，本次未改变远程 Runtime 生产路径。 + 2026-09-09 外部知识库适配基础验证： - 使用本机未入库的测试配置累计执行 107 次只读请求，其中更新后的最终归档脚本执行 19 次；脚本 diff --git a/scripts/external-knowledge-probe.mjs b/scripts/external-knowledge-probe.mjs index 6a3cd71a..8369ad77 100644 --- a/scripts/external-knowledge-probe.mjs +++ b/scripts/external-knowledge-probe.mjs @@ -78,7 +78,10 @@ function baseUrl(provider, input) { } url.pathname = url.pathname.replace(/\/+$/, '') if (provider === 'dify') url.pathname = url.pathname.replace(/\/v1$/, '') - if (provider === 'fastgpt' && !url.pathname.endsWith('/api')) url.pathname += '/api' + if (provider === 'fastgpt') { + url.pathname = url.pathname.replace(/\/api\/core\/dataset$/, '/api') + if (!url.pathname.endsWith('/api')) url.pathname += '/api' + } if (provider === 'ragflow') url.pathname = url.pathname.replace(/\/api\/v1$/, '') return url } @@ -97,8 +100,9 @@ async function boundedJson(response, maximumBytes = 5_000_000) { async function request(context, name, url, init = {}) { const startedAt = performance.now() + let response try { - const response = await fetch(url, { + response = await fetch(url, { ...init, redirect: 'error', signal: AbortSignal.timeout(15_000), @@ -113,9 +117,14 @@ async function request(context, name, url, init = {}) { name, method: init.method ?? 'GET', path: url.pathname + .replace(/(\/datasets\/)[^/]+/, '$1{id}') .replace(/[0-9a-f]{8}-[0-9a-f-]{27,}/gi, '{id}') .replace(/[0-9a-f]{24}/gi, '{id}'), status: response.status, + businessCode: Number.isFinite(body?.code) || [ + 'invalid_param', 'unauthorized', 'forbidden', 'not_found', + 'dataset_not_found', 'internal_server_error', 'rate_limit_exceeded' + ].includes(body?.code) ? body.code : undefined, durationMs: Math.round(performance.now() - startedAt), responseKeys: objectKeys(body) }) @@ -125,16 +134,59 @@ async function request(context, name, url, init = {}) { name, method: init.method ?? 'GET', path: url.pathname + .replace(/(\/datasets\/)[^/]+/, '$1{id}') .replace(/[0-9a-f]{8}-[0-9a-f-]{27,}/gi, '{id}') .replace(/[0-9a-f]{24}/gi, '{id}'), - status: 'failed', + status: response?.status ?? 'failed', durationMs: Math.round(performance.now() - startedAt), - error: error?.name ?? 'Error' + error: ['Error', 'TypeError', 'SyntaxError', 'TimeoutError', 'AbortError'].includes(error?.name) + ? error.name : 'Error', + causeCode: [ + 'ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'ENOTFOUND', 'EAI_AGAIN', + 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_SOCKET', 'CERT_HAS_EXPIRED', + 'CERT_NOT_YET_VALID', 'DEPTH_ZERO_SELF_SIGNED_CERT', + 'SELF_SIGNED_CERT_IN_CHAIN', 'UNABLE_TO_VERIFY_LEAF_SIGNATURE', + 'UNABLE_TO_GET_ISSUER_CERT_LOCALLY', 'ERR_TLS_CERT_ALTNAME_INVALID', + 'ERR_SSL_WRONG_VERSION_NUMBER' + ].includes(error?.cause?.code) ? error.cause.code : undefined }) - return undefined + return response ? { response, body: undefined } : undefined } } +function responseFailure(result, businessCode) { + if (!result) return { success: false, failure: 'request-failed' } + if (!result.response.ok) return { success: false, failure: 'http-error' } + if (result.body === undefined) return { success: false, failure: 'request-failed' } + if (businessCode !== undefined && asRecord(result.body)?.code !== businessCode) { + return { success: false, failure: 'business-error' } + } + return undefined +} + +function difyOutcome(result, kind) { + const failure = responseFailure(result) + if (failure) return failure + const body = asRecord(result.body) + const valid = kind === 'detail' + ? typeof body?.id === 'string' && body.id.length > 0 + : kind === 'catalog' + ? Array.isArray(body?.data) && body.data.every((item) => typeof asRecord(item)?.id === 'string') + : Array.isArray(body?.records) && body.records.every((item) => asRecord(asRecord(item)?.segment)) + return valid ? { success: true } : { success: false, failure: 'invalid-shape' } +} + +function ragflowOutcome(result, kind) { + const failure = responseFailure(result, 0) + if (failure) return failure + const data = asRecord(result.body)?.data + const valid = kind === 'retrieval' + ? Array.isArray(asRecord(data)?.chunks) && data.chunks.every((item) => asRecord(item)) + : Array.isArray(data) && data.every((item) => typeof asRecord(item)?.id === 'string') && + (kind !== 'detail' || data.length > 0) + return valid ? { success: true } : { success: false, failure: 'invalid-shape' } +} + function resultShape(items, scoreKey) { const scores = finiteNumbers(items.map((item) => asRecord(item)?.[scoreKey])) return { @@ -192,7 +244,8 @@ async function probeDify(configuration_) { catalogUrl.searchParams.set('limit', '100') const catalog = await request(context, 'catalog', catalogUrl) const catalogBody = asRecord(catalog?.body) - const items = Array.isArray(catalogBody?.data) ? catalogBody.data : [] + const catalogOutcome = difyOutcome(catalog, 'catalog') + const items = catalogOutcome.success ? catalogBody.data : [] const first = asRecord(items[0]) let detail const retrievals = [] @@ -209,8 +262,9 @@ async function probeDify(configuration_) { { method: 'POST', body: JSON.stringify({ query }) } ) const retrievalBody = asRecord(retrieval?.body) - const records = Array.isArray(retrievalBody?.records) ? retrievalBody.records : [] - retrievals.push({ mode: 'dataset-defaults', ...difyResultShape(records) }) + const outcome = difyOutcome(retrieval, 'retrieval') + retrievals.push({ mode: 'dataset-defaults', ...outcome, + ...(outcome.success ? difyResultShape(retrievalBody.records) : {}) }) if (extended && asRecord(first.retrieval_model_dict)) { const override = await request( context, @@ -225,8 +279,9 @@ async function probeDify(configuration_) { } ) const overrideBody = asRecord(override?.body) - const overrideRecords = Array.isArray(overrideBody?.records) ? overrideBody.records : [] - retrievals.push({ mode: 'explicit-current-config', ...difyResultShape(overrideRecords) }) + const overrideOutcome = difyOutcome(override, 'retrieval') + retrievals.push({ mode: 'explicit-current-config', ...overrideOutcome, + ...(overrideOutcome.success ? difyResultShape(overrideBody.records) : {}) }) } } return { @@ -240,24 +295,27 @@ async function probeDify(configuration_) { : undefined }, catalog: { - count: items.length, - total: Number.isFinite(catalogBody?.total) ? catalogBody.total : undefined, - itemKeys: unionKeys(items), - retrievalModelKeys: unionKeys(items.map((item) => asRecord(item)?.retrieval_model_dict)), - indexingTechniques: [...new Set(items.map((item) => asRecord(item)?.indexing_technique).filter((value) => typeof value === 'string'))].sort(), - metadataSchemaCount: items.filter((item) => { - const value = asRecord(item)?.doc_metadata - return Array.isArray(value) && value.length > 0 - }).length, - multimodalCount: items.filter((item) => asRecord(item)?.is_multimodal === true).length, - pipelineConfiguredCount: items.filter((item) => typeof asRecord(item)?.pipeline_id === 'string').length, - summaryIndexConfiguredCount: items.filter((item) => asRecord(item)?.summary_index_setting !== null && asRecord(item)?.summary_index_setting !== undefined).length + ...catalogOutcome, + ...(catalogOutcome.success ? { + count: items.length, + total: Number.isFinite(catalogBody?.total) ? catalogBody.total : undefined, + itemKeys: unionKeys(items), + retrievalModelKeys: unionKeys(items.map((item) => asRecord(item)?.retrieval_model_dict)), + indexingTechniques: [...new Set(items.map((item) => asRecord(item)?.indexing_technique).filter((value) => typeof value === 'string'))].sort(), + metadataSchemaCount: items.filter((item) => { + const value = asRecord(item)?.doc_metadata + return Array.isArray(value) && value.length > 0 + }).length, + multimodalCount: items.filter((item) => asRecord(item)?.is_multimodal === true).length, + pipelineConfiguredCount: items.filter((item) => typeof asRecord(item)?.pipeline_id === 'string').length, + summaryIndexConfiguredCount: items.filter((item) => asRecord(item)?.summary_index_setting !== null && asRecord(item)?.summary_index_setting !== undefined).length + } : {}) }, - detail: { keys: objectKeys(detail?.body) }, + detail: { ...(typeof first?.id === 'string' ? difyOutcome(detail, 'detail') : { success: false, failure: 'not-probed' }), keys: objectKeys(detail?.body) }, retrievals, features: { - datasetDefaults: true, - detailSupported: detail?.response.ok === true, + datasetDefaults: retrievals.some((item) => item.mode === 'dataset-defaults' && item.success), + detailSupported: difyOutcome(detail, 'detail').success, retrievalOverrideObserved: items.some((item) => asRecord(item)?.retrieval_model_dict !== undefined), retrievalOverrideProbed: retrievals.some((item) => item.mode === 'explicit-current-config'), nonEmptyResultObserved: retrievals.some((item) => item.count > 0) @@ -267,10 +325,9 @@ async function probeDify(configuration_) { } function fastGptListResult(result) { - if (!result) return { success: false, failure: 'request-failed' } - if (!result.response.ok) return { success: false, failure: 'http-error' } + const failure = responseFailure(result, 200) + if (failure) return failure const body = asRecord(result.body) - if (body?.code !== 200) return { success: false, failure: 'business-error' } const data = asRecord(body.data) const list = Array.isArray(body.data) ? body.data : data?.list if (!Array.isArray(list)) return { success: false, failure: 'invalid-shape' } @@ -279,9 +336,23 @@ function fastGptListResult(result) { function fastGptRetrievalShape(result) { const { list, ...outcome } = fastGptListResult(result) + if (!outcome.success) return outcome + const data = asRecord(result.body.data) + const arrayScores = list.flatMap((item) => Array.isArray(asRecord(item)?.score) ? item.score : []) + const scoreValues = finiteNumbers(arrayScores.map((item) => asRecord(item)?.value)) return { ...outcome, - ...(list ? resultShape(list, 'score') : {}) + envelope: Array.isArray(result.body.data) ? 'data-array' : 'data-list', + ...resultShape(list, 'score'), + scoreTypes: [...new Set(list.flatMap((item) => { + const score = asRecord(item)?.score + return Number.isFinite(score) ? ['number'] : Array.isArray(score) ? ['array'] : [] + }))].sort(), + scoreItemKeys: unionKeys(arrayScores), + scoreValueRange: scoreValues.length > 0 + ? { minimum: Math.min(...scoreValues), maximum: Math.max(...scoreValues) } + : undefined, + rerankEnabled: typeof data?.usingReRank === 'boolean' ? data.usingReRank : undefined } } @@ -292,7 +363,7 @@ async function probeFastGpt(configuration_) { context, 'catalog', endpoint(base, '/core/dataset/list'), - { method: 'POST', body: JSON.stringify({ parentId: '' }) } + { method: 'POST', body: JSON.stringify({ parentId: null }) } ) const catalogBody = asRecord(catalog?.body) const rawData = catalogBody?.data @@ -375,7 +446,7 @@ async function probeFastGpt(configuration_) { features: { searchModes: [...new Set(retrievals.filter((item) => item.success && !item.usingRerank).map((item) => item.searchMode))], rerankProbed: retrievals.some((item) => item.usingRerank), - rerankSucceeded: retrievals.some((item) => item.usingRerank && item.success), + rerankRequestSucceeded: retrievals.some((item) => item.usingRerank && item.success), extensionQueryProbed: false }, operations: context.operations @@ -390,7 +461,8 @@ async function probeRagflow(configuration_) { catalogUrl.searchParams.set('page_size', '100') const catalog = await request(context, 'catalog', catalogUrl) const catalogBody = asRecord(catalog?.body) - const items = Array.isArray(catalogBody?.data) ? catalogBody.data : [] + const catalogOutcome = ragflowOutcome(catalog, 'catalog') + const items = catalogOutcome.success ? catalogBody.data : [] const graphConfiguredItems = items.filter((item) => { const record = asRecord(item) const graph = asRecord(asRecord(record?.parser_config)?.graphrag) @@ -440,8 +512,9 @@ async function probeRagflow(configuration_) { } ) const data = asRecord(asRecord(result?.body)?.data) - const chunks = Array.isArray(data?.chunks) ? data.chunks : [] - retrievals.push({ feature: feature.name, ...resultShape(chunks, 'similarity') }) + const outcome = ragflowOutcome(result, 'retrieval') + retrievals.push({ feature: feature.name, ...outcome, + ...(outcome.success ? resultShape(data.chunks, 'similarity') : {}) }) } } const detailBody = asRecord(detail?.body) @@ -457,19 +530,22 @@ async function probeRagflow(configuration_) { : undefined }, catalog: { - count: items.length, - total: Number.isFinite(catalogBody?.total_datasets) ? catalogBody.total_datasets : undefined, - itemKeys: unionKeys(items), - parserConfigKeys: unionKeys(items.map((item) => asRecord(item)?.parser_config)), - graphConfiguredCount: graphConfiguredItems.length, - graphCompletedCount: graphCompletedItems.length, - knowledgeCompilationConfiguredCount: compilationConfiguredItems.length + ...catalogOutcome, + ...(catalogOutcome.success ? { + count: items.length, + total: Number.isFinite(catalogBody?.total_datasets) ? catalogBody.total_datasets : undefined, + itemKeys: unionKeys(items), + parserConfigKeys: unionKeys(items.map((item) => asRecord(item)?.parser_config)), + graphConfiguredCount: graphConfiguredItems.length, + graphCompletedCount: graphCompletedItems.length, + knowledgeCompilationConfiguredCount: compilationConfiguredItems.length + } : {}) }, - detail: { keys: objectKeys(detailData) }, + detail: { ...(typeof first?.id === 'string' ? ragflowOutcome(detail, 'detail') : { success: false, failure: 'not-probed' }), keys: objectKeys(detailData) }, retrievals, features: { - graphRetrievalProbed: extended, - knowledgeCompilationProbed: extended, + graphRetrievalProbed: retrievals.some((item) => item.feature === 'graph'), + knowledgeCompilationProbed: retrievals.some((item) => item.feature === 'knowledge-compilation'), metadataConditionProbed: false }, operations: context.operations diff --git a/tests/external-knowledge-probe-fastgpt.test.ts b/tests/external-knowledge-probe-fastgpt.test.ts index 6ba74033..053c9cbb 100644 --- a/tests/external-knowledge-probe-fastgpt.test.ts +++ b/tests/external-knowledge-probe-fastgpt.test.ts @@ -20,7 +20,7 @@ type ProviderReport = { count?: number contentPresent?: boolean }> - features: { searchModes: string[]; rerankProbed: boolean; rerankSucceeded: boolean } + features: { searchModes: string[]; rerankProbed: boolean; rerankRequestSucceeded: boolean } operations: Array<{ name: string; status: number | 'failed' }> } @@ -40,8 +40,10 @@ async function probe( globalThis.fetch = async (url, init) => { const path = new URL(url).pathname; let fixture; - if (path.endsWith('/list')) fixture = catalog; - else if (path.endsWith('/detail')) fixture = {body: {code: 200, data: {}}}; + if (path === '/prefix/api/core/dataset/list') { + if (JSON.parse(init.body).parentId !== null) throw new Error('Invalid root parent'); + fixture = catalog; + } else if (path.endsWith('/detail')) fixture = {body: {code: 200, data: {}}}; else if (path.endsWith('/searchTest')) { const input = JSON.parse(init.body); fixture = fixtures[input.usingReRank ? 'rerank' : input.searchMode]; @@ -60,7 +62,7 @@ async function probe( { env: { ...process.env, - GOODBUDDY_FASTGPT_BASE_URL: 'http://fixture.invalid', + GOODBUDDY_FASTGPT_BASE_URL: 'http://fixture.invalid/prefix/api/core/dataset/', GOODBUDDY_FASTGPT_API_KEY: 'fixture-only-key' }, timeout: 10_000 @@ -88,7 +90,7 @@ describe('FastGPT probe success evidence', () => { embedding: fixture, fullTextRecall: fixture, mixedRecall: fixture, rerank: fixture }) expect(report.features).toEqual({ - searchModes: [], rerankProbed: true, rerankSucceeded: false, + searchModes: [], rerankProbed: true, rerankRequestSucceeded: false, extensionQueryProbed: false }) expect(report.retrievals).toHaveLength(4) @@ -108,7 +110,7 @@ describe('FastGPT probe success evidence', () => { }) expect(report.features).toMatchObject({ searchModes: ['embedding', 'fullTextRecall'], - rerankProbed: true, rerankSucceeded: true + rerankProbed: true, rerankRequestSucceeded: true }) expect(report.retrievals[0]).toMatchObject({ success: true, count: 0, contentPresent: false }) expect(report.retrievals[1]).toMatchObject({ success: true, count: 0 }) @@ -126,8 +128,31 @@ describe('FastGPT probe success evidence', () => { expect(report.catalog).not.toHaveProperty('count') expect(report.retrievals).toEqual([]) expect(report.features).toMatchObject({ - searchModes: [], rerankProbed: false, rerankSucceeded: false + searchModes: [], rerankProbed: false, rerankRequestSucceeded: false }) expect(report.operations).toHaveLength(1) }) + + it('reports both retrieval envelopes, score structures and actual rerank response separately', async () => { + const report = await probe({ + embedding: { body: { code: 200, data: [{ score: 0.6, q: 'private text' }] } }, + fullTextRecall: { body: { code: 200, data: { list: [], usingReRank: true } } }, + mixedRecall: success(), + rerank: { body: { code: 200, data: { + usingReRank: false, + list: [{ score: [{ type: 'private score type', value: 0.2, index: 1 }], q: 'private text' }] + } } } + }) + expect(report.retrievals[0]).toMatchObject({ + envelope: 'data-array', scoreTypes: ['number'], scoreRange: { minimum: 0.6, maximum: 0.6 } + }) + expect(report.retrievals[3]).toMatchObject({ + envelope: 'data-list', scoreTypes: ['array'], scoreItemKeys: ['index', 'type', 'value'], + scoreValueRange: { minimum: 0.2, maximum: 0.2 }, rerankEnabled: false + }) + expect(report.retrievals[1]).toMatchObject({ rerankEnabled: true }) + expect(report.retrievals[0]).not.toHaveProperty('rerankEnabled') + expect(report.features.rerankRequestSucceeded).toBe(true) + expect(JSON.stringify(report)).not.toContain('private') + }) }) diff --git a/tests/external-knowledge-probe-outcomes.test.ts b/tests/external-knowledge-probe-outcomes.test.ts new file mode 100644 index 00000000..41feb7a0 --- /dev/null +++ b/tests/external-knowledge-probe-outcomes.test.ts @@ -0,0 +1,127 @@ +import { execFile } from 'node:child_process' +import { resolve } from 'node:path' +import { pathToFileURL } from 'node:url' +import { promisify } from 'node:util' +import { describe, expect, it } from 'vitest' + +type Fixture = { status?: number; body?: unknown; raw?: string; causeCode?: string } +type Outcome = { success: boolean; failure?: string; count?: number } +type Report = { + catalog: Outcome + detail: Outcome + retrievals: Outcome[] + features: Record + operations: Array<{ status: number | 'failed'; businessCode?: number | string; causeCode?: string }> +} + +async function probe(provider: 'dify' | 'ragflow', fixtures: Fixture[]): Promise { + const script = ` + const fixtures = ${JSON.stringify(fixtures)}; + globalThis.fetch = async () => { + const fixture = fixtures.shift(); + if (!fixture) throw new Error('Unexpected request'); + if (fixture.causeCode) throw new TypeError('private network message', { + cause: { code: fixture.causeCode, message: 'private cause' } + }); + return new Response(fixture.raw ?? JSON.stringify(fixture.body), { status: fixture.status ?? 200 }); + }; + process.argv = [process.execPath, 'probe', '--provider=${provider}', '--extended']; + await import(${JSON.stringify(pathToFileURL(resolve('scripts/external-knowledge-probe.mjs')).href)}); + ` + const { stdout } = await promisify(execFile)(process.execPath, ['--input-type=module', '--eval', script], { + env: { + ...process.env, + [`GOODBUDDY_${provider.toUpperCase()}_BASE_URL`]: 'https://fixture.invalid', + [`GOODBUDDY_${provider.toUpperCase()}_API_KEY`]: 'fixture-only-key' + }, + timeout: 10_000 + }) + expect(stdout).not.toMatch(/private|fixture.invalid|fixture-only-key|fixture-dataset/) + return JSON.parse(stdout).providers[0] as Report +} + +describe.each(['dify', 'ragflow'] as const)('%s probe outcomes', (provider) => { + const catalog = { body: { code: 0, data: [{ id: 'fixture-dataset', retrieval_model_dict: {} }] } } + const detail = { body: provider === 'dify' ? { id: 'fixture-dataset' } : catalog.body } + const empty = { body: provider === 'dify' ? { records: [] } : { code: 0, data: { chunks: [] } } } + + it('reports successful empty catalogs without claiming uncalled features', async () => { + const report = await probe(provider, [{ body: { code: 0, data: [] } }]) + expect(report.catalog).toMatchObject({ success: true, count: 0 }) + expect(report.detail).toMatchObject({ success: false, failure: 'not-probed' }) + expect(report.retrievals).toEqual([]) + expect(Object.values(report.features)).not.toContain(true) + expect(report.operations).toHaveLength(1) + }) + + it.each([ + [{ status: 403, body: { code: 'forbidden', message: 'private message' } }, 'http-error'], + [{ status: 502, raw: 'private body' }, 'http-error'], + [{ body: { code: 0, data: {} } }, 'invalid-shape'], + [{ raw: '{broken' }, 'request-failed'] + ] as const)('does not turn catalog failure into an empty catalog: %j', async (fixture, failure) => { + const report = await probe(provider, [fixture]) + expect(report.catalog).toMatchObject({ success: false, failure }) + expect(report.catalog).not.toHaveProperty('count') + expect(report.detail).toMatchObject({ success: false, failure: 'not-probed' }) + expect(report.retrievals).toEqual([]) + expect(Object.values(report.features)).not.toContain(true) + expect(report.operations[0]).toMatchObject({ status: (fixture as Fixture).status ?? 200 }) + }) + + it('distinguishes successful empty retrievals from failed detail and retrievals', async () => { + const failure = { status: 500, body: { code: 500, message: 'private error' } } + const report = await probe(provider, [catalog, failure, empty, failure, failure]) + expect(report.catalog).toMatchObject({ success: true, count: 1 }) + expect(report.detail).toMatchObject({ success: false, failure: 'http-error' }) + expect(report.retrievals[0]).toMatchObject({ success: true, count: 0 }) + for (const retrieval of report.retrievals.slice(1)) { + expect(retrieval).toMatchObject({ success: false, failure: 'http-error' }) + expect(retrieval).not.toHaveProperty('count') + } + expect(report.operations[1]).toMatchObject({ status: 500, businessCode: 500 }) + }) + + it('validates detail and retrieval shapes', async () => { + const invalid = { body: { code: 0, records: [null], data: { chunks: null } } } + const report = await probe(provider, [catalog, invalid, invalid, invalid, invalid]) + expect(report.detail).toMatchObject({ success: false, failure: 'invalid-shape' }) + expect(report.retrievals.every((item) => !item.success && item.failure === 'invalid-shape')).toBe(true) + }) + + it('recognizes valid detail responses', async () => { + const report = await probe(provider, [catalog, detail, empty, empty, empty]) + expect(report.detail.success).toBe(true) + }) + + it.each(['UNABLE_TO_VERIFY_LEAF_SIGNATURE', 'ERR_TLS_CERT_ALTNAME_INVALID', 'private-code'])( + 'only exposes allowlisted network cause codes: %s', async (causeCode) => { + const report = await probe(provider, [{ causeCode }]) + expect(report.operations[0]).toMatchObject({ status: 'failed' }) + expect(report.operations[0]?.causeCode).toBe(causeCode === 'private-code' ? undefined : causeCode) + expect(report.catalog).toMatchObject({ success: false, failure: 'request-failed' }) + } + ) + + it.each(['forbidden', 'private-code'])('only exposes known string business codes: %s', async (code) => { + const report = await probe(provider, [{ status: 403, body: { code, message: 'private message' } }]) + expect(report.operations[0]).toMatchObject({ status: 403 }) + expect(report.operations[0]?.businessCode).toBe(code === 'forbidden' ? code : undefined) + }) +}) + +it('rejects RAGFlow nonzero business codes in catalog, detail and retrieval', async () => { + const failure = { body: { code: 102, data: [{ id: 'fixture-dataset' }] } } + const catalogReport = await probe('ragflow', [failure]) + expect(catalogReport.catalog).toMatchObject({ success: false, failure: 'business-error' }) + expect(catalogReport.catalog).not.toHaveProperty('count') + const report = await probe('ragflow', [ + { body: { code: 0, data: [{ id: 'fixture-dataset' }] } }, + failure, failure, failure, failure + ]) + expect(report.detail).toMatchObject({ success: false, failure: 'business-error' }) + for (const retrieval of report.retrievals) { + expect(retrieval).toMatchObject({ success: false, failure: 'business-error' }) + expect(retrieval).not.toHaveProperty('count') + } +}) diff --git a/tests/external-knowledge-probe-report.test.ts b/tests/external-knowledge-probe-report.test.ts index 7ce931c3..3f97dba8 100644 --- a/tests/external-knowledge-probe-report.test.ts +++ b/tests/external-knowledge-probe-report.test.ts @@ -23,18 +23,22 @@ type ProbeReport = { providers: ProbeProvider[] } -async function report(): Promise { +async function report(filename = 'external-knowledge-probe-baseline.json'): Promise { return JSON.parse( await readFile( - resolve('docs/features/knowledge-base/external-knowledge-probe-baseline.json'), + resolve('docs/features/knowledge-base', filename), 'utf8' ) ) as ProbeReport } describe('external knowledge probe baseline', () => { - it('contains no endpoint, credential, remote identifier, or mutation operation', async () => { - const value = await report() + it.each([ + 'external-knowledge-probe-baseline.json', + 'external-knowledge-probe-2026-09-10-strict.json', + 'external-knowledge-probe-2026-09-10-extended.json' + ])('%s contains no endpoint, credential, remote identifier, or mutation operation', async (filename) => { + const value = await report(filename) const serialized = JSON.stringify(value) expect(serialized).not.toMatch(/https?:\/\//)