fix: bound document extraction

This commit is contained in:
lofyer
2026-08-13 04:46:30 +08:00
parent 5b579ae100
commit e3b5702767
13 changed files with 1106 additions and 194 deletions
+29
View File
@@ -139,4 +139,33 @@ describe('DocumentOcrBroker', () => {
broker.dispose() broker.dispose()
await expect(active).rejects.toThrow('OCR 解析已取消') await expect(active).rejects.toThrow('OCR 解析已取消')
}) })
it('rejects OCR sections outside the requested page set', async () => {
const send = vi.fn()
const broker = new DocumentOcrBroker({
isDestroyed: vi.fn(() => false),
webContents: { send }
} as never)
const pending = broker.recognize(request())
const dispatched = send.mock.calls.find(
([channel]) => channel === ipcChannels.documentParsingOcrRequest
)?.[1] as { requestId: string }
broker.respond({
requestId: dispatched.requestId,
sections: [
{
locator: '第 2 页',
pageNumber: 2,
content: 'wrong page',
confidence: 0.9
}
],
pageCount: 2,
warnings: []
})
await expect(pending).rejects.toThrow('OCR 响应页码无效')
broker.dispose()
})
}) })
+17
View File
@@ -99,6 +99,23 @@ export class DocumentOcrBroker {
return return
} }
if (result.success) { if (result.success) {
if (
pending.request.mimeType === 'application/pdf' &&
result.data.sections.some(
(section) =>
section.pageNumber === undefined ||
section.pageNumber > result.data.pageCount ||
(
pending.request.pageNumbers !== undefined &&
!pending.request.pageNumbers.includes(section.pageNumber)
)
)
) {
this.finishRequest(requestId, () =>
pending.reject(new Error('OCR 响应页码无效'))
)
return
}
this.finishRequest(requestId, () => this.finishRequest(requestId, () =>
pending.resolve(result.data) pending.resolve(result.data)
) )
+112 -1
View File
@@ -57,6 +57,7 @@ function createService(overrides?: {
requestId: string requestId: string
sections: Array<{ sections: Array<{
locator: string locator: string
pageNumber?: number
content: string content: string
confidence: number confidence: number
}> }>
@@ -75,6 +76,7 @@ function createService(overrides?: {
sections: [ sections: [
{ {
locator: '第 1 页', locator: '第 1 页',
pageNumber: 1,
content: '扫描件识别正文', content: '扫描件识别正文',
confidence: 0.93 confidence: 0.93
} }
@@ -136,7 +138,9 @@ describe('DocumentParsingService', () => {
locator: '第 1 页', locator: '第 1 页',
content: '扫描件识别正文', content: '扫描件识别正文',
method: 'ocr', method: 'ocr',
confidence: 0.93 confidence: 0.93,
pageNumber: 1,
blockKind: 'text'
} }
]) ])
expect(recognize).toHaveBeenCalledWith( expect(recognize).toHaveBeenCalledWith(
@@ -203,6 +207,41 @@ describe('DocumentParsingService', () => {
]) ])
}) })
it('does not silently index a partial mixed PDF when OCR fails', async () => {
const { service } = createService({
recognize: async () => {
throw new Error('OCR runtime failed')
}
})
await expect(
service.parse(
'mixed.pdf',
createPdfFixture('Native PDF body text', ''),
'knowledge-index'
)
).rejects.toThrow('OCR runtime failed')
})
it('does not silently index a mixed PDF when OCR returns no text', async () => {
const { service } = createService({
recognize: async () => ({
requestId: crypto.randomUUID(),
sections: [],
pageCount: 2,
warnings: ['第 2 页未识别到文字']
})
})
await expect(
service.parse(
'mixed.pdf',
createPdfFixture('Native PDF body text', ''),
'knowledge-index'
)
).rejects.toThrow('第 2 页未识别到可索引文本')
})
it('limits the number of pages sent to OCR rather than total PDF pages', async () => { it('limits the number of pages sent to OCR rather than total PDF pages', async () => {
const { recognize, service } = createService({ const { recognize, service } = createService({
settings: { maximumPages: 1 } settings: { maximumPages: 1 }
@@ -259,4 +298,76 @@ describe('DocumentParsingService', () => {
).rejects.toThrow('请先安装并校验') ).rejects.toThrow('请先安装并校验')
expect(settingsStore.update).not.toHaveBeenCalled() expect(settingsStore.update).not.toHaveBeenCalled()
}) })
it('rejects oversized non-PDF input through the unified service', async () => {
const { service } = createService()
await expect(
service.parse(
'large.txt',
Buffer.alloc(20 * 1024 * 1024 + 1),
'knowledge-index'
)
).rejects.toThrow('20MB')
})
it('bounds OCR output before returning parsed sections', async () => {
const { service } = createService({
recognize: async () => ({
requestId: crypto.randomUUID(),
sections: [
{
locator: '第 1 页',
pageNumber: 1,
content: 'x'.repeat(1_000_000),
confidence: 0.9
},
{
locator: '第 2 页',
pageNumber: 2,
content: 'y'.repeat(1_000_000),
confidence: 0.9
},
{
locator: '第 3 页',
pageNumber: 3,
content: 'z'.repeat(1_000_000),
confidence: 0.9
},
{
locator: '第 4 页',
pageNumber: 4,
content: 'a'.repeat(1_000_000),
confidence: 0.9
},
{
locator: '第 5 页',
pageNumber: 5,
content: 'b'.repeat(1_000_000),
confidence: 0.9
}
],
pageCount: 5,
warnings: []
}),
settings: {
chatWorkflow: 'high-fidelity',
maximumPages: 5
}
})
const parsed = await service.parse(
'large-ocr.pdf',
createPdfFixture('', '', '', '', ''),
'chat-attachment'
)
expect(parsed.content.length).toBeLessThanOrEqual(5_000_000)
expect(
parsed.sections.map((section) => section.content).join('\n\n')
).toBe(parsed.content)
expect(parsed.warnings).toContain(
'文档提取文本超过 5,000,000 字符,已截断'
)
})
}) })
+126 -24
View File
@@ -3,6 +3,8 @@ import {
documentParsingDiagnosticSchema, documentParsingDiagnosticSchema,
documentParsingSettingsUpdateSchema, documentParsingSettingsUpdateSchema,
documentParsingSnapshotSchema, documentParsingSnapshotSchema,
maximumDocumentExtractedCharacters,
maximumDocumentParsingWarnings,
type DocumentParsingDiagnostic, type DocumentParsingDiagnostic,
type DocumentParsingPurpose, type DocumentParsingPurpose,
type DocumentParsingSettings, type DocumentParsingSettings,
@@ -12,6 +14,7 @@ import type { DocumentOcrBroker } from './document-ocr-broker'
import type { DocumentOcrModelManager } from './document-ocr-model-manager' import type { DocumentOcrModelManager } from './document-ocr-model-manager'
import type { DocumentParsingSettingsStore } from './document-parsing-settings-store' import type { DocumentParsingSettingsStore } from './document-parsing-settings-store'
import { import {
assertDocumentBuffer,
DocumentTextUnavailableError, DocumentTextUnavailableError,
extractPdfTextPages, extractPdfTextPages,
parseDocument, parseDocument,
@@ -81,20 +84,57 @@ function buildPdfDocument(
pageCount: number, pageCount: number,
warnings: string[] = [] warnings: string[] = []
): ParsedDocument { ): ParsedDocument {
const content = sections const truncationWarning =
'文档提取文本超过 5,000,000 字符,已截断'
const boundedWarnings = [
...new Set(
warnings.filter((warning) => warning !== truncationWarning)
)
]
const limitedSections: ParsedSection[] = []
let remaining = maximumDocumentExtractedCharacters
let truncated = false
for (const section of sections) {
const separatorLength = limitedSections.length > 0 ? 2 : 0
if (remaining <= separatorLength) {
truncated = true
break
}
const content = section.content.slice(0, remaining - separatorLength)
if (content) {
limitedSections.push(
content === section.content ? section : { ...section, content }
)
remaining -= separatorLength + content.length
}
if (content.length < section.content.length) {
truncated = true
break
}
}
if (limitedSections.length < sections.length) {
truncated = true
}
const content = limitedSections
.map((section) => section.content) .map((section) => section.content)
.join('\n\n') .join('\n\n')
.slice(0, 5_000_000)
if (!content) { if (!content) {
throw new DocumentTextUnavailableError() throw new DocumentTextUnavailableError()
} }
const documentWarnings =
truncated || warnings.includes(truncationWarning)
? [
...boundedWarnings.slice(0, maximumDocumentParsingWarnings - 1),
truncationWarning
]
: boundedWarnings.slice(0, maximumDocumentParsingWarnings)
return { return {
title: name.replace(/\.[^.]+$/u, ''), title: name.replace(/\.[^.]+$/u, ''),
sourceFormat: '.pdf', sourceFormat: '.pdf',
content, content,
sections, sections: limitedSections,
pageCount, pageCount,
warnings warnings: documentWarnings
} }
} }
@@ -104,7 +144,9 @@ function nativePdfSections(pages: PdfTextPage[]): ParsedSection[] {
.map((page) => ({ .map((page) => ({
locator: `${page.pageNumber}`, locator: `${page.pageNumber}`,
content: page.content, content: page.content,
method: 'native' as const method: 'native' as const,
pageNumber: page.pageNumber,
blockKind: 'text' as const
})) }))
} }
@@ -158,12 +200,14 @@ export class DocumentParsingService {
signal signal
) => { ) => {
ensureNotAborted(signal) ensureNotAborted(signal)
assertDocumentBuffer(buffer)
if (extname(name).toLowerCase() !== '.pdf') { if (extname(name).toLowerCase() !== '.pdf') {
return parseDocument(name, buffer) return parseDocument(name, buffer, signal)
} }
const settings = await this.settingsStore.get() const settings = await this.settingsStore.get()
const pages = await extractPdfTextPages(buffer) const extracted = await extractPdfTextPages(buffer, { signal })
const { pages } = extracted
ensureNotAborted(signal) ensureNotAborted(signal)
const mode = effectiveOcrMode(settings, purpose) const mode = effectiveOcrMode(settings, purpose)
const pagesWithoutUsefulText = pages const pagesWithoutUsefulText = pages
@@ -179,13 +223,19 @@ export class DocumentParsingService {
if (mode === 'disabled') { if (mode === 'disabled') {
const native = nativePdfSections(pages) const native = nativePdfSections(pages)
if (native.length > 0) { if (native.length > 0) {
const warnings = [
...(pagesWithoutUsefulText.length > 0
? ['部分页面没有有效文本,当前工作流未启用 OCR']
: []),
...(extracted.truncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: [])
]
return buildPdfDocument( return buildPdfDocument(
name, name,
native, native,
pages.length, extracted.pageCount,
pagesWithoutUsefulText.length > 0 warnings
? ['部分页面没有有效文本,当前工作流未启用 OCR']
: []
) )
} }
throw new DocumentTextUnavailableError( throw new DocumentTextUnavailableError(
@@ -196,7 +246,10 @@ export class DocumentParsingService {
return buildPdfDocument( return buildPdfDocument(
name, name,
nativePdfSections(pages), nativePdfSections(pages),
pages.length extracted.pageCount,
extracted.truncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: []
) )
} }
if (ocrPageNumbers.length > settings.maximumPages) { if (ocrPageNumbers.length > settings.maximumPages) {
@@ -211,11 +264,20 @@ export class DocumentParsingService {
if (!modelStatus.available || !modelStatus.verified) { if (!modelStatus.available || !modelStatus.verified) {
if ( if (
mode === 'auto' && mode === 'auto' &&
purpose !== 'knowledge-index' &&
native.some((section) => hasUsefulText(section.content)) native.some((section) => hasUsefulText(section.content))
) { ) {
return buildPdfDocument(name, native, pages.length, [ return buildPdfDocument(
`本地 OCR 不可用,已保留 PDF 文本层内容:${modelStatus.detail}` name,
]) native,
extracted.pageCount,
[
`本地 OCR 不可用,已保留 PDF 文本层内容:${modelStatus.detail}`,
...(extracted.truncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: [])
]
)
} }
throw new Error(modelStatus.detail) throw new Error(modelStatus.detail)
} }
@@ -238,23 +300,45 @@ export class DocumentParsingService {
ensureNotAborted(signal) ensureNotAborted(signal)
if ( if (
mode === 'auto' && mode === 'auto' &&
purpose !== 'knowledge-index' &&
native.some((section) => hasUsefulText(section.content)) native.some((section) => hasUsefulText(section.content))
) { ) {
const detail = const detail =
error instanceof Error ? error.message : '本地 OCR 识别失败' error instanceof Error ? error.message : '本地 OCR 识别失败'
return buildPdfDocument(name, native, pages.length, [ return buildPdfDocument(
`本地 OCR 失败,已保留 PDF 文本层内容:${detail}` name,
]) native,
extracted.pageCount,
[
`本地 OCR 失败,已保留 PDF 文本层内容:${detail}`,
...(extracted.truncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: [])
]
)
} }
throw error throw error
} }
ensureNotAborted(signal) ensureNotAborted(signal)
const ocrByLocator = new Map( const ocrByPageNumber = new Map(
ocr.sections.map((section) => [section.locator, section]) ocr.sections.flatMap((section) =>
section.pageNumber === undefined
? []
: [[section.pageNumber, section] as const]
)
) )
const missingOcrPage = ocrPageNumbers.find(
(pageNumber) => !ocrByPageNumber.has(pageNumber)
)
if (
missingOcrPage !== undefined &&
purpose === 'knowledge-index'
) {
throw new Error(`${missingOcrPage} 页未识别到可索引文本`)
}
const merged = pages.flatMap((page): ParsedSection[] => { const merged = pages.flatMap((page): ParsedSection[] => {
const locator = `${page.pageNumber}` const locator = `${page.pageNumber}`
const recognized = ocrByLocator.get(locator) const recognized = ocrByPageNumber.get(page.pageNumber)
if ( if (
recognized && recognized &&
(mode === 'always' || !hasUsefulText(page.content)) (mode === 'always' || !hasUsefulText(page.content))
@@ -264,15 +348,33 @@ export class DocumentParsingService {
locator, locator,
content: recognized.content, content: recognized.content,
method: 'ocr', method: 'ocr',
confidence: recognized.confidence confidence: recognized.confidence,
pageNumber: page.pageNumber,
blockKind: 'text'
} }
] ]
} }
return page.content return page.content
? [{ locator, content: page.content, method: 'native' }] ? [{
locator,
content: page.content,
method: 'native',
pageNumber: page.pageNumber,
blockKind: 'text'
}]
: [] : []
}) })
return buildPdfDocument(name, merged, pages.length, ocr.warnings) return buildPdfDocument(
name,
merged,
extracted.pageCount,
[
...ocr.warnings,
...(extracted.truncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: [])
]
)
} }
async diagnose( async diagnose(
+16
View File
@@ -804,8 +804,13 @@ describe('registerIpcHandlers document parsing', () => {
temporaryDirectories.push(directory) temporaryDirectories.push(directory)
const diagnosticPath = join(directory, 'diagnostic.pdf') const diagnosticPath = join(directory, 'diagnostic.pdf')
const artifactPath = join(directory, 'artifact.pdf') const artifactPath = join(directory, 'artifact.pdf')
const oversizedArtifactPath = join(directory, 'oversized.pdf')
await writeFile(diagnosticPath, 'diagnostic') await writeFile(diagnosticPath, 'diagnostic')
await writeFile(artifactPath, 'artifact') await writeFile(artifactPath, 'artifact')
await writeFile(
oversizedArtifactPath,
Buffer.alloc(20 * 1024 * 1024 + 1)
)
const diagnostic = { const diagnostic = {
fileName: 'diagnostic.pdf', fileName: 'diagnostic.pdf',
sourceFormat: 'PDF', sourceFormat: 'PDF',
@@ -923,6 +928,17 @@ describe('registerIpcHandlers document parsing', () => {
'artifact-import' 'artifact-import'
) )
electronMocks.showOpenDialog.mockResolvedValueOnce({
canceled: false,
filePaths: [oversizedArtifactPath]
})
await expect(
electronMocks.handlers.get(
ipcChannels.artifactsImportFiles
)?.(event)
).rejects.toThrow('超过大小限制')
expect(documentParsingService.parse).toHaveBeenCalledOnce()
await dispose() await dispose()
}) })
}) })
+38 -5
View File
@@ -5,12 +5,19 @@ import {
ipcMain, ipcMain,
shell shell
} from 'electron' } from 'electron'
import { lstat, mkdir, readFile, realpath, stat } from 'node:fs/promises' import {
lstat,
mkdir,
readFile,
realpath,
stat
} from 'node:fs/promises'
import { randomUUID } from 'node:crypto' import { randomUUID } from 'node:crypto'
import { homedir } from 'node:os' import { homedir } from 'node:os'
import { basename, extname, isAbsolute, join } from 'node:path' import { basename, extname, isAbsolute, join } from 'node:path'
import { z } from 'zod' import { z } from 'zod'
import { formatShortcutForDisplay } from '../shared/shortcut' import { formatShortcutForDisplay } from '../shared/shortcut'
import { readBoundedFile } from './workspace-file-access'
import { import {
approvalDecisionSchema, approvalDecisionSchema,
agentQuestionResponseSchema, agentQuestionResponseSchema,
@@ -473,6 +480,19 @@ function assertTrustedSender(
} }
} }
async function readArtifactImportFile(
path: string,
maximumBytes: number,
label: string
): Promise<Buffer> {
return readBoundedFile(
path,
maximumBytes,
`${label}超过大小限制`,
`${label}不是普通文件`
)
}
function getKnowledgeSnapshot( function getKnowledgeSnapshot(
service: KnowledgeService, service: KnowledgeService,
selectedLibraryId?: string selectedLibraryId?: string
@@ -3420,14 +3440,15 @@ export function registerIpcHandlers(
const artifacts: AssistantArtifact[] = [] const artifacts: AssistantArtifact[] = []
for (const filePath of result.filePaths.slice(0, 10)) { for (const filePath of result.filePaths.slice(0, 10)) {
const canonicalPath = await realpath(filePath) const canonicalPath = await realpath(filePath)
const file = await readFile(canonicalPath)
const extension = extname(canonicalPath).toLowerCase() const extension = extname(canonicalPath).toLowerCase()
const name = basename(canonicalPath) const name = basename(canonicalPath)
const imageMimeType = imageMimeTypes[extension] const imageMimeType = imageMimeTypes[extension]
if (imageMimeType) { if (imageMimeType) {
if (file.byteLength > 3 * 1024 * 1024) { const file = await readArtifactImportFile(
throw new Error(`图片“${name}”超过 3MB 预览限制`) canonicalPath,
} 3 * 1024 * 1024,
`图片“${name}`
)
artifacts.push( artifacts.push(
assistantDatabase.createImageArtifact({ assistantDatabase.createImageArtifact({
projectId, projectId,
@@ -3439,6 +3460,11 @@ export function registerIpcHandlers(
continue continue
} }
if (extension === '.html' || extension === '.htm') { if (extension === '.html' || extension === '.htm') {
const file = await readArtifactImportFile(
canonicalPath,
5 * 1024 * 1024,
`文件“${name}`
)
artifacts.push( artifacts.push(
assistantDatabase.createInlineArtifact({ assistantDatabase.createInlineArtifact({
projectId, projectId,
@@ -3450,6 +3476,13 @@ export function registerIpcHandlers(
) )
continue continue
} }
const file = await readArtifactImportFile(
canonicalPath,
extension === '.pdf'
? 20 * 1024 * 1024
: 5 * 1024 * 1024,
`文件“${name}`
)
const parsed = documentParsingService const parsed = documentParsingService
? await documentParsingService.parse( ? await documentParsingService.parse(
name, name,
@@ -20,9 +20,14 @@ describe('PDF extraction in Electron main', () => {
promise: Promise.resolve({ promise: Promise.resolve({
numPages: 1, numPages: 1,
getPage: vi.fn(async () => ({ getPage: vi.fn(async () => ({
getTextContent: vi.fn(async () => ({ streamTextContent: vi.fn(() =>
items: [{ str: 'PDF body text' }] new ReadableStream({
})), start(controller) {
controller.enqueue({ items: [{ str: 'PDF body text' }] })
controller.close()
}
})
),
cleanup cleanup
})) }))
}), }),
@@ -31,12 +36,14 @@ describe('PDF extraction in Electron main', () => {
await expect( await expect(
extractPdfTextPages(Buffer.from('synthetic PDF')) extractPdfTextPages(Buffer.from('synthetic PDF'))
).resolves.toEqual([ ).resolves.toEqual({
{ pageCount: 1,
truncated: false,
pages: [{
pageNumber: 1, pageNumber: 1,
content: 'PDF body text' content: 'PDF body text'
} }]
]) })
expect(getDocument).toHaveBeenCalledWith({ expect(getDocument).toHaveBeenCalledWith({
data: expect.any(Uint8Array), data: expect.any(Uint8Array),
disableFontFace: true, disableFontFace: true,
@@ -55,26 +62,33 @@ describe('PDF extraction in Electron main', () => {
promise: Promise.resolve({ promise: Promise.resolve({
numPages: 1, numPages: 1,
getPage: vi.fn(async () => ({ getPage: vi.fn(async () => ({
getTextContent: vi.fn(async () => ({ streamTextContent: vi.fn(() =>
items: [ new ReadableStream({
{ start(controller) {
str: 'first', controller.enqueue({
hasEOL: true, items: [
transform: [1, 0, 0, 1, 10, 100], {
height: 10 str: 'first',
}, hasEOL: true,
{ transform: [1, 0, 0, 1, 10, 100],
str: 'second', height: 10
transform: [1, 0, 0, 1, 10, 80], },
height: 10 {
}, str: 'second',
{ transform: [1, 0, 0, 1, 10, 80],
str: 'line', height: 10
transform: [1, 0, 0, 1, 50, 80], },
height: 10 {
str: 'line',
transform: [1, 0, 0, 1, 50, 80],
height: 10
}
]
})
controller.close()
} }
] })
})), ),
cleanup cleanup
})) }))
}), }),
@@ -83,11 +97,127 @@ describe('PDF extraction in Electron main', () => {
await expect( await expect(
extractPdfTextPages(Buffer.from('synthetic PDF')) extractPdfTextPages(Buffer.from('synthetic PDF'))
).resolves.toEqual([ ).resolves.toEqual({
{ pageCount: 1,
truncated: false,
pages: [{
pageNumber: 1, pageNumber: 1,
content: 'first\nsecond line' content: 'first\nsecond line'
} }]
]) })
})
it('stops extracting pages at the aggregate character limit', async () => {
const getPage = vi.fn(async (pageNumber: number) => ({
streamTextContent: vi.fn(() =>
new ReadableStream({
start(controller) {
controller.enqueue({
items: [{ str: pageNumber === 1 ? 'first' : 'second' }]
})
controller.close()
}
})
),
cleanup: vi.fn()
}))
const destroy = vi.fn(async () => undefined)
getDocument.mockReturnValue({
promise: Promise.resolve({ numPages: 2, getPage }),
destroy
})
await expect(
extractPdfTextPages(Buffer.from('synthetic PDF'), {
maximumCharacters: 5
})
).resolves.toEqual({
pageCount: 2,
truncated: true,
pages: [{ pageNumber: 1, content: 'first' }]
})
expect(getPage).toHaveBeenCalledOnce()
})
it('cancels the PDF text stream after reaching the character limit', async () => {
let pulls = 0
const cancel = vi.fn()
const streamTextContent = vi.fn(() =>
new ReadableStream({
pull(controller) {
pulls += 1
controller.enqueue({ items: [{ str: 'abcde' }] })
},
cancel
})
)
const destroy = vi.fn(async () => undefined)
getDocument.mockReturnValue({
promise: Promise.resolve({
numPages: 1,
getPage: vi.fn(async () => ({
streamTextContent,
cleanup: vi.fn()
}))
}),
destroy
})
await expect(
extractPdfTextPages(Buffer.from('synthetic PDF'), {
maximumCharacters: 5
})
).resolves.toEqual({
pageCount: 1,
truncated: true,
pages: [{ pageNumber: 1, content: 'abcde' }]
})
expect(pulls).toBeLessThanOrEqual(2)
expect(cancel).toHaveBeenCalled()
})
it('rejects oversized PDFs before reading pages', async () => {
const getPage = vi.fn()
const destroy = vi.fn(async () => undefined)
getDocument.mockReturnValue({
promise: Promise.resolve({
numPages: 3,
getPage
}),
destroy
})
await expect(
extractPdfTextPages(Buffer.from('synthetic PDF'), {
maximumPages: 2
})
).rejects.toThrow('超过 2 页限制')
expect(getPage).not.toHaveBeenCalled()
expect(destroy).toHaveBeenCalledOnce()
})
it('destroys PDF loading when extraction is cancelled', async () => {
let resolveLoading: ((value: {
numPages: number
getPage: ReturnType<typeof vi.fn>
}) => void) | undefined
const destroy = vi.fn(async () => undefined)
getDocument.mockReturnValue({
promise: new Promise((resolve) => {
resolveLoading = resolve
}),
destroy
})
const controller = new AbortController()
const extraction = extractPdfTextPages(
Buffer.from('synthetic PDF'),
{ signal: controller.signal }
)
controller.abort(new Error('cancel PDF extraction'))
resolveLoading?.({ numPages: 0, getPage: vi.fn() })
await expect(extraction).rejects.toThrow('cancel PDF extraction')
expect(destroy).toHaveBeenCalled()
}) })
}) })
@@ -136,6 +136,53 @@ describe('document parser', () => {
await expect( await expect(
parseDocument('expanded.docx', Buffer.from(expandedArchive)) parseDocument('expanded.docx', Buffer.from(expandedArchive))
).rejects.toThrow('损坏') ).rejects.toThrow('损坏')
await expect(
parseDocument('invalid.txt', Buffer.from([0xc3, 0x28]))
).rejects.toThrow('UTF-8')
})
it('keeps extracted sections consistent with the document character limit', async () => {
const parsed = await parseDocument(
'large.txt',
Buffer.from('x'.repeat(5_000_100))
)
expect(parsed.content).toHaveLength(5_000_000)
expect(parsed.sections).toEqual([
{
locator: '全文',
content: parsed.content
}
])
expect(parsed.warnings).toEqual([
'文档提取文本超过 5,000,000 字符,已截断'
])
})
it('rejects chunk output that exceeds the database limit', () => {
expect(() =>
chunkDocumentAdvanced(
{
title: 'Too many chunks',
sourceFormat: '.txt',
content: '',
sections: Array.from({ length: 10_001 }, (_, index) => ({
locator: `section-${index}`,
content: 'content'
})),
warnings: []
},
{
version: 1,
mode: 'fixed',
targetCharacters: 400,
overlapCharacters: 0,
parentCharacters: 1_600,
childCharacters: 300,
contextualIndexingEnabled: false
}
)
).toThrow('超过 10,000 个分区')
}) })
it('preserves headings and creates recall-only children with parent context', () => { it('preserves headings and creates recall-only children with parent context', () => {
+380 -104
View File
@@ -5,6 +5,10 @@ import type {
KnowledgeChunkingSettings, KnowledgeChunkingSettings,
KnowledgeChunkRole KnowledgeChunkRole
} from '../../shared/knowledge-contracts' } from '../../shared/knowledge-contracts'
import {
maximumDocumentExtractedCharacters,
maximumPdfPageCount
} from '../../shared/document-parsing-contracts'
export type ParsedSection = { export type ParsedSection = {
locator: string locator: string
@@ -40,9 +44,10 @@ export type DocumentChunk = {
export type DocumentBlockKind = 'text' | 'table' | 'slide' export type DocumentBlockKind = 'text' | 'table' | 'slide'
const maximumDocumentBytes = 20 * 1024 * 1024 const maximumDocumentBytes = 20 * 1024 * 1024
const maximumExtractedCharacters = 5_000_000
const maximumHeadingCharacters = 512 const maximumHeadingCharacters = 512
const maximumHeadingDepth = 6 const maximumHeadingDepth = 6
export const maximumDocumentChunks = 10_000
const maximumDocumentSections = 10_000
export const maximumChunkContextPrefixCharacters = 512 export const maximumChunkContextPrefixCharacters = 512
const textExtensions = new Set([ const textExtensions = new Set([
'.c', '.c',
@@ -112,7 +117,12 @@ function extractXmlText(xml: string): string {
} }
function decodeText(buffer: Buffer): string { function decodeText(buffer: Buffer): string {
const content = buffer.toString('utf8') let content: string
try {
content = new TextDecoder('utf-8', { fatal: true }).decode(buffer)
} catch (error) {
throw new Error('文件不是受支持的 UTF-8 文本', { cause: error })
}
const nullCount = [...content.slice(0, 8_192)].filter( const nullCount = [...content.slice(0, 8_192)].filter(
(character) => character.charCodeAt(0) === 0 (character) => character.charCodeAt(0) === 0
).length ).length
@@ -328,12 +338,18 @@ function parseOfficeArchive(
} }
async function parsePdf( async function parsePdf(
buffer: Buffer buffer: Buffer,
): Promise<{ sections: ParsedSection[]; pageCount: number }> { signal?: AbortSignal
const pages = await extractPdfTextPages(buffer) ): Promise<{
sections: ParsedSection[]
pageCount: number
truncated: boolean
}> {
const extracted = await extractPdfTextPages(buffer, { signal })
return { return {
pageCount: pages.length, pageCount: extracted.pageCount,
sections: pages truncated: extracted.truncated,
sections: extracted.pages
.filter((page) => page.content.length > 0) .filter((page) => page.content.length > 0)
.map((page) => ({ .map((page) => ({
locator: `${page.pageNumber}`, locator: `${page.pageNumber}`,
@@ -349,6 +365,18 @@ export type PdfTextPage = {
content: string content: string
} }
export type PdfTextExtraction = {
pages: PdfTextPage[]
pageCount: number
truncated: boolean
}
export type PdfTextExtractionOptions = {
maximumPages?: number
maximumCharacters?: number
signal?: AbortSignal
}
export class DocumentTextUnavailableError extends Error { export class DocumentTextUnavailableError extends Error {
constructor(message = '文档中没有可索引的文本内容') { constructor(message = '文档中没有可索引的文本内容') {
super(message) super(message)
@@ -356,31 +384,43 @@ export class DocumentTextUnavailableError extends Error {
} }
} }
function reconstructPdfText( type PdfTextItem = {
items: readonly unknown[] str?: string
): string { hasEOL?: boolean
const lines: string[] = [] transform?: ArrayLike<number>
let line: string[] = [] height?: number
let previousY: number | undefined }
let previousHeight = 0
const flush = (): void => {
const value = line.join(' ').replace(/[ \t]+/gu, ' ').trim()
if (value) {
lines.push(value)
}
line = []
}
type PdfTextReconstructionState = {
lines: string[]
line: string[]
lineCharacterCount: number
characterCount: number
previousY?: number
previousHeight: number
}
function flushPdfTextLine(state: PdfTextReconstructionState): void {
if (state.lineCharacterCount > 0) {
const value = state.line.join(' ')
state.lines.push(value)
state.characterCount +=
(state.lines.length > 1 ? 1 : 0) + value.length
}
state.line = []
state.lineCharacterCount = 0
}
function consumePdfTextItems(
state: PdfTextReconstructionState,
items: readonly unknown[],
maximumCharacters: number
): 'complete' | 'limit' | 'truncated' {
for (const candidate of items) { for (const candidate of items) {
if (typeof candidate !== 'object' || candidate === null) { if (typeof candidate !== 'object' || candidate === null) {
continue continue
} }
const item = candidate as { const item = candidate as PdfTextItem
str?: string
hasEOL?: boolean
transform?: ArrayLike<number>
height?: number
}
const value = typeof item.str === 'string' ? item.str.trim() : '' const value = typeof item.str === 'string' ? item.str.trim() : ''
const y = const y =
item.transform && Number.isFinite(item.transform[5]) item.transform && Number.isFinite(item.transform[5])
@@ -391,33 +431,130 @@ function reconstructPdfText(
? Math.abs(item.height) ? Math.abs(item.height)
: 0 : 0
const coordinateLineBreak = const coordinateLineBreak =
line.length > 0 && state.line.length > 0 &&
y !== undefined && y !== undefined &&
previousY !== undefined && state.previousY !== undefined &&
Math.abs(y - previousY) > Math.abs(y - state.previousY) >
Math.max(3, previousHeight * 0.8, height * 0.8) Math.max(3, state.previousHeight * 0.8, height * 0.8)
if (coordinateLineBreak) { if (coordinateLineBreak) {
flush() flushPdfTextLine(state)
} }
if (value) { if (value) {
line.push(value) const lineSeparator = state.line.length > 0 ? 1 : 0
const documentSeparator =
state.line.length === 0 && state.lines.length > 0 ? 1 : 0
const available =
maximumCharacters -
state.characterCount -
state.lineCharacterCount -
lineSeparator -
documentSeparator
if (available <= 0) {
return 'truncated'
}
const limited = value.slice(0, available)
state.line.push(limited)
state.lineCharacterCount += lineSeparator + limited.length
if (limited.length < value.length) {
return 'truncated'
}
} }
if (item.hasEOL) { if (item.hasEOL) {
flush() flushPdfTextLine(state)
previousY = undefined state.previousY = undefined
previousHeight = 0 state.previousHeight = 0
} else if (y !== undefined) { } else if (y !== undefined) {
previousY = y state.previousY = y
previousHeight = height state.previousHeight = height
}
if (
state.characterCount +
state.lineCharacterCount +
(state.line.length > 0 && state.lines.length > 0 ? 1 : 0) >=
maximumCharacters
) {
return 'limit'
} }
} }
flush() return 'complete'
return lines.join('\n').replace(/\n{3,}/gu, '\n\n').trim() }
async function reconstructPdfTextStream(
stream: ReadableStream,
maximumCharacters: number,
signal?: AbortSignal
): Promise<{ content: string; truncated: boolean }> {
const reader = stream.getReader()
const state: PdfTextReconstructionState = {
lines: [],
line: [],
lineCharacterCount: 0,
characterCount: 0,
previousHeight: 0
}
let truncated = false
try {
while (true) {
signal?.throwIfAborted()
const result = await reader.read()
if (result.done) {
break
}
const chunk = result.value as {
items?: readonly unknown[]
}
const consumption = consumePdfTextItems(
state,
chunk.items ?? [],
maximumCharacters
)
if (consumption === 'truncated') {
truncated = true
break
}
if (consumption === 'limit') {
const lookahead = await reader.read()
truncated = !lookahead.done
break
}
}
} finally {
await reader.cancel().catch(() => undefined)
reader.releaseLock()
}
flushPdfTextLine(state)
return {
content: state.lines.join('\n'),
truncated
}
} }
export async function extractPdfTextPages( export async function extractPdfTextPages(
buffer: Buffer buffer: Buffer,
): Promise<PdfTextPage[]> { options: PdfTextExtractionOptions = {}
): Promise<PdfTextExtraction> {
const maximumPages = options.maximumPages ?? maximumPdfPageCount
const maximumCharacters =
options.maximumCharacters ?? maximumDocumentExtractedCharacters
if (
!Number.isSafeInteger(maximumPages) ||
maximumPages < 1 ||
maximumPages > maximumPdfPageCount
) {
throw new RangeError(
`PDF page limit must be between 1 and ${maximumPdfPageCount}`
)
}
if (
!Number.isSafeInteger(maximumCharacters) ||
maximumCharacters < 1 ||
maximumCharacters > maximumDocumentExtractedCharacters
) {
throw new RangeError(
`PDF character limit must be between 1 and ${maximumDocumentExtractedCharacters}`
)
}
options.signal?.throwIfAborted()
const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs') const pdfjs = await import('pdfjs-dist/legacy/build/pdf.mjs')
const loadingTask = pdfjs.getDocument({ const loadingTask = pdfjs.getDocument({
data: new Uint8Array(buffer), data: new Uint8Array(buffer),
@@ -429,40 +566,121 @@ export async function extractPdfTextPages(
useSystemFonts: false, useSystemFonts: false,
useWorkerFetch: false useWorkerFetch: false
}) })
const document = await loadingTask.promise const abortLoading = (): void => {
void loadingTask.destroy()
}
options.signal?.addEventListener('abort', abortLoading, { once: true })
const pages: PdfTextPage[] = [] const pages: PdfTextPage[] = []
let remainingCharacters = maximumCharacters
let truncated = false
try { try {
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) { const document = await loadingTask.promise
const page = await document.getPage(pageNumber) options.signal?.throwIfAborted()
const text = await page.getTextContent() if (document.numPages > maximumPages) {
const content = reconstructPdfText(text.items) throw new Error(
pages.push({ pageNumber, content }) `PDF 有 ${document.numPages} 页,超过 ${maximumPages} 页限制`
page.cleanup() )
} }
for (let pageNumber = 1; pageNumber <= document.numPages; pageNumber += 1) {
options.signal?.throwIfAborted()
const page = await document.getPage(pageNumber)
try {
const reconstructed = await reconstructPdfTextStream(
page.streamTextContent(),
remainingCharacters,
options.signal
)
pages.push({ pageNumber, content: reconstructed.content })
remainingCharacters -= reconstructed.content.length
if (reconstructed.truncated || remainingCharacters === 0) {
truncated =
reconstructed.truncated || pageNumber < document.numPages
break
}
} finally {
page.cleanup()
}
}
return { pages, pageCount: document.numPages, truncated }
} finally { } finally {
options.signal?.removeEventListener('abort', abortLoading)
await loadingTask.destroy() await loadingTask.destroy()
} }
return pages
} }
export async function parseDocument( export function assertDocumentBuffer(buffer: Buffer): void {
name: string,
buffer: Buffer
): Promise<ParsedDocument> {
if (buffer.byteLength === 0) { if (buffer.byteLength === 0) {
throw new Error('文档内容为空') throw new Error('文档内容为空')
} }
if (buffer.byteLength > maximumDocumentBytes) { if (buffer.byteLength > maximumDocumentBytes) {
throw new Error('单个文档不能超过 20MB') throw new Error('单个文档不能超过 20MB')
} }
}
function limitParsedSections(
sections: readonly ParsedSection[]
): {
content: string
sections: ParsedSection[]
truncated: boolean
} {
if (sections.length > maximumDocumentSections) {
throw new Error(
`文档包含超过 ${maximumDocumentSections.toLocaleString('en-US')} 个分区`
)
}
const limited: ParsedSection[] = []
let remaining = maximumDocumentExtractedCharacters
let truncated = false
for (const section of sections) {
if (!section.content) {
continue
}
const separatorLength = limited.length > 0 ? 2 : 0
if (remaining <= separatorLength) {
truncated = true
break
}
const maximumSectionLength = remaining - separatorLength
const content = section.content.slice(0, maximumSectionLength)
if (content) {
limited.push(
content === section.content ? section : { ...section, content }
)
remaining -= separatorLength + content.length
}
if (content.length < section.content.length) {
truncated = true
break
}
}
if (limited.length < sections.filter((section) => section.content).length) {
truncated = true
}
return {
content: limited.map((section) => section.content).join('\n\n'),
sections: limited,
truncated
}
}
export async function parseDocument(
name: string,
buffer: Buffer,
signal?: AbortSignal
): Promise<ParsedDocument> {
assertDocumentBuffer(buffer)
signal?.throwIfAborted()
const extension = extname(name).toLowerCase() const extension = extname(name).toLowerCase()
let sections: ParsedSection[] let sections: ParsedSection[]
let pageCount: number | undefined let pageCount: number | undefined
let extractionTruncated = false
if (extension === '.pdf') { if (extension === '.pdf') {
const parsedPdf = await parsePdf(buffer) const parsedPdf = await parsePdf(buffer, signal)
sections = parsedPdf.sections sections = parsedPdf.sections
pageCount = parsedPdf.pageCount pageCount = parsedPdf.pageCount
extractionTruncated = parsedPdf.truncated
} else if (['.docx', '.xlsx', '.pptx'].includes(extension)) { } else if (['.docx', '.xlsx', '.pptx'].includes(extension)) {
sections = parseOfficeArchive(buffer, extension) sections = parseOfficeArchive(buffer, extension)
} else if (['.html', '.htm'].includes(extension)) { } else if (['.html', '.htm'].includes(extension)) {
@@ -481,19 +699,19 @@ export async function parseDocument(
throw new Error(`不支持的文档类型:${extension || '未知'}`) throw new Error(`不支持的文档类型:${extension || '未知'}`)
} }
const content = sections signal?.throwIfAborted()
.map((section) => section.content) const limited = limitParsedSections(sections)
.join('\n\n') if (!limited.content) {
.slice(0, maximumExtractedCharacters)
if (!content) {
throw new DocumentTextUnavailableError() throw new DocumentTextUnavailableError()
} }
return { return {
title: name.replace(/\.[^.]+$/, ''), title: name.replace(/\.[^.]+$/, ''),
sourceFormat: extension || 'unknown', sourceFormat: extension || 'unknown',
content, content: limited.content,
sections, sections: limited.sections,
warnings: [], warnings: limited.truncated || extractionTruncated
? ['文档提取文本超过 5,000,000 字符,已截断']
: [],
...(pageCount === undefined ? {} : { pageCount }) ...(pageCount === undefined ? {} : { pageCount })
} }
} }
@@ -501,8 +719,14 @@ export async function parseDocument(
function splitNatural( function splitNatural(
content: string, content: string,
maximumLength: number, maximumLength: number,
overlap: number overlap: number,
maximumParts = maximumDocumentChunks
): string[] { ): string[] {
if (maximumParts < 1 && content.trim()) {
throw new Error(
`文档分块超过 ${maximumDocumentChunks.toLocaleString('en-US')} 个,请增大分块长度或缩小文档`
)
}
const chunks: string[] = [] const chunks: string[] = []
let offset = 0 let offset = 0
while (offset < content.length) { while (offset < content.length) {
@@ -524,6 +748,11 @@ function splitNatural(
} }
const value = content.slice(offset, end).trim() const value = content.slice(offset, end).trim()
if (value) { if (value) {
if (chunks.length >= maximumParts) {
throw new Error(
`文档分块超过 ${maximumDocumentChunks.toLocaleString('en-US')} 个,请增大分块长度或缩小文档`
)
}
chunks.push(value) chunks.push(value)
} }
if (end >= content.length) { if (end >= content.length) {
@@ -578,9 +807,20 @@ function chunkMetadata(
} }
function structuredSections(document: ParsedDocument): StructuredSection[] { function structuredSections(document: ParsedDocument): StructuredSection[] {
return document.sections.flatMap((section) => { const result: StructuredSection[] = []
const lines = section.content.split(/\r?\n/u) const append = (section: StructuredSection): void => {
const result: StructuredSection[] = [] if (!section.content) {
return
}
if (result.length >= maximumDocumentSections) {
throw new Error(
`文档包含超过 ${maximumDocumentSections.toLocaleString('en-US')} 个结构分区`
)
}
result.push(section)
}
for (const section of document.sections) {
const sectionStart = result.length
let heading: string | undefined let heading: string | undefined
let headingPath = section.headingPath let headingPath = section.headingPath
? [...section.headingPath].slice(0, maximumHeadingDepth) ? [...section.headingPath].slice(0, maximumHeadingDepth)
@@ -590,7 +830,7 @@ function structuredSections(document: ParsedDocument): StructuredSection[] {
const flush = (): void => { const flush = (): void => {
const content = body.join('\n').trim() const content = body.join('\n').trim()
if (content) { if (content) {
result.push({ append({
locator: heading locator: heading
? `${section.locator} · ${heading}`.slice(0, 8_192) ? `${section.locator} · ${heading}`.slice(0, 8_192)
: section.locator, : section.locator,
@@ -602,7 +842,16 @@ function structuredSections(document: ParsedDocument): StructuredSection[] {
} }
body = [] body = []
} }
for (const line of lines) { let offset = 0
while (offset <= section.content.length) {
const lineEnd = section.content.indexOf('\n', offset)
const rawLine = section.content.slice(
offset,
lineEnd < 0 ? section.content.length : lineEnd
)
const line = rawLine.endsWith('\r')
? rawLine.slice(0, -1)
: rawLine
const match = /^\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line) const match = /^\s{0,3}(#{1,6})\s+(.+?)\s*#*\s*$/u.exec(line)
if (match) { if (match) {
flush() flush()
@@ -617,18 +866,21 @@ function structuredSections(document: ParsedDocument): StructuredSection[] {
} else { } else {
body.push(line) body.push(line)
} }
if (lineEnd < 0) {
break
}
offset = lineEnd + 1
} }
flush() flush()
return result.length > 0 if (result.length === sectionStart) {
? result append({
: [ locator: section.locator,
{ content: section.content.trim(),
locator: section.locator, ...sectionMetadata(section)
content: section.content.trim(), })
...sectionMetadata(section) }
} }
] return result
}).filter((section) => section.content.length > 0)
} }
function normalizeContextValue(value: string, maximumLength: number): string { function normalizeContextValue(value: string, maximumLength: number): string {
@@ -696,38 +948,55 @@ export function chunkDocumentAdvanced(
document: ParsedDocument, document: ParsedDocument,
settings: KnowledgeChunkingSettings settings: KnowledgeChunkingSettings
): DocumentChunk[] { ): DocumentChunk[] {
if (document.sections.length > maximumDocumentSections) {
throw new Error(
`文档包含超过 ${maximumDocumentSections.toLocaleString('en-US')} 个分区`
)
}
if (settings.mode === 'fixed') { if (settings.mode === 'fixed') {
return document.sections.flatMap((section) => const result: DocumentChunk[] = []
splitNatural( for (const section of document.sections) {
const parts = splitNatural(
section.content, section.content,
settings.targetCharacters, settings.targetCharacters,
settings.overlapCharacters settings.overlapCharacters,
).map((content) => ({ maximumDocumentChunks - result.length
position: 0, )
locator: section.locator, for (const content of parts) {
content, result.push({
...chunkMetadata(section), position: result.length,
role: 'standalone' as const locator: section.locator,
})) content,
).map((chunk, position) => ({ ...chunk, position })) ...chunkMetadata(section),
role: 'standalone'
})
}
}
return result
} }
const sections = structuredSections(document) const sections = structuredSections(document)
if (settings.mode === 'structure') { if (settings.mode === 'structure') {
return sections.flatMap((section) => const result: DocumentChunk[] = []
splitNatural( for (const section of sections) {
const parts = splitNatural(
section.content, section.content,
settings.targetCharacters, settings.targetCharacters,
settings.overlapCharacters settings.overlapCharacters,
).map((content) => ({ maximumDocumentChunks - result.length
position: 0, )
locator: section.locator, for (const content of parts) {
heading: section.heading, result.push({
content, position: result.length,
...chunkMetadata(section), locator: section.locator,
role: 'standalone' as const heading: section.heading,
})) content,
).map((chunk, position) => ({ ...chunk, position })) ...chunkMetadata(section),
role: 'standalone'
})
}
}
return result
} }
const result: DocumentChunk[] = [] const result: DocumentChunk[] = []
@@ -735,8 +1004,14 @@ export function chunkDocumentAdvanced(
for (const parentContent of splitNatural( for (const parentContent of splitNatural(
section.content, section.content,
settings.parentCharacters, settings.parentCharacters,
0 0,
maximumDocumentChunks - result.length
)) { )) {
if (result.length >= maximumDocumentChunks) {
throw new Error(
`文档分块超过 ${maximumDocumentChunks.toLocaleString('en-US')} 个,请增大分块长度或缩小文档`
)
}
const parentPosition = result.length const parentPosition = result.length
result.push({ result.push({
position: parentPosition, position: parentPosition,
@@ -753,7 +1028,8 @@ export function chunkDocumentAdvanced(
for (const childContent of splitNatural( for (const childContent of splitNatural(
parentContent, parentContent,
settings.childCharacters, settings.childCharacters,
childOverlap childOverlap,
maximumDocumentChunks - result.length
)) { )) {
result.push({ result.push({
position: result.length, position: result.length,
+60
View File
@@ -0,0 +1,60 @@
import {
mkdtemp,
open,
rm,
writeFile
} from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { describe, expect, it, vi } from 'vitest'
import { readBoundedFile } from './workspace-file-access'
type ReadMethod = (
this: Awaited<ReturnType<typeof open>>,
buffer: Buffer,
offset: number,
length: number,
position: number
) => Promise<{ bytesRead: number; buffer: Buffer }>
describe('workspace file access', () => {
it('continues reading after a short file-handle read', async () => {
const directory = await mkdtemp(join(tmpdir(), 'goodbuddy-short-read-'))
const path = join(directory, 'short-read.txt')
const content = Buffer.from('hello')
await writeFile(path, content)
const probe = await open(path, 'r')
const prototype = Object.getPrototypeOf(probe) as {
read: ReadMethod
}
const originalRead = prototype.read
await probe.close()
const read = vi
.spyOn(prototype, 'read')
.mockImplementation(function (
this: Awaited<ReturnType<typeof open>>,
buffer,
offset,
length,
position
) {
return originalRead.call(
this,
buffer,
offset,
Math.min(length, position === 0 ? 2 : 3),
position
)
})
try {
await expect(
readBoundedFile(path, 5, 'too large')
).resolves.toEqual(content)
expect(read).toHaveBeenCalledTimes(3)
} finally {
read.mockRestore()
await rm(directory, { recursive: true, force: true })
}
})
})
+39 -12
View File
@@ -52,27 +52,54 @@ export async function readBoundedUtf8File(
tooLargeMessage: string, tooLargeMessage: string,
invalidUtf8Message: string invalidUtf8Message: string
): Promise<{ content: string; size: number }> { ): Promise<{ content: string; size: number }> {
const data = await readBoundedFile(
filePath,
maximumBytes,
tooLargeMessage
)
try {
return {
content: new TextDecoder('utf-8', { fatal: true }).decode(data),
size: data.byteLength
}
} catch (error) {
throw new Error(invalidUtf8Message, { cause: error })
}
}
export async function readBoundedFile(
filePath: string,
maximumBytes: number,
tooLargeMessage: string,
invalidFileMessage = tooLargeMessage
): Promise<Buffer> {
const handle = await open(filePath, 'r') const handle = await open(filePath, 'r')
try { try {
const metadata = await handle.stat() const metadata = await handle.stat()
if (!metadata.isFile()) {
throw new Error(invalidFileMessage)
}
if (metadata.size > maximumBytes) { if (metadata.size > maximumBytes) {
throw new Error(tooLargeMessage) throw new Error(tooLargeMessage)
} }
const data = Buffer.alloc(metadata.size + 1) const data = Buffer.alloc(metadata.size + 1)
const result = await handle.read(data, 0, data.length, 0) let bytesRead = 0
if (result.bytesRead > maximumBytes) { while (bytesRead < data.length) {
throw new Error(tooLargeMessage) const result = await handle.read(
} data,
try { bytesRead,
return { data.length - bytesRead,
content: new TextDecoder('utf-8', { fatal: true }).decode( bytesRead
data.subarray(0, result.bytesRead) )
), if (result.bytesRead === 0) {
size: result.bytesRead break
}
bytesRead += result.bytesRead
if (bytesRead > maximumBytes) {
throw new Error(tooLargeMessage)
} }
} catch (error) {
throw new Error(invalidUtf8Message, { cause: error })
} }
return data.subarray(0, bytesRead)
} finally { } finally {
await handle.close() await handle.close()
} }
+38 -11
View File
@@ -10,6 +10,12 @@ import type {
DocumentOcrRequest, DocumentOcrRequest,
DocumentOcrResult DocumentOcrResult
} from '../../shared/document-parsing-contracts' } from '../../shared/document-parsing-contracts'
import {
maximumDocumentExtractedCharacters,
maximumDocumentOcrSectionCharacters,
maximumDocumentParsingWarnings,
maximumPdfPageCount
} from '../../shared/document-parsing-contracts'
import { createWorkerPdfLoadingParameters } from './document-ocr-pdf' import { createWorkerPdfLoadingParameters } from './document-ocr-pdf'
type InitializeMessage = { type InitializeMessage = {
@@ -85,16 +91,21 @@ async function initialize(assets: DocumentOcrAssets): Promise<void> {
async function recognizeImage( async function recognizeImage(
data: ArrayBuffer, data: ArrayBuffer,
locator: string locator: string,
pageNumber?: number
): Promise<DocumentOcrResult['sections'][number] | undefined> { ): Promise<DocumentOcrResult['sections'][number] | undefined> {
if (!service?.isInitialized()) { if (!service?.isInitialized()) {
throw new Error('本地 OCR 模型尚未初始化') throw new Error('本地 OCR 模型尚未初始化')
} }
const result = await service.recognize(data) const result = await service.recognize(data)
const content = result.text.replace(/\n{3,}/gu, '\n\n').trim() const content = result.text.replace(/\n{3,}/gu, '\n\n').trim()
if (content.length > maximumDocumentOcrSectionCharacters) {
throw new Error('单页 OCR 输出超过字符限制')
}
return content return content
? { ? {
locator, locator,
...(pageNumber === undefined ? {} : { pageNumber }),
content, content,
confidence: result.confidence confidence: result.confidence
} }
@@ -152,6 +163,18 @@ async function recognizePdf(
createWorkerPdfLoadingParameters(request.data) createWorkerPdfLoadingParameters(request.data)
) )
const document = await loadingTask.promise const document = await loadingTask.promise
if (document.numPages > maximumPdfPageCount) {
await loadingTask.destroy()
throw new Error(
`PDF 有 ${document.numPages} 页,超过 ${maximumPdfPageCount} 页限制`
)
}
if (!request.pageNumbers && document.numPages > request.maximumPages) {
await loadingTask.destroy()
throw new Error(
`PDF 有 ${document.numPages} 页需要 OCR,超过 ${request.maximumPages} 页限制`
)
}
const selectedPages = new Set( const selectedPages = new Set(
request.pageNumbers ?? request.pageNumbers ??
Array.from( Array.from(
@@ -174,15 +197,11 @@ async function recognizePdf(
} }
const sections: DocumentOcrResult['sections'] = [] const sections: DocumentOcrResult['sections'] = []
const warnings: string[] = [] const warnings: string[] = []
let extractedCharacters = 0
try { try {
for ( for (const pageNumber of [...selectedPages].sort(
let pageNumber = 1; (left, right) => left - right
pageNumber <= document.numPages; )) {
pageNumber += 1
) {
if (!selectedPages.has(pageNumber)) {
continue
}
worker.postMessage({ worker.postMessage({
type: 'progress', type: 'progress',
requestId: request.requestId, requestId: request.requestId,
@@ -192,11 +211,19 @@ async function recognizePdf(
try { try {
const section = await recognizeImage( const section = await recognizeImage(
await renderPdfPage(page), await renderPdfPage(page),
`${pageNumber}` `${pageNumber}`,
pageNumber
) )
if (section) { if (section) {
extractedCharacters += section.content.length
if (
extractedCharacters >
maximumDocumentExtractedCharacters
) {
throw new Error('OCR 输出超过文档字符限制')
}
sections.push(section) sections.push(section)
} else { } else if (warnings.length < maximumDocumentParsingWarnings) {
warnings.push(`${pageNumber} 页未识别到文字`) warnings.push(`${pageNumber} 页未识别到文字`)
} }
} finally { } finally {
+44 -7
View File
@@ -1,5 +1,10 @@
import { z } from 'zod' import { z } from 'zod'
export const maximumDocumentExtractedCharacters = 5_000_000
export const maximumDocumentOcrSectionCharacters = 1_000_000
export const maximumDocumentParsingWarnings = 20
export const maximumPdfPageCount = 10_000
export const documentParsingPurposeSchema = z.enum([ export const documentParsingPurposeSchema = z.enum([
'chat-attachment', 'chat-attachment',
'knowledge-index', 'knowledge-index',
@@ -201,13 +206,15 @@ export const documentParsingDiagnosticSchema = z
.object({ .object({
fileName: z.string().trim().min(1).max(500), fileName: z.string().trim().min(1).max(500),
sourceFormat: z.string().trim().min(1).max(32), sourceFormat: z.string().trim().min(1).max(32),
pageCount: z.number().int().nonnegative().max(10_000), pageCount: z.number().int().nonnegative().max(maximumPdfPageCount),
ocrPageCount: z.number().int().nonnegative().max(10_000), ocrPageCount: z.number().int().nonnegative().max(maximumPdfPageCount),
characterCount: z.number().int().nonnegative().safe(), characterCount: z.number().int().nonnegative().safe(),
method: z.enum(['native', 'ocr', 'mixed']), method: z.enum(['native', 'ocr', 'mixed']),
durationMs: z.number().int().nonnegative().safe(), durationMs: z.number().int().nonnegative().safe(),
preview: z.string().max(2_000), preview: z.string().max(2_000),
warnings: z.array(z.string().trim().min(1).max(500)).max(20) warnings: z
.array(z.string().trim().min(1).max(500))
.max(maximumDocumentParsingWarnings)
}) })
.strict() .strict()
@@ -239,7 +246,7 @@ export const documentOcrRequestSchema = z
), ),
maximumPages: z.number().int().min(1).max(500), maximumPages: z.number().int().min(1).max(500),
pageNumbers: z pageNumbers: z
.array(z.number().int().min(1).max(10_000)) .array(z.number().int().min(1).max(maximumPdfPageCount))
.min(1) .min(1)
.max(500) .max(500)
.optional(), .optional(),
@@ -271,7 +278,17 @@ export const documentOcrRequestSchema = z
export const documentOcrSectionSchema = z export const documentOcrSectionSchema = z
.object({ .object({
locator: z.string().trim().min(1).max(500), locator: z.string().trim().min(1).max(500),
content: z.string().trim().min(1).max(1_000_000), pageNumber: z
.number()
.int()
.min(1)
.max(maximumPdfPageCount)
.optional(),
content: z
.string()
.trim()
.min(1)
.max(maximumDocumentOcrSectionCharacters),
confidence: z.number().min(0).max(1) confidence: z.number().min(0).max(1)
}) })
.strict() .strict()
@@ -280,10 +297,30 @@ export const documentOcrResultSchema = z
.object({ .object({
requestId: z.string().uuid(), requestId: z.string().uuid(),
sections: z.array(documentOcrSectionSchema).max(500), sections: z.array(documentOcrSectionSchema).max(500),
pageCount: z.number().int().nonnegative().max(10_000), pageCount: z
warnings: z.array(z.string().trim().min(1).max(500)).max(20) .number()
.int()
.nonnegative()
.max(maximumPdfPageCount),
warnings: z
.array(z.string().trim().min(1).max(500))
.max(maximumDocumentParsingWarnings)
}) })
.strict() .strict()
.superRefine((result, context) => {
let characters = 0
for (const section of result.sections) {
characters += section.content.length
if (characters > maximumDocumentExtractedCharacters) {
context.addIssue({
code: 'custom',
path: ['sections'],
message: 'OCR 输出超过文档字符限制'
})
return
}
}
})
export const documentOcrFailureSchema = z export const documentOcrFailureSchema = z
.object({ .object({