import { createHash } from 'node:crypto' import { copyFile, lstat, mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises' import { resolve } from 'node:path' import { documentOcrAssetsSchema, documentOcrModelCatalogEntrySchema, documentOcrModelCatalogViewEntrySchema, documentOcrModelProgressSnapshotSchema, documentOcrModelSnapshotSchema, documentParsingModelStatusSchema, installedDocumentOcrModelSchema, localOcrModelIdSchema, type DocumentOcrAssets, type DocumentOcrModelCatalogEntry, type DocumentOcrModelCatalogViewEntry, type DocumentOcrModelFile, type DocumentOcrModelOperation, type DocumentOcrModelProgressSnapshot, type DocumentOcrModelSnapshot, type InstalledDocumentOcrModel } from '../shared/document-parsing-contracts' import { MODEL_DOWNLOAD_SOURCES, getMaximumModelPackageBytes, getModelDownloadAvailability, getModelPackageFingerprints, modelPackageFingerprintMatches, resolveModelDownloadPackage, type ModelDownloadSource, type ResolvedModelArtifactFile } from '../shared/model-download-contracts' import { DOCUMENT_OCR_MODEL_CATALOG } from './document-ocr-model-catalog' import { exportModelArchive, extractModelArchive } from './model-archive' import { fetchModelDownloadResponse } from './model-download-transport' import { MODEL_PARTIAL_SUFFIX, attachModelAbortSignal, cleanupStaleModelInstallArtifacts, createModelStagingDirectory, ensureModelOperationNotAborted, hashModelFile, managedModelChild, writeModelBuffer } from './model-package-utils' import { isMissingFileError } from './settings-file-utils' const DEFAULT_MAX_FILE_BYTES = 96 * 1024 * 1024 const MANIFEST_FILE_NAME = 'manifest.json' const MAXIMUM_ARCHIVE_BYTES = 512 * 1024 * 1024 const ARCHIVE_OVERHEAD_BYTES = 1024 * 1024 const executableExtensionPattern = /\.(?:app|bat|bin|cmd|com|cpl|dll|dmg|exe|hta|inf|ins|iso|jar|js|jse|lnk|msi|msp|mst|pif|ps1|reg|scr|sh|sys|vb|vbe|vbs|ws|wsc|wsf|wsh)$/iu type ActiveOperation = { controller: AbortController progress: DocumentOcrModelOperation } type ActiveVerification = { generation: number promise: Promise } export type DocumentOcrModelManagerOptions = { userDataDirectory: string fetch: typeof fetch catalog?: readonly DocumentOcrModelCatalogEntry[] getDownloadSource?: () => | ModelDownloadSource | Promise maxFileBytes?: number } function cloneCatalogEntry( entry: DocumentOcrModelCatalogEntry ): DocumentOcrModelCatalogEntry { return documentOcrModelCatalogEntrySchema.parse(entry) } function toCatalogView(entry: DocumentOcrModelCatalogEntry) { const { repositoryUrls, files, ...metadata } = entry void repositoryUrls return documentOcrModelCatalogViewEntrySchema.parse({ ...metadata, files: files.map((file) => ({ name: file.name, role: file.role, size: file.size, sha256: file.sha256 })), downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) => getModelDownloadAvailability(files, source) ) }) } function documentOcrPackageMatches( entry: DocumentOcrModelCatalogEntry, files: readonly { name: string role: string size: number sha256: string }[] ): boolean { return getModelPackageFingerprints(entry.files).some( (fingerprint) => modelPackageFingerprintMatches(fingerprint, files) ) } function safeChild(parent: string, name: string): string { return managedModelChild( parent, name, 'OCR 模型路径超出受管目录' ) } function toArrayBuffer(buffer: Buffer): ArrayBuffer { return Uint8Array.from(buffer).buffer } function parseYamlScalar(value: string): string { if (value.startsWith("'") && value.endsWith("'")) { return value.slice(1, -1).replace(/''/gu, "'") } if (value.startsWith('"') && value.endsWith('"')) { return JSON.parse(value) as string } return value } export function extractPaddleCharacterDictionary(source: string): string { const characters: string[] = [] let readingDictionary = false for (const line of source.replace(/\r/gu, '').split('\n')) { if (line === ' character_dict:') { readingDictionary = true continue } if (!readingDictionary) { continue } const match = /^ {2}- (.*)$/u.exec(line) if (!match) { break } const character = parseYamlScalar(match[1]!) if (!character) { throw new Error('OCR 字符字典包含空条目') } characters.push(character) } if (characters.length < 100) { throw new Error('OCR 字符字典格式无效') } return `${characters.join('\n')}\n` } export class DocumentOcrModelManager { readonly rootDirectory: string private readonly transport: typeof fetch private readonly catalog: DocumentOcrModelCatalogEntry[] private readonly catalogViews: DocumentOcrModelCatalogViewEntry[] private readonly getDownloadSource: () => | ModelDownloadSource | Promise private readonly maxFileBytes: number private readonly operations = new Map() private readonly verifiedModels = new Map() private readonly verificationGenerations = new Map() constructor(options: DocumentOcrModelManagerOptions) { if (!options.userDataDirectory.trim()) { throw new Error('userDataDirectory is required') } this.rootDirectory = resolve( options.userDataDirectory, 'models', 'document-ocr' ) this.transport = options.fetch this.getDownloadSource = options.getDownloadSource ?? (() => 'modelscope') this.catalog = (options.catalog ?? DOCUMENT_OCR_MODEL_CATALOG).map( cloneCatalogEntry ) if ( new Set(this.catalog.map((entry) => entry.id)).size !== this.catalog.length ) { throw new Error('OCR 模型目录包含重复 ID') } this.catalogViews = this.catalog.map(toCatalogView) this.maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES if ( !Number.isSafeInteger(this.maxFileBytes) || this.maxFileBytes <= 0 || this.maxFileBytes > 512 * 1024 * 1024 ) { throw new RangeError('maxFileBytes must be a positive safe integer') } } async getSnapshot(): Promise { await this.ensureRoot() await this.cleanupStaleArtifacts() const [selectedDownloadSource, installed] = await Promise.all([ this.getDownloadSource(), this.readInstalled() ]) return documentOcrModelSnapshotSchema.parse({ rootDirectory: this.rootDirectory, selectedDownloadSource, catalog: this.catalogViews, installed, operations: [...this.operations.values()].map((operation) => ({ ...operation.progress })) }) } getProgressSnapshot(): DocumentOcrModelProgressSnapshot { return documentOcrModelProgressSnapshotSchema.parse({ operations: [...this.operations.values()].map((operation) => ({ ...operation.progress })) }) } async getStatus( modelId: string ): Promise> { const id = localOcrModelIdSchema.parse(modelId) const entry = this.catalog.find((candidate) => candidate.id === id) if (!entry) { return documentParsingModelStatusSchema.parse({ id, displayName: id, available: false, verified: false, runtime: 'onnxruntime-web-wasm', detail: '当前版本不再提供此 OCR 模型,请选择其他模型' }) } try { await this.getVerifiedStatus(entry) return documentParsingModelStatusSchema.parse({ id: entry.id, displayName: entry.displayName, available: true, verified: true, runtime: entry.runtime, detail: '模型已安装并通过 SHA-256 校验,可离线使用' }) } catch { return documentParsingModelStatusSchema.parse({ id: entry.id, displayName: entry.displayName, available: false, verified: false, runtime: entry.runtime, detail: '模型尚未安装或校验失败,请从当前模型下载源获取' }) } } getAssets(modelId: string): Promise { return this.loadVerifiedAssets(this.requireCatalogEntry(modelId)) } getRepositoryUrl( modelId: string, source: ModelDownloadSource ): string { const entry = this.requireCatalogEntry(modelId) resolveModelDownloadPackage(entry.files, source) const repositoryUrl = entry.repositoryUrls[source] if (!repositoryUrl) { throw new Error('当前下载源暂不提供此 OCR 模型的仓库') } return repositoryUrl } async install( modelId: string, downloadSource?: ModelDownloadSource, externalSignal?: AbortSignal ): Promise { const entry = this.requireCatalogEntry(modelId) const selectedDownloadSource = downloadSource ?? (await this.getDownloadSource()) const resolvedPackage = resolveModelDownloadPackage( entry.files, selectedDownloadSource ) const operation = this.beginOperation( entry.id, 'download', resolvedPackage.totalBytes, resolvedPackage.source ) const detachAbort = this.attachExternalSignal( externalSignal, operation.controller ) let stagingDirectory: string | undefined try { await this.ensureRoot() await this.assertNotInstalled(entry.id) stagingDirectory = await this.createStagingDirectory(entry.id) for (const file of resolvedPackage.files) { ensureModelOperationNotAborted(operation.controller.signal) operation.progress.phase = 'transferring' operation.progress.currentFile = file.name await this.downloadFile( file, safeChild(stagingDirectory, file.name), operation, operation.controller.signal ) } operation.progress.phase = 'installing' operation.progress.currentFile = null const installed = await this.createInstalledManifest( entry, 'download', stagingDirectory, operation.controller.signal ) ensureModelOperationNotAborted(operation.controller.signal) await rename(stagingDirectory, this.modelDirectory(entry.id)) stagingDirectory = undefined this.invalidateVerification(entry.id) return installed } finally { detachAbort() this.operations.delete(entry.id) if (stagingDirectory) { await rm(stagingDirectory, { recursive: true, force: true }) } } } async registerLocalDirectory( modelId: string, sourceDirectory: string, externalSignal?: AbortSignal ): Promise { const entry = this.requireCatalogEntry(modelId) const source = resolve(sourceDirectory) const operation = this.beginOperation(entry.id, 'import', null) const detachAbort = this.attachExternalSignal( externalSignal, operation.controller ) let stagingDirectory: string | undefined try { await this.ensureRoot() await this.assertNotInstalled(entry.id) await this.validateLocalDirectory( source, entry, operation.controller.signal ) stagingDirectory = await this.createStagingDirectory(entry.id) operation.progress.phase = 'transferring' for (const file of entry.files) { ensureModelOperationNotAborted(operation.controller.signal) operation.progress.currentFile = file.name const sourceFile = safeChild(source, file.name) const destination = safeChild(stagingDirectory, file.name) await copyFile(sourceFile, destination) operation.progress.completedBytes += (await stat(destination)).size } operation.progress.totalBytes = operation.progress.completedBytes operation.progress.phase = 'installing' operation.progress.currentFile = null const installed = await this.createInstalledManifest( entry, 'local', stagingDirectory, operation.controller.signal ) ensureModelOperationNotAborted(operation.controller.signal) await rename(stagingDirectory, this.modelDirectory(entry.id)) stagingDirectory = undefined this.invalidateVerification(entry.id) return installed } finally { detachAbort() this.operations.delete(entry.id) if (stagingDirectory) { await rm(stagingDirectory, { recursive: true, force: true }) } } } async exportArchive( modelId: string, destinationPath: string ): Promise { const entry = this.requireCatalogEntry(modelId) await this.ensureRoot() const installed = (await this.readInstalled()).find( (model) => model.id === entry.id ) if (!installed) { throw new Error('只能导出已安装的 OCR 模型') } if (!documentOcrPackageMatches(entry, installed.files)) { throw new Error('OCR 模型文件与当前模型目录不匹配') } const directory = this.modelDirectory(entry.id) const files = [] for (const expected of entry.files) { const recorded = installed.files.find( (file) => file.name === expected.name && file.role === expected.role ) if (!recorded) { throw new Error(`OCR 模型文件校验失败:${expected.name}`) } files.push({ name: expected.name, role: expected.role, size: recorded.size, sha256: recorded.sha256 }) } await exportModelArchive({ destinationPath, sourceDirectory: directory, descriptor: { kind: 'document-ocr', modelId: entry.id, displayName: entry.displayName, files } }) } async importArchive( modelId: string, archivePath: string ): Promise { const entry = this.requireCatalogEntry(modelId) const expectedTotal = getMaximumModelPackageBytes(entry.files) const operation = this.beginOperation( entry.id, 'import', expectedTotal ) let stagingDirectory: string | undefined try { await this.ensureRoot() await this.assertNotInstalled(entry.id) stagingDirectory = await this.createStagingDirectory(entry.id) operation.progress.phase = 'transferring' const descriptor = await extractModelArchive({ archivePath, destinationDirectory: stagingDirectory, expectedKind: 'document-ocr', expectedModelId: entry.id, expectedFiles: entry.files.map((file) => ({ name: file.name, role: file.role })), maximumArchiveBytes: Math.min( MAXIMUM_ARCHIVE_BYTES, expectedTotal + ARCHIVE_OVERHEAD_BYTES ), maximumFileBytes: this.maxFileBytes, maximumTotalBytes: expectedTotal + ARCHIVE_OVERHEAD_BYTES, signal: operation.controller.signal, onProgress: (completedBytes) => { operation.progress.completedBytes = completedBytes } }) if (!documentOcrPackageMatches(entry, descriptor.files)) { throw new Error('OCR 模型 ZIP 与当前模型目录不匹配') } operation.progress.totalBytes = descriptor.files.reduce( (total, file) => total + file.size, 0 ) operation.progress.phase = 'installing' operation.progress.currentFile = null const installed = installedDocumentOcrModelSchema.parse({ id: entry.id, displayName: entry.displayName, source: 'local', installedAt: new Date().toISOString(), files: descriptor.files }) await writeFile( safeChild(stagingDirectory, MANIFEST_FILE_NAME), `${JSON.stringify(installed, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' } ) ensureModelOperationNotAborted(operation.controller.signal) await rename(stagingDirectory, this.modelDirectory(entry.id)) stagingDirectory = undefined this.invalidateVerification(entry.id) return installed } finally { this.operations.delete(entry.id) if (stagingDirectory) { await rm(stagingDirectory, { recursive: true, force: true }) } } } cancel(modelId: string): boolean { const id = localOcrModelIdSchema.parse(modelId) const operation = this.operations.get(id) if (!operation) { return false } operation.controller.abort() return true } async remove(modelId: string): Promise { const id = localOcrModelIdSchema.parse(modelId) this.cancel(id) this.invalidateVerification(id) await rm(this.modelDirectory(id), { recursive: true, force: true }) } dispose(): void { for (const operation of this.operations.values()) { operation.controller.abort() } this.operations.clear() this.verifiedModels.clear() this.verificationGenerations.clear() } private async ensureRoot(): Promise { await mkdir(this.rootDirectory, { recursive: true }) } private modelDirectory(modelId: string): string { return safeChild( this.rootDirectory, localOcrModelIdSchema.parse(modelId) ) } private requireCatalogEntry( modelId: string ): DocumentOcrModelCatalogEntry { const id = localOcrModelIdSchema.parse(modelId) const entry = this.catalog.find((candidate) => candidate.id === id) if (!entry) { throw new Error('未知的 OCR 模型') } return entry } private beginOperation( modelId: string, kind: DocumentOcrModelOperation['kind'], totalBytes: number | null, downloadSource?: ModelDownloadSource ): ActiveOperation { if (this.operations.has(modelId)) { throw new Error('该 OCR 模型已有进行中的操作') } const operation: ActiveOperation = { controller: new AbortController(), progress: { modelId: localOcrModelIdSchema.parse(modelId), kind, phase: 'preparing', currentFile: null, completedBytes: 0, totalBytes, ...(downloadSource ? { downloadSource } : {}) } } this.operations.set(modelId, operation) return operation } private attachExternalSignal( signal: AbortSignal | undefined, controller: AbortController ): () => void { return attachModelAbortSignal(signal, controller) } private async assertNotInstalled(modelId: string): Promise { try { await lstat(this.modelDirectory(modelId)) throw new Error('OCR 模型已安装') } catch (error) { if (isMissingFileError(error)) { return } throw error } } private async createStagingDirectory(modelId: string): Promise { return createModelStagingDirectory( this.rootDirectory, modelId, 'OCR 模型路径超出受管目录' ) } private async downloadFile( file: ResolvedModelArtifactFile, destination: string, operation: ActiveOperation, signal: AbortSignal ): Promise { if (file.size > this.maxFileBytes) { throw new RangeError(`OCR 模型文件过大:${file.name}`) } const response = await fetchModelDownloadResponse({ transport: this.transport, initialUrl: file.target.url, redirectHosts: file.target.redirectHosts, signal, modelLabel: 'OCR 模型' }) if (!response.ok) { await response.body?.cancel().catch(() => undefined) throw new Error(`OCR 模型下载失败:HTTP ${response.status}`) } if (!response.body) { throw new Error('OCR 模型下载响应没有内容') } const declaredLength = response.headers.get('content-length') if ( declaredLength !== null && Number(declaredLength) !== file.size ) { await response.body.cancel().catch(() => undefined) throw new Error(`OCR 模型文件大小不匹配:${file.name}`) } const partialPath = `${destination}${MODEL_PARTIAL_SUFFIX}` const handle = await open(partialPath, 'wx') const reader = response.body.getReader() const hash = createHash('sha256') let written = 0 try { while (true) { ensureModelOperationNotAborted(signal) const result = await reader.read() if (result.done) { break } if ( written + result.value.byteLength > file.size || written + result.value.byteLength > this.maxFileBytes ) { await reader.cancel() throw new RangeError(`OCR 模型文件过大:${file.name}`) } const persistedBytes = await writeModelBuffer( handle, result.value, (persisted) => { hash.update(persisted) operation.progress.completedBytes += persisted.byteLength } ) written += persistedBytes } } catch (error) { await reader.cancel().catch(() => undefined) throw error } finally { await handle.close() } if ( written !== file.size || hash.digest('hex') !== file.sha256 ) { throw new Error(`OCR 模型文件校验失败:${file.name}`) } await rename(partialPath, destination) } private async validateLocalDirectory( sourceDirectory: string, entry: DocumentOcrModelCatalogEntry, signal: AbortSignal ): Promise { const sourceInfo = await lstat(sourceDirectory) if (!sourceInfo.isDirectory() || sourceInfo.isSymbolicLink()) { throw new Error('本地 OCR 模型来源必须是普通目录') } const entries = await readdir(sourceDirectory, { withFileTypes: true }) for (const localEntry of entries) { ensureModelOperationNotAborted(signal) if ( localEntry.isSymbolicLink() || executableExtensionPattern.test(localEntry.name) ) { throw new Error('本地 OCR 模型目录包含不安全文件') } } const actualFiles: InstalledDocumentOcrModel['files'] = [] for (const file of entry.files) { ensureModelOperationNotAborted(signal) const path = safeChild(sourceDirectory, file.name) const info = await lstat(path) if (!info.isFile() || info.isSymbolicLink()) { throw new Error(`OCR 模型文件必须是普通文件:${file.name}`) } if ( info.size <= 0 || info.size > this.maxFileBytes ) { throw new RangeError(`OCR 模型文件大小无效:${file.name}`) } actualFiles.push({ name: file.name, role: file.role, ...(await hashModelFile(path, signal)) }) } if (!documentOcrPackageMatches(entry, actualFiles)) { throw new Error( '本地 OCR 模型文件校验失败:与当前模型目录不匹配' ) } } private async createInstalledManifest( entry: DocumentOcrModelCatalogEntry, source: InstalledDocumentOcrModel['source'], stagingDirectory: string, signal: AbortSignal ): Promise { const files = [] for (const file of entry.files) { ensureModelOperationNotAborted(signal) files.push({ name: file.name, role: file.role, ...(await hashModelFile( safeChild(stagingDirectory, file.name), signal )) }) } const manifest = installedDocumentOcrModelSchema.parse({ id: entry.id, displayName: entry.displayName, source, installedAt: new Date().toISOString(), files }) if (!documentOcrPackageMatches(entry, manifest.files)) { throw new Error('OCR 模型文件与当前模型目录不匹配') } await writeFile( safeChild(stagingDirectory, MANIFEST_FILE_NAME), `${JSON.stringify(manifest, null, 2)}\n`, { encoding: 'utf8', flag: 'wx' } ) return manifest } private async readInstalled(): Promise { const entries = await readdir(this.rootDirectory, { withFileTypes: true }) const installed: InstalledDocumentOcrModel[] = [] for (const entry of entries) { if ( !entry.isDirectory() || entry.name.startsWith('.install-') || !localOcrModelIdSchema.safeParse(entry.name).success ) { continue } try { const manifest = installedDocumentOcrModelSchema.parse( JSON.parse( await readFile( safeChild( this.modelDirectory(entry.name), MANIFEST_FILE_NAME ), 'utf8' ) ) as unknown ) if (manifest.id === entry.name) { installed.push(manifest) } } catch { // Ignore incomplete or externally modified model directories. } } return installed } private async readInstalledManifest( entry: DocumentOcrModelCatalogEntry ): Promise { const directory = this.modelDirectory(entry.id) const manifest = installedDocumentOcrModelSchema.parse( JSON.parse( await readFile( safeChild(directory, MANIFEST_FILE_NAME), 'utf8' ) ) as unknown ) if (manifest.id !== entry.id) { throw new Error('OCR 模型清单 ID 不匹配') } return manifest } private async verifyInstalledModel( entry: DocumentOcrModelCatalogEntry ): Promise { const directory = this.modelDirectory(entry.id) const manifest = await this.readInstalledManifest(entry) if (!documentOcrPackageMatches(entry, manifest.files)) { throw new Error('OCR 模型文件与当前模型目录不匹配') } for (const file of entry.files) { const installed = manifest.files.find( (candidate) => candidate.name === file.name && candidate.role === file.role ) const actual = await hashModelFile( safeChild(directory, file.name) ) if ( !installed || actual.size !== installed.size || actual.sha256 !== installed.sha256 ) { throw new Error(`OCR 模型文件校验失败:${file.name}`) } } } private getVerifiedStatus( entry: DocumentOcrModelCatalogEntry ): Promise { const generation = this.verificationGenerations.get(entry.id) ?? 0 const active = this.verifiedModels.get(entry.id) if (active?.generation === generation) { return active.promise } const verification = this.verifyInstalledModel(entry).then(() => { if ( (this.verificationGenerations.get(entry.id) ?? 0) !== generation ) { throw new Error('OCR 模型在校验期间已发生变化') } }) const tracked = verification.finally(() => { if (this.verifiedModels.get(entry.id)?.promise === tracked) { this.verifiedModels.delete(entry.id) } }) this.verifiedModels.set(entry.id, { generation, promise: tracked }) return tracked } private invalidateVerification(modelId: string): void { this.verificationGenerations.set( modelId, (this.verificationGenerations.get(modelId) ?? 0) + 1 ) this.verifiedModels.delete(modelId) } private async loadVerifiedAssets( entry: DocumentOcrModelCatalogEntry ): Promise { const directory = this.modelDirectory(entry.id) const manifest = await this.readInstalledManifest(entry) if (!documentOcrPackageMatches(entry, manifest.files)) { throw new Error('OCR 模型文件与当前模型目录不匹配') } const loaded = new Map< DocumentOcrModelFile['role'], ArrayBuffer >() for (const file of entry.files) { const installed = manifest.files.find( (candidate) => candidate.name === file.name && candidate.role === file.role ) const path = safeChild(directory, file.name) const contents = await readFile(path) const actual = { size: contents.byteLength, sha256: createHash('sha256').update(contents).digest('hex') } if ( !installed || actual.size !== installed.size || actual.sha256 !== installed.sha256 ) { throw new Error(`OCR 模型文件校验失败:${file.name}`) } loaded.set( file.role, file.role === 'dictionary' ? toArrayBuffer( Buffer.from( extractPaddleCharacterDictionary( contents.toString('utf8') ), 'utf8' ) ) : toArrayBuffer(contents) ) } return documentOcrAssetsSchema.parse({ modelId: entry.id, detection: loaded.get('detection'), recognition: loaded.get('recognition'), dictionary: loaded.get('dictionary') }) } private cleanupStaleArtifacts(): Promise { return cleanupStaleModelInstallArtifacts({ rootDirectory: this.rootDirectory, isModelId: (value) => localOcrModelIdSchema.safeParse(value).success, activeModelIds: new Set(this.operations.keys()), partialFileNames: new Set( this.catalog.flatMap((entry) => entry.files.map((file) => file.name) ) ), escapeMessage: 'OCR 模型路径超出受管目录' }) } }