mesalogo
a0977645a3
Selecting ModelScope could leave supported speech models unavailable because their fixed targets were incomplete and the official LFS redirect host was not allowed. Speech models now resolve complete per-source artifact fingerprints while preserving integrity checks, ZIP compatibility, and source isolation. Release note: 修复选择 ModelScope 后部分语音模型仍无法下载的问题；六个语音模型现均可从已验证的固定 ModelScope 来源下载。
978 lines
29 KiB
TypeScript
978 lines
29 KiB
TypeScript
import { createHash } from 'node:crypto'
|
|
import {
|
|
copyFile,
|
|
lstat,
|
|
mkdir,
|
|
open,
|
|
readFile,
|
|
readdir,
|
|
rename,
|
|
rm,
|
|
stat,
|
|
writeFile
|
|
} from 'node:fs/promises'
|
|
import { resolve } from 'node:path'
|
|
import {
|
|
documentOcrAssetsSchema,
|
|
documentOcrModelCatalogEntrySchema,
|
|
documentOcrModelCatalogViewEntrySchema,
|
|
documentOcrModelProgressSnapshotSchema,
|
|
documentOcrModelSnapshotSchema,
|
|
documentParsingModelStatusSchema,
|
|
installedDocumentOcrModelSchema,
|
|
localOcrModelIdSchema,
|
|
type DocumentOcrAssets,
|
|
type DocumentOcrModelCatalogEntry,
|
|
type DocumentOcrModelCatalogViewEntry,
|
|
type DocumentOcrModelFile,
|
|
type DocumentOcrModelOperation,
|
|
type DocumentOcrModelProgressSnapshot,
|
|
type DocumentOcrModelSnapshot,
|
|
type InstalledDocumentOcrModel
|
|
} from '../shared/document-parsing-contracts'
|
|
import {
|
|
MODEL_DOWNLOAD_SOURCES,
|
|
getMaximumModelPackageBytes,
|
|
getModelDownloadAvailability,
|
|
getModelPackageFingerprints,
|
|
modelPackageFingerprintMatches,
|
|
resolveModelDownloadPackage,
|
|
type ModelDownloadSource,
|
|
type ResolvedModelArtifactFile
|
|
} from '../shared/model-download-contracts'
|
|
import { DOCUMENT_OCR_MODEL_CATALOG } from './document-ocr-model-catalog'
|
|
import {
|
|
exportModelArchive,
|
|
extractModelArchive
|
|
} from './model-archive'
|
|
import { fetchModelDownloadResponse } from './model-download-transport'
|
|
import {
|
|
MODEL_PARTIAL_SUFFIX,
|
|
attachModelAbortSignal,
|
|
cleanupStaleModelInstallArtifacts,
|
|
createModelStagingDirectory,
|
|
ensureModelOperationNotAborted,
|
|
hashModelFile,
|
|
managedModelChild,
|
|
writeModelBuffer
|
|
} from './model-package-utils'
|
|
import { isMissingFileError } from './settings-file-utils'
|
|
|
|
const DEFAULT_MAX_FILE_BYTES = 96 * 1024 * 1024
|
|
const MANIFEST_FILE_NAME = 'manifest.json'
|
|
const MAXIMUM_ARCHIVE_BYTES = 512 * 1024 * 1024
|
|
const ARCHIVE_OVERHEAD_BYTES = 1024 * 1024
|
|
const executableExtensionPattern =
|
|
/\.(?:app|bat|bin|cmd|com|cpl|dll|dmg|exe|hta|inf|ins|iso|jar|js|jse|lnk|msi|msp|mst|pif|ps1|reg|scr|sh|sys|vb|vbe|vbs|ws|wsc|wsf|wsh)$/iu
|
|
|
|
type ActiveOperation = {
|
|
controller: AbortController
|
|
progress: DocumentOcrModelOperation
|
|
}
|
|
|
|
type ActiveVerification = {
|
|
generation: number
|
|
promise: Promise<void>
|
|
}
|
|
|
|
export type DocumentOcrModelManagerOptions = {
|
|
userDataDirectory: string
|
|
fetch: typeof fetch
|
|
catalog?: readonly DocumentOcrModelCatalogEntry[]
|
|
getDownloadSource?: () =>
|
|
| ModelDownloadSource
|
|
| Promise<ModelDownloadSource>
|
|
maxFileBytes?: number
|
|
}
|
|
|
|
function cloneCatalogEntry(
|
|
entry: DocumentOcrModelCatalogEntry
|
|
): DocumentOcrModelCatalogEntry {
|
|
return documentOcrModelCatalogEntrySchema.parse(entry)
|
|
}
|
|
|
|
function toCatalogView(entry: DocumentOcrModelCatalogEntry) {
|
|
const { repositoryUrls, files, ...metadata } = entry
|
|
void repositoryUrls
|
|
return documentOcrModelCatalogViewEntrySchema.parse({
|
|
...metadata,
|
|
files: files.map((file) => ({
|
|
name: file.name,
|
|
role: file.role,
|
|
size: file.size,
|
|
sha256: file.sha256
|
|
})),
|
|
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
|
|
getModelDownloadAvailability(files, source)
|
|
)
|
|
})
|
|
}
|
|
|
|
function documentOcrPackageMatches(
|
|
entry: DocumentOcrModelCatalogEntry,
|
|
files: readonly {
|
|
name: string
|
|
role: string
|
|
size: number
|
|
sha256: string
|
|
}[]
|
|
): boolean {
|
|
return getModelPackageFingerprints(entry.files).some(
|
|
(fingerprint) =>
|
|
modelPackageFingerprintMatches(fingerprint, files)
|
|
)
|
|
}
|
|
|
|
function safeChild(parent: string, name: string): string {
|
|
return managedModelChild(
|
|
parent,
|
|
name,
|
|
'OCR 模型路径超出受管目录'
|
|
)
|
|
}
|
|
|
|
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
|
return Uint8Array.from(buffer).buffer
|
|
}
|
|
|
|
function parseYamlScalar(value: string): string {
|
|
if (value.startsWith("'") && value.endsWith("'")) {
|
|
return value.slice(1, -1).replace(/''/gu, "'")
|
|
}
|
|
if (value.startsWith('"') && value.endsWith('"')) {
|
|
return JSON.parse(value) as string
|
|
}
|
|
return value
|
|
}
|
|
|
|
export function extractPaddleCharacterDictionary(source: string): string {
|
|
const characters: string[] = []
|
|
let readingDictionary = false
|
|
for (const line of source.replace(/\r/gu, '').split('\n')) {
|
|
if (line === ' character_dict:') {
|
|
readingDictionary = true
|
|
continue
|
|
}
|
|
if (!readingDictionary) {
|
|
continue
|
|
}
|
|
const match = /^ {2}- (.*)$/u.exec(line)
|
|
if (!match) {
|
|
break
|
|
}
|
|
const character = parseYamlScalar(match[1]!)
|
|
if (!character) {
|
|
throw new Error('OCR 字符字典包含空条目')
|
|
}
|
|
characters.push(character)
|
|
}
|
|
if (characters.length < 100) {
|
|
throw new Error('OCR 字符字典格式无效')
|
|
}
|
|
return `${characters.join('\n')}\n`
|
|
}
|
|
|
|
export class DocumentOcrModelManager {
|
|
readonly rootDirectory: string
|
|
|
|
private readonly transport: typeof fetch
|
|
private readonly catalog: DocumentOcrModelCatalogEntry[]
|
|
private readonly catalogViews: DocumentOcrModelCatalogViewEntry[]
|
|
private readonly getDownloadSource: () =>
|
|
| ModelDownloadSource
|
|
| Promise<ModelDownloadSource>
|
|
private readonly maxFileBytes: number
|
|
private readonly operations = new Map<string, ActiveOperation>()
|
|
private readonly verifiedModels = new Map<string, ActiveVerification>()
|
|
private readonly verificationGenerations = new Map<string, number>()
|
|
|
|
constructor(options: DocumentOcrModelManagerOptions) {
|
|
if (!options.userDataDirectory.trim()) {
|
|
throw new Error('userDataDirectory is required')
|
|
}
|
|
this.rootDirectory = resolve(
|
|
options.userDataDirectory,
|
|
'models',
|
|
'document-ocr'
|
|
)
|
|
this.transport = options.fetch
|
|
this.getDownloadSource =
|
|
options.getDownloadSource ?? (() => 'modelscope')
|
|
this.catalog = (options.catalog ?? DOCUMENT_OCR_MODEL_CATALOG).map(
|
|
cloneCatalogEntry
|
|
)
|
|
if (
|
|
new Set(this.catalog.map((entry) => entry.id)).size !==
|
|
this.catalog.length
|
|
) {
|
|
throw new Error('OCR 模型目录包含重复 ID')
|
|
}
|
|
this.catalogViews = this.catalog.map(toCatalogView)
|
|
this.maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES
|
|
if (
|
|
!Number.isSafeInteger(this.maxFileBytes) ||
|
|
this.maxFileBytes <= 0 ||
|
|
this.maxFileBytes > 512 * 1024 * 1024
|
|
) {
|
|
throw new RangeError('maxFileBytes must be a positive safe integer')
|
|
}
|
|
}
|
|
|
|
async getSnapshot(): Promise<DocumentOcrModelSnapshot> {
|
|
await this.ensureRoot()
|
|
await this.cleanupStaleArtifacts()
|
|
const [selectedDownloadSource, installed] = await Promise.all([
|
|
this.getDownloadSource(),
|
|
this.readInstalled()
|
|
])
|
|
return documentOcrModelSnapshotSchema.parse({
|
|
rootDirectory: this.rootDirectory,
|
|
selectedDownloadSource,
|
|
catalog: this.catalogViews,
|
|
installed,
|
|
operations: [...this.operations.values()].map((operation) => ({
|
|
...operation.progress
|
|
}))
|
|
})
|
|
}
|
|
|
|
getProgressSnapshot(): DocumentOcrModelProgressSnapshot {
|
|
return documentOcrModelProgressSnapshotSchema.parse({
|
|
operations: [...this.operations.values()].map((operation) => ({
|
|
...operation.progress
|
|
}))
|
|
})
|
|
}
|
|
|
|
async getStatus(
|
|
modelId: string
|
|
): Promise<ReturnType<typeof documentParsingModelStatusSchema.parse>> {
|
|
const id = localOcrModelIdSchema.parse(modelId)
|
|
const entry = this.catalog.find((candidate) => candidate.id === id)
|
|
if (!entry) {
|
|
return documentParsingModelStatusSchema.parse({
|
|
id,
|
|
displayName: id,
|
|
available: false,
|
|
verified: false,
|
|
runtime: 'onnxruntime-web-wasm',
|
|
detail: '当前版本不再提供此 OCR 模型，请选择其他模型'
|
|
})
|
|
}
|
|
try {
|
|
await this.getVerifiedStatus(entry)
|
|
return documentParsingModelStatusSchema.parse({
|
|
id: entry.id,
|
|
displayName: entry.displayName,
|
|
available: true,
|
|
verified: true,
|
|
runtime: entry.runtime,
|
|
detail: '模型已安装并通过 SHA-256 校验，可离线使用'
|
|
})
|
|
} catch {
|
|
return documentParsingModelStatusSchema.parse({
|
|
id: entry.id,
|
|
displayName: entry.displayName,
|
|
available: false,
|
|
verified: false,
|
|
runtime: entry.runtime,
|
|
detail: '模型尚未安装或校验失败，请从当前模型下载源获取'
|
|
})
|
|
}
|
|
}
|
|
|
|
getAssets(modelId: string): Promise<DocumentOcrAssets> {
|
|
return this.loadVerifiedAssets(this.requireCatalogEntry(modelId))
|
|
}
|
|
|
|
getRepositoryUrl(
|
|
modelId: string,
|
|
source: ModelDownloadSource
|
|
): string {
|
|
const entry = this.requireCatalogEntry(modelId)
|
|
resolveModelDownloadPackage(entry.files, source)
|
|
const repositoryUrl = entry.repositoryUrls[source]
|
|
if (!repositoryUrl) {
|
|
throw new Error('当前下载源暂不提供此 OCR 模型的仓库')
|
|
}
|
|
return repositoryUrl
|
|
}
|
|
|
|
async install(
|
|
modelId: string,
|
|
downloadSource?: ModelDownloadSource,
|
|
externalSignal?: AbortSignal
|
|
): Promise<InstalledDocumentOcrModel> {
|
|
const entry = this.requireCatalogEntry(modelId)
|
|
const selectedDownloadSource =
|
|
downloadSource ?? (await this.getDownloadSource())
|
|
const resolvedPackage = resolveModelDownloadPackage(
|
|
entry.files,
|
|
selectedDownloadSource
|
|
)
|
|
const operation = this.beginOperation(
|
|
entry.id,
|
|
'download',
|
|
resolvedPackage.totalBytes,
|
|
resolvedPackage.source
|
|
)
|
|
const detachAbort = this.attachExternalSignal(
|
|
externalSignal,
|
|
operation.controller
|
|
)
|
|
let stagingDirectory: string | undefined
|
|
try {
|
|
await this.ensureRoot()
|
|
await this.assertNotInstalled(entry.id)
|
|
stagingDirectory = await this.createStagingDirectory(entry.id)
|
|
for (const file of resolvedPackage.files) {
|
|
ensureModelOperationNotAborted(operation.controller.signal)
|
|
operation.progress.phase = 'transferring'
|
|
operation.progress.currentFile = file.name
|
|
await this.downloadFile(
|
|
file,
|
|
safeChild(stagingDirectory, file.name),
|
|
operation,
|
|
operation.controller.signal
|
|
)
|
|
}
|
|
operation.progress.phase = 'installing'
|
|
operation.progress.currentFile = null
|
|
const installed = await this.createInstalledManifest(
|
|
entry,
|
|
'download',
|
|
stagingDirectory,
|
|
operation.controller.signal
|
|
)
|
|
ensureModelOperationNotAborted(operation.controller.signal)
|
|
await rename(stagingDirectory, this.modelDirectory(entry.id))
|
|
stagingDirectory = undefined
|
|
this.invalidateVerification(entry.id)
|
|
return installed
|
|
} finally {
|
|
detachAbort()
|
|
this.operations.delete(entry.id)
|
|
if (stagingDirectory) {
|
|
await rm(stagingDirectory, { recursive: true, force: true })
|
|
}
|
|
}
|
|
}
|
|
|
|
async registerLocalDirectory(
|
|
modelId: string,
|
|
sourceDirectory: string,
|
|
externalSignal?: AbortSignal
|
|
): Promise<InstalledDocumentOcrModel> {
|
|
const entry = this.requireCatalogEntry(modelId)
|
|
const source = resolve(sourceDirectory)
|
|
const operation = this.beginOperation(entry.id, 'import', null)
|
|
const detachAbort = this.attachExternalSignal(
|
|
externalSignal,
|
|
operation.controller
|
|
)
|
|
let stagingDirectory: string | undefined
|
|
try {
|
|
await this.ensureRoot()
|
|
await this.assertNotInstalled(entry.id)
|
|
await this.validateLocalDirectory(
|
|
source,
|
|
entry,
|
|
operation.controller.signal
|
|
)
|
|
stagingDirectory = await this.createStagingDirectory(entry.id)
|
|
operation.progress.phase = 'transferring'
|
|
for (const file of entry.files) {
|
|
ensureModelOperationNotAborted(operation.controller.signal)
|
|
operation.progress.currentFile = file.name
|
|
const sourceFile = safeChild(source, file.name)
|
|
const destination = safeChild(stagingDirectory, file.name)
|
|
await copyFile(sourceFile, destination)
|
|
operation.progress.completedBytes +=
|
|
(await stat(destination)).size
|
|
}
|
|
operation.progress.totalBytes =
|
|
operation.progress.completedBytes
|
|
operation.progress.phase = 'installing'
|
|
operation.progress.currentFile = null
|
|
const installed = await this.createInstalledManifest(
|
|
entry,
|
|
'local',
|
|
stagingDirectory,
|
|
operation.controller.signal
|
|
)
|
|
ensureModelOperationNotAborted(operation.controller.signal)
|
|
await rename(stagingDirectory, this.modelDirectory(entry.id))
|
|
stagingDirectory = undefined
|
|
this.invalidateVerification(entry.id)
|
|
return installed
|
|
} finally {
|
|
detachAbort()
|
|
this.operations.delete(entry.id)
|
|
if (stagingDirectory) {
|
|
await rm(stagingDirectory, { recursive: true, force: true })
|
|
}
|
|
}
|
|
}
|
|
|
|
async exportArchive(
|
|
modelId: string,
|
|
destinationPath: string
|
|
): Promise<void> {
|
|
const entry = this.requireCatalogEntry(modelId)
|
|
await this.ensureRoot()
|
|
const installed = (await this.readInstalled()).find(
|
|
(model) => model.id === entry.id
|
|
)
|
|
if (!installed) {
|
|
throw new Error('只能导出已安装的 OCR 模型')
|
|
}
|
|
if (!documentOcrPackageMatches(entry, installed.files)) {
|
|
throw new Error('OCR 模型文件与当前模型目录不匹配')
|
|
}
|
|
const directory = this.modelDirectory(entry.id)
|
|
const files = []
|
|
for (const expected of entry.files) {
|
|
const recorded = installed.files.find(
|
|
(file) =>
|
|
file.name === expected.name &&
|
|
file.role === expected.role
|
|
)
|
|
if (!recorded) {
|
|
throw new Error(`OCR 模型文件校验失败：${expected.name}`)
|
|
}
|
|
files.push({
|
|
name: expected.name,
|
|
role: expected.role,
|
|
size: recorded.size,
|
|
sha256: recorded.sha256
|
|
})
|
|
}
|
|
await exportModelArchive({
|
|
destinationPath,
|
|
sourceDirectory: directory,
|
|
descriptor: {
|
|
kind: 'document-ocr',
|
|
modelId: entry.id,
|
|
displayName: entry.displayName,
|
|
files
|
|
}
|
|
})
|
|
}
|
|
|
|
async importArchive(
|
|
modelId: string,
|
|
archivePath: string
|
|
): Promise<InstalledDocumentOcrModel> {
|
|
const entry = this.requireCatalogEntry(modelId)
|
|
const expectedTotal = getMaximumModelPackageBytes(entry.files)
|
|
const operation = this.beginOperation(
|
|
entry.id,
|
|
'import',
|
|
expectedTotal
|
|
)
|
|
let stagingDirectory: string | undefined
|
|
try {
|
|
await this.ensureRoot()
|
|
await this.assertNotInstalled(entry.id)
|
|
stagingDirectory = await this.createStagingDirectory(entry.id)
|
|
operation.progress.phase = 'transferring'
|
|
const descriptor = await extractModelArchive({
|
|
archivePath,
|
|
destinationDirectory: stagingDirectory,
|
|
expectedKind: 'document-ocr',
|
|
expectedModelId: entry.id,
|
|
expectedFiles: entry.files.map((file) => ({
|
|
name: file.name,
|
|
role: file.role
|
|
})),
|
|
maximumArchiveBytes: Math.min(
|
|
MAXIMUM_ARCHIVE_BYTES,
|
|
expectedTotal + ARCHIVE_OVERHEAD_BYTES
|
|
),
|
|
maximumFileBytes: this.maxFileBytes,
|
|
maximumTotalBytes: expectedTotal + ARCHIVE_OVERHEAD_BYTES,
|
|
signal: operation.controller.signal,
|
|
onProgress: (completedBytes) => {
|
|
operation.progress.completedBytes = completedBytes
|
|
}
|
|
})
|
|
if (!documentOcrPackageMatches(entry, descriptor.files)) {
|
|
throw new Error('OCR 模型 ZIP 与当前模型目录不匹配')
|
|
}
|
|
operation.progress.totalBytes = descriptor.files.reduce(
|
|
(total, file) => total + file.size,
|
|
0
|
|
)
|
|
operation.progress.phase = 'installing'
|
|
operation.progress.currentFile = null
|
|
const installed = installedDocumentOcrModelSchema.parse({
|
|
id: entry.id,
|
|
displayName: entry.displayName,
|
|
source: 'local',
|
|
installedAt: new Date().toISOString(),
|
|
files: descriptor.files
|
|
})
|
|
await writeFile(
|
|
safeChild(stagingDirectory, MANIFEST_FILE_NAME),
|
|
`${JSON.stringify(installed, null, 2)}\n`,
|
|
{ encoding: 'utf8', flag: 'wx' }
|
|
)
|
|
ensureModelOperationNotAborted(operation.controller.signal)
|
|
await rename(stagingDirectory, this.modelDirectory(entry.id))
|
|
stagingDirectory = undefined
|
|
this.invalidateVerification(entry.id)
|
|
return installed
|
|
} finally {
|
|
this.operations.delete(entry.id)
|
|
if (stagingDirectory) {
|
|
await rm(stagingDirectory, { recursive: true, force: true })
|
|
}
|
|
}
|
|
}
|
|
|
|
cancel(modelId: string): boolean {
|
|
const id = localOcrModelIdSchema.parse(modelId)
|
|
const operation = this.operations.get(id)
|
|
if (!operation) {
|
|
return false
|
|
}
|
|
operation.controller.abort()
|
|
return true
|
|
}
|
|
|
|
async remove(modelId: string): Promise<void> {
|
|
const id = localOcrModelIdSchema.parse(modelId)
|
|
this.cancel(id)
|
|
this.invalidateVerification(id)
|
|
await rm(this.modelDirectory(id), {
|
|
recursive: true,
|
|
force: true
|
|
})
|
|
}
|
|
|
|
dispose(): void {
|
|
for (const operation of this.operations.values()) {
|
|
operation.controller.abort()
|
|
}
|
|
this.operations.clear()
|
|
this.verifiedModels.clear()
|
|
this.verificationGenerations.clear()
|
|
}
|
|
|
|
private async ensureRoot(): Promise<void> {
|
|
await mkdir(this.rootDirectory, { recursive: true })
|
|
}
|
|
|
|
private modelDirectory(modelId: string): string {
|
|
return safeChild(
|
|
this.rootDirectory,
|
|
localOcrModelIdSchema.parse(modelId)
|
|
)
|
|
}
|
|
|
|
private requireCatalogEntry(
|
|
modelId: string
|
|
): DocumentOcrModelCatalogEntry {
|
|
const id = localOcrModelIdSchema.parse(modelId)
|
|
const entry = this.catalog.find((candidate) => candidate.id === id)
|
|
if (!entry) {
|
|
throw new Error('未知的 OCR 模型')
|
|
}
|
|
return entry
|
|
}
|
|
|
|
private beginOperation(
|
|
modelId: string,
|
|
kind: DocumentOcrModelOperation['kind'],
|
|
totalBytes: number | null,
|
|
downloadSource?: ModelDownloadSource
|
|
): ActiveOperation {
|
|
if (this.operations.has(modelId)) {
|
|
throw new Error('该 OCR 模型已有进行中的操作')
|
|
}
|
|
const operation: ActiveOperation = {
|
|
controller: new AbortController(),
|
|
progress: {
|
|
modelId: localOcrModelIdSchema.parse(modelId),
|
|
kind,
|
|
phase: 'preparing',
|
|
currentFile: null,
|
|
completedBytes: 0,
|
|
totalBytes,
|
|
...(downloadSource ? { downloadSource } : {})
|
|
}
|
|
}
|
|
this.operations.set(modelId, operation)
|
|
return operation
|
|
}
|
|
|
|
private attachExternalSignal(
|
|
signal: AbortSignal | undefined,
|
|
controller: AbortController
|
|
): () => void {
|
|
return attachModelAbortSignal(signal, controller)
|
|
}
|
|
|
|
private async assertNotInstalled(modelId: string): Promise<void> {
|
|
try {
|
|
await lstat(this.modelDirectory(modelId))
|
|
throw new Error('OCR 模型已安装')
|
|
} catch (error) {
|
|
if (isMissingFileError(error)) {
|
|
return
|
|
}
|
|
throw error
|
|
}
|
|
}
|
|
|
|
private async createStagingDirectory(modelId: string): Promise<string> {
|
|
return createModelStagingDirectory(
|
|
this.rootDirectory,
|
|
modelId,
|
|
'OCR 模型路径超出受管目录'
|
|
)
|
|
}
|
|
|
|
private async downloadFile(
|
|
file: ResolvedModelArtifactFile<DocumentOcrModelFile['role']>,
|
|
destination: string,
|
|
operation: ActiveOperation,
|
|
signal: AbortSignal
|
|
): Promise<void> {
|
|
if (file.size > this.maxFileBytes) {
|
|
throw new RangeError(`OCR 模型文件过大：${file.name}`)
|
|
}
|
|
const response = await fetchModelDownloadResponse({
|
|
transport: this.transport,
|
|
initialUrl: file.target.url,
|
|
redirectHosts: file.target.redirectHosts,
|
|
signal,
|
|
modelLabel: 'OCR 模型'
|
|
})
|
|
if (!response.ok) {
|
|
await response.body?.cancel().catch(() => undefined)
|
|
throw new Error(`OCR 模型下载失败：HTTP ${response.status}`)
|
|
}
|
|
if (!response.body) {
|
|
throw new Error('OCR 模型下载响应没有内容')
|
|
}
|
|
const declaredLength = response.headers.get('content-length')
|
|
if (
|
|
declaredLength !== null &&
|
|
Number(declaredLength) !== file.size
|
|
) {
|
|
await response.body.cancel().catch(() => undefined)
|
|
throw new Error(`OCR 模型文件大小不匹配：${file.name}`)
|
|
}
|
|
|
|
const partialPath = `${destination}${MODEL_PARTIAL_SUFFIX}`
|
|
const handle = await open(partialPath, 'wx')
|
|
const reader = response.body.getReader()
|
|
const hash = createHash('sha256')
|
|
let written = 0
|
|
try {
|
|
while (true) {
|
|
ensureModelOperationNotAborted(signal)
|
|
const result = await reader.read()
|
|
if (result.done) {
|
|
break
|
|
}
|
|
if (
|
|
written + result.value.byteLength > file.size ||
|
|
written + result.value.byteLength > this.maxFileBytes
|
|
) {
|
|
await reader.cancel()
|
|
throw new RangeError(`OCR 模型文件过大：${file.name}`)
|
|
}
|
|
const persistedBytes = await writeModelBuffer(
|
|
handle,
|
|
result.value,
|
|
(persisted) => {
|
|
hash.update(persisted)
|
|
operation.progress.completedBytes += persisted.byteLength
|
|
}
|
|
)
|
|
written += persistedBytes
|
|
}
|
|
} catch (error) {
|
|
await reader.cancel().catch(() => undefined)
|
|
throw error
|
|
} finally {
|
|
await handle.close()
|
|
}
|
|
if (
|
|
written !== file.size ||
|
|
hash.digest('hex') !== file.sha256
|
|
) {
|
|
throw new Error(`OCR 模型文件校验失败：${file.name}`)
|
|
}
|
|
await rename(partialPath, destination)
|
|
}
|
|
|
|
private async validateLocalDirectory(
|
|
sourceDirectory: string,
|
|
entry: DocumentOcrModelCatalogEntry,
|
|
signal: AbortSignal
|
|
): Promise<void> {
|
|
const sourceInfo = await lstat(sourceDirectory)
|
|
if (!sourceInfo.isDirectory() || sourceInfo.isSymbolicLink()) {
|
|
throw new Error('本地 OCR 模型来源必须是普通目录')
|
|
}
|
|
const entries = await readdir(sourceDirectory, { withFileTypes: true })
|
|
for (const localEntry of entries) {
|
|
ensureModelOperationNotAborted(signal)
|
|
if (
|
|
localEntry.isSymbolicLink() ||
|
|
executableExtensionPattern.test(localEntry.name)
|
|
) {
|
|
throw new Error('本地 OCR 模型目录包含不安全文件')
|
|
}
|
|
}
|
|
const actualFiles: InstalledDocumentOcrModel['files'] = []
|
|
for (const file of entry.files) {
|
|
ensureModelOperationNotAborted(signal)
|
|
const path = safeChild(sourceDirectory, file.name)
|
|
const info = await lstat(path)
|
|
if (!info.isFile() || info.isSymbolicLink()) {
|
|
throw new Error(`OCR 模型文件必须是普通文件：${file.name}`)
|
|
}
|
|
if (
|
|
info.size <= 0 ||
|
|
info.size > this.maxFileBytes
|
|
) {
|
|
throw new RangeError(`OCR 模型文件大小无效：${file.name}`)
|
|
}
|
|
actualFiles.push({
|
|
name: file.name,
|
|
role: file.role,
|
|
...(await hashModelFile(path, signal))
|
|
})
|
|
}
|
|
if (!documentOcrPackageMatches(entry, actualFiles)) {
|
|
throw new Error(
|
|
'本地 OCR 模型文件校验失败：与当前模型目录不匹配'
|
|
)
|
|
}
|
|
}
|
|
|
|
private async createInstalledManifest(
|
|
entry: DocumentOcrModelCatalogEntry,
|
|
source: InstalledDocumentOcrModel['source'],
|
|
stagingDirectory: string,
|
|
signal: AbortSignal
|
|
): Promise<InstalledDocumentOcrModel> {
|
|
const files = []
|
|
for (const file of entry.files) {
|
|
ensureModelOperationNotAborted(signal)
|
|
files.push({
|
|
name: file.name,
|
|
role: file.role,
|
|
...(await hashModelFile(
|
|
safeChild(stagingDirectory, file.name),
|
|
signal
|
|
))
|
|
})
|
|
}
|
|
const manifest = installedDocumentOcrModelSchema.parse({
|
|
id: entry.id,
|
|
displayName: entry.displayName,
|
|
source,
|
|
installedAt: new Date().toISOString(),
|
|
files
|
|
})
|
|
if (!documentOcrPackageMatches(entry, manifest.files)) {
|
|
throw new Error('OCR 模型文件与当前模型目录不匹配')
|
|
}
|
|
await writeFile(
|
|
safeChild(stagingDirectory, MANIFEST_FILE_NAME),
|
|
`${JSON.stringify(manifest, null, 2)}\n`,
|
|
{ encoding: 'utf8', flag: 'wx' }
|
|
)
|
|
return manifest
|
|
}
|
|
|
|
private async readInstalled(): Promise<InstalledDocumentOcrModel[]> {
|
|
const entries = await readdir(this.rootDirectory, {
|
|
withFileTypes: true
|
|
})
|
|
const installed: InstalledDocumentOcrModel[] = []
|
|
for (const entry of entries) {
|
|
if (
|
|
!entry.isDirectory() ||
|
|
entry.name.startsWith('.install-') ||
|
|
!localOcrModelIdSchema.safeParse(entry.name).success
|
|
) {
|
|
continue
|
|
}
|
|
try {
|
|
const manifest = installedDocumentOcrModelSchema.parse(
|
|
JSON.parse(
|
|
await readFile(
|
|
safeChild(
|
|
this.modelDirectory(entry.name),
|
|
MANIFEST_FILE_NAME
|
|
),
|
|
'utf8'
|
|
)
|
|
) as unknown
|
|
)
|
|
if (manifest.id === entry.name) {
|
|
installed.push(manifest)
|
|
}
|
|
} catch {
|
|
// Ignore incomplete or externally modified model directories.
|
|
}
|
|
}
|
|
return installed
|
|
}
|
|
|
|
private async readInstalledManifest(
|
|
entry: DocumentOcrModelCatalogEntry
|
|
): Promise<InstalledDocumentOcrModel> {
|
|
const directory = this.modelDirectory(entry.id)
|
|
const manifest = installedDocumentOcrModelSchema.parse(
|
|
JSON.parse(
|
|
await readFile(
|
|
safeChild(directory, MANIFEST_FILE_NAME),
|
|
'utf8'
|
|
)
|
|
) as unknown
|
|
)
|
|
if (manifest.id !== entry.id) {
|
|
throw new Error('OCR 模型清单 ID 不匹配')
|
|
}
|
|
return manifest
|
|
}
|
|
|
|
private async verifyInstalledModel(
|
|
entry: DocumentOcrModelCatalogEntry
|
|
): Promise<void> {
|
|
const directory = this.modelDirectory(entry.id)
|
|
const manifest = await this.readInstalledManifest(entry)
|
|
if (!documentOcrPackageMatches(entry, manifest.files)) {
|
|
throw new Error('OCR 模型文件与当前模型目录不匹配')
|
|
}
|
|
for (const file of entry.files) {
|
|
const installed = manifest.files.find(
|
|
(candidate) =>
|
|
candidate.name === file.name &&
|
|
candidate.role === file.role
|
|
)
|
|
const actual = await hashModelFile(
|
|
safeChild(directory, file.name)
|
|
)
|
|
if (
|
|
!installed ||
|
|
actual.size !== installed.size ||
|
|
actual.sha256 !== installed.sha256
|
|
) {
|
|
throw new Error(`OCR 模型文件校验失败：${file.name}`)
|
|
}
|
|
}
|
|
}
|
|
|
|
private getVerifiedStatus(
|
|
entry: DocumentOcrModelCatalogEntry
|
|
): Promise<void> {
|
|
const generation = this.verificationGenerations.get(entry.id) ?? 0
|
|
const active = this.verifiedModels.get(entry.id)
|
|
if (active?.generation === generation) {
|
|
return active.promise
|
|
}
|
|
const verification = this.verifyInstalledModel(entry).then(() => {
|
|
if (
|
|
(this.verificationGenerations.get(entry.id) ?? 0) !==
|
|
generation
|
|
) {
|
|
throw new Error('OCR 模型在校验期间已发生变化')
|
|
}
|
|
})
|
|
const tracked = verification.finally(() => {
|
|
if (this.verifiedModels.get(entry.id)?.promise === tracked) {
|
|
this.verifiedModels.delete(entry.id)
|
|
}
|
|
})
|
|
this.verifiedModels.set(entry.id, {
|
|
generation,
|
|
promise: tracked
|
|
})
|
|
return tracked
|
|
}
|
|
|
|
private invalidateVerification(modelId: string): void {
|
|
this.verificationGenerations.set(
|
|
modelId,
|
|
(this.verificationGenerations.get(modelId) ?? 0) + 1
|
|
)
|
|
this.verifiedModels.delete(modelId)
|
|
}
|
|
|
|
private async loadVerifiedAssets(
|
|
entry: DocumentOcrModelCatalogEntry
|
|
): Promise<DocumentOcrAssets> {
|
|
const directory = this.modelDirectory(entry.id)
|
|
const manifest = await this.readInstalledManifest(entry)
|
|
if (!documentOcrPackageMatches(entry, manifest.files)) {
|
|
throw new Error('OCR 模型文件与当前模型目录不匹配')
|
|
}
|
|
const loaded = new Map<
|
|
DocumentOcrModelFile['role'],
|
|
ArrayBuffer
|
|
>()
|
|
for (const file of entry.files) {
|
|
const installed = manifest.files.find(
|
|
(candidate) =>
|
|
candidate.name === file.name &&
|
|
candidate.role === file.role
|
|
)
|
|
const path = safeChild(directory, file.name)
|
|
const contents = await readFile(path)
|
|
const actual = {
|
|
size: contents.byteLength,
|
|
sha256: createHash('sha256').update(contents).digest('hex')
|
|
}
|
|
if (
|
|
!installed ||
|
|
actual.size !== installed.size ||
|
|
actual.sha256 !== installed.sha256
|
|
) {
|
|
throw new Error(`OCR 模型文件校验失败：${file.name}`)
|
|
}
|
|
loaded.set(
|
|
file.role,
|
|
file.role === 'dictionary'
|
|
? toArrayBuffer(
|
|
Buffer.from(
|
|
extractPaddleCharacterDictionary(
|
|
contents.toString('utf8')
|
|
),
|
|
'utf8'
|
|
)
|
|
)
|
|
: toArrayBuffer(contents)
|
|
)
|
|
}
|
|
return documentOcrAssetsSchema.parse({
|
|
modelId: entry.id,
|
|
detection: loaded.get('detection'),
|
|
recognition: loaded.get('recognition'),
|
|
dictionary: loaded.get('dictionary')
|
|
})
|
|
}
|
|
|
|
private cleanupStaleArtifacts(): Promise<void> {
|
|
return cleanupStaleModelInstallArtifacts({
|
|
rootDirectory: this.rootDirectory,
|
|
isModelId: (value) =>
|
|
localOcrModelIdSchema.safeParse(value).success,
|
|
activeModelIds: new Set(this.operations.keys()),
|
|
partialFileNames: new Set(
|
|
this.catalog.flatMap((entry) =>
|
|
entry.files.map((file) => file.name)
|
|
)
|
|
),
|
|
escapeMessage: 'OCR 模型路径超出受管目录'
|
|
})
|
|
}
|
|
}
|