feat: add explicit managed model download sources

Managed speech and OCR downloads previously used catalog-specific source URLs without a global selection. Platform Features now lets users choose ModelScope by default or Hugging Face, while Main validates and freezes that source for each download.

Verified coverage remains explicit: downloads never mix artifacts or silently switch sources, and installed models plus ZIP imports stay source-independent.

Release note: 可在“设置 → 平台功能 → 通用设置”中选择 ModelScope 或 Hugging Face 作为后续语音输入与 OCR 模型下载源;缺少完整已验证文件或下载失败时不会静默换源。
This commit is contained in:
mesalogo
2026-08-19 13:43:16 +08:00
parent c41d7e8b96
commit 43e1d162dc
44 changed files with 6083 additions and 839 deletions
+99 -74
View File
@@ -15,26 +15,35 @@ import { dirname, resolve } from 'node:path'
import {
documentOcrAssetsSchema,
documentOcrModelCatalogEntrySchema,
documentOcrModelCatalogViewEntrySchema,
documentOcrModelSnapshotSchema,
documentParsingModelStatusSchema,
installedDocumentOcrModelSchema,
localOcrModelIdSchema,
type DocumentOcrAssets,
type DocumentOcrModelCatalogEntry,
type DocumentOcrModelCatalogViewEntry,
type DocumentOcrModelFile,
type DocumentOcrModelOperation,
type DocumentOcrModelSnapshot,
type InstalledDocumentOcrModel
} from '../shared/document-parsing-contracts'
import {
MODEL_DOWNLOAD_SOURCES,
getModelDownloadAvailability,
resolveModelDownloadPackage,
type ModelDownloadSource,
type ResolvedModelArtifactFile
} from '../shared/model-download-contracts'
import { DOCUMENT_OCR_MODEL_CATALOG } from './document-ocr-model-catalog'
import {
exportModelArchive,
extractModelArchive
} from './model-archive'
import { fetchModelDownloadResponse } from './model-download-transport'
const DEFAULT_MAX_FILE_BYTES = 96 * 1024 * 1024
const MANIFEST_FILE_NAME = 'manifest.json'
const MAX_REDIRECTS = 3
const PARTIAL_SUFFIX = '.partial'
const MAXIMUM_ARCHIVE_BYTES = 512 * 1024 * 1024
const ARCHIVE_OVERHEAD_BYTES = 1024 * 1024
@@ -50,6 +59,9 @@ export type DocumentOcrModelManagerOptions = {
userDataDirectory: string
fetch: typeof fetch
catalog?: readonly DocumentOcrModelCatalogEntry[]
getDownloadSource?: () =>
| ModelDownloadSource
| Promise<ModelDownloadSource>
maxFileBytes?: number
}
@@ -69,6 +81,23 @@ function cloneCatalogEntry(
return documentOcrModelCatalogEntrySchema.parse(entry)
}
function toCatalogView(entry: DocumentOcrModelCatalogEntry) {
const { repositoryUrls, files, ...metadata } = entry
void repositoryUrls
return documentOcrModelCatalogViewEntrySchema.parse({
...metadata,
files: files.map((file) => ({
name: file.name,
role: file.role,
size: file.size,
sha256: file.sha256
})),
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
getModelDownloadAvailability(files, source)
)
})
}
function safeChild(parent: string, name: string): string {
const child = resolve(parent, name)
if (dirname(child) !== resolve(parent)) {
@@ -77,14 +106,6 @@ function safeChild(parent: string, name: string): string {
return child
}
function validateDownloadUrl(value: string): URL {
const url = new URL(value)
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
throw new Error('OCR 模型下载地址必须使用 HTTP 或 HTTPS')
}
return url
}
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
return Uint8Array.from(buffer).buffer
}
@@ -157,6 +178,10 @@ export class DocumentOcrModelManager {
private readonly transport: typeof fetch
private readonly catalog: DocumentOcrModelCatalogEntry[]
private readonly catalogViews: DocumentOcrModelCatalogViewEntry[]
private readonly getDownloadSource: () =>
| ModelDownloadSource
| Promise<ModelDownloadSource>
private readonly maxFileBytes: number
private readonly operations = new Map<string, ActiveOperation>()
private readonly verifiedModels = new Map<string, Promise<void>>()
@@ -171,6 +196,8 @@ export class DocumentOcrModelManager {
'document-ocr'
)
this.transport = options.fetch
this.getDownloadSource =
options.getDownloadSource ?? (() => 'modelscope')
this.catalog = (options.catalog ?? DOCUMENT_OCR_MODEL_CATALOG).map(
cloneCatalogEntry
)
@@ -180,6 +207,7 @@ export class DocumentOcrModelManager {
) {
throw new Error('OCR 模型目录包含重复 ID')
}
this.catalogViews = this.catalog.map(toCatalogView)
this.maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES
if (
!Number.isSafeInteger(this.maxFileBytes) ||
@@ -192,10 +220,15 @@ export class DocumentOcrModelManager {
async getSnapshot(): Promise<DocumentOcrModelSnapshot> {
await this.ensureRoot()
const [selectedDownloadSource, installed] = await Promise.all([
this.getDownloadSource(),
this.readInstalled()
])
return documentOcrModelSnapshotSchema.parse({
rootDirectory: this.rootDirectory,
catalog: this.catalog.map(cloneCatalogEntry),
installed: await this.readInstalled(),
selectedDownloadSource,
catalog: this.catalogViews,
installed,
operations: [...this.operations.values()].map((operation) => ({
...operation.progress
}))
@@ -234,7 +267,7 @@ export class DocumentOcrModelManager {
available: false,
verified: false,
runtime: entry.runtime,
detail: '模型尚未安装或校验失败,请从 ModelScope 下载'
detail: '模型尚未安装或校验失败,请从当前模型下载源获取'
})
}
}
@@ -243,19 +276,37 @@ export class DocumentOcrModelManager {
return this.loadVerifiedAssets(this.requireCatalogEntry(modelId))
}
getRepositoryUrl(
modelId: string,
source: ModelDownloadSource
): string {
const entry = this.requireCatalogEntry(modelId)
resolveModelDownloadPackage(entry.files, source)
const repositoryUrl = entry.repositoryUrls[source]
if (!repositoryUrl) {
throw new Error('当前下载源暂不提供此 OCR 模型的仓库')
}
return repositoryUrl
}
async install(
modelId: string,
downloadSource?: ModelDownloadSource,
externalSignal?: AbortSignal
): Promise<InstalledDocumentOcrModel> {
const entry = this.requireCatalogEntry(modelId)
const totalBytes = entry.files.reduce(
(total, file) => total + file.download.size,
0
const selectedDownloadSource =
downloadSource ?? (await this.getDownloadSource())
const resolvedPackage = resolveModelDownloadPackage(
entry.files,
selectedDownloadSource
)
const operation = this.beginOperation(
entry.id,
'download',
resolvedPackage.totalBytes,
resolvedPackage.source
)
if (!Number.isSafeInteger(totalBytes)) {
throw new RangeError('OCR 模型总大小超出安全范围')
}
const operation = this.beginOperation(entry.id, 'download', totalBytes)
const detachAbort = this.attachExternalSignal(
externalSignal,
operation.controller
@@ -265,7 +316,7 @@ export class DocumentOcrModelManager {
await this.ensureRoot()
await this.assertNotInstalled(entry.id)
stagingDirectory = await this.createStagingDirectory(entry.id)
for (const file of entry.files) {
for (const file of resolvedPackage.files) {
ensureNotAborted(operation.controller.signal)
operation.progress.phase = 'transferring'
operation.progress.currentFile = file.name
@@ -376,8 +427,8 @@ export class DocumentOcrModelManager {
)
if (
!recorded ||
recorded.size !== expected.download.size ||
recorded.sha256 !== expected.download.sha256
recorded.size !== expected.size ||
recorded.sha256 !== expected.sha256
) {
throw new Error(`OCR 模型文件校验失败:${expected.name}`)
}
@@ -406,7 +457,7 @@ export class DocumentOcrModelManager {
): Promise<InstalledDocumentOcrModel> {
const entry = this.requireCatalogEntry(modelId)
const expectedTotal = entry.files.reduce(
(total, file) => total + file.download.size,
(total, file) => total + file.size,
0
)
const operation = this.beginOperation(
@@ -448,8 +499,8 @@ export class DocumentOcrModelManager {
)
if (
!archived ||
archived.size !== expected.download.size ||
archived.sha256 !== expected.download.sha256
archived.size !== expected.size ||
archived.sha256 !== expected.sha256
) {
throw new Error(
`OCR 模型 ZIP 与当前模型目录不匹配:${expected.name}`
@@ -536,7 +587,8 @@ export class DocumentOcrModelManager {
private beginOperation(
modelId: string,
kind: DocumentOcrModelOperation['kind'],
totalBytes: number | null
totalBytes: number | null,
downloadSource?: ModelDownloadSource
): ActiveOperation {
if (this.operations.has(modelId)) {
throw new Error('该 OCR 模型已有进行中的操作')
@@ -549,7 +601,8 @@ export class DocumentOcrModelManager {
phase: 'preparing',
currentFile: null,
completedBytes: 0,
totalBytes
totalBytes,
...(downloadSource ? { downloadSource } : {})
}
}
this.operations.set(modelId, operation)
@@ -597,50 +650,22 @@ export class DocumentOcrModelManager {
return directory
}
private async fetchFollowingRedirects(
initialUrl: string,
signal: AbortSignal
): Promise<Response> {
let url = validateDownloadUrl(initialUrl)
for (let redirectCount = 0; ; redirectCount += 1) {
ensureNotAborted(signal)
const response = await this.transport(url, {
method: 'GET',
redirect: 'manual',
credentials: 'omit',
cache: 'no-store',
signal
})
if ([301, 302, 303, 307, 308].includes(response.status)) {
if (redirectCount >= MAX_REDIRECTS) {
await response.body?.cancel().catch(() => undefined)
throw new Error('OCR 模型下载重定向次数过多')
}
const location = response.headers.get('location')
await response.body?.cancel().catch(() => undefined)
if (!location) {
throw new Error('OCR 模型下载重定向缺少地址')
}
url = validateDownloadUrl(new URL(location, url).toString())
continue
}
return response
}
}
private async downloadFile(
file: DocumentOcrModelFile,
file: ResolvedModelArtifactFile<DocumentOcrModelFile['role']>,
destination: string,
operation: ActiveOperation,
signal: AbortSignal
): Promise<void> {
if (file.download.size > this.maxFileBytes) {
if (file.size > this.maxFileBytes) {
throw new RangeError(`OCR 模型文件过大:${file.name}`)
}
const response = await this.fetchFollowingRedirects(
file.download.url,
signal
)
const response = await fetchModelDownloadResponse({
transport: this.transport,
initialUrl: file.target.url,
redirectHosts: file.target.redirectHosts,
signal,
modelLabel: 'OCR 模型'
})
if (!response.ok) {
await response.body?.cancel().catch(() => undefined)
throw new Error(`OCR 模型下载失败:HTTP ${response.status}`)
@@ -651,7 +676,7 @@ export class DocumentOcrModelManager {
const declaredLength = response.headers.get('content-length')
if (
declaredLength !== null &&
Number(declaredLength) !== file.download.size
Number(declaredLength) !== file.size
) {
await response.body.cancel().catch(() => undefined)
throw new Error(`OCR 模型文件大小不匹配:${file.name}`)
@@ -671,7 +696,7 @@ export class DocumentOcrModelManager {
}
written += result.value.byteLength
if (
written > file.download.size ||
written > file.size ||
written > this.maxFileBytes
) {
await reader.cancel()
@@ -688,8 +713,8 @@ export class DocumentOcrModelManager {
await handle.close()
}
if (
written !== file.download.size ||
hash.digest('hex') !== file.download.sha256
written !== file.size ||
hash.digest('hex') !== file.sha256
) {
throw new Error(`OCR 模型文件校验失败:${file.name}`)
}
@@ -724,8 +749,8 @@ export class DocumentOcrModelManager {
}
const actual = await hashFile(path, signal)
if (
actual.size !== file.download.size ||
actual.sha256 !== file.download.sha256
actual.size !== file.size ||
actual.sha256 !== file.sha256
) {
throw new Error(`本地 OCR 模型文件校验失败:${file.name}`)
}
@@ -832,8 +857,8 @@ export class DocumentOcrModelManager {
const actual = await hashFile(safeChild(directory, file.name))
if (
!installed ||
actual.size !== file.download.size ||
actual.sha256 !== file.download.sha256 ||
actual.size !== file.size ||
actual.sha256 !== file.sha256 ||
actual.size !== installed.size ||
actual.sha256 !== installed.sha256
) {
@@ -879,8 +904,8 @@ export class DocumentOcrModelManager {
}
if (
!installed ||
actual.size !== file.download.size ||
actual.sha256 !== file.download.sha256 ||
actual.size !== file.size ||
actual.sha256 !== file.sha256 ||
actual.size !== installed.size ||
actual.sha256 !== installed.sha256
) {