feat: add explicit managed model download sources
Managed speech and OCR downloads previously used catalog-specific source URLs without a global selection. Platform Features now lets users choose ModelScope by default or Hugging Face, while Main validates and freezes that source for each download. Verified coverage remains explicit: downloads never mix artifacts or silently switch sources, and installed models plus ZIP imports stay source-independent. Release note: 可在“设置 → 平台功能 → 通用设置”中选择 ModelScope 或 Hugging Face 作为后续语音输入与 OCR 模型下载源；缺少完整已验证文件或下载失败时不会静默换源。
This commit is contained in:
mesalogo
@@ -63,6 +63,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -70,14 +71,16 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
})
|
||||
expect(JSON.parse(await readFile(filePath, 'utf8'))).toEqual({
|
||||
version: 6,
|
||||
version: 7,
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined',
|
||||
@@ -102,6 +105,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -124,6 +128,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -146,6 +151,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -168,6 +174,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'after-save-manual',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -194,9 +201,10 @@ describe('ApplicationSettingsStore', () => {
|
||||
new ApplicationSettingsStore(filePath).getLastSeenReleaseNotesVersion()
|
||||
).resolves.toBe('0.8.18')
|
||||
expect(JSON.parse(await readFile(filePath, 'utf8'))).toEqual({
|
||||
version: 6,
|
||||
version: 7,
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'after-save-manual',
|
||||
magicNoteCommentFormat: 'narrative',
|
||||
@@ -222,6 +230,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'after-save-auto',
|
||||
magicNoteCommentFormat: 'structured'
|
||||
@@ -231,6 +240,39 @@ describe('ApplicationSettingsStore', () => {
|
||||
)
|
||||
})
|
||||
|
||||
it('lazily migrates version 6 to the default ModelScope source', async () => {
|
||||
const { filePath, store } = await createStore()
|
||||
const versionSix = {
|
||||
version: 6,
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'mirror',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'after-save-auto',
|
||||
magicNoteCommentFormat: 'structured',
|
||||
lastSeenReleaseNotesVersion: '0.8.18'
|
||||
}
|
||||
await writeFile(filePath, JSON.stringify(versionSix), 'utf8')
|
||||
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'mirror',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'after-save-auto',
|
||||
magicNoteCommentFormat: 'structured'
|
||||
})
|
||||
expect(JSON.parse(await readFile(filePath, 'utf8'))).toEqual(
|
||||
versionSix
|
||||
)
|
||||
|
||||
await store.update({ modelDownloadSource: 'hugging-face' })
|
||||
expect(JSON.parse(await readFile(filePath, 'utf8'))).toEqual({
|
||||
...versionSix,
|
||||
version: 7,
|
||||
modelDownloadSource: 'hugging-face'
|
||||
})
|
||||
})
|
||||
|
||||
it('strictly rejects incomplete full settings', () => {
|
||||
for (const input of [
|
||||
{},
|
||||
@@ -254,6 +296,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
for (const input of [
|
||||
{},
|
||||
{ checkUpdatesOnStartup: 'true' },
|
||||
{ modelDownloadSource: 'automatic' },
|
||||
{ anotherSetting: true },
|
||||
null
|
||||
]) {
|
||||
@@ -274,6 +317,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -387,14 +431,16 @@ describe('ApplicationSettingsStore', () => {
|
||||
await expect(store.get()).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
})
|
||||
expect(JSON.parse(await readFile(filePath, 'utf8'))).toEqual({
|
||||
version: 6,
|
||||
version: 7,
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined',
|
||||
@@ -419,6 +465,7 @@ describe('ApplicationSettingsStore', () => {
|
||||
).resolves.toEqual({
|
||||
checkUpdatesOnStartup: false,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: true,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
|
||||
@@ -20,7 +20,7 @@ export {
|
||||
} from '../shared/application-settings-contracts'
|
||||
export type { ApplicationSettings } from '../shared/application-settings-contracts'
|
||||
|
||||
const CURRENT_SETTINGS_VERSION = 6
|
||||
const CURRENT_SETTINGS_VERSION = 7
|
||||
|
||||
const legacyStoredApplicationSettingsSchema = z
|
||||
.object({
|
||||
@@ -47,20 +47,28 @@ const versionThreeStoredApplicationSettingsSchema = z
|
||||
.strict()
|
||||
|
||||
const versionFourStoredApplicationSettingsSchema = applicationSettingsSchema
|
||||
.omit({ updateSource: true })
|
||||
.omit({ updateSource: true, modelDownloadSource: true })
|
||||
.extend({
|
||||
version: z.literal(4)
|
||||
})
|
||||
.strict()
|
||||
|
||||
const versionFiveStoredApplicationSettingsSchema = applicationSettingsSchema
|
||||
.omit({ updateSource: true })
|
||||
.omit({ updateSource: true, modelDownloadSource: true })
|
||||
.extend({
|
||||
version: z.literal(5),
|
||||
lastSeenReleaseNotesVersion: releaseVersionSchema.nullable()
|
||||
})
|
||||
.strict()
|
||||
|
||||
const versionSixStoredApplicationSettingsSchema = applicationSettingsSchema
|
||||
.omit({ modelDownloadSource: true })
|
||||
.extend({
|
||||
version: z.literal(6),
|
||||
lastSeenReleaseNotesVersion: releaseVersionSchema.nullable()
|
||||
})
|
||||
.strict()
|
||||
|
||||
const storedApplicationSettingsSchema = applicationSettingsSchema
|
||||
.extend({
|
||||
version: z.literal(CURRENT_SETTINGS_VERSION),
|
||||
@@ -75,6 +83,7 @@ type StoredApplicationSettings = z.infer<
|
||||
export const defaultApplicationSettings: ApplicationSettings = {
|
||||
checkUpdatesOnStartup: true,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined'
|
||||
@@ -131,13 +140,24 @@ export class ApplicationSettingsStore {
|
||||
)
|
||||
const result = storedApplicationSettingsSchema.safeParse(parsed)
|
||||
if (!result.success) {
|
||||
const versionSixResult =
|
||||
versionSixStoredApplicationSettingsSchema.safeParse(parsed)
|
||||
if (versionSixResult.success) {
|
||||
this.settings = {
|
||||
...versionSixResult.data,
|
||||
version: CURRENT_SETTINGS_VERSION,
|
||||
modelDownloadSource: 'modelscope'
|
||||
}
|
||||
return this.settings
|
||||
}
|
||||
const versionFiveResult =
|
||||
versionFiveStoredApplicationSettingsSchema.safeParse(parsed)
|
||||
if (versionFiveResult.success) {
|
||||
this.settings = {
|
||||
...versionFiveResult.data,
|
||||
version: CURRENT_SETTINGS_VERSION,
|
||||
updateSource: 'github'
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope'
|
||||
}
|
||||
return this.settings
|
||||
}
|
||||
@@ -148,6 +168,7 @@ export class ApplicationSettingsStore {
|
||||
...versionFourResult.data,
|
||||
version: CURRENT_SETTINGS_VERSION,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
lastSeenReleaseNotesVersion: null
|
||||
}
|
||||
return this.settings
|
||||
@@ -159,6 +180,7 @@ export class ApplicationSettingsStore {
|
||||
...versionThreeResult.data,
|
||||
version: CURRENT_SETTINGS_VERSION,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNoteCommentFormat: 'combined',
|
||||
lastSeenReleaseNotesVersion: null
|
||||
}
|
||||
@@ -171,6 +193,7 @@ export class ApplicationSettingsStore {
|
||||
...versionTwoResult.data,
|
||||
version: CURRENT_SETTINGS_VERSION,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined',
|
||||
lastSeenReleaseNotesVersion: null
|
||||
@@ -185,6 +208,7 @@ export class ApplicationSettingsStore {
|
||||
checkUpdatesOnStartup:
|
||||
legacyResult.data.checkUpdatesOnStartup,
|
||||
updateSource: 'github',
|
||||
modelDownloadSource: 'modelscope',
|
||||
magicNotesEnabled: false,
|
||||
magicNoteCommentMode: 'immediate',
|
||||
magicNoteCommentFormat: 'combined',
|
||||
@@ -225,6 +249,7 @@ export class ApplicationSettingsStore {
|
||||
return {
|
||||
checkUpdatesOnStartup: stored.checkUpdatesOnStartup,
|
||||
updateSource: stored.updateSource,
|
||||
modelDownloadSource: stored.modelDownloadSource,
|
||||
magicNotesEnabled: stored.magicNotesEnabled,
|
||||
magicNoteCommentMode: stored.magicNoteCommentMode,
|
||||
magicNoteCommentFormat: stored.magicNoteCommentFormat,
|
||||
@@ -257,6 +282,7 @@ export class ApplicationSettingsStore {
|
||||
return {
|
||||
checkUpdatesOnStartup: next.checkUpdatesOnStartup,
|
||||
updateSource: next.updateSource,
|
||||
modelDownloadSource: next.modelDownloadSource,
|
||||
magicNotesEnabled: next.magicNotesEnabled,
|
||||
magicNoteCommentMode: next.magicNoteCommentMode,
|
||||
magicNoteCommentFormat: next.magicNoteCommentFormat
|
||||
|
||||
@@ -2,19 +2,66 @@ import {
|
||||
documentOcrModelCatalogEntrySchema,
|
||||
type DocumentOcrModelCatalogEntry
|
||||
} from '../shared/document-parsing-contracts'
|
||||
import {
|
||||
huggingFaceTarget,
|
||||
modelScopeTarget
|
||||
} from './model-download-targets'
|
||||
|
||||
const detectionRevision =
|
||||
'7d7f5d128d9309ebf6de4f21f404dd583afdbae3'
|
||||
const recognitionRevision =
|
||||
'afba04b618200c5f4824531c6e42c957c6439d9a'
|
||||
const smallDetectionRevision =
|
||||
'956a0b620a4017cc04056c692be1703b0025d028'
|
||||
const smallRecognitionRevision =
|
||||
'296d43bc0ebced0fd9c605174aa5962e49810ab6'
|
||||
const mediumDetectionRevision =
|
||||
'c317b40325be40bfaaff58c8dcece2a075294f8a'
|
||||
const mediumRecognitionRevision =
|
||||
'db5d610d492a14e3c34dc1fd4e9339bd369f79e6'
|
||||
const repositories = {
|
||||
tinyDetection: 'PaddlePaddle/PP-OCRv6_tiny_det_onnx',
|
||||
tinyRecognition: 'PaddlePaddle/PP-OCRv6_tiny_rec_onnx',
|
||||
smallDetection: 'PaddlePaddle/PP-OCRv6_small_det_onnx',
|
||||
smallRecognition: 'PaddlePaddle/PP-OCRv6_small_rec_onnx',
|
||||
mediumDetection: 'PaddlePaddle/PP-OCRv6_medium_det_onnx',
|
||||
mediumRecognition: 'PaddlePaddle/PP-OCRv6_medium_rec_onnx'
|
||||
} as const
|
||||
|
||||
const modelScopeRevisions = {
|
||||
tinyDetection: '7d7f5d128d9309ebf6de4f21f404dd583afdbae3',
|
||||
tinyRecognition: 'afba04b618200c5f4824531c6e42c957c6439d9a',
|
||||
smallDetection: '956a0b620a4017cc04056c692be1703b0025d028',
|
||||
smallRecognition: '296d43bc0ebced0fd9c605174aa5962e49810ab6',
|
||||
mediumDetection: 'c317b40325be40bfaaff58c8dcece2a075294f8a',
|
||||
mediumRecognition: 'db5d610d492a14e3c34dc1fd4e9339bd369f79e6'
|
||||
} as const
|
||||
|
||||
const huggingFaceRevisions = {
|
||||
tinyDetection: '2ba1506c0380b8f0b03dd142459aac66d4421f6c',
|
||||
tinyRecognition: '2612ab37152ae0a677521bae4e1e3d4fb4cf7c30',
|
||||
smallDetection: '28fe5895c24fd108c19eb3e8479f4ab385fbfc62',
|
||||
smallRecognition: 'b8f84f0b80c529de40b4fbb3544b84fa7233a513',
|
||||
mediumDetection: '61323801669c338b7891481ec7bac61ce31b576a',
|
||||
mediumRecognition: '50c7eacafc52fa7bcf4194e8cd08e46f8558504b'
|
||||
} as const
|
||||
|
||||
type RepositoryKey = keyof typeof repositories
|
||||
|
||||
function targets(
|
||||
repositoryKey: RepositoryKey,
|
||||
file: string
|
||||
) {
|
||||
const repository = repositories[repositoryKey]
|
||||
return {
|
||||
modelscope: modelScopeTarget(
|
||||
repository,
|
||||
modelScopeRevisions[repositoryKey],
|
||||
file
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
repository,
|
||||
huggingFaceRevisions[repositoryKey],
|
||||
file
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
function repositoryUrls(repositoryKey: RepositoryKey) {
|
||||
const repository = repositories[repositoryKey]
|
||||
return {
|
||||
modelscope: `https://modelscope.cn/models/${repository}`,
|
||||
'hugging-face': `https://huggingface.co/${repository}`
|
||||
}
|
||||
}
|
||||
|
||||
export const DOCUMENT_OCR_MODEL_CATALOG: readonly DocumentOcrModelCatalogEntry[] =
|
||||
documentOcrModelCatalogEntrySchema.array().parse([
|
||||
@@ -28,54 +75,37 @@ export const DOCUMENT_OCR_MODEL_CATALOG: readonly DocumentOcrModelCatalogEntry[]
|
||||
quality: 'basic',
|
||||
speed: 'fast',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_tiny_rec_onnx',
|
||||
repositoryUrls: repositoryUrls('tinyRecognition'),
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice:
|
||||
'检测与识别模型由 PaddlePaddle 在 ModelScope 发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
'检测与识别模型由 PaddlePaddle 官方发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
url: 'https://github.com/PaddlePaddle/PaddleOCR/blob/main/LICENSE'
|
||||
},
|
||||
files: [
|
||||
{
|
||||
name: 'detection.onnx',
|
||||
role: 'detection',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_tiny_det_onnx/resolve/' +
|
||||
`${detectionRevision}/inference.onnx`,
|
||||
size: 1_780_590,
|
||||
sha256:
|
||||
'193bab7a04fca699a6c82e6abb5b81bdb28177f0abd4062552b04908dafb19f8'
|
||||
}
|
||||
size: 1_780_590,
|
||||
sha256:
|
||||
'193bab7a04fca699a6c82e6abb5b81bdb28177f0abd4062552b04908dafb19f8',
|
||||
targets: targets('tinyDetection', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'recognition.onnx',
|
||||
role: 'recognition',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_tiny_rec_onnx/resolve/' +
|
||||
`${recognitionRevision}/inference.onnx`,
|
||||
size: 4_462_639,
|
||||
sha256:
|
||||
'9ef676d6ed3c88256a2d92c640c44f25b0c40947e111b14b8be8f594091563e6'
|
||||
}
|
||||
size: 4_462_639,
|
||||
sha256:
|
||||
'9ef676d6ed3c88256a2d92c640c44f25b0c40947e111b14b8be8f594091563e6',
|
||||
targets: targets('tinyRecognition', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'dictionary.yml',
|
||||
role: 'dictionary',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_tiny_rec_onnx/resolve/' +
|
||||
`${recognitionRevision}/inference.yml`,
|
||||
size: 55_571,
|
||||
sha256:
|
||||
'66170210bad538e83fff3c4a3867e547d6bf20b50d64b20347c4b913f3034ea1'
|
||||
}
|
||||
size: 55_571,
|
||||
sha256:
|
||||
'66170210bad538e83fff3c4a3867e547d6bf20b50d64b20347c4b913f3034ea1',
|
||||
targets: targets('tinyRecognition', 'inference.yml')
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -89,54 +119,37 @@ export const DOCUMENT_OCR_MODEL_CATALOG: readonly DocumentOcrModelCatalogEntry[]
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_small_rec_onnx',
|
||||
repositoryUrls: repositoryUrls('smallRecognition'),
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice:
|
||||
'检测与识别模型由 PaddlePaddle 在 ModelScope 发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
'检测与识别模型由 PaddlePaddle 官方发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
url: 'https://github.com/PaddlePaddle/PaddleOCR/blob/main/LICENSE'
|
||||
},
|
||||
files: [
|
||||
{
|
||||
name: 'detection.onnx',
|
||||
role: 'detection',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_small_det_onnx/resolve/' +
|
||||
`${smallDetectionRevision}/inference.onnx`,
|
||||
size: 9_880_512,
|
||||
sha256:
|
||||
'd73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e'
|
||||
}
|
||||
size: 9_880_512,
|
||||
sha256:
|
||||
'd73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e',
|
||||
targets: targets('smallDetection', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'recognition.onnx',
|
||||
role: 'recognition',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_small_rec_onnx/resolve/' +
|
||||
`${smallRecognitionRevision}/inference.onnx`,
|
||||
size: 21_159_378,
|
||||
sha256:
|
||||
'5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634'
|
||||
}
|
||||
size: 21_159_378,
|
||||
sha256:
|
||||
'5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634',
|
||||
targets: targets('smallRecognition', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'dictionary.yml',
|
||||
role: 'dictionary',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_small_rec_onnx/resolve/' +
|
||||
`${smallRecognitionRevision}/inference.yml`,
|
||||
size: 150_579,
|
||||
sha256:
|
||||
'ab078671bb49f06228eadccd34f1bb501e157f7a047095ffb943ba81512c77d1'
|
||||
}
|
||||
size: 150_579,
|
||||
sha256:
|
||||
'ab078671bb49f06228eadccd34f1bb501e157f7a047095ffb943ba81512c77d1',
|
||||
targets: targets('smallRecognition', 'inference.yml')
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -150,54 +163,37 @@ export const DOCUMENT_OCR_MODEL_CATALOG: readonly DocumentOcrModelCatalogEntry[]
|
||||
quality: 'high',
|
||||
speed: 'slow',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_medium_rec_onnx',
|
||||
repositoryUrls: repositoryUrls('mediumRecognition'),
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice:
|
||||
'检测与识别模型由 PaddlePaddle 在 ModelScope 发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
'检测与识别模型由 PaddlePaddle 官方发布，使用前请阅读模型仓库及 PaddleOCR 的许可证说明。',
|
||||
url: 'https://github.com/PaddlePaddle/PaddleOCR/blob/main/LICENSE'
|
||||
},
|
||||
files: [
|
||||
{
|
||||
name: 'detection.onnx',
|
||||
role: 'detection',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_medium_det_onnx/resolve/' +
|
||||
`${mediumDetectionRevision}/inference.onnx`,
|
||||
size: 62_032_837,
|
||||
sha256:
|
||||
'eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1'
|
||||
}
|
||||
size: 62_032_837,
|
||||
sha256:
|
||||
'eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1',
|
||||
targets: targets('mediumDetection', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'recognition.onnx',
|
||||
role: 'recognition',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_medium_rec_onnx/resolve/' +
|
||||
`${mediumRecognitionRevision}/inference.onnx`,
|
||||
size: 76_554_979,
|
||||
sha256:
|
||||
'9c09abf0957f7968c7586464b7397b84ad2387a0497a351af40e9acc71b673ba'
|
||||
}
|
||||
size: 76_554_979,
|
||||
sha256:
|
||||
'9c09abf0957f7968c7586464b7397b84ad2387a0497a351af40e9acc71b673ba',
|
||||
targets: targets('mediumRecognition', 'inference.onnx')
|
||||
},
|
||||
{
|
||||
name: 'dictionary.yml',
|
||||
role: 'dictionary',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/PaddlePaddle/' +
|
||||
'PP-OCRv6_medium_rec_onnx/resolve/' +
|
||||
`${mediumRecognitionRevision}/inference.yml`,
|
||||
size: 150_580,
|
||||
sha256:
|
||||
'991b700facf5b50a7de193468207d5f4255b538dde0d312ae3b7c7a9b6873129'
|
||||
}
|
||||
size: 150_580,
|
||||
sha256:
|
||||
'991b700facf5b50a7de193468207d5f4255b538dde0d312ae3b7c7a9b6873129',
|
||||
targets: targets('mediumRecognition', 'inference.yml')
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -70,8 +70,12 @@ function catalog(
|
||||
quality: 'balanced',
|
||||
speed: 'fast',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_tiny_rec_onnx',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
'https://modelscope.cn/models/example/test-model',
|
||||
'hugging-face':
|
||||
'https://huggingface.co/example/test-model'
|
||||
},
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice: 'Test license notice.',
|
||||
@@ -80,10 +84,27 @@ function catalog(
|
||||
files: files.map((file) => ({
|
||||
name: file.name,
|
||||
role: file.role,
|
||||
download: {
|
||||
url: `https://modelscope.cn/models/example/resolve/revision/${file.name}`,
|
||||
size: file.bytes.byteLength,
|
||||
sha256: sha256(file.bytes)
|
||||
size: file.bytes.byteLength,
|
||||
sha256: sha256(file.bytes),
|
||||
targets: {
|
||||
modelscope: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/${file.name}`,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/test-model',
|
||||
revision: 'a'.repeat(40),
|
||||
redirectHosts: []
|
||||
},
|
||||
'hugging-face': {
|
||||
url:
|
||||
'https://huggingface.co/example/test-model/' +
|
||||
`resolve/${'b'.repeat(40)}/${file.name}`,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/example/test-model',
|
||||
revision: 'b'.repeat(40),
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
}))
|
||||
}
|
||||
@@ -124,10 +145,12 @@ async function createManager(
|
||||
throw new Error('Test OCR catalog is empty')
|
||||
}
|
||||
const files = new Map(
|
||||
entry.files.map((file) => [
|
||||
file.download.url,
|
||||
modelBytes[file.role]
|
||||
])
|
||||
entry.files.flatMap((file) =>
|
||||
Object.values(file.targets).map((target) => [
|
||||
target.url,
|
||||
modelBytes[file.role]
|
||||
] as const)
|
||||
)
|
||||
)
|
||||
const transport = vi.fn(async (input: string | URL | Request) => {
|
||||
const url =
|
||||
@@ -174,7 +197,7 @@ describe('DocumentOcrModelManager', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('uses immutable SHA-256 verified ModelScope catalog files', () => {
|
||||
it('uses immutable byte-identical ModelScope and Hugging Face files', () => {
|
||||
expect(DOCUMENT_OCR_MODEL_CATALOG).toHaveLength(3)
|
||||
expect(
|
||||
new Set(DOCUMENT_OCR_MODEL_CATALOG.map((entry) => entry.id)).size
|
||||
@@ -186,20 +209,28 @@ describe('DocumentOcrModelManager', () => {
|
||||
).toEqual(['pp-ocrv6-small'])
|
||||
|
||||
for (const entry of DOCUMENT_OCR_MODEL_CATALOG) {
|
||||
expect(entry.repositoryUrls.modelscope).toMatch(
|
||||
/^https:\/\/modelscope\.cn\/models\/PaddlePaddle\//u
|
||||
)
|
||||
expect(entry.repositoryUrls['hugging-face']).toMatch(
|
||||
/^https:\/\/huggingface\.co\/PaddlePaddle\//u
|
||||
)
|
||||
for (const file of entry.files) {
|
||||
expect(file.download.url).toMatch(
|
||||
expect(file.targets.modelscope?.url).toMatch(
|
||||
/^https:\/\/modelscope\.cn\/models\/PaddlePaddle\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
expect(file.download.sha256).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(file.download.size).toBeGreaterThan(0)
|
||||
expect(file.targets['hugging-face']?.url).toMatch(
|
||||
/^https:\/\/huggingface\.co\/PaddlePaddle\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
expect(file.sha256).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(file.size).toBeGreaterThan(0)
|
||||
}
|
||||
}
|
||||
|
||||
expect(
|
||||
DOCUMENT_OCR_MODEL_CATALOG.find(
|
||||
(entry) => entry.id === 'pp-ocrv6-small'
|
||||
)
|
||||
).toMatchObject({
|
||||
const small = DOCUMENT_OCR_MODEL_CATALOG.find(
|
||||
(entry) => entry.id === 'pp-ocrv6-small'
|
||||
)
|
||||
expect(small).toMatchObject({
|
||||
languages: ['50 种语言'],
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
@@ -207,30 +238,29 @@ describe('DocumentOcrModelManager', () => {
|
||||
files: [
|
||||
{
|
||||
role: 'detection',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_small_det_onnx/resolve/956a0b620a4017cc04056c692be1703b0025d028/inference.onnx',
|
||||
size: 9_880_512,
|
||||
sha256:
|
||||
'd73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e'
|
||||
size: 9_880_512,
|
||||
sha256:
|
||||
'd73e0058b7a8086bbd57f3d10b8bcd4ff95363f67e06e2762b5e814fe9c9410e',
|
||||
targets: {
|
||||
modelscope: {
|
||||
revision: '956a0b620a4017cc04056c692be1703b0025d028'
|
||||
},
|
||||
'hugging-face': {
|
||||
revision: '28fe5895c24fd108c19eb3e8479f4ab385fbfc62'
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
role: 'recognition',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_small_rec_onnx/resolve/296d43bc0ebced0fd9c605174aa5962e49810ab6/inference.onnx',
|
||||
size: 21_159_378,
|
||||
sha256:
|
||||
'5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634'
|
||||
}
|
||||
size: 21_159_378,
|
||||
sha256:
|
||||
'5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634'
|
||||
},
|
||||
{
|
||||
role: 'dictionary',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_small_rec_onnx/resolve/296d43bc0ebced0fd9c605174aa5962e49810ab6/inference.yml',
|
||||
size: 150_579,
|
||||
sha256:
|
||||
'ab078671bb49f06228eadccd34f1bb501e157f7a047095ffb943ba81512c77d1'
|
||||
}
|
||||
size: 150_579,
|
||||
sha256:
|
||||
'ab078671bb49f06228eadccd34f1bb501e157f7a047095ffb943ba81512c77d1'
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -246,30 +276,21 @@ describe('DocumentOcrModelManager', () => {
|
||||
files: [
|
||||
{
|
||||
role: 'detection',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_medium_det_onnx/resolve/c317b40325be40bfaaff58c8dcece2a075294f8a/inference.onnx',
|
||||
size: 62_032_837,
|
||||
sha256:
|
||||
'eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1'
|
||||
}
|
||||
size: 62_032_837,
|
||||
sha256:
|
||||
'eb13b44b25bb36f89528b68720af8a61d9cf381176107f465db1757b65d086e1'
|
||||
},
|
||||
{
|
||||
role: 'recognition',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_medium_rec_onnx/resolve/db5d610d492a14e3c34dc1fd4e9339bd369f79e6/inference.onnx',
|
||||
size: 76_554_979,
|
||||
sha256:
|
||||
'9c09abf0957f7968c7586464b7397b84ad2387a0497a351af40e9acc71b673ba'
|
||||
}
|
||||
size: 76_554_979,
|
||||
sha256:
|
||||
'9c09abf0957f7968c7586464b7397b84ad2387a0497a351af40e9acc71b673ba'
|
||||
},
|
||||
{
|
||||
role: 'dictionary',
|
||||
download: {
|
||||
url: 'https://modelscope.cn/models/PaddlePaddle/PP-OCRv6_medium_rec_onnx/resolve/db5d610d492a14e3c34dc1fd4e9339bd369f79e6/inference.yml',
|
||||
size: 150_580,
|
||||
sha256:
|
||||
'991b700facf5b50a7de193468207d5f4255b538dde0d312ae3b7c7a9b6873129'
|
||||
}
|
||||
size: 150_580,
|
||||
sha256:
|
||||
'991b700facf5b50a7de193468207d5f4255b538dde0d312ae3b7c7a9b6873129'
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -296,6 +317,58 @@ describe('DocumentOcrModelManager', () => {
|
||||
expect(new TextDecoder().decode(assets.dictionary)).toContain(
|
||||
"!\n\"\n'\n"
|
||||
)
|
||||
const snapshot = await manager.getSnapshot()
|
||||
expect(snapshot.selectedDownloadSource).toBe('modelscope')
|
||||
expect(snapshot.catalog[0]?.files[0]).not.toHaveProperty('targets')
|
||||
expect(JSON.stringify(snapshot.catalog)).not.toContain('/resolve/')
|
||||
})
|
||||
|
||||
it('downloads the same canonical package from Hugging Face', async () => {
|
||||
const { manager } = await createManager()
|
||||
|
||||
await expect(
|
||||
manager.install('pp-ocrv6-tiny', 'hugging-face')
|
||||
).resolves.toMatchObject({
|
||||
id: 'pp-ocrv6-tiny',
|
||||
source: 'download'
|
||||
})
|
||||
})
|
||||
|
||||
it('does not request another source when selected coverage is missing', async () => {
|
||||
const directory = await mkdtemp(
|
||||
join(tmpdir(), 'goodbuddy-document-ocr-model-')
|
||||
)
|
||||
temporaryDirectories.push(directory)
|
||||
const detection = Buffer.from('detection')
|
||||
const recognition = Buffer.from('recognition')
|
||||
const dictionary = dictionaryYaml()
|
||||
const sourceCatalog = catalog(
|
||||
detection,
|
||||
recognition,
|
||||
dictionary
|
||||
).map((entry) => ({
|
||||
...entry,
|
||||
repositoryUrls: {
|
||||
modelscope: entry.repositoryUrls.modelscope
|
||||
},
|
||||
files: entry.files.map((file) => ({
|
||||
...file,
|
||||
targets: {
|
||||
modelscope: file.targets.modelscope
|
||||
}
|
||||
}))
|
||||
}))
|
||||
const transport = vi.fn<typeof fetch>()
|
||||
const manager = new DocumentOcrModelManager({
|
||||
userDataDirectory: directory,
|
||||
fetch: transport,
|
||||
catalog: sourceCatalog
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('pp-ocrv6-tiny', 'hugging-face')
|
||||
).rejects.toThrow('当前下载源')
|
||||
expect(transport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('rejects an imported model whose hash does not match', async () => {
|
||||
|
||||
@@ -15,26 +15,35 @@ import { dirname, resolve } from 'node:path'
|
||||
import {
|
||||
documentOcrAssetsSchema,
|
||||
documentOcrModelCatalogEntrySchema,
|
||||
documentOcrModelCatalogViewEntrySchema,
|
||||
documentOcrModelSnapshotSchema,
|
||||
documentParsingModelStatusSchema,
|
||||
installedDocumentOcrModelSchema,
|
||||
localOcrModelIdSchema,
|
||||
type DocumentOcrAssets,
|
||||
type DocumentOcrModelCatalogEntry,
|
||||
type DocumentOcrModelCatalogViewEntry,
|
||||
type DocumentOcrModelFile,
|
||||
type DocumentOcrModelOperation,
|
||||
type DocumentOcrModelSnapshot,
|
||||
type InstalledDocumentOcrModel
|
||||
} from '../shared/document-parsing-contracts'
|
||||
import {
|
||||
MODEL_DOWNLOAD_SOURCES,
|
||||
getModelDownloadAvailability,
|
||||
resolveModelDownloadPackage,
|
||||
type ModelDownloadSource,
|
||||
type ResolvedModelArtifactFile
|
||||
} from '../shared/model-download-contracts'
|
||||
import { DOCUMENT_OCR_MODEL_CATALOG } from './document-ocr-model-catalog'
|
||||
import {
|
||||
exportModelArchive,
|
||||
extractModelArchive
|
||||
} from './model-archive'
|
||||
import { fetchModelDownloadResponse } from './model-download-transport'
|
||||
|
||||
const DEFAULT_MAX_FILE_BYTES = 96 * 1024 * 1024
|
||||
const MANIFEST_FILE_NAME = 'manifest.json'
|
||||
const MAX_REDIRECTS = 3
|
||||
const PARTIAL_SUFFIX = '.partial'
|
||||
const MAXIMUM_ARCHIVE_BYTES = 512 * 1024 * 1024
|
||||
const ARCHIVE_OVERHEAD_BYTES = 1024 * 1024
|
||||
@@ -50,6 +59,9 @@ export type DocumentOcrModelManagerOptions = {
|
||||
userDataDirectory: string
|
||||
fetch: typeof fetch
|
||||
catalog?: readonly DocumentOcrModelCatalogEntry[]
|
||||
getDownloadSource?: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
maxFileBytes?: number
|
||||
}
|
||||
|
||||
@@ -69,6 +81,23 @@ function cloneCatalogEntry(
|
||||
return documentOcrModelCatalogEntrySchema.parse(entry)
|
||||
}
|
||||
|
||||
function toCatalogView(entry: DocumentOcrModelCatalogEntry) {
|
||||
const { repositoryUrls, files, ...metadata } = entry
|
||||
void repositoryUrls
|
||||
return documentOcrModelCatalogViewEntrySchema.parse({
|
||||
...metadata,
|
||||
files: files.map((file) => ({
|
||||
name: file.name,
|
||||
role: file.role,
|
||||
size: file.size,
|
||||
sha256: file.sha256
|
||||
})),
|
||||
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
|
||||
getModelDownloadAvailability(files, source)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function safeChild(parent: string, name: string): string {
|
||||
const child = resolve(parent, name)
|
||||
if (dirname(child) !== resolve(parent)) {
|
||||
@@ -77,14 +106,6 @@ function safeChild(parent: string, name: string): string {
|
||||
return child
|
||||
}
|
||||
|
||||
function validateDownloadUrl(value: string): URL {
|
||||
const url = new URL(value)
|
||||
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
||||
throw new Error('OCR 模型下载地址必须使用 HTTP 或 HTTPS')
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
function toArrayBuffer(buffer: Buffer): ArrayBuffer {
|
||||
return Uint8Array.from(buffer).buffer
|
||||
}
|
||||
@@ -157,6 +178,10 @@ export class DocumentOcrModelManager {
|
||||
|
||||
private readonly transport: typeof fetch
|
||||
private readonly catalog: DocumentOcrModelCatalogEntry[]
|
||||
private readonly catalogViews: DocumentOcrModelCatalogViewEntry[]
|
||||
private readonly getDownloadSource: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
private readonly maxFileBytes: number
|
||||
private readonly operations = new Map<string, ActiveOperation>()
|
||||
private readonly verifiedModels = new Map<string, Promise<void>>()
|
||||
@@ -171,6 +196,8 @@ export class DocumentOcrModelManager {
|
||||
'document-ocr'
|
||||
)
|
||||
this.transport = options.fetch
|
||||
this.getDownloadSource =
|
||||
options.getDownloadSource ?? (() => 'modelscope')
|
||||
this.catalog = (options.catalog ?? DOCUMENT_OCR_MODEL_CATALOG).map(
|
||||
cloneCatalogEntry
|
||||
)
|
||||
@@ -180,6 +207,7 @@ export class DocumentOcrModelManager {
|
||||
) {
|
||||
throw new Error('OCR 模型目录包含重复 ID')
|
||||
}
|
||||
this.catalogViews = this.catalog.map(toCatalogView)
|
||||
this.maxFileBytes = options.maxFileBytes ?? DEFAULT_MAX_FILE_BYTES
|
||||
if (
|
||||
!Number.isSafeInteger(this.maxFileBytes) ||
|
||||
@@ -192,10 +220,15 @@ export class DocumentOcrModelManager {
|
||||
|
||||
async getSnapshot(): Promise<DocumentOcrModelSnapshot> {
|
||||
await this.ensureRoot()
|
||||
const [selectedDownloadSource, installed] = await Promise.all([
|
||||
this.getDownloadSource(),
|
||||
this.readInstalled()
|
||||
])
|
||||
return documentOcrModelSnapshotSchema.parse({
|
||||
rootDirectory: this.rootDirectory,
|
||||
catalog: this.catalog.map(cloneCatalogEntry),
|
||||
installed: await this.readInstalled(),
|
||||
selectedDownloadSource,
|
||||
catalog: this.catalogViews,
|
||||
installed,
|
||||
operations: [...this.operations.values()].map((operation) => ({
|
||||
...operation.progress
|
||||
}))
|
||||
@@ -234,7 +267,7 @@ export class DocumentOcrModelManager {
|
||||
available: false,
|
||||
verified: false,
|
||||
runtime: entry.runtime,
|
||||
detail: '模型尚未安装或校验失败，请从 ModelScope 下载'
|
||||
detail: '模型尚未安装或校验失败，请从当前模型下载源获取'
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -243,19 +276,37 @@ export class DocumentOcrModelManager {
|
||||
return this.loadVerifiedAssets(this.requireCatalogEntry(modelId))
|
||||
}
|
||||
|
||||
getRepositoryUrl(
|
||||
modelId: string,
|
||||
source: ModelDownloadSource
|
||||
): string {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
resolveModelDownloadPackage(entry.files, source)
|
||||
const repositoryUrl = entry.repositoryUrls[source]
|
||||
if (!repositoryUrl) {
|
||||
throw new Error('当前下载源暂不提供此 OCR 模型的仓库')
|
||||
}
|
||||
return repositoryUrl
|
||||
}
|
||||
|
||||
async install(
|
||||
modelId: string,
|
||||
downloadSource?: ModelDownloadSource,
|
||||
externalSignal?: AbortSignal
|
||||
): Promise<InstalledDocumentOcrModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
const totalBytes = entry.files.reduce(
|
||||
(total, file) => total + file.download.size,
|
||||
0
|
||||
const selectedDownloadSource =
|
||||
downloadSource ?? (await this.getDownloadSource())
|
||||
const resolvedPackage = resolveModelDownloadPackage(
|
||||
entry.files,
|
||||
selectedDownloadSource
|
||||
)
|
||||
const operation = this.beginOperation(
|
||||
entry.id,
|
||||
'download',
|
||||
resolvedPackage.totalBytes,
|
||||
resolvedPackage.source
|
||||
)
|
||||
if (!Number.isSafeInteger(totalBytes)) {
|
||||
throw new RangeError('OCR 模型总大小超出安全范围')
|
||||
}
|
||||
const operation = this.beginOperation(entry.id, 'download', totalBytes)
|
||||
const detachAbort = this.attachExternalSignal(
|
||||
externalSignal,
|
||||
operation.controller
|
||||
@@ -265,7 +316,7 @@ export class DocumentOcrModelManager {
|
||||
await this.ensureRoot()
|
||||
await this.assertNotInstalled(entry.id)
|
||||
stagingDirectory = await this.createStagingDirectory(entry.id)
|
||||
for (const file of entry.files) {
|
||||
for (const file of resolvedPackage.files) {
|
||||
ensureNotAborted(operation.controller.signal)
|
||||
operation.progress.phase = 'transferring'
|
||||
operation.progress.currentFile = file.name
|
||||
@@ -376,8 +427,8 @@ export class DocumentOcrModelManager {
|
||||
)
|
||||
if (
|
||||
!recorded ||
|
||||
recorded.size !== expected.download.size ||
|
||||
recorded.sha256 !== expected.download.sha256
|
||||
recorded.size !== expected.size ||
|
||||
recorded.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(`OCR 模型文件校验失败：${expected.name}`)
|
||||
}
|
||||
@@ -406,7 +457,7 @@ export class DocumentOcrModelManager {
|
||||
): Promise<InstalledDocumentOcrModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
const expectedTotal = entry.files.reduce(
|
||||
(total, file) => total + file.download.size,
|
||||
(total, file) => total + file.size,
|
||||
0
|
||||
)
|
||||
const operation = this.beginOperation(
|
||||
@@ -448,8 +499,8 @@ export class DocumentOcrModelManager {
|
||||
)
|
||||
if (
|
||||
!archived ||
|
||||
archived.size !== expected.download.size ||
|
||||
archived.sha256 !== expected.download.sha256
|
||||
archived.size !== expected.size ||
|
||||
archived.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(
|
||||
`OCR 模型 ZIP 与当前模型目录不匹配：${expected.name}`
|
||||
@@ -536,7 +587,8 @@ export class DocumentOcrModelManager {
|
||||
private beginOperation(
|
||||
modelId: string,
|
||||
kind: DocumentOcrModelOperation['kind'],
|
||||
totalBytes: number | null
|
||||
totalBytes: number | null,
|
||||
downloadSource?: ModelDownloadSource
|
||||
): ActiveOperation {
|
||||
if (this.operations.has(modelId)) {
|
||||
throw new Error('该 OCR 模型已有进行中的操作')
|
||||
@@ -549,7 +601,8 @@ export class DocumentOcrModelManager {
|
||||
phase: 'preparing',
|
||||
currentFile: null,
|
||||
completedBytes: 0,
|
||||
totalBytes
|
||||
totalBytes,
|
||||
...(downloadSource ? { downloadSource } : {})
|
||||
}
|
||||
}
|
||||
this.operations.set(modelId, operation)
|
||||
@@ -597,50 +650,22 @@ export class DocumentOcrModelManager {
|
||||
return directory
|
||||
}
|
||||
|
||||
private async fetchFollowingRedirects(
|
||||
initialUrl: string,
|
||||
signal: AbortSignal
|
||||
): Promise<Response> {
|
||||
let url = validateDownloadUrl(initialUrl)
|
||||
for (let redirectCount = 0; ; redirectCount += 1) {
|
||||
ensureNotAborted(signal)
|
||||
const response = await this.transport(url, {
|
||||
method: 'GET',
|
||||
redirect: 'manual',
|
||||
credentials: 'omit',
|
||||
cache: 'no-store',
|
||||
signal
|
||||
})
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
if (redirectCount >= MAX_REDIRECTS) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error('OCR 模型下载重定向次数过多')
|
||||
}
|
||||
const location = response.headers.get('location')
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
if (!location) {
|
||||
throw new Error('OCR 模型下载重定向缺少地址')
|
||||
}
|
||||
url = validateDownloadUrl(new URL(location, url).toString())
|
||||
continue
|
||||
}
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadFile(
|
||||
file: DocumentOcrModelFile,
|
||||
file: ResolvedModelArtifactFile<DocumentOcrModelFile['role']>,
|
||||
destination: string,
|
||||
operation: ActiveOperation,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
if (file.download.size > this.maxFileBytes) {
|
||||
if (file.size > this.maxFileBytes) {
|
||||
throw new RangeError(`OCR 模型文件过大：${file.name}`)
|
||||
}
|
||||
const response = await this.fetchFollowingRedirects(
|
||||
file.download.url,
|
||||
signal
|
||||
)
|
||||
const response = await fetchModelDownloadResponse({
|
||||
transport: this.transport,
|
||||
initialUrl: file.target.url,
|
||||
redirectHosts: file.target.redirectHosts,
|
||||
signal,
|
||||
modelLabel: 'OCR 模型'
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(`OCR 模型下载失败：HTTP ${response.status}`)
|
||||
@@ -651,7 +676,7 @@ export class DocumentOcrModelManager {
|
||||
const declaredLength = response.headers.get('content-length')
|
||||
if (
|
||||
declaredLength !== null &&
|
||||
Number(declaredLength) !== file.download.size
|
||||
Number(declaredLength) !== file.size
|
||||
) {
|
||||
await response.body.cancel().catch(() => undefined)
|
||||
throw new Error(`OCR 模型文件大小不匹配：${file.name}`)
|
||||
@@ -671,7 +696,7 @@ export class DocumentOcrModelManager {
|
||||
}
|
||||
written += result.value.byteLength
|
||||
if (
|
||||
written > file.download.size ||
|
||||
written > file.size ||
|
||||
written > this.maxFileBytes
|
||||
) {
|
||||
await reader.cancel()
|
||||
@@ -688,8 +713,8 @@ export class DocumentOcrModelManager {
|
||||
await handle.close()
|
||||
}
|
||||
if (
|
||||
written !== file.download.size ||
|
||||
hash.digest('hex') !== file.download.sha256
|
||||
written !== file.size ||
|
||||
hash.digest('hex') !== file.sha256
|
||||
) {
|
||||
throw new Error(`OCR 模型文件校验失败：${file.name}`)
|
||||
}
|
||||
@@ -724,8 +749,8 @@ export class DocumentOcrModelManager {
|
||||
}
|
||||
const actual = await hashFile(path, signal)
|
||||
if (
|
||||
actual.size !== file.download.size ||
|
||||
actual.sha256 !== file.download.sha256
|
||||
actual.size !== file.size ||
|
||||
actual.sha256 !== file.sha256
|
||||
) {
|
||||
throw new Error(`本地 OCR 模型文件校验失败：${file.name}`)
|
||||
}
|
||||
@@ -832,8 +857,8 @@ export class DocumentOcrModelManager {
|
||||
const actual = await hashFile(safeChild(directory, file.name))
|
||||
if (
|
||||
!installed ||
|
||||
actual.size !== file.download.size ||
|
||||
actual.sha256 !== file.download.sha256 ||
|
||||
actual.size !== file.size ||
|
||||
actual.sha256 !== file.sha256 ||
|
||||
actual.size !== installed.size ||
|
||||
actual.sha256 !== installed.sha256
|
||||
) {
|
||||
@@ -879,8 +904,8 @@ export class DocumentOcrModelManager {
|
||||
}
|
||||
if (
|
||||
!installed ||
|
||||
actual.size !== file.download.size ||
|
||||
actual.sha256 !== file.download.sha256 ||
|
||||
actual.size !== file.size ||
|
||||
actual.sha256 !== file.sha256 ||
|
||||
actual.size !== installed.size ||
|
||||
actual.sha256 !== installed.sha256
|
||||
) {
|
||||
|
||||
+6
-2
@@ -440,7 +440,9 @@ if (hasSingleInstanceLock) {
|
||||
)
|
||||
documentOcrModelManager = new DocumentOcrModelManager({
|
||||
userDataDirectory: app.getPath('userData'),
|
||||
fetch: globalThis.fetch
|
||||
fetch: globalThis.fetch,
|
||||
getDownloadSource: async () =>
|
||||
(await applicationSettingsStore.get()).modelDownloadSource
|
||||
})
|
||||
documentOcrBroker = new DocumentOcrBroker(mainWindow)
|
||||
const documentParsingService = new DocumentParsingService(
|
||||
@@ -456,7 +458,9 @@ if (hasSingleInstanceLock) {
|
||||
})
|
||||
const speechModelManager = new SpeechModelManager({
|
||||
userDataDirectory: app.getPath('userData'),
|
||||
fetch: globalThis.fetch
|
||||
fetch: globalThis.fetch,
|
||||
getDownloadSource: async () =>
|
||||
(await applicationSettingsStore.get()).modelDownloadSource
|
||||
})
|
||||
const speechTranscriptionService = new SpeechTranscriptionService(
|
||||
speechModelManager
|
||||
|
||||
@@ -430,6 +430,179 @@ describe('registerIpcHandlers update source routing', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('registerIpcHandlers model download source routing', () => {
|
||||
afterEach(() => {
|
||||
electronMocks.handlers.clear()
|
||||
vi.clearAllMocks()
|
||||
})
|
||||
|
||||
it('uses the persisted source and rejects stale renderer requests', async () => {
|
||||
const webContents = {
|
||||
mainFrame: { url: 'file:///goodbuddy/index.html' },
|
||||
getURL: vi.fn(() => 'file:///goodbuddy/index.html'),
|
||||
isDestroyed: vi.fn(() => false),
|
||||
send: vi.fn()
|
||||
}
|
||||
const window = {
|
||||
webContents,
|
||||
isDestroyed: vi.fn(() => false),
|
||||
isMaximized: vi.fn(() => false),
|
||||
on: vi.fn(),
|
||||
removeListener: vi.fn()
|
||||
}
|
||||
const event = {
|
||||
sender: webContents,
|
||||
senderFrame: webContents.mainFrame
|
||||
}
|
||||
const speechSnapshot = {
|
||||
rootDirectory: 'C:\\models\\speech',
|
||||
selectedDownloadSource: 'hugging-face',
|
||||
catalog: [],
|
||||
installed: [],
|
||||
operations: [],
|
||||
selectedModelId: null
|
||||
}
|
||||
const ocrSnapshot = {
|
||||
settings: {},
|
||||
status: {},
|
||||
ocrModels: {
|
||||
rootDirectory: 'C:\\models\\ocr',
|
||||
selectedDownloadSource: 'hugging-face',
|
||||
catalog: [],
|
||||
installed: [],
|
||||
operations: []
|
||||
}
|
||||
}
|
||||
const speechModelManager = {
|
||||
install: vi.fn(async () => undefined),
|
||||
getSnapshot: vi.fn(async () => speechSnapshot),
|
||||
getRepositoryUrl: vi.fn(
|
||||
() => 'https://huggingface.co/example/speech'
|
||||
)
|
||||
}
|
||||
const documentOcrModelManager = {
|
||||
install: vi.fn(async () => undefined),
|
||||
getRepositoryUrl: vi.fn(
|
||||
() => 'https://huggingface.co/example/ocr'
|
||||
)
|
||||
}
|
||||
const documentParsingService = {
|
||||
snapshot: vi.fn(async () => ocrSnapshot)
|
||||
}
|
||||
const applicationSettingsStore = {
|
||||
get: vi.fn(async () => ({
|
||||
modelDownloadSource: 'hugging-face'
|
||||
}))
|
||||
}
|
||||
const dispose = registerIpcHandlers(
|
||||
window as never,
|
||||
{ capability: 'text' } as never,
|
||||
'CommandOrControl+Shift+Space',
|
||||
{} as never,
|
||||
{} as never,
|
||||
{ clear: vi.fn() } as never,
|
||||
{} as never,
|
||||
{ claimDueSchedules: vi.fn(() => []) } as never,
|
||||
{ clear: vi.fn() } as never,
|
||||
{} as never,
|
||||
vi.fn(async () => undefined),
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
applicationSettingsStore as never,
|
||||
undefined,
|
||||
speechModelManager as never,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
documentParsingService as never,
|
||||
documentOcrModelManager as never
|
||||
)
|
||||
|
||||
await expect(
|
||||
electronMocks.handlers.get(ipcChannels.speechModelsInstall)?.(
|
||||
event,
|
||||
{
|
||||
modelId: 'speech-model',
|
||||
expectedDownloadSource: 'hugging-face'
|
||||
}
|
||||
)
|
||||
).resolves.toEqual(speechSnapshot)
|
||||
expect(speechModelManager.install).toHaveBeenCalledWith(
|
||||
'speech-model',
|
||||
'hugging-face'
|
||||
)
|
||||
|
||||
await expect(
|
||||
electronMocks.handlers.get(ipcChannels.speechModelsInstall)?.(
|
||||
event,
|
||||
{
|
||||
modelId: 'speech-model',
|
||||
expectedDownloadSource: 'modelscope'
|
||||
}
|
||||
)
|
||||
).rejects.toThrow('模型下载源已变化')
|
||||
expect(speechModelManager.install).toHaveBeenCalledTimes(1)
|
||||
|
||||
applicationSettingsStore.get.mockRejectedValueOnce(
|
||||
new Error('settings unavailable')
|
||||
)
|
||||
await expect(
|
||||
electronMocks.handlers.get(ipcChannels.speechModelsInstall)?.(
|
||||
event,
|
||||
{
|
||||
modelId: 'speech-model',
|
||||
expectedDownloadSource: 'hugging-face'
|
||||
}
|
||||
)
|
||||
).rejects.toThrow('settings unavailable')
|
||||
expect(speechModelManager.install).toHaveBeenCalledTimes(1)
|
||||
|
||||
await expect(
|
||||
electronMocks.handlers.get(
|
||||
ipcChannels.documentOcrModelsInstall
|
||||
)?.(event, {
|
||||
modelId: 'ocr-model',
|
||||
expectedDownloadSource: 'hugging-face'
|
||||
})
|
||||
).resolves.toEqual(ocrSnapshot)
|
||||
expect(documentOcrModelManager.install).toHaveBeenCalledWith(
|
||||
'ocr-model',
|
||||
'hugging-face'
|
||||
)
|
||||
|
||||
await expect(
|
||||
electronMocks.handlers.get(
|
||||
ipcChannels.speechModelsOpenRepository
|
||||
)?.(event, { modelId: 'speech-model' })
|
||||
).resolves.toBeUndefined()
|
||||
expect(speechModelManager.getRepositoryUrl).toHaveBeenCalledWith(
|
||||
'speech-model',
|
||||
'hugging-face'
|
||||
)
|
||||
expect(electronMocks.openExternal).toHaveBeenLastCalledWith(
|
||||
'https://huggingface.co/example/speech'
|
||||
)
|
||||
|
||||
await expect(
|
||||
electronMocks.handlers.get(
|
||||
ipcChannels.documentOcrModelsOpenRepository
|
||||
)?.(event, { modelId: 'ocr-model' })
|
||||
).resolves.toBeUndefined()
|
||||
expect(
|
||||
documentOcrModelManager.getRepositoryUrl
|
||||
).toHaveBeenCalledWith('ocr-model', 'hugging-face')
|
||||
expect(electronMocks.openExternal).toHaveBeenLastCalledWith(
|
||||
'https://huggingface.co/example/ocr'
|
||||
)
|
||||
|
||||
await dispose()
|
||||
})
|
||||
})
|
||||
|
||||
vi.mock('electron', () => ({
|
||||
app: {
|
||||
getName: vi.fn(() => 'GoodBuddy'),
|
||||
|
||||
+43
-24
@@ -109,6 +109,7 @@ import { applicationSettingsUpdateSchema } from '../shared/application-settings-
|
||||
import { releaseNotesAcknowledgeSchema } from '../shared/release-notes-contracts'
|
||||
import {
|
||||
speechModelActionInputSchema,
|
||||
speechModelInstallInputSchema,
|
||||
speechModelSelectionInputSchema
|
||||
} from '../shared/speech-model-contracts'
|
||||
import {
|
||||
@@ -119,6 +120,7 @@ import {
|
||||
} from '../shared/embedding-contracts'
|
||||
import {
|
||||
documentOcrModelActionInputSchema,
|
||||
documentOcrModelInstallInputSchema,
|
||||
documentOcrFailureSchema,
|
||||
documentOcrResultSchema,
|
||||
documentParsingSettingsUpdateSchema,
|
||||
@@ -3513,16 +3515,25 @@ export function registerIpcHandlers(
|
||||
|
||||
registerHandler(
|
||||
ipcChannels.documentOcrModelsInstall,
|
||||
(event, input: unknown) => {
|
||||
async (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
if (!documentOcrModelManager || !documentParsingService) {
|
||||
if (
|
||||
!documentOcrModelManager ||
|
||||
!documentParsingService ||
|
||||
!applicationSettingsStore
|
||||
) {
|
||||
throw new Error('本地 OCR 模型服务不可用')
|
||||
}
|
||||
const { modelId } =
|
||||
documentOcrModelActionInputSchema.parse(input)
|
||||
const { modelId, expectedDownloadSource } =
|
||||
documentOcrModelInstallInputSchema.parse(input)
|
||||
const { modelDownloadSource: selectedDownloadSource } =
|
||||
await applicationSettingsStore.get()
|
||||
if (selectedDownloadSource !== expectedDownloadSource) {
|
||||
throw new Error('模型下载源已变化，请刷新后重试')
|
||||
}
|
||||
return trackExecution(
|
||||
documentOcrModelManager
|
||||
.install(modelId)
|
||||
.install(modelId, selectedDownloadSource)
|
||||
.then(() => documentParsingService.snapshot())
|
||||
)
|
||||
}
|
||||
@@ -3611,19 +3622,19 @@ export function registerIpcHandlers(
|
||||
ipcChannels.documentOcrModelsOpenRepository,
|
||||
async (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
if (!documentOcrModelManager) {
|
||||
if (!documentOcrModelManager || !applicationSettingsStore) {
|
||||
throw new Error('本地 OCR 模型服务不可用')
|
||||
}
|
||||
const { modelId } =
|
||||
documentOcrModelActionInputSchema.parse(input)
|
||||
const snapshot = await documentOcrModelManager.getSnapshot()
|
||||
const entry = snapshot.catalog.find(
|
||||
(candidate) => candidate.id === modelId
|
||||
const { modelDownloadSource: selectedDownloadSource } =
|
||||
await applicationSettingsStore.get()
|
||||
await shell.openExternal(
|
||||
documentOcrModelManager.getRepositoryUrl(
|
||||
modelId,
|
||||
selectedDownloadSource
|
||||
)
|
||||
)
|
||||
if (!entry) {
|
||||
throw new Error('未知的 OCR 模型')
|
||||
}
|
||||
await shell.openExternal(entry.repositoryUrl)
|
||||
}
|
||||
)
|
||||
|
||||
@@ -3759,15 +3770,21 @@ export function registerIpcHandlers(
|
||||
|
||||
registerHandler(
|
||||
ipcChannels.speechModelsInstall,
|
||||
(event, input: unknown) => {
|
||||
async (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
if (!speechModelManager) {
|
||||
if (!speechModelManager || !applicationSettingsStore) {
|
||||
throw new Error('语音模型服务不可用')
|
||||
}
|
||||
const { modelId } = speechModelActionInputSchema.parse(input)
|
||||
const { modelId, expectedDownloadSource } =
|
||||
speechModelInstallInputSchema.parse(input)
|
||||
const { modelDownloadSource: selectedDownloadSource } =
|
||||
await applicationSettingsStore.get()
|
||||
if (selectedDownloadSource !== expectedDownloadSource) {
|
||||
throw new Error('模型下载源已变化，请刷新后重试')
|
||||
}
|
||||
return trackExecution(
|
||||
speechModelManager
|
||||
.install(modelId)
|
||||
.install(modelId, selectedDownloadSource)
|
||||
.then(() => speechModelManager.getSnapshot())
|
||||
)
|
||||
}
|
||||
@@ -3862,16 +3879,18 @@ export function registerIpcHandlers(
|
||||
ipcChannels.speechModelsOpenRepository,
|
||||
async (event, input: unknown) => {
|
||||
assertTrustedSender(event, window)
|
||||
if (!speechModelManager) {
|
||||
if (!speechModelManager || !applicationSettingsStore) {
|
||||
throw new Error('语音模型服务不可用')
|
||||
}
|
||||
const { modelId } = speechModelActionInputSchema.parse(input)
|
||||
const snapshot = await speechModelManager.getSnapshot()
|
||||
const entry = snapshot.catalog.find((item) => item.id === modelId)
|
||||
if (!entry) {
|
||||
throw new Error('未知的语音模型')
|
||||
}
|
||||
await shell.openExternal(entry.repositoryUrl)
|
||||
const { modelDownloadSource: selectedDownloadSource } =
|
||||
await applicationSettingsStore.get()
|
||||
await shell.openExternal(
|
||||
speechModelManager.getRepositoryUrl(
|
||||
modelId,
|
||||
selectedDownloadSource
|
||||
)
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import {
|
||||
MODEL_DOWNLOAD_REDIRECT_HOSTS,
|
||||
type ModelArtifactTarget
|
||||
} from '../shared/model-download-contracts'
|
||||
|
||||
export function modelScopeTarget(
|
||||
repository: string,
|
||||
revision: string,
|
||||
file: string
|
||||
): ModelArtifactTarget {
|
||||
const repositoryUrl = `https://modelscope.cn/models/${repository}`
|
||||
return {
|
||||
url: `${repositoryUrl}/resolve/${revision}/${file}`,
|
||||
repositoryUrl,
|
||||
revision,
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
|
||||
export function huggingFaceTarget(
|
||||
repository: string,
|
||||
revision: string,
|
||||
file: string
|
||||
): ModelArtifactTarget {
|
||||
const repositoryUrl = `https://huggingface.co/${repository}`
|
||||
return {
|
||||
url: `${repositoryUrl}/resolve/${revision}/${file}`,
|
||||
repositoryUrl,
|
||||
revision,
|
||||
redirectHosts: [
|
||||
...MODEL_DOWNLOAD_REDIRECT_HOSTS['hugging-face']
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
const MAX_REDIRECTS = 3
|
||||
const redirectStatuses = new Set([301, 302, 303, 307, 308])
|
||||
|
||||
function validateDownloadUrl(value: string, modelLabel: string): URL {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'https:' ||
|
||||
(url.port !== '' && url.port !== '443') ||
|
||||
url.username ||
|
||||
url.password ||
|
||||
url.hash
|
||||
) {
|
||||
throw new Error(
|
||||
`${modelLabel}下载地址必须是使用标准端口、无凭据和 Fragment 的 HTTPS URL`
|
||||
)
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
export async function fetchModelDownloadResponse(options: {
|
||||
transport: typeof fetch
|
||||
initialUrl: string
|
||||
redirectHosts: readonly string[]
|
||||
signal: AbortSignal
|
||||
modelLabel: string
|
||||
}): Promise<Response> {
|
||||
let url = validateDownloadUrl(options.initialUrl, options.modelLabel)
|
||||
const initialHost = url.hostname
|
||||
const allowedRedirectHosts = new Set(options.redirectHosts)
|
||||
for (let redirectCount = 0; ; redirectCount += 1) {
|
||||
if (options.signal.aborted) {
|
||||
throw new DOMException('The operation was aborted', 'AbortError')
|
||||
}
|
||||
const response = await options.transport(url, {
|
||||
method: 'GET',
|
||||
redirect: 'manual',
|
||||
credentials: 'omit',
|
||||
cache: 'no-store',
|
||||
signal: options.signal
|
||||
})
|
||||
if (!redirectStatuses.has(response.status)) {
|
||||
return response
|
||||
}
|
||||
if (redirectCount >= MAX_REDIRECTS) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(
|
||||
`${options.modelLabel}下载重定向次数过多`
|
||||
)
|
||||
}
|
||||
const location = response.headers.get('location')
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
if (!location) {
|
||||
throw new Error(
|
||||
`${options.modelLabel}下载重定向缺少地址`
|
||||
)
|
||||
}
|
||||
const nextUrl = validateDownloadUrl(
|
||||
new URL(location, url).toString(),
|
||||
options.modelLabel
|
||||
)
|
||||
if (
|
||||
nextUrl.hostname !== initialHost &&
|
||||
!allowedRedirectHosts.has(nextUrl.hostname)
|
||||
) {
|
||||
throw new Error(
|
||||
`${options.modelLabel}下载重定向到未声明的主机`
|
||||
)
|
||||
}
|
||||
url = nextUrl
|
||||
}
|
||||
}
|
||||
@@ -2,11 +2,33 @@ import {
|
||||
speechModelCatalogEntrySchema,
|
||||
type SpeechModelCatalogEntry
|
||||
} from '../../shared/speech-model-contracts'
|
||||
import {
|
||||
huggingFaceTarget,
|
||||
modelScopeTarget
|
||||
} from '../model-download-targets'
|
||||
|
||||
const senseVoiceModelScopeRepository =
|
||||
'pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue'
|
||||
const senseVoiceModelScopeRevision =
|
||||
'73eca47697f980daa3d16112404174b6b950b514'
|
||||
const senseVoiceHuggingFaceRepository =
|
||||
'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17'
|
||||
const senseVoiceHuggingFaceRevision =
|
||||
'2365baeacb507f821a0c8120fcee3d484dba7a07'
|
||||
|
||||
const whisperTinyModelScopeRepository =
|
||||
'pengzhendong/sherpa-onnx-whisper-tiny'
|
||||
const whisperTinyModelScopeRevision =
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f'
|
||||
const whisperTinyHuggingFaceRepository =
|
||||
'csukuangfj/sherpa-onnx-whisper-tiny'
|
||||
const whisperTinyHuggingFaceRevision =
|
||||
'65176e2deb88badc814a94058666cadccc29b61c'
|
||||
|
||||
/**
|
||||
* This catalog intentionally contains metadata only. Model weights are never
|
||||
* bundled with GoodBuddy. Entries remain manual-only until every downloadable
|
||||
* file has a pinned revision, byte size, and independently verified SHA-256.
|
||||
* Model weights are never bundled with GoodBuddy. Canonical file identity is
|
||||
* source-independent; a source target is included only after its bytes match
|
||||
* the declared size and SHA-256.
|
||||
*/
|
||||
export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
speechModelCatalogEntrySchema.array().parse([
|
||||
@@ -21,13 +43,16 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'fast',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
`https://modelscope.cn/models/${senseVoiceModelScopeRepository}`,
|
||||
'hugging-face':
|
||||
`https://huggingface.co/${senseVoiceHuggingFaceRepository}`
|
||||
},
|
||||
license: {
|
||||
name: '模型仓库自定义许可（Model License）',
|
||||
notice:
|
||||
'SenseVoiceSmall 权重采用模型仓库声明的自定义 MODEL LICENSE，并非 Apache-2.0 或 MIT；导入和使用前请阅读完整许可条款。',
|
||||
'SenseVoiceSmall 权重采用上游声明的自定义 MODEL LICENSE，并非 Apache-2.0 或 MIT；导入和使用前请阅读完整许可条款。',
|
||||
url: 'https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE'
|
||||
},
|
||||
manualOnly: false,
|
||||
@@ -35,29 +60,39 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
|
||||
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
|
||||
'model.int8.onnx',
|
||||
size: 239_233_841,
|
||||
sha256:
|
||||
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51'
|
||||
size: 239_233_841,
|
||||
sha256:
|
||||
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
senseVoiceModelScopeRepository,
|
||||
senseVoiceModelScopeRevision,
|
||||
'model.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
senseVoiceHuggingFaceRepository,
|
||||
senseVoiceHuggingFaceRevision,
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
|
||||
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
|
||||
'tokens.txt',
|
||||
size: 315_894,
|
||||
sha256:
|
||||
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc'
|
||||
size: 315_894,
|
||||
sha256:
|
||||
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
senseVoiceModelScopeRepository,
|
||||
senseVoiceModelScopeRevision,
|
||||
'tokens.txt'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
senseVoiceHuggingFaceRepository,
|
||||
senseVoiceHuggingFaceRevision,
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -73,9 +108,12 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'basic',
|
||||
speed: 'fast',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
`https://modelscope.cn/models/${whisperTinyModelScopeRepository}`,
|
||||
'hugging-face':
|
||||
`https://huggingface.co/${whisperTinyHuggingFaceRepository}`
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -87,43 +125,58 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'tiny-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-encoder.int8.onnx',
|
||||
size: 12_937_772,
|
||||
sha256:
|
||||
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434'
|
||||
size: 12_937_772,
|
||||
sha256:
|
||||
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-encoder.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tiny-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-decoder.int8.onnx',
|
||||
size: 89_855_401,
|
||||
sha256:
|
||||
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925'
|
||||
size: 89_855_401,
|
||||
sha256:
|
||||
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-decoder.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tiny-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-tokens.txt'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -139,9 +192,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'fast',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -156,29 +210,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
|
||||
'model.int8.onnx',
|
||||
size: 223_385_835,
|
||||
sha256:
|
||||
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7'
|
||||
size: 223_385_835,
|
||||
sha256:
|
||||
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
|
||||
'tokens.txt',
|
||||
size: 75_756,
|
||||
sha256:
|
||||
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6'
|
||||
size: 75_756,
|
||||
sha256:
|
||||
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -194,9 +248,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en'
|
||||
},
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice:
|
||||
@@ -211,29 +266,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
|
||||
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
|
||||
'model.int8.onnx',
|
||||
size: 244_684_152,
|
||||
sha256:
|
||||
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144'
|
||||
size: 244_684_152,
|
||||
sha256:
|
||||
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
'8d90151338178bb433354c9fb677bd3acb8023cd',
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
|
||||
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
|
||||
'tokens.txt',
|
||||
size: 118_931,
|
||||
sha256:
|
||||
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492'
|
||||
size: 118_931,
|
||||
sha256:
|
||||
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
'8d90151338178bb433354c9fb677bd3acb8023cd',
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -249,8 +304,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -262,43 +319,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'small-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-encoder.int8.onnx',
|
||||
size: 112_442_483,
|
||||
sha256:
|
||||
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9'
|
||||
size: 112_442_483,
|
||||
sha256:
|
||||
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'small-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-decoder.int8.onnx',
|
||||
size: 262_226_114,
|
||||
sha256:
|
||||
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee'
|
||||
size: 262_226_114,
|
||||
sha256:
|
||||
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'small-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -314,8 +371,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'slow',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -327,43 +386,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'medium-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-encoder.int8.onnx',
|
||||
size: 374_196_283,
|
||||
sha256:
|
||||
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e'
|
||||
size: 374_196_283,
|
||||
sha256:
|
||||
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'medium-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-decoder.int8.onnx',
|
||||
size: 571_059_257,
|
||||
sha256:
|
||||
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747'
|
||||
size: 571_059_257,
|
||||
sha256:
|
||||
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'medium-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,12 +35,36 @@ function sha256(value: Uint8Array): string {
|
||||
}
|
||||
|
||||
function manualCatalog(): SpeechModelCatalogEntry[] {
|
||||
return SPEECH_MODEL_CATALOG.map((entry) => ({
|
||||
const entry = SPEECH_MODEL_CATALOG.find(
|
||||
(candidate) => candidate.id === 'sensevoice-small-int8'
|
||||
)
|
||||
if (!entry) {
|
||||
throw new Error('SenseVoice test catalog entry is missing')
|
||||
}
|
||||
const modelBytes = new TextEncoder().encode('model')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
return [{
|
||||
...entry,
|
||||
manualOnly: true,
|
||||
manualReason: entry.manualReason ?? '测试使用本地目录导入。',
|
||||
files: entry.files.map(({ name, role }) => ({ name, role }))
|
||||
}))
|
||||
manualReason: '测试使用本地目录导入。',
|
||||
repositoryUrls: {},
|
||||
files: [
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes),
|
||||
targets: {}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes),
|
||||
targets: {}
|
||||
}
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
function downloadableCatalog(
|
||||
@@ -58,8 +82,12 @@ function downloadableCatalog(
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
'hugging-face':
|
||||
'https://huggingface.co/example/download-test-model'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice: 'Test-only model metadata.',
|
||||
@@ -70,23 +98,53 @@ function downloadableCatalog(
|
||||
{
|
||||
name: 'model.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/model.onnx`,
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes)
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes),
|
||||
targets: {
|
||||
modelscope: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/model.onnx`,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
revision: 'a'.repeat(40),
|
||||
redirectHosts: []
|
||||
},
|
||||
'hugging-face': {
|
||||
url:
|
||||
'https://huggingface.co/example/download-test-model/' +
|
||||
`resolve/${'b'.repeat(40)}/model.onnx`,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/example/download-test-model',
|
||||
revision: 'b'.repeat(40),
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/tokens.txt`,
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes)
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes),
|
||||
targets: {
|
||||
modelscope: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/tokens.txt`,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
revision: 'a'.repeat(40),
|
||||
redirectHosts: []
|
||||
},
|
||||
'hugging-face': {
|
||||
url:
|
||||
'https://huggingface.co/example/download-test-model/' +
|
||||
`resolve/${'b'.repeat(40)}/tokens.txt`,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/example/download-test-model',
|
||||
revision: 'b'.repeat(40),
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -131,10 +189,19 @@ describe('speech model catalog', () => {
|
||||
license: { name: 'MIT License' }
|
||||
})
|
||||
expect(
|
||||
senseVoice?.files.every((file) => file.download !== undefined)
|
||||
senseVoice?.files.every(
|
||||
(file) =>
|
||||
file.targets.modelscope !== undefined &&
|
||||
file.targets['hugging-face'] !== undefined
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
whisper?.files.every(
|
||||
(file) =>
|
||||
file.targets.modelscope !== undefined &&
|
||||
file.targets['hugging-face'] !== undefined
|
||||
)
|
||||
).toBe(true)
|
||||
expect(whisper?.files.every((file) => file.download !== undefined))
|
||||
.toBe(true)
|
||||
expect(whisper?.files.map((file) => file.name)).toEqual([
|
||||
'tiny-encoder.int8.onnx',
|
||||
'tiny-decoder.int8.onnx',
|
||||
@@ -163,15 +230,21 @@ describe('speech model catalog', () => {
|
||||
})
|
||||
expect(SPEECH_MODEL_CATALOG).toHaveLength(6)
|
||||
for (const entry of SPEECH_MODEL_CATALOG) {
|
||||
expect(entry.repositoryUrl).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models\/|huggingface\.co\/)/u
|
||||
)
|
||||
for (const file of entry.files) {
|
||||
expect(file.download?.url).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
expect(file.sha256).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(file.size).toBeGreaterThan(0)
|
||||
for (const target of Object.values(file.targets)) {
|
||||
expect(target?.url).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(
|
||||
paraformerBilingual?.files.some(
|
||||
(file) => file.targets.modelscope !== undefined
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -242,10 +315,14 @@ describe('SpeechModelManager downloads', () => {
|
||||
).toBe(false)
|
||||
|
||||
await manager.select('download-test-model')
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
const snapshot = await manager.snapshot()
|
||||
expect(snapshot).toMatchObject({
|
||||
selectedDownloadSource: 'modelscope',
|
||||
selectedModelId: 'download-test-model',
|
||||
operations: []
|
||||
})
|
||||
expect(snapshot.catalog[0]?.files[0]).not.toHaveProperty('targets')
|
||||
expect(JSON.stringify(snapshot.catalog)).not.toContain('/resolve/')
|
||||
await manager.remove('download-test-model')
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
selectedModelId: null,
|
||||
@@ -253,7 +330,63 @@ describe('SpeechModelManager downloads', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts arbitrary HTTP hosts and cross-host redirects', async () => {
|
||||
it('freezes the operation source when the global setting changes', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
let selectedSource: 'modelscope' | 'hugging-face' = 'modelscope'
|
||||
let releaseFirstRequest: (() => void) | undefined
|
||||
let markFirstRequestStarted: (() => void) | undefined
|
||||
const firstRequestStarted = new Promise<void>((resolveStarted) => {
|
||||
markFirstRequestStarted = resolveStarted
|
||||
})
|
||||
const firstRequestGate = new Promise<void>((resolveRequest) => {
|
||||
releaseFirstRequest = resolveRequest
|
||||
})
|
||||
let requestCount = 0
|
||||
const transport = vi.fn<typeof fetch>(async (input) => {
|
||||
requestCount += 1
|
||||
if (requestCount === 1) {
|
||||
markFirstRequestStarted?.()
|
||||
await firstRequestGate
|
||||
}
|
||||
const bytes = String(input).endsWith('model.onnx')
|
||||
? modelBytes
|
||||
: tokenBytes
|
||||
return new Response(bytes, {
|
||||
headers: { 'content-length': String(bytes.byteLength) }
|
||||
})
|
||||
})
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes, tokenBytes),
|
||||
fetch: transport,
|
||||
getDownloadSource: () => selectedSource
|
||||
})
|
||||
|
||||
const installing = manager.install('download-test-model')
|
||||
await firstRequestStarted
|
||||
selectedSource = 'hugging-face'
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
selectedDownloadSource: 'hugging-face',
|
||||
operations: [
|
||||
{
|
||||
modelId: 'download-test-model',
|
||||
kind: 'download',
|
||||
downloadSource: 'modelscope'
|
||||
}
|
||||
]
|
||||
})
|
||||
releaseFirstRequest?.()
|
||||
await installing
|
||||
expect(
|
||||
transport.mock.calls.every(
|
||||
([input]) => new URL(String(input)).hostname === 'modelscope.cn'
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('follows only source-declared HTTPS redirect hosts', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
@@ -261,24 +394,24 @@ describe('SpeechModelManager downloads', () => {
|
||||
...entry,
|
||||
files: entry.files.map((file) => ({
|
||||
...file,
|
||||
download: file.download
|
||||
? {
|
||||
...file.download,
|
||||
url: file.download.url.replace(
|
||||
'https://modelscope.cn',
|
||||
'http://models.internal.example'
|
||||
)
|
||||
}
|
||||
: undefined
|
||||
targets: {
|
||||
...file.targets,
|
||||
'hugging-face': file.targets['hugging-face']
|
||||
? {
|
||||
...file.targets['hugging-face'],
|
||||
redirectHosts: ['cdn-lfs.hf.co']
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
}))
|
||||
}))
|
||||
const transport = vi.fn<typeof fetch>(async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.hostname === 'models.internal.example') {
|
||||
if (url.hostname === 'huggingface.co') {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
location: `https://cdn.example.net${url.pathname}`
|
||||
location: `https://cdn-lfs.hf.co${url.pathname}`
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -296,19 +429,91 @@ describe('SpeechModelManager downloads', () => {
|
||||
})
|
||||
|
||||
await expect(
|
||||
redirected.install('download-test-model')
|
||||
redirected.install('download-test-model', 'hugging-face')
|
||||
).resolves.toMatchObject({ id: 'download-test-model' })
|
||||
expect(transport).toHaveBeenCalledTimes(4)
|
||||
expect(
|
||||
transport.mock.calls.map(([input]) => new URL(String(input)).hostname)
|
||||
).toEqual([
|
||||
'models.internal.example',
|
||||
'cdn.example.net',
|
||||
'models.internal.example',
|
||||
'cdn.example.net'
|
||||
'huggingface.co',
|
||||
'cdn-lfs.hf.co',
|
||||
'huggingface.co',
|
||||
'cdn-lfs.hf.co'
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects undeclared redirect hosts without following them', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const transport = vi.fn<typeof fetch>(async () =>
|
||||
new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
location: 'https://untrusted.example/model.onnx'
|
||||
}
|
||||
})
|
||||
)
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes),
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model')
|
||||
).rejects.toThrow('未声明的主机')
|
||||
expect(transport).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not request another source when selected coverage is missing', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const catalog = downloadableCatalog(modelBytes).map((entry) => ({
|
||||
...entry,
|
||||
repositoryUrls: {
|
||||
'hugging-face': entry.repositoryUrls['hugging-face']
|
||||
},
|
||||
files: entry.files.map((file) => ({
|
||||
...file,
|
||||
targets: {
|
||||
'hugging-face': file.targets['hugging-face']
|
||||
}
|
||||
}))
|
||||
}))
|
||||
const transport = vi.fn<typeof fetch>()
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog,
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model', 'modelscope')
|
||||
).rejects.toThrow('当前下载源')
|
||||
expect(transport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not request another source after a download failure', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const transport = vi.fn<typeof fetch>(
|
||||
async () => new Response(null, { status: 503 })
|
||||
)
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes),
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model', 'modelscope')
|
||||
).rejects.toThrow('HTTP 503')
|
||||
expect(transport).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
new URL(String(transport.mock.calls[0]?.[0])).hostname
|
||||
).toBe('modelscope.cn')
|
||||
})
|
||||
|
||||
it('rejects bad digests without installing', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
|
||||
@@ -16,22 +16,31 @@ import { z } from 'zod'
|
||||
import {
|
||||
installedSpeechModelSchema,
|
||||
speechModelCatalogEntrySchema,
|
||||
speechModelCatalogViewEntrySchema,
|
||||
speechModelIdSchema,
|
||||
speechModelSnapshotSchema,
|
||||
type InstalledSpeechModel,
|
||||
type SpeechModelCatalogEntry,
|
||||
type SpeechModelCatalogViewEntry,
|
||||
type SpeechModelFileSpec,
|
||||
type SpeechModelOperation,
|
||||
type SpeechModelSnapshot
|
||||
} from '../../shared/speech-model-contracts'
|
||||
import {
|
||||
MODEL_DOWNLOAD_SOURCES,
|
||||
getModelDownloadAvailability,
|
||||
resolveModelDownloadPackage,
|
||||
type ModelDownloadSource,
|
||||
type ResolvedModelArtifactFile
|
||||
} from '../../shared/model-download-contracts'
|
||||
import { SPEECH_MODEL_CATALOG } from './speech-model-catalog'
|
||||
import {
|
||||
exportModelArchive,
|
||||
extractModelArchive
|
||||
} from '../model-archive'
|
||||
import { fetchModelDownloadResponse } from '../model-download-transport'
|
||||
|
||||
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024
|
||||
const MAX_REDIRECTS = 3
|
||||
const MANIFEST_FILE_NAME = 'manifest.json'
|
||||
const SELECTION_FILE_NAME = '.selection.json'
|
||||
const PARTIAL_SUFFIX = '.partial'
|
||||
@@ -56,6 +65,9 @@ export type SpeechModelManagerOptions = {
|
||||
userDataDirectory: string
|
||||
fetch: typeof fetch
|
||||
catalog?: readonly SpeechModelCatalogEntry[]
|
||||
getDownloadSource?: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
maxFileBytes?: number
|
||||
}
|
||||
|
||||
@@ -72,6 +84,23 @@ function cloneCatalogEntry(
|
||||
return speechModelCatalogEntrySchema.parse(entry)
|
||||
}
|
||||
|
||||
function toCatalogView(entry: SpeechModelCatalogEntry) {
|
||||
const { repositoryUrls, files, ...metadata } = entry
|
||||
void repositoryUrls
|
||||
return speechModelCatalogViewEntrySchema.parse({
|
||||
...metadata,
|
||||
files: files.map((file) => ({
|
||||
name: file.name,
|
||||
role: file.role,
|
||||
size: file.size,
|
||||
sha256: file.sha256
|
||||
})),
|
||||
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
|
||||
getModelDownloadAvailability(files, source)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function abortError(): DOMException {
|
||||
return new DOMException('The operation was aborted', 'AbortError')
|
||||
}
|
||||
@@ -102,17 +131,6 @@ function safeChild(parent: string, name: string): string {
|
||||
return child
|
||||
}
|
||||
|
||||
function validateDownloadUrl(value: string): URL {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'http:' &&
|
||||
url.protocol !== 'https:'
|
||||
) {
|
||||
throw new Error('模型下载地址必须使用 HTTP 或 HTTPS')
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
async function hashFile(
|
||||
path: string,
|
||||
signal?: AbortSignal
|
||||
@@ -147,6 +165,10 @@ export class SpeechModelManager {
|
||||
|
||||
private readonly transport: typeof fetch
|
||||
private readonly catalog: SpeechModelCatalogEntry[]
|
||||
private readonly catalogViews: SpeechModelCatalogViewEntry[]
|
||||
private readonly getDownloadSource: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
private readonly maxFileBytes: number
|
||||
private readonly operations = new Map<string, ActiveOperation>()
|
||||
|
||||
@@ -160,23 +182,31 @@ export class SpeechModelManager {
|
||||
'speech'
|
||||
)
|
||||
this.transport = options.fetch
|
||||
this.getDownloadSource =
|
||||
options.getDownloadSource ?? (() => 'modelscope')
|
||||
this.catalog = (options.catalog ?? SPEECH_MODEL_CATALOG).map(
|
||||
cloneCatalogEntry
|
||||
)
|
||||
if (new Set(this.catalog.map((entry) => entry.id)).size !== this.catalog.length) {
|
||||
throw new Error('语音模型目录包含重复 ID')
|
||||
}
|
||||
this.catalogViews = this.catalog.map(toCatalogView)
|
||||
this.maxFileBytes = validateMaximumBytes(options.maxFileBytes)
|
||||
}
|
||||
|
||||
async snapshot(): Promise<SpeechModelSnapshot> {
|
||||
await this.ensureRoot()
|
||||
const installed = await this.readInstalled()
|
||||
const selected = await this.readSelection()
|
||||
const [installed, selected, selectedDownloadSource] =
|
||||
await Promise.all([
|
||||
this.readInstalled(),
|
||||
this.readSelection(),
|
||||
this.getDownloadSource()
|
||||
])
|
||||
const installedIds = new Set(installed.map((model) => model.id))
|
||||
return speechModelSnapshotSchema.parse({
|
||||
rootDirectory: this.rootDirectory,
|
||||
catalog: this.catalog.map(cloneCatalogEntry),
|
||||
selectedDownloadSource,
|
||||
catalog: this.catalogViews,
|
||||
installed,
|
||||
operations: [...this.operations.values()].map((operation) => ({
|
||||
...operation.progress
|
||||
@@ -190,6 +220,19 @@ export class SpeechModelManager {
|
||||
return this.snapshot()
|
||||
}
|
||||
|
||||
getRepositoryUrl(
|
||||
modelId: string,
|
||||
source: ModelDownloadSource
|
||||
): string {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
resolveModelDownloadPackage(entry.files, source)
|
||||
const repositoryUrl = entry.repositoryUrls[source]
|
||||
if (!repositoryUrl) {
|
||||
throw new Error('当前下载源暂不提供此模型的仓库')
|
||||
}
|
||||
return repositoryUrl
|
||||
}
|
||||
|
||||
async getSelectedRuntimeModel(): Promise<
|
||||
SelectedSpeechRuntimeModel | undefined
|
||||
> {
|
||||
@@ -216,6 +259,7 @@ export class SpeechModelManager {
|
||||
|
||||
async install(
|
||||
modelId: string,
|
||||
downloadSource?: ModelDownloadSource,
|
||||
externalSignal?: AbortSignal
|
||||
): Promise<InstalledSpeechModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
@@ -224,27 +268,17 @@ export class SpeechModelManager {
|
||||
entry.manualReason ?? '该模型只能从本地目录导入'
|
||||
)
|
||||
}
|
||||
const downloadableFiles = entry.files.filter(
|
||||
(
|
||||
file
|
||||
): file is SpeechModelFileSpec & {
|
||||
download: NonNullable<SpeechModelFileSpec['download']>
|
||||
} => file.download !== undefined
|
||||
const selectedDownloadSource =
|
||||
downloadSource ?? (await this.getDownloadSource())
|
||||
const resolvedPackage = resolveModelDownloadPackage(
|
||||
entry.files,
|
||||
selectedDownloadSource
|
||||
)
|
||||
if (downloadableFiles.length !== entry.files.length) {
|
||||
throw new Error('模型下载元数据不完整')
|
||||
}
|
||||
const totalBytes = downloadableFiles.reduce(
|
||||
(total, file) => total + file.download.size,
|
||||
0
|
||||
)
|
||||
if (!Number.isSafeInteger(totalBytes)) {
|
||||
throw new RangeError('模型总大小超出安全范围')
|
||||
}
|
||||
const operation = this.beginOperation(
|
||||
entry.id,
|
||||
'download',
|
||||
totalBytes
|
||||
resolvedPackage.totalBytes,
|
||||
resolvedPackage.source
|
||||
)
|
||||
const detachExternalAbort = this.attachExternalSignal(
|
||||
externalSignal,
|
||||
@@ -255,7 +289,7 @@ export class SpeechModelManager {
|
||||
await this.ensureRoot()
|
||||
await this.assertNotInstalled(entry.id)
|
||||
stagingDirectory = await this.createStagingDirectory(entry.id)
|
||||
for (const file of downloadableFiles) {
|
||||
for (const file of resolvedPackage.files) {
|
||||
ensureNotAborted(operation.controller.signal)
|
||||
operation.progress.phase = 'transferring'
|
||||
operation.progress.currentFile = file.name
|
||||
@@ -409,9 +443,8 @@ export class SpeechModelManager {
|
||||
!recorded ||
|
||||
recorded.size <= 0 ||
|
||||
recorded.size > this.maxFileBytes ||
|
||||
(expected.download &&
|
||||
(recorded.size !== expected.download.size ||
|
||||
recorded.sha256 !== expected.download.sha256))
|
||||
recorded.size !== expected.size ||
|
||||
recorded.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(`语音模型文件不可导出：${expected.name}`)
|
||||
}
|
||||
@@ -440,8 +473,7 @@ export class SpeechModelManager {
|
||||
): Promise<InstalledSpeechModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
const expectedTotal = entry.files.reduce(
|
||||
(total, file) =>
|
||||
total + (file.download?.size ?? this.maxFileBytes),
|
||||
(total, file) => total + file.size,
|
||||
0
|
||||
)
|
||||
const maximumTotalBytes = Math.min(
|
||||
@@ -488,9 +520,8 @@ export class SpeechModelManager {
|
||||
if (
|
||||
!archived ||
|
||||
archived.size > this.maxFileBytes ||
|
||||
(expected.download &&
|
||||
(archived.size !== expected.download.size ||
|
||||
archived.sha256 !== expected.download.sha256))
|
||||
archived.size !== expected.size ||
|
||||
archived.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(
|
||||
`语音模型 ZIP 与当前模型目录不匹配：${expected.name}`
|
||||
@@ -544,7 +575,8 @@ export class SpeechModelManager {
|
||||
private beginOperation(
|
||||
modelId: string,
|
||||
kind: SpeechModelOperation['kind'],
|
||||
totalBytes: number | null
|
||||
totalBytes: number | null,
|
||||
downloadSource?: ModelDownloadSource
|
||||
): ActiveOperation {
|
||||
if (this.operations.has(modelId)) {
|
||||
throw new Error('该模型已有进行中的操作')
|
||||
@@ -557,7 +589,8 @@ export class SpeechModelManager {
|
||||
phase: 'preparing',
|
||||
currentFile: null,
|
||||
completedBytes: 0,
|
||||
totalBytes
|
||||
totalBytes,
|
||||
...(downloadSource ? { downloadSource } : {})
|
||||
}
|
||||
}
|
||||
this.operations.set(modelId, operation)
|
||||
@@ -605,55 +638,25 @@ export class SpeechModelManager {
|
||||
return directory
|
||||
}
|
||||
|
||||
private async fetchFollowingRedirects(
|
||||
initialUrl: string,
|
||||
signal: AbortSignal
|
||||
): Promise<Response> {
|
||||
let url = validateDownloadUrl(initialUrl)
|
||||
for (let redirectCount = 0; ; redirectCount += 1) {
|
||||
ensureNotAborted(signal)
|
||||
const response = await this.transport(url, {
|
||||
method: 'GET',
|
||||
redirect: 'manual',
|
||||
credentials: 'omit',
|
||||
cache: 'no-store',
|
||||
signal
|
||||
})
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
if (redirectCount >= MAX_REDIRECTS) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error('模型下载重定向次数过多')
|
||||
}
|
||||
const location = response.headers.get('location')
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
if (!location) {
|
||||
throw new Error('模型下载重定向缺少地址')
|
||||
}
|
||||
url = validateDownloadUrl(new URL(location, url).toString())
|
||||
continue
|
||||
}
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadFile(
|
||||
file: SpeechModelFileSpec & {
|
||||
download: NonNullable<SpeechModelFileSpec['download']>
|
||||
},
|
||||
file: ResolvedModelArtifactFile<SpeechModelFileSpec['role']>,
|
||||
destination: string,
|
||||
operation: ActiveOperation,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
if (
|
||||
file.download.size > this.maxFileBytes ||
|
||||
file.download.size <= 0
|
||||
file.size > this.maxFileBytes ||
|
||||
file.size <= 0
|
||||
) {
|
||||
throw new RangeError(`模型文件大小超出限制：${file.name}`)
|
||||
}
|
||||
const response = await this.fetchFollowingRedirects(
|
||||
file.download.url,
|
||||
signal
|
||||
)
|
||||
const response = await fetchModelDownloadResponse({
|
||||
transport: this.transport,
|
||||
initialUrl: file.target.url,
|
||||
redirectHosts: file.target.redirectHosts,
|
||||
signal,
|
||||
modelLabel: '模型'
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(`模型下载失败：HTTP ${response.status}`)
|
||||
@@ -666,7 +669,7 @@ export class SpeechModelManager {
|
||||
const parsedLength = Number(declaredLength)
|
||||
if (
|
||||
!Number.isSafeInteger(parsedLength) ||
|
||||
parsedLength !== file.download.size
|
||||
parsedLength !== file.size
|
||||
) {
|
||||
await response.body.cancel().catch(() => undefined)
|
||||
throw new Error(`模型文件大小不匹配：${file.name}`)
|
||||
@@ -687,7 +690,7 @@ export class SpeechModelManager {
|
||||
}
|
||||
written += result.value.byteLength
|
||||
if (
|
||||
written > file.download.size ||
|
||||
written > file.size ||
|
||||
written > this.maxFileBytes
|
||||
) {
|
||||
await reader.cancel()
|
||||
@@ -703,10 +706,10 @@ export class SpeechModelManager {
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
if (written !== file.download.size) {
|
||||
if (written !== file.size) {
|
||||
throw new Error(`模型文件大小不匹配：${file.name}`)
|
||||
}
|
||||
if (hash.digest('hex') !== file.download.sha256) {
|
||||
if (hash.digest('hex') !== file.sha256) {
|
||||
throw new Error(`模型文件校验失败：${file.name}`)
|
||||
}
|
||||
await rename(partialPath, destination)
|
||||
@@ -741,10 +744,9 @@ export class SpeechModelManager {
|
||||
throw new RangeError(`模型文件大小无效：${expectedFile.name}`)
|
||||
}
|
||||
if (
|
||||
expectedFile.download &&
|
||||
(sourceFileInfo.size !== expectedFile.download.size ||
|
||||
(await hashFile(sourceFile, signal)).sha256 !==
|
||||
expectedFile.download.sha256)
|
||||
sourceFileInfo.size !== expectedFile.size ||
|
||||
(await hashFile(sourceFile, signal)).sha256 !==
|
||||
expectedFile.sha256
|
||||
) {
|
||||
throw new Error(`本地模型文件校验失败：${expectedFile.name}`)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user