feat: add explicit managed model download sources

Managed speech and OCR downloads previously used catalog-specific source URLs without a global selection. Platform Features now lets users choose ModelScope by default or Hugging Face, while Main validates and freezes that source for each download.

Verified coverage remains explicit: downloads never mix artifacts or silently switch sources, and installed models plus ZIP imports stay source-independent.

Release note: 可在“设置 → 平台功能 → 通用设置”中选择 ModelScope 或 Hugging Face 作为后续语音输入与 OCR 模型下载源;缺少完整已验证文件或下载失败时不会静默换源。
This commit is contained in:
mesalogo
2026-08-19 13:43:16 +08:00
parent c41d7e8b96
commit 43e1d162dc
44 changed files with 6083 additions and 839 deletions
+214 -155
View File
@@ -2,11 +2,33 @@ import {
speechModelCatalogEntrySchema,
type SpeechModelCatalogEntry
} from '../../shared/speech-model-contracts'
import {
huggingFaceTarget,
modelScopeTarget
} from '../model-download-targets'
const senseVoiceModelScopeRepository =
'pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue'
const senseVoiceModelScopeRevision =
'73eca47697f980daa3d16112404174b6b950b514'
const senseVoiceHuggingFaceRepository =
'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17'
const senseVoiceHuggingFaceRevision =
'2365baeacb507f821a0c8120fcee3d484dba7a07'
const whisperTinyModelScopeRepository =
'pengzhendong/sherpa-onnx-whisper-tiny'
const whisperTinyModelScopeRevision =
'33a655645234f82ce833cf27b689d9c2212e693f'
const whisperTinyHuggingFaceRepository =
'csukuangfj/sherpa-onnx-whisper-tiny'
const whisperTinyHuggingFaceRevision =
'65176e2deb88badc814a94058666cadccc29b61c'
/**
* This catalog intentionally contains metadata only. Model weights are never
* bundled with GoodBuddy. Entries remain manual-only until every downloadable
* file has a pinned revision, byte size, and independently verified SHA-256.
* Model weights are never bundled with GoodBuddy. Canonical file identity is
* source-independent; a source target is included only after its bytes match
* the declared size and SHA-256.
*/
export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
speechModelCatalogEntrySchema.array().parse([
@@ -21,13 +43,16 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'high',
speed: 'fast',
recommended: true,
repositoryUrl:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue',
repositoryUrls: {
modelscope:
`https://modelscope.cn/models/${senseVoiceModelScopeRepository}`,
'hugging-face':
`https://huggingface.co/${senseVoiceHuggingFaceRepository}`
},
license: {
name: '模型仓库自定义许可(Model License)',
notice:
'SenseVoiceSmall 权重采用模型仓库声明的自定义 MODEL LICENSE,并非 Apache-2.0 或 MIT;导入和使用前请阅读完整许可条款。',
'SenseVoiceSmall 权重采用上游声明的自定义 MODEL LICENSE,并非 Apache-2.0 或 MIT;导入和使用前请阅读完整许可条款。',
url: 'https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE'
},
manualOnly: false,
@@ -35,29 +60,39 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'model.int8.onnx',
role: 'model',
download: {
url:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
'model.int8.onnx',
size: 239_233_841,
sha256:
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51'
size: 239_233_841,
sha256:
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51',
targets: {
modelscope: modelScopeTarget(
senseVoiceModelScopeRepository,
senseVoiceModelScopeRevision,
'model.int8.onnx'
),
'hugging-face': huggingFaceTarget(
senseVoiceHuggingFaceRepository,
senseVoiceHuggingFaceRevision,
'model.int8.onnx'
)
}
},
{
name: 'tokens.txt',
role: 'tokens',
download: {
url:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
'tokens.txt',
size: 315_894,
sha256:
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc'
size: 315_894,
sha256:
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc',
targets: {
modelscope: modelScopeTarget(
senseVoiceModelScopeRepository,
senseVoiceModelScopeRevision,
'tokens.txt'
),
'hugging-face': huggingFaceTarget(
senseVoiceHuggingFaceRepository,
senseVoiceHuggingFaceRevision,
'tokens.txt'
)
}
}
]
@@ -73,9 +108,12 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'basic',
speed: 'fast',
recommended: false,
repositoryUrl:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-whisper-tiny',
repositoryUrls: {
modelscope:
`https://modelscope.cn/models/${whisperTinyModelScopeRepository}`,
'hugging-face':
`https://huggingface.co/${whisperTinyHuggingFaceRepository}`
},
license: {
name: 'MIT License',
notice:
@@ -87,43 +125,58 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'tiny-encoder.int8.onnx',
role: 'encoder',
download: {
url:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-whisper-tiny/resolve/' +
'33a655645234f82ce833cf27b689d9c2212e693f/' +
'tiny-encoder.int8.onnx',
size: 12_937_772,
sha256:
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434'
size: 12_937_772,
sha256:
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434',
targets: {
modelscope: modelScopeTarget(
whisperTinyModelScopeRepository,
whisperTinyModelScopeRevision,
'tiny-encoder.int8.onnx'
),
'hugging-face': huggingFaceTarget(
whisperTinyHuggingFaceRepository,
whisperTinyHuggingFaceRevision,
'tiny-encoder.int8.onnx'
)
}
},
{
name: 'tiny-decoder.int8.onnx',
role: 'decoder',
download: {
url:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-whisper-tiny/resolve/' +
'33a655645234f82ce833cf27b689d9c2212e693f/' +
'tiny-decoder.int8.onnx',
size: 89_855_401,
sha256:
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925'
size: 89_855_401,
sha256:
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925',
targets: {
modelscope: modelScopeTarget(
whisperTinyModelScopeRepository,
whisperTinyModelScopeRevision,
'tiny-decoder.int8.onnx'
),
'hugging-face': huggingFaceTarget(
whisperTinyHuggingFaceRepository,
whisperTinyHuggingFaceRevision,
'tiny-decoder.int8.onnx'
)
}
},
{
name: 'tiny-tokens.txt',
role: 'tokens',
download: {
url:
'https://modelscope.cn/models/pengzhendong/' +
'sherpa-onnx-whisper-tiny/resolve/' +
'33a655645234f82ce833cf27b689d9c2212e693f/' +
'tiny-tokens.txt',
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
targets: {
modelscope: modelScopeTarget(
whisperTinyModelScopeRepository,
whisperTinyModelScopeRevision,
'tiny-tokens.txt'
),
'hugging-face': huggingFaceTarget(
whisperTinyHuggingFaceRepository,
whisperTinyHuggingFaceRevision,
'tiny-tokens.txt'
)
}
}
]
@@ -139,9 +192,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'high',
speed: 'fast',
recommended: true,
repositoryUrl:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-bilingual-zh-en',
repositoryUrls: {
'hugging-face':
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en'
},
license: {
name: 'MIT License',
notice:
@@ -156,29 +210,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'model.int8.onnx',
role: 'model',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
'model.int8.onnx',
size: 223_385_835,
sha256:
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7'
size: 223_385_835,
sha256:
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
'model.int8.onnx'
)
}
},
{
name: 'tokens.txt',
role: 'tokens',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
'tokens.txt',
size: 75_756,
sha256:
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6'
size: 75_756,
sha256:
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
'tokens.txt'
)
}
}
]
@@ -194,9 +248,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'high',
speed: 'balanced',
recommended: false,
repositoryUrl:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
repositoryUrls: {
'hugging-face':
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en'
},
license: {
name: 'Apache License 2.0',
notice:
@@ -211,29 +266,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'model.int8.onnx',
role: 'model',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
'model.int8.onnx',
size: 244_684_152,
sha256:
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144'
size: 244_684_152,
sha256:
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
'8d90151338178bb433354c9fb677bd3acb8023cd',
'model.int8.onnx'
)
}
},
{
name: 'tokens.txt',
role: 'tokens',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
'tokens.txt',
size: 118_931,
sha256:
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492'
size: 118_931,
sha256:
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
'8d90151338178bb433354c9fb677bd3acb8023cd',
'tokens.txt'
)
}
}
]
@@ -249,8 +304,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'balanced',
speed: 'balanced',
recommended: false,
repositoryUrl:
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small',
repositoryUrls: {
'hugging-face':
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small'
},
license: {
name: 'MIT License',
notice:
@@ -262,43 +319,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'small-encoder.int8.onnx',
role: 'encoder',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-small/resolve/' +
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
'small-encoder.int8.onnx',
size: 112_442_483,
sha256:
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9'
size: 112_442_483,
sha256:
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-small',
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
'small-encoder.int8.onnx'
)
}
},
{
name: 'small-decoder.int8.onnx',
role: 'decoder',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-small/resolve/' +
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
'small-decoder.int8.onnx',
size: 262_226_114,
sha256:
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee'
size: 262_226_114,
sha256:
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-small',
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
'small-decoder.int8.onnx'
)
}
},
{
name: 'small-tokens.txt',
role: 'tokens',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-small/resolve/' +
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
'small-tokens.txt',
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-small',
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
'small-tokens.txt'
)
}
}
]
@@ -314,8 +371,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
quality: 'high',
speed: 'slow',
recommended: false,
repositoryUrl:
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium',
repositoryUrls: {
'hugging-face':
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium'
},
license: {
name: 'MIT License',
notice:
@@ -327,43 +386,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
{
name: 'medium-encoder.int8.onnx',
role: 'encoder',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-medium/resolve/' +
'8c31d28503847560985df21f90e14f0c736e075e/' +
'medium-encoder.int8.onnx',
size: 374_196_283,
sha256:
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e'
size: 374_196_283,
sha256:
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-medium',
'8c31d28503847560985df21f90e14f0c736e075e',
'medium-encoder.int8.onnx'
)
}
},
{
name: 'medium-decoder.int8.onnx',
role: 'decoder',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-medium/resolve/' +
'8c31d28503847560985df21f90e14f0c736e075e/' +
'medium-decoder.int8.onnx',
size: 571_059_257,
sha256:
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747'
size: 571_059_257,
sha256:
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-medium',
'8c31d28503847560985df21f90e14f0c736e075e',
'medium-decoder.int8.onnx'
)
}
},
{
name: 'medium-tokens.txt',
role: 'tokens',
download: {
url:
'https://huggingface.co/csukuangfj/' +
'sherpa-onnx-whisper-medium/resolve/' +
'8c31d28503847560985df21f90e14f0c736e075e/' +
'medium-tokens.txt',
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
size: 816_730,
sha256:
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
targets: {
'hugging-face': huggingFaceTarget(
'csukuangfj/sherpa-onnx-whisper-medium',
'8c31d28503847560985df21f90e14f0c736e075e',
'medium-tokens.txt'
)
}
}
]
+250 -45
View File
@@ -35,12 +35,36 @@ function sha256(value: Uint8Array): string {
}
function manualCatalog(): SpeechModelCatalogEntry[] {
return SPEECH_MODEL_CATALOG.map((entry) => ({
const entry = SPEECH_MODEL_CATALOG.find(
(candidate) => candidate.id === 'sensevoice-small-int8'
)
if (!entry) {
throw new Error('SenseVoice test catalog entry is missing')
}
const modelBytes = new TextEncoder().encode('model')
const tokenBytes = new TextEncoder().encode('tokens')
return [{
...entry,
manualOnly: true,
manualReason: entry.manualReason ?? '测试使用本地目录导入。',
files: entry.files.map(({ name, role }) => ({ name, role }))
}))
manualReason: '测试使用本地目录导入。',
repositoryUrls: {},
files: [
{
name: 'model.int8.onnx',
role: 'model',
size: modelBytes.byteLength,
sha256: sha256(modelBytes),
targets: {}
},
{
name: 'tokens.txt',
role: 'tokens',
size: tokenBytes.byteLength,
sha256: sha256(tokenBytes),
targets: {}
}
]
}]
}
function downloadableCatalog(
@@ -58,8 +82,12 @@ function downloadableCatalog(
quality: 'balanced',
speed: 'balanced',
recommended: false,
repositoryUrl:
'https://modelscope.cn/models/example/download-test-model',
repositoryUrls: {
modelscope:
'https://modelscope.cn/models/example/download-test-model',
'hugging-face':
'https://huggingface.co/example/download-test-model'
},
license: {
name: 'MIT License',
notice: 'Test-only model metadata.',
@@ -70,23 +98,53 @@ function downloadableCatalog(
{
name: 'model.onnx',
role: 'model',
download: {
url:
'https://modelscope.cn/models/example/download-test-model/' +
`resolve/${'a'.repeat(40)}/model.onnx`,
size: modelBytes.byteLength,
sha256: sha256(modelBytes)
size: modelBytes.byteLength,
sha256: sha256(modelBytes),
targets: {
modelscope: {
url:
'https://modelscope.cn/models/example/download-test-model/' +
`resolve/${'a'.repeat(40)}/model.onnx`,
repositoryUrl:
'https://modelscope.cn/models/example/download-test-model',
revision: 'a'.repeat(40),
redirectHosts: []
},
'hugging-face': {
url:
'https://huggingface.co/example/download-test-model/' +
`resolve/${'b'.repeat(40)}/model.onnx`,
repositoryUrl:
'https://huggingface.co/example/download-test-model',
revision: 'b'.repeat(40),
redirectHosts: []
}
}
},
{
name: 'tokens.txt',
role: 'tokens',
download: {
url:
'https://modelscope.cn/models/example/download-test-model/' +
`resolve/${'a'.repeat(40)}/tokens.txt`,
size: tokenBytes.byteLength,
sha256: sha256(tokenBytes)
size: tokenBytes.byteLength,
sha256: sha256(tokenBytes),
targets: {
modelscope: {
url:
'https://modelscope.cn/models/example/download-test-model/' +
`resolve/${'a'.repeat(40)}/tokens.txt`,
repositoryUrl:
'https://modelscope.cn/models/example/download-test-model',
revision: 'a'.repeat(40),
redirectHosts: []
},
'hugging-face': {
url:
'https://huggingface.co/example/download-test-model/' +
`resolve/${'b'.repeat(40)}/tokens.txt`,
repositoryUrl:
'https://huggingface.co/example/download-test-model',
revision: 'b'.repeat(40),
redirectHosts: []
}
}
}
]
@@ -131,10 +189,19 @@ describe('speech model catalog', () => {
license: { name: 'MIT License' }
})
expect(
senseVoice?.files.every((file) => file.download !== undefined)
senseVoice?.files.every(
(file) =>
file.targets.modelscope !== undefined &&
file.targets['hugging-face'] !== undefined
)
).toBe(true)
expect(
whisper?.files.every(
(file) =>
file.targets.modelscope !== undefined &&
file.targets['hugging-face'] !== undefined
)
).toBe(true)
expect(whisper?.files.every((file) => file.download !== undefined))
.toBe(true)
expect(whisper?.files.map((file) => file.name)).toEqual([
'tiny-encoder.int8.onnx',
'tiny-decoder.int8.onnx',
@@ -163,15 +230,21 @@ describe('speech model catalog', () => {
})
expect(SPEECH_MODEL_CATALOG).toHaveLength(6)
for (const entry of SPEECH_MODEL_CATALOG) {
expect(entry.repositoryUrl).toMatch(
/^https:\/\/(?:modelscope\.cn\/models\/|huggingface\.co\/)/u
)
for (const file of entry.files) {
expect(file.download?.url).toMatch(
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
)
expect(file.sha256).toMatch(/^[a-f0-9]{64}$/u)
expect(file.size).toBeGreaterThan(0)
for (const target of Object.values(file.targets)) {
expect(target?.url).toMatch(
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
)
}
}
}
expect(
paraformerBilingual?.files.some(
(file) => file.targets.modelscope !== undefined
)
).toBe(false)
})
})
@@ -242,10 +315,14 @@ describe('SpeechModelManager downloads', () => {
).toBe(false)
await manager.select('download-test-model')
await expect(manager.snapshot()).resolves.toMatchObject({
const snapshot = await manager.snapshot()
expect(snapshot).toMatchObject({
selectedDownloadSource: 'modelscope',
selectedModelId: 'download-test-model',
operations: []
})
expect(snapshot.catalog[0]?.files[0]).not.toHaveProperty('targets')
expect(JSON.stringify(snapshot.catalog)).not.toContain('/resolve/')
await manager.remove('download-test-model')
await expect(manager.snapshot()).resolves.toMatchObject({
selectedModelId: null,
@@ -253,7 +330,63 @@ describe('SpeechModelManager downloads', () => {
})
})
it('accepts arbitrary HTTP hosts and cross-host redirects', async () => {
it('freezes the operation source when the global setting changes', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
const tokenBytes = new TextEncoder().encode('tokens')
let selectedSource: 'modelscope' | 'hugging-face' = 'modelscope'
let releaseFirstRequest: (() => void) | undefined
let markFirstRequestStarted: (() => void) | undefined
const firstRequestStarted = new Promise<void>((resolveStarted) => {
markFirstRequestStarted = resolveStarted
})
const firstRequestGate = new Promise<void>((resolveRequest) => {
releaseFirstRequest = resolveRequest
})
let requestCount = 0
const transport = vi.fn<typeof fetch>(async (input) => {
requestCount += 1
if (requestCount === 1) {
markFirstRequestStarted?.()
await firstRequestGate
}
const bytes = String(input).endsWith('model.onnx')
? modelBytes
: tokenBytes
return new Response(bytes, {
headers: { 'content-length': String(bytes.byteLength) }
})
})
const manager = new SpeechModelManager({
userDataDirectory: userData,
catalog: downloadableCatalog(modelBytes, tokenBytes),
fetch: transport,
getDownloadSource: () => selectedSource
})
const installing = manager.install('download-test-model')
await firstRequestStarted
selectedSource = 'hugging-face'
await expect(manager.snapshot()).resolves.toMatchObject({
selectedDownloadSource: 'hugging-face',
operations: [
{
modelId: 'download-test-model',
kind: 'download',
downloadSource: 'modelscope'
}
]
})
releaseFirstRequest?.()
await installing
expect(
transport.mock.calls.every(
([input]) => new URL(String(input)).hostname === 'modelscope.cn'
)
).toBe(true)
})
it('follows only source-declared HTTPS redirect hosts', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
const tokenBytes = new TextEncoder().encode('tokens')
@@ -261,24 +394,24 @@ describe('SpeechModelManager downloads', () => {
...entry,
files: entry.files.map((file) => ({
...file,
download: file.download
? {
...file.download,
url: file.download.url.replace(
'https://modelscope.cn',
'http://models.internal.example'
)
}
: undefined
targets: {
...file.targets,
'hugging-face': file.targets['hugging-face']
? {
...file.targets['hugging-face'],
redirectHosts: ['cdn-lfs.hf.co']
}
: undefined
}
}))
}))
const transport = vi.fn<typeof fetch>(async (input) => {
const url = new URL(String(input))
if (url.hostname === 'models.internal.example') {
if (url.hostname === 'huggingface.co') {
return new Response(null, {
status: 302,
headers: {
location: `https://cdn.example.net${url.pathname}`
location: `https://cdn-lfs.hf.co${url.pathname}`
}
})
}
@@ -296,19 +429,91 @@ describe('SpeechModelManager downloads', () => {
})
await expect(
redirected.install('download-test-model')
redirected.install('download-test-model', 'hugging-face')
).resolves.toMatchObject({ id: 'download-test-model' })
expect(transport).toHaveBeenCalledTimes(4)
expect(
transport.mock.calls.map(([input]) => new URL(String(input)).hostname)
).toEqual([
'models.internal.example',
'cdn.example.net',
'models.internal.example',
'cdn.example.net'
'huggingface.co',
'cdn-lfs.hf.co',
'huggingface.co',
'cdn-lfs.hf.co'
])
})
it('rejects undeclared redirect hosts without following them', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
const transport = vi.fn<typeof fetch>(async () =>
new Response(null, {
status: 302,
headers: {
location: 'https://untrusted.example/model.onnx'
}
})
)
const manager = new SpeechModelManager({
userDataDirectory: userData,
catalog: downloadableCatalog(modelBytes),
fetch: transport
})
await expect(
manager.install('download-test-model')
).rejects.toThrow('未声明的主机')
expect(transport).toHaveBeenCalledTimes(1)
})
it('does not request another source when selected coverage is missing', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
const catalog = downloadableCatalog(modelBytes).map((entry) => ({
...entry,
repositoryUrls: {
'hugging-face': entry.repositoryUrls['hugging-face']
},
files: entry.files.map((file) => ({
...file,
targets: {
'hugging-face': file.targets['hugging-face']
}
}))
}))
const transport = vi.fn<typeof fetch>()
const manager = new SpeechModelManager({
userDataDirectory: userData,
catalog,
fetch: transport
})
await expect(
manager.install('download-test-model', 'modelscope')
).rejects.toThrow('当前下载源')
expect(transport).not.toHaveBeenCalled()
})
it('does not request another source after a download failure', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
const transport = vi.fn<typeof fetch>(
async () => new Response(null, { status: 503 })
)
const manager = new SpeechModelManager({
userDataDirectory: userData,
catalog: downloadableCatalog(modelBytes),
fetch: transport
})
await expect(
manager.install('download-test-model', 'modelscope')
).rejects.toThrow('HTTP 503')
expect(transport).toHaveBeenCalledTimes(1)
expect(
new URL(String(transport.mock.calls[0]?.[0])).hostname
).toBe('modelscope.cn')
})
it('rejects bad digests without installing', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('expected')
+93 -91
View File
@@ -16,22 +16,31 @@ import { z } from 'zod'
import {
installedSpeechModelSchema,
speechModelCatalogEntrySchema,
speechModelCatalogViewEntrySchema,
speechModelIdSchema,
speechModelSnapshotSchema,
type InstalledSpeechModel,
type SpeechModelCatalogEntry,
type SpeechModelCatalogViewEntry,
type SpeechModelFileSpec,
type SpeechModelOperation,
type SpeechModelSnapshot
} from '../../shared/speech-model-contracts'
import {
MODEL_DOWNLOAD_SOURCES,
getModelDownloadAvailability,
resolveModelDownloadPackage,
type ModelDownloadSource,
type ResolvedModelArtifactFile
} from '../../shared/model-download-contracts'
import { SPEECH_MODEL_CATALOG } from './speech-model-catalog'
import {
exportModelArchive,
extractModelArchive
} from '../model-archive'
import { fetchModelDownloadResponse } from '../model-download-transport'
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024
const MAX_REDIRECTS = 3
const MANIFEST_FILE_NAME = 'manifest.json'
const SELECTION_FILE_NAME = '.selection.json'
const PARTIAL_SUFFIX = '.partial'
@@ -56,6 +65,9 @@ export type SpeechModelManagerOptions = {
userDataDirectory: string
fetch: typeof fetch
catalog?: readonly SpeechModelCatalogEntry[]
getDownloadSource?: () =>
| ModelDownloadSource
| Promise<ModelDownloadSource>
maxFileBytes?: number
}
@@ -72,6 +84,23 @@ function cloneCatalogEntry(
return speechModelCatalogEntrySchema.parse(entry)
}
function toCatalogView(entry: SpeechModelCatalogEntry) {
const { repositoryUrls, files, ...metadata } = entry
void repositoryUrls
return speechModelCatalogViewEntrySchema.parse({
...metadata,
files: files.map((file) => ({
name: file.name,
role: file.role,
size: file.size,
sha256: file.sha256
})),
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
getModelDownloadAvailability(files, source)
)
})
}
function abortError(): DOMException {
return new DOMException('The operation was aborted', 'AbortError')
}
@@ -102,17 +131,6 @@ function safeChild(parent: string, name: string): string {
return child
}
function validateDownloadUrl(value: string): URL {
const url = new URL(value)
if (
url.protocol !== 'http:' &&
url.protocol !== 'https:'
) {
throw new Error('模型下载地址必须使用 HTTP 或 HTTPS')
}
return url
}
async function hashFile(
path: string,
signal?: AbortSignal
@@ -147,6 +165,10 @@ export class SpeechModelManager {
private readonly transport: typeof fetch
private readonly catalog: SpeechModelCatalogEntry[]
private readonly catalogViews: SpeechModelCatalogViewEntry[]
private readonly getDownloadSource: () =>
| ModelDownloadSource
| Promise<ModelDownloadSource>
private readonly maxFileBytes: number
private readonly operations = new Map<string, ActiveOperation>()
@@ -160,23 +182,31 @@ export class SpeechModelManager {
'speech'
)
this.transport = options.fetch
this.getDownloadSource =
options.getDownloadSource ?? (() => 'modelscope')
this.catalog = (options.catalog ?? SPEECH_MODEL_CATALOG).map(
cloneCatalogEntry
)
if (new Set(this.catalog.map((entry) => entry.id)).size !== this.catalog.length) {
throw new Error('语音模型目录包含重复 ID')
}
this.catalogViews = this.catalog.map(toCatalogView)
this.maxFileBytes = validateMaximumBytes(options.maxFileBytes)
}
async snapshot(): Promise<SpeechModelSnapshot> {
await this.ensureRoot()
const installed = await this.readInstalled()
const selected = await this.readSelection()
const [installed, selected, selectedDownloadSource] =
await Promise.all([
this.readInstalled(),
this.readSelection(),
this.getDownloadSource()
])
const installedIds = new Set(installed.map((model) => model.id))
return speechModelSnapshotSchema.parse({
rootDirectory: this.rootDirectory,
catalog: this.catalog.map(cloneCatalogEntry),
selectedDownloadSource,
catalog: this.catalogViews,
installed,
operations: [...this.operations.values()].map((operation) => ({
...operation.progress
@@ -190,6 +220,19 @@ export class SpeechModelManager {
return this.snapshot()
}
getRepositoryUrl(
modelId: string,
source: ModelDownloadSource
): string {
const entry = this.requireCatalogEntry(modelId)
resolveModelDownloadPackage(entry.files, source)
const repositoryUrl = entry.repositoryUrls[source]
if (!repositoryUrl) {
throw new Error('当前下载源暂不提供此模型的仓库')
}
return repositoryUrl
}
async getSelectedRuntimeModel(): Promise<
SelectedSpeechRuntimeModel | undefined
> {
@@ -216,6 +259,7 @@ export class SpeechModelManager {
async install(
modelId: string,
downloadSource?: ModelDownloadSource,
externalSignal?: AbortSignal
): Promise<InstalledSpeechModel> {
const entry = this.requireCatalogEntry(modelId)
@@ -224,27 +268,17 @@ export class SpeechModelManager {
entry.manualReason ?? '该模型只能从本地目录导入'
)
}
const downloadableFiles = entry.files.filter(
(
file
): file is SpeechModelFileSpec & {
download: NonNullable<SpeechModelFileSpec['download']>
} => file.download !== undefined
const selectedDownloadSource =
downloadSource ?? (await this.getDownloadSource())
const resolvedPackage = resolveModelDownloadPackage(
entry.files,
selectedDownloadSource
)
if (downloadableFiles.length !== entry.files.length) {
throw new Error('模型下载元数据不完整')
}
const totalBytes = downloadableFiles.reduce(
(total, file) => total + file.download.size,
0
)
if (!Number.isSafeInteger(totalBytes)) {
throw new RangeError('模型总大小超出安全范围')
}
const operation = this.beginOperation(
entry.id,
'download',
totalBytes
resolvedPackage.totalBytes,
resolvedPackage.source
)
const detachExternalAbort = this.attachExternalSignal(
externalSignal,
@@ -255,7 +289,7 @@ export class SpeechModelManager {
await this.ensureRoot()
await this.assertNotInstalled(entry.id)
stagingDirectory = await this.createStagingDirectory(entry.id)
for (const file of downloadableFiles) {
for (const file of resolvedPackage.files) {
ensureNotAborted(operation.controller.signal)
operation.progress.phase = 'transferring'
operation.progress.currentFile = file.name
@@ -409,9 +443,8 @@ export class SpeechModelManager {
!recorded ||
recorded.size <= 0 ||
recorded.size > this.maxFileBytes ||
(expected.download &&
(recorded.size !== expected.download.size ||
recorded.sha256 !== expected.download.sha256))
recorded.size !== expected.size ||
recorded.sha256 !== expected.sha256
) {
throw new Error(`语音模型文件不可导出:${expected.name}`)
}
@@ -440,8 +473,7 @@ export class SpeechModelManager {
): Promise<InstalledSpeechModel> {
const entry = this.requireCatalogEntry(modelId)
const expectedTotal = entry.files.reduce(
(total, file) =>
total + (file.download?.size ?? this.maxFileBytes),
(total, file) => total + file.size,
0
)
const maximumTotalBytes = Math.min(
@@ -488,9 +520,8 @@ export class SpeechModelManager {
if (
!archived ||
archived.size > this.maxFileBytes ||
(expected.download &&
(archived.size !== expected.download.size ||
archived.sha256 !== expected.download.sha256))
archived.size !== expected.size ||
archived.sha256 !== expected.sha256
) {
throw new Error(
`语音模型 ZIP 与当前模型目录不匹配:${expected.name}`
@@ -544,7 +575,8 @@ export class SpeechModelManager {
private beginOperation(
modelId: string,
kind: SpeechModelOperation['kind'],
totalBytes: number | null
totalBytes: number | null,
downloadSource?: ModelDownloadSource
): ActiveOperation {
if (this.operations.has(modelId)) {
throw new Error('该模型已有进行中的操作')
@@ -557,7 +589,8 @@ export class SpeechModelManager {
phase: 'preparing',
currentFile: null,
completedBytes: 0,
totalBytes
totalBytes,
...(downloadSource ? { downloadSource } : {})
}
}
this.operations.set(modelId, operation)
@@ -605,55 +638,25 @@ export class SpeechModelManager {
return directory
}
private async fetchFollowingRedirects(
initialUrl: string,
signal: AbortSignal
): Promise<Response> {
let url = validateDownloadUrl(initialUrl)
for (let redirectCount = 0; ; redirectCount += 1) {
ensureNotAborted(signal)
const response = await this.transport(url, {
method: 'GET',
redirect: 'manual',
credentials: 'omit',
cache: 'no-store',
signal
})
if ([301, 302, 303, 307, 308].includes(response.status)) {
if (redirectCount >= MAX_REDIRECTS) {
await response.body?.cancel().catch(() => undefined)
throw new Error('模型下载重定向次数过多')
}
const location = response.headers.get('location')
await response.body?.cancel().catch(() => undefined)
if (!location) {
throw new Error('模型下载重定向缺少地址')
}
url = validateDownloadUrl(new URL(location, url).toString())
continue
}
return response
}
}
private async downloadFile(
file: SpeechModelFileSpec & {
download: NonNullable<SpeechModelFileSpec['download']>
},
file: ResolvedModelArtifactFile<SpeechModelFileSpec['role']>,
destination: string,
operation: ActiveOperation,
signal: AbortSignal
): Promise<void> {
if (
file.download.size > this.maxFileBytes ||
file.download.size <= 0
file.size > this.maxFileBytes ||
file.size <= 0
) {
throw new RangeError(`模型文件大小超出限制:${file.name}`)
}
const response = await this.fetchFollowingRedirects(
file.download.url,
signal
)
const response = await fetchModelDownloadResponse({
transport: this.transport,
initialUrl: file.target.url,
redirectHosts: file.target.redirectHosts,
signal,
modelLabel: '模型'
})
if (!response.ok) {
await response.body?.cancel().catch(() => undefined)
throw new Error(`模型下载失败:HTTP ${response.status}`)
@@ -666,7 +669,7 @@ export class SpeechModelManager {
const parsedLength = Number(declaredLength)
if (
!Number.isSafeInteger(parsedLength) ||
parsedLength !== file.download.size
parsedLength !== file.size
) {
await response.body.cancel().catch(() => undefined)
throw new Error(`模型文件大小不匹配:${file.name}`)
@@ -687,7 +690,7 @@ export class SpeechModelManager {
}
written += result.value.byteLength
if (
written > file.download.size ||
written > file.size ||
written > this.maxFileBytes
) {
await reader.cancel()
@@ -703,10 +706,10 @@ export class SpeechModelManager {
} finally {
await handle.close()
}
if (written !== file.download.size) {
if (written !== file.size) {
throw new Error(`模型文件大小不匹配:${file.name}`)
}
if (hash.digest('hex') !== file.download.sha256) {
if (hash.digest('hex') !== file.sha256) {
throw new Error(`模型文件校验失败:${file.name}`)
}
await rename(partialPath, destination)
@@ -741,10 +744,9 @@ export class SpeechModelManager {
throw new RangeError(`模型文件大小无效:${expectedFile.name}`)
}
if (
expectedFile.download &&
(sourceFileInfo.size !== expectedFile.download.size ||
(await hashFile(sourceFile, signal)).sha256 !==
expectedFile.download.sha256)
sourceFileInfo.size !== expectedFile.size ||
(await hashFile(sourceFile, signal)).sha256 !==
expectedFile.sha256
) {
throw new Error(`本地模型文件校验失败:${expectedFile.name}`)
}