feat: add explicit managed model download sources
Managed speech and OCR downloads previously used catalog-specific source URLs without a global selection. Platform Features now lets users choose ModelScope by default or Hugging Face, while Main validates and freezes that source for each download. Verified coverage remains explicit: downloads never mix artifacts or silently switch sources, and installed models plus ZIP imports stay source-independent. Release note: 可在“设置 → 平台功能 → 通用设置”中选择 ModelScope 或 Hugging Face 作为后续语音输入与 OCR 模型下载源;缺少完整已验证文件或下载失败时不会静默换源。
This commit is contained in:
@@ -2,11 +2,33 @@ import {
|
||||
speechModelCatalogEntrySchema,
|
||||
type SpeechModelCatalogEntry
|
||||
} from '../../shared/speech-model-contracts'
|
||||
import {
|
||||
huggingFaceTarget,
|
||||
modelScopeTarget
|
||||
} from '../model-download-targets'
|
||||
|
||||
const senseVoiceModelScopeRepository =
|
||||
'pengzhendong/sherpa-onnx-sense-voice-zh-en-ja-ko-yue'
|
||||
const senseVoiceModelScopeRevision =
|
||||
'73eca47697f980daa3d16112404174b6b950b514'
|
||||
const senseVoiceHuggingFaceRepository =
|
||||
'csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17'
|
||||
const senseVoiceHuggingFaceRevision =
|
||||
'2365baeacb507f821a0c8120fcee3d484dba7a07'
|
||||
|
||||
const whisperTinyModelScopeRepository =
|
||||
'pengzhendong/sherpa-onnx-whisper-tiny'
|
||||
const whisperTinyModelScopeRevision =
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f'
|
||||
const whisperTinyHuggingFaceRepository =
|
||||
'csukuangfj/sherpa-onnx-whisper-tiny'
|
||||
const whisperTinyHuggingFaceRevision =
|
||||
'65176e2deb88badc814a94058666cadccc29b61c'
|
||||
|
||||
/**
|
||||
* This catalog intentionally contains metadata only. Model weights are never
|
||||
* bundled with GoodBuddy. Entries remain manual-only until every downloadable
|
||||
* file has a pinned revision, byte size, and independently verified SHA-256.
|
||||
* Model weights are never bundled with GoodBuddy. Canonical file identity is
|
||||
* source-independent; a source target is included only after its bytes match
|
||||
* the declared size and SHA-256.
|
||||
*/
|
||||
export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
speechModelCatalogEntrySchema.array().parse([
|
||||
@@ -21,13 +43,16 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'fast',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
`https://modelscope.cn/models/${senseVoiceModelScopeRepository}`,
|
||||
'hugging-face':
|
||||
`https://huggingface.co/${senseVoiceHuggingFaceRepository}`
|
||||
},
|
||||
license: {
|
||||
name: '模型仓库自定义许可(Model License)',
|
||||
notice:
|
||||
'SenseVoiceSmall 权重采用模型仓库声明的自定义 MODEL LICENSE,并非 Apache-2.0 或 MIT;导入和使用前请阅读完整许可条款。',
|
||||
'SenseVoiceSmall 权重采用上游声明的自定义 MODEL LICENSE,并非 Apache-2.0 或 MIT;导入和使用前请阅读完整许可条款。',
|
||||
url: 'https://github.com/modelscope/FunASR/blob/main/MODEL_LICENSE'
|
||||
},
|
||||
manualOnly: false,
|
||||
@@ -35,29 +60,39 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
|
||||
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
|
||||
'model.int8.onnx',
|
||||
size: 239_233_841,
|
||||
sha256:
|
||||
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51'
|
||||
size: 239_233_841,
|
||||
sha256:
|
||||
'c71f0ce00bec95b07744e116345e33d8cbbe08cef896382cf907bf4b51a2cd51',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
senseVoiceModelScopeRepository,
|
||||
senseVoiceModelScopeRevision,
|
||||
'model.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
senseVoiceHuggingFaceRepository,
|
||||
senseVoiceHuggingFaceRevision,
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-sense-voice-zh-en-ja-ko-yue/' +
|
||||
'resolve/73eca47697f980daa3d16112404174b6b950b514/' +
|
||||
'tokens.txt',
|
||||
size: 315_894,
|
||||
sha256:
|
||||
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc'
|
||||
size: 315_894,
|
||||
sha256:
|
||||
'f449eb28dc567533d7fa59be34e2abca8784f771850c78a47fb731a31429a1dc',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
senseVoiceModelScopeRepository,
|
||||
senseVoiceModelScopeRevision,
|
||||
'tokens.txt'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
senseVoiceHuggingFaceRepository,
|
||||
senseVoiceHuggingFaceRevision,
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -73,9 +108,12 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'basic',
|
||||
speed: 'fast',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
`https://modelscope.cn/models/${whisperTinyModelScopeRepository}`,
|
||||
'hugging-face':
|
||||
`https://huggingface.co/${whisperTinyHuggingFaceRepository}`
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -87,43 +125,58 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'tiny-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-encoder.int8.onnx',
|
||||
size: 12_937_772,
|
||||
sha256:
|
||||
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434'
|
||||
size: 12_937_772,
|
||||
sha256:
|
||||
'd24fb083ae3b1041fc24e97971d60e280c9342201fbb67b0ab428a8b4a51a434',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-encoder.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tiny-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-decoder.int8.onnx',
|
||||
size: 89_855_401,
|
||||
sha256:
|
||||
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925'
|
||||
size: 89_855_401,
|
||||
sha256:
|
||||
'd2fece8dd42771f1df975c6c0445770d0c292bf7547c2cae04a6c0cc57540925',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-decoder.int8.onnx'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tiny-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/pengzhendong/' +
|
||||
'sherpa-onnx-whisper-tiny/resolve/' +
|
||||
'33a655645234f82ce833cf27b689d9c2212e693f/' +
|
||||
'tiny-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
modelscope: modelScopeTarget(
|
||||
whisperTinyModelScopeRepository,
|
||||
whisperTinyModelScopeRevision,
|
||||
'tiny-tokens.txt'
|
||||
),
|
||||
'hugging-face': huggingFaceTarget(
|
||||
whisperTinyHuggingFaceRepository,
|
||||
whisperTinyHuggingFaceRevision,
|
||||
'tiny-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -139,9 +192,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'fast',
|
||||
recommended: true,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -156,29 +210,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
|
||||
'model.int8.onnx',
|
||||
size: 223_385_835,
|
||||
sha256:
|
||||
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7'
|
||||
size: 223_385_835,
|
||||
sha256:
|
||||
'9ada9127ca5b82320385ac12340eb8b05dee64fd45cf8cf593ec693826ec2fd7',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-bilingual-zh-en/resolve/' +
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b/' +
|
||||
'tokens.txt',
|
||||
size: 75_756,
|
||||
sha256:
|
||||
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6'
|
||||
size: 75_756,
|
||||
sha256:
|
||||
'59aba8873a2ed1e122c25fee421e25f283b63290efbde85c1f01a853d83cb6e6',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-bilingual-zh-en',
|
||||
'4b891f7b5c73d874e607797a4b0578fd4c35dd4b',
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -194,9 +248,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en'
|
||||
},
|
||||
license: {
|
||||
name: 'Apache License 2.0',
|
||||
notice:
|
||||
@@ -211,29 +266,29 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
|
||||
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
|
||||
'model.int8.onnx',
|
||||
size: 244_684_152,
|
||||
sha256:
|
||||
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144'
|
||||
size: 244_684_152,
|
||||
sha256:
|
||||
'eb3cdd288f535cf73258f491cdd7d68ad5a00aee135c0bba4c0884ea8d926144',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
'8d90151338178bb433354c9fb677bd3acb8023cd',
|
||||
'model.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-paraformer-trilingual-zh-cantonese-en/' +
|
||||
'resolve/8d90151338178bb433354c9fb677bd3acb8023cd/' +
|
||||
'tokens.txt',
|
||||
size: 118_931,
|
||||
sha256:
|
||||
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492'
|
||||
size: 118_931,
|
||||
sha256:
|
||||
'8e4593d7a2eb2404ff82976b5494265e9a06283ca4d5e8605bf7b4fed557a492',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-paraformer-trilingual-zh-cantonese-en',
|
||||
'8d90151338178bb433354c9fb677bd3acb8023cd',
|
||||
'tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -249,8 +304,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-small'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -262,43 +319,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'small-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-encoder.int8.onnx',
|
||||
size: 112_442_483,
|
||||
sha256:
|
||||
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9'
|
||||
size: 112_442_483,
|
||||
sha256:
|
||||
'4cbe7b22fa9026b843b60a68640c747de05bafb1a11b57edc0e66c232d9f33a9',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'small-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-decoder.int8.onnx',
|
||||
size: 262_226_114,
|
||||
sha256:
|
||||
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee'
|
||||
size: 262_226_114,
|
||||
sha256:
|
||||
'acad50b5c782696e91b55914cc5ab4f756f1532f76e22aa6fc615f39fb69a8ee',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'small-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-small/resolve/' +
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309/' +
|
||||
'small-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-small',
|
||||
'8f3c18b358db4d1f2fc1eae49d75cd20989e4309',
|
||||
'small-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -314,8 +371,10 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
quality: 'high',
|
||||
speed: 'slow',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium',
|
||||
repositoryUrls: {
|
||||
'hugging-face':
|
||||
'https://huggingface.co/csukuangfj/sherpa-onnx-whisper-medium'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice:
|
||||
@@ -327,43 +386,43 @@ export const SPEECH_MODEL_CATALOG: readonly SpeechModelCatalogEntry[] =
|
||||
{
|
||||
name: 'medium-encoder.int8.onnx',
|
||||
role: 'encoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-encoder.int8.onnx',
|
||||
size: 374_196_283,
|
||||
sha256:
|
||||
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e'
|
||||
size: 374_196_283,
|
||||
sha256:
|
||||
'1c54582b4d829de0089f6cb63bbbdb3bf7555398bacaf855fbecf1a84dfd193e',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-encoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'medium-decoder.int8.onnx',
|
||||
role: 'decoder',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-decoder.int8.onnx',
|
||||
size: 571_059_257,
|
||||
sha256:
|
||||
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747'
|
||||
size: 571_059_257,
|
||||
sha256:
|
||||
'595d00a338a365a7bfa0ca7f296cabc639583bef770ab6130df90f49a6412747',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-decoder.int8.onnx'
|
||||
)
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'medium-tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://huggingface.co/csukuangfj/' +
|
||||
'sherpa-onnx-whisper-medium/resolve/' +
|
||||
'8c31d28503847560985df21f90e14f0c736e075e/' +
|
||||
'medium-tokens.txt',
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126'
|
||||
size: 816_730,
|
||||
sha256:
|
||||
'b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126',
|
||||
targets: {
|
||||
'hugging-face': huggingFaceTarget(
|
||||
'csukuangfj/sherpa-onnx-whisper-medium',
|
||||
'8c31d28503847560985df21f90e14f0c736e075e',
|
||||
'medium-tokens.txt'
|
||||
)
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,12 +35,36 @@ function sha256(value: Uint8Array): string {
|
||||
}
|
||||
|
||||
function manualCatalog(): SpeechModelCatalogEntry[] {
|
||||
return SPEECH_MODEL_CATALOG.map((entry) => ({
|
||||
const entry = SPEECH_MODEL_CATALOG.find(
|
||||
(candidate) => candidate.id === 'sensevoice-small-int8'
|
||||
)
|
||||
if (!entry) {
|
||||
throw new Error('SenseVoice test catalog entry is missing')
|
||||
}
|
||||
const modelBytes = new TextEncoder().encode('model')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
return [{
|
||||
...entry,
|
||||
manualOnly: true,
|
||||
manualReason: entry.manualReason ?? '测试使用本地目录导入。',
|
||||
files: entry.files.map(({ name, role }) => ({ name, role }))
|
||||
}))
|
||||
manualReason: '测试使用本地目录导入。',
|
||||
repositoryUrls: {},
|
||||
files: [
|
||||
{
|
||||
name: 'model.int8.onnx',
|
||||
role: 'model',
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes),
|
||||
targets: {}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes),
|
||||
targets: {}
|
||||
}
|
||||
]
|
||||
}]
|
||||
}
|
||||
|
||||
function downloadableCatalog(
|
||||
@@ -58,8 +82,12 @@ function downloadableCatalog(
|
||||
quality: 'balanced',
|
||||
speed: 'balanced',
|
||||
recommended: false,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
repositoryUrls: {
|
||||
modelscope:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
'hugging-face':
|
||||
'https://huggingface.co/example/download-test-model'
|
||||
},
|
||||
license: {
|
||||
name: 'MIT License',
|
||||
notice: 'Test-only model metadata.',
|
||||
@@ -70,23 +98,53 @@ function downloadableCatalog(
|
||||
{
|
||||
name: 'model.onnx',
|
||||
role: 'model',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/model.onnx`,
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes)
|
||||
size: modelBytes.byteLength,
|
||||
sha256: sha256(modelBytes),
|
||||
targets: {
|
||||
modelscope: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/model.onnx`,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
revision: 'a'.repeat(40),
|
||||
redirectHosts: []
|
||||
},
|
||||
'hugging-face': {
|
||||
url:
|
||||
'https://huggingface.co/example/download-test-model/' +
|
||||
`resolve/${'b'.repeat(40)}/model.onnx`,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/example/download-test-model',
|
||||
revision: 'b'.repeat(40),
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: 'tokens.txt',
|
||||
role: 'tokens',
|
||||
download: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/tokens.txt`,
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes)
|
||||
size: tokenBytes.byteLength,
|
||||
sha256: sha256(tokenBytes),
|
||||
targets: {
|
||||
modelscope: {
|
||||
url:
|
||||
'https://modelscope.cn/models/example/download-test-model/' +
|
||||
`resolve/${'a'.repeat(40)}/tokens.txt`,
|
||||
repositoryUrl:
|
||||
'https://modelscope.cn/models/example/download-test-model',
|
||||
revision: 'a'.repeat(40),
|
||||
redirectHosts: []
|
||||
},
|
||||
'hugging-face': {
|
||||
url:
|
||||
'https://huggingface.co/example/download-test-model/' +
|
||||
`resolve/${'b'.repeat(40)}/tokens.txt`,
|
||||
repositoryUrl:
|
||||
'https://huggingface.co/example/download-test-model',
|
||||
revision: 'b'.repeat(40),
|
||||
redirectHosts: []
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -131,10 +189,19 @@ describe('speech model catalog', () => {
|
||||
license: { name: 'MIT License' }
|
||||
})
|
||||
expect(
|
||||
senseVoice?.files.every((file) => file.download !== undefined)
|
||||
senseVoice?.files.every(
|
||||
(file) =>
|
||||
file.targets.modelscope !== undefined &&
|
||||
file.targets['hugging-face'] !== undefined
|
||||
)
|
||||
).toBe(true)
|
||||
expect(
|
||||
whisper?.files.every(
|
||||
(file) =>
|
||||
file.targets.modelscope !== undefined &&
|
||||
file.targets['hugging-face'] !== undefined
|
||||
)
|
||||
).toBe(true)
|
||||
expect(whisper?.files.every((file) => file.download !== undefined))
|
||||
.toBe(true)
|
||||
expect(whisper?.files.map((file) => file.name)).toEqual([
|
||||
'tiny-encoder.int8.onnx',
|
||||
'tiny-decoder.int8.onnx',
|
||||
@@ -163,15 +230,21 @@ describe('speech model catalog', () => {
|
||||
})
|
||||
expect(SPEECH_MODEL_CATALOG).toHaveLength(6)
|
||||
for (const entry of SPEECH_MODEL_CATALOG) {
|
||||
expect(entry.repositoryUrl).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models\/|huggingface\.co\/)/u
|
||||
)
|
||||
for (const file of entry.files) {
|
||||
expect(file.download?.url).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
expect(file.sha256).toMatch(/^[a-f0-9]{64}$/u)
|
||||
expect(file.size).toBeGreaterThan(0)
|
||||
for (const target of Object.values(file.targets)) {
|
||||
expect(target?.url).toMatch(
|
||||
/^https:\/\/(?:modelscope\.cn\/models|huggingface\.co)\/[^/]+\/[^/]+\/resolve\/[a-f0-9]{40}\/[^/]+$/u
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(
|
||||
paraformerBilingual?.files.some(
|
||||
(file) => file.targets.modelscope !== undefined
|
||||
)
|
||||
).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -242,10 +315,14 @@ describe('SpeechModelManager downloads', () => {
|
||||
).toBe(false)
|
||||
|
||||
await manager.select('download-test-model')
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
const snapshot = await manager.snapshot()
|
||||
expect(snapshot).toMatchObject({
|
||||
selectedDownloadSource: 'modelscope',
|
||||
selectedModelId: 'download-test-model',
|
||||
operations: []
|
||||
})
|
||||
expect(snapshot.catalog[0]?.files[0]).not.toHaveProperty('targets')
|
||||
expect(JSON.stringify(snapshot.catalog)).not.toContain('/resolve/')
|
||||
await manager.remove('download-test-model')
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
selectedModelId: null,
|
||||
@@ -253,7 +330,63 @@ describe('SpeechModelManager downloads', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('accepts arbitrary HTTP hosts and cross-host redirects', async () => {
|
||||
it('freezes the operation source when the global setting changes', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
let selectedSource: 'modelscope' | 'hugging-face' = 'modelscope'
|
||||
let releaseFirstRequest: (() => void) | undefined
|
||||
let markFirstRequestStarted: (() => void) | undefined
|
||||
const firstRequestStarted = new Promise<void>((resolveStarted) => {
|
||||
markFirstRequestStarted = resolveStarted
|
||||
})
|
||||
const firstRequestGate = new Promise<void>((resolveRequest) => {
|
||||
releaseFirstRequest = resolveRequest
|
||||
})
|
||||
let requestCount = 0
|
||||
const transport = vi.fn<typeof fetch>(async (input) => {
|
||||
requestCount += 1
|
||||
if (requestCount === 1) {
|
||||
markFirstRequestStarted?.()
|
||||
await firstRequestGate
|
||||
}
|
||||
const bytes = String(input).endsWith('model.onnx')
|
||||
? modelBytes
|
||||
: tokenBytes
|
||||
return new Response(bytes, {
|
||||
headers: { 'content-length': String(bytes.byteLength) }
|
||||
})
|
||||
})
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes, tokenBytes),
|
||||
fetch: transport,
|
||||
getDownloadSource: () => selectedSource
|
||||
})
|
||||
|
||||
const installing = manager.install('download-test-model')
|
||||
await firstRequestStarted
|
||||
selectedSource = 'hugging-face'
|
||||
await expect(manager.snapshot()).resolves.toMatchObject({
|
||||
selectedDownloadSource: 'hugging-face',
|
||||
operations: [
|
||||
{
|
||||
modelId: 'download-test-model',
|
||||
kind: 'download',
|
||||
downloadSource: 'modelscope'
|
||||
}
|
||||
]
|
||||
})
|
||||
releaseFirstRequest?.()
|
||||
await installing
|
||||
expect(
|
||||
transport.mock.calls.every(
|
||||
([input]) => new URL(String(input)).hostname === 'modelscope.cn'
|
||||
)
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it('follows only source-declared HTTPS redirect hosts', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const tokenBytes = new TextEncoder().encode('tokens')
|
||||
@@ -261,24 +394,24 @@ describe('SpeechModelManager downloads', () => {
|
||||
...entry,
|
||||
files: entry.files.map((file) => ({
|
||||
...file,
|
||||
download: file.download
|
||||
? {
|
||||
...file.download,
|
||||
url: file.download.url.replace(
|
||||
'https://modelscope.cn',
|
||||
'http://models.internal.example'
|
||||
)
|
||||
}
|
||||
: undefined
|
||||
targets: {
|
||||
...file.targets,
|
||||
'hugging-face': file.targets['hugging-face']
|
||||
? {
|
||||
...file.targets['hugging-face'],
|
||||
redirectHosts: ['cdn-lfs.hf.co']
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
}))
|
||||
}))
|
||||
const transport = vi.fn<typeof fetch>(async (input) => {
|
||||
const url = new URL(String(input))
|
||||
if (url.hostname === 'models.internal.example') {
|
||||
if (url.hostname === 'huggingface.co') {
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
location: `https://cdn.example.net${url.pathname}`
|
||||
location: `https://cdn-lfs.hf.co${url.pathname}`
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -296,19 +429,91 @@ describe('SpeechModelManager downloads', () => {
|
||||
})
|
||||
|
||||
await expect(
|
||||
redirected.install('download-test-model')
|
||||
redirected.install('download-test-model', 'hugging-face')
|
||||
).resolves.toMatchObject({ id: 'download-test-model' })
|
||||
expect(transport).toHaveBeenCalledTimes(4)
|
||||
expect(
|
||||
transport.mock.calls.map(([input]) => new URL(String(input)).hostname)
|
||||
).toEqual([
|
||||
'models.internal.example',
|
||||
'cdn.example.net',
|
||||
'models.internal.example',
|
||||
'cdn.example.net'
|
||||
'huggingface.co',
|
||||
'cdn-lfs.hf.co',
|
||||
'huggingface.co',
|
||||
'cdn-lfs.hf.co'
|
||||
])
|
||||
})
|
||||
|
||||
it('rejects undeclared redirect hosts without following them', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const transport = vi.fn<typeof fetch>(async () =>
|
||||
new Response(null, {
|
||||
status: 302,
|
||||
headers: {
|
||||
location: 'https://untrusted.example/model.onnx'
|
||||
}
|
||||
})
|
||||
)
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes),
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model')
|
||||
).rejects.toThrow('未声明的主机')
|
||||
expect(transport).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('does not request another source when selected coverage is missing', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const catalog = downloadableCatalog(modelBytes).map((entry) => ({
|
||||
...entry,
|
||||
repositoryUrls: {
|
||||
'hugging-face': entry.repositoryUrls['hugging-face']
|
||||
},
|
||||
files: entry.files.map((file) => ({
|
||||
...file,
|
||||
targets: {
|
||||
'hugging-face': file.targets['hugging-face']
|
||||
}
|
||||
}))
|
||||
}))
|
||||
const transport = vi.fn<typeof fetch>()
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog,
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model', 'modelscope')
|
||||
).rejects.toThrow('当前下载源')
|
||||
expect(transport).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('does not request another source after a download failure', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
const transport = vi.fn<typeof fetch>(
|
||||
async () => new Response(null, { status: 503 })
|
||||
)
|
||||
const manager = new SpeechModelManager({
|
||||
userDataDirectory: userData,
|
||||
catalog: downloadableCatalog(modelBytes),
|
||||
fetch: transport
|
||||
})
|
||||
|
||||
await expect(
|
||||
manager.install('download-test-model', 'modelscope')
|
||||
).rejects.toThrow('HTTP 503')
|
||||
expect(transport).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
new URL(String(transport.mock.calls[0]?.[0])).hostname
|
||||
).toBe('modelscope.cn')
|
||||
})
|
||||
|
||||
it('rejects bad digests without installing', async () => {
|
||||
const userData = await temporaryDirectory()
|
||||
const modelBytes = new TextEncoder().encode('expected')
|
||||
|
||||
@@ -16,22 +16,31 @@ import { z } from 'zod'
|
||||
import {
|
||||
installedSpeechModelSchema,
|
||||
speechModelCatalogEntrySchema,
|
||||
speechModelCatalogViewEntrySchema,
|
||||
speechModelIdSchema,
|
||||
speechModelSnapshotSchema,
|
||||
type InstalledSpeechModel,
|
||||
type SpeechModelCatalogEntry,
|
||||
type SpeechModelCatalogViewEntry,
|
||||
type SpeechModelFileSpec,
|
||||
type SpeechModelOperation,
|
||||
type SpeechModelSnapshot
|
||||
} from '../../shared/speech-model-contracts'
|
||||
import {
|
||||
MODEL_DOWNLOAD_SOURCES,
|
||||
getModelDownloadAvailability,
|
||||
resolveModelDownloadPackage,
|
||||
type ModelDownloadSource,
|
||||
type ResolvedModelArtifactFile
|
||||
} from '../../shared/model-download-contracts'
|
||||
import { SPEECH_MODEL_CATALOG } from './speech-model-catalog'
|
||||
import {
|
||||
exportModelArchive,
|
||||
extractModelArchive
|
||||
} from '../model-archive'
|
||||
import { fetchModelDownloadResponse } from '../model-download-transport'
|
||||
|
||||
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024
|
||||
const MAX_REDIRECTS = 3
|
||||
const MANIFEST_FILE_NAME = 'manifest.json'
|
||||
const SELECTION_FILE_NAME = '.selection.json'
|
||||
const PARTIAL_SUFFIX = '.partial'
|
||||
@@ -56,6 +65,9 @@ export type SpeechModelManagerOptions = {
|
||||
userDataDirectory: string
|
||||
fetch: typeof fetch
|
||||
catalog?: readonly SpeechModelCatalogEntry[]
|
||||
getDownloadSource?: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
maxFileBytes?: number
|
||||
}
|
||||
|
||||
@@ -72,6 +84,23 @@ function cloneCatalogEntry(
|
||||
return speechModelCatalogEntrySchema.parse(entry)
|
||||
}
|
||||
|
||||
function toCatalogView(entry: SpeechModelCatalogEntry) {
|
||||
const { repositoryUrls, files, ...metadata } = entry
|
||||
void repositoryUrls
|
||||
return speechModelCatalogViewEntrySchema.parse({
|
||||
...metadata,
|
||||
files: files.map((file) => ({
|
||||
name: file.name,
|
||||
role: file.role,
|
||||
size: file.size,
|
||||
sha256: file.sha256
|
||||
})),
|
||||
downloadAvailability: MODEL_DOWNLOAD_SOURCES.map((source) =>
|
||||
getModelDownloadAvailability(files, source)
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
function abortError(): DOMException {
|
||||
return new DOMException('The operation was aborted', 'AbortError')
|
||||
}
|
||||
@@ -102,17 +131,6 @@ function safeChild(parent: string, name: string): string {
|
||||
return child
|
||||
}
|
||||
|
||||
function validateDownloadUrl(value: string): URL {
|
||||
const url = new URL(value)
|
||||
if (
|
||||
url.protocol !== 'http:' &&
|
||||
url.protocol !== 'https:'
|
||||
) {
|
||||
throw new Error('模型下载地址必须使用 HTTP 或 HTTPS')
|
||||
}
|
||||
return url
|
||||
}
|
||||
|
||||
async function hashFile(
|
||||
path: string,
|
||||
signal?: AbortSignal
|
||||
@@ -147,6 +165,10 @@ export class SpeechModelManager {
|
||||
|
||||
private readonly transport: typeof fetch
|
||||
private readonly catalog: SpeechModelCatalogEntry[]
|
||||
private readonly catalogViews: SpeechModelCatalogViewEntry[]
|
||||
private readonly getDownloadSource: () =>
|
||||
| ModelDownloadSource
|
||||
| Promise<ModelDownloadSource>
|
||||
private readonly maxFileBytes: number
|
||||
private readonly operations = new Map<string, ActiveOperation>()
|
||||
|
||||
@@ -160,23 +182,31 @@ export class SpeechModelManager {
|
||||
'speech'
|
||||
)
|
||||
this.transport = options.fetch
|
||||
this.getDownloadSource =
|
||||
options.getDownloadSource ?? (() => 'modelscope')
|
||||
this.catalog = (options.catalog ?? SPEECH_MODEL_CATALOG).map(
|
||||
cloneCatalogEntry
|
||||
)
|
||||
if (new Set(this.catalog.map((entry) => entry.id)).size !== this.catalog.length) {
|
||||
throw new Error('语音模型目录包含重复 ID')
|
||||
}
|
||||
this.catalogViews = this.catalog.map(toCatalogView)
|
||||
this.maxFileBytes = validateMaximumBytes(options.maxFileBytes)
|
||||
}
|
||||
|
||||
async snapshot(): Promise<SpeechModelSnapshot> {
|
||||
await this.ensureRoot()
|
||||
const installed = await this.readInstalled()
|
||||
const selected = await this.readSelection()
|
||||
const [installed, selected, selectedDownloadSource] =
|
||||
await Promise.all([
|
||||
this.readInstalled(),
|
||||
this.readSelection(),
|
||||
this.getDownloadSource()
|
||||
])
|
||||
const installedIds = new Set(installed.map((model) => model.id))
|
||||
return speechModelSnapshotSchema.parse({
|
||||
rootDirectory: this.rootDirectory,
|
||||
catalog: this.catalog.map(cloneCatalogEntry),
|
||||
selectedDownloadSource,
|
||||
catalog: this.catalogViews,
|
||||
installed,
|
||||
operations: [...this.operations.values()].map((operation) => ({
|
||||
...operation.progress
|
||||
@@ -190,6 +220,19 @@ export class SpeechModelManager {
|
||||
return this.snapshot()
|
||||
}
|
||||
|
||||
getRepositoryUrl(
|
||||
modelId: string,
|
||||
source: ModelDownloadSource
|
||||
): string {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
resolveModelDownloadPackage(entry.files, source)
|
||||
const repositoryUrl = entry.repositoryUrls[source]
|
||||
if (!repositoryUrl) {
|
||||
throw new Error('当前下载源暂不提供此模型的仓库')
|
||||
}
|
||||
return repositoryUrl
|
||||
}
|
||||
|
||||
async getSelectedRuntimeModel(): Promise<
|
||||
SelectedSpeechRuntimeModel | undefined
|
||||
> {
|
||||
@@ -216,6 +259,7 @@ export class SpeechModelManager {
|
||||
|
||||
async install(
|
||||
modelId: string,
|
||||
downloadSource?: ModelDownloadSource,
|
||||
externalSignal?: AbortSignal
|
||||
): Promise<InstalledSpeechModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
@@ -224,27 +268,17 @@ export class SpeechModelManager {
|
||||
entry.manualReason ?? '该模型只能从本地目录导入'
|
||||
)
|
||||
}
|
||||
const downloadableFiles = entry.files.filter(
|
||||
(
|
||||
file
|
||||
): file is SpeechModelFileSpec & {
|
||||
download: NonNullable<SpeechModelFileSpec['download']>
|
||||
} => file.download !== undefined
|
||||
const selectedDownloadSource =
|
||||
downloadSource ?? (await this.getDownloadSource())
|
||||
const resolvedPackage = resolveModelDownloadPackage(
|
||||
entry.files,
|
||||
selectedDownloadSource
|
||||
)
|
||||
if (downloadableFiles.length !== entry.files.length) {
|
||||
throw new Error('模型下载元数据不完整')
|
||||
}
|
||||
const totalBytes = downloadableFiles.reduce(
|
||||
(total, file) => total + file.download.size,
|
||||
0
|
||||
)
|
||||
if (!Number.isSafeInteger(totalBytes)) {
|
||||
throw new RangeError('模型总大小超出安全范围')
|
||||
}
|
||||
const operation = this.beginOperation(
|
||||
entry.id,
|
||||
'download',
|
||||
totalBytes
|
||||
resolvedPackage.totalBytes,
|
||||
resolvedPackage.source
|
||||
)
|
||||
const detachExternalAbort = this.attachExternalSignal(
|
||||
externalSignal,
|
||||
@@ -255,7 +289,7 @@ export class SpeechModelManager {
|
||||
await this.ensureRoot()
|
||||
await this.assertNotInstalled(entry.id)
|
||||
stagingDirectory = await this.createStagingDirectory(entry.id)
|
||||
for (const file of downloadableFiles) {
|
||||
for (const file of resolvedPackage.files) {
|
||||
ensureNotAborted(operation.controller.signal)
|
||||
operation.progress.phase = 'transferring'
|
||||
operation.progress.currentFile = file.name
|
||||
@@ -409,9 +443,8 @@ export class SpeechModelManager {
|
||||
!recorded ||
|
||||
recorded.size <= 0 ||
|
||||
recorded.size > this.maxFileBytes ||
|
||||
(expected.download &&
|
||||
(recorded.size !== expected.download.size ||
|
||||
recorded.sha256 !== expected.download.sha256))
|
||||
recorded.size !== expected.size ||
|
||||
recorded.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(`语音模型文件不可导出:${expected.name}`)
|
||||
}
|
||||
@@ -440,8 +473,7 @@ export class SpeechModelManager {
|
||||
): Promise<InstalledSpeechModel> {
|
||||
const entry = this.requireCatalogEntry(modelId)
|
||||
const expectedTotal = entry.files.reduce(
|
||||
(total, file) =>
|
||||
total + (file.download?.size ?? this.maxFileBytes),
|
||||
(total, file) => total + file.size,
|
||||
0
|
||||
)
|
||||
const maximumTotalBytes = Math.min(
|
||||
@@ -488,9 +520,8 @@ export class SpeechModelManager {
|
||||
if (
|
||||
!archived ||
|
||||
archived.size > this.maxFileBytes ||
|
||||
(expected.download &&
|
||||
(archived.size !== expected.download.size ||
|
||||
archived.sha256 !== expected.download.sha256))
|
||||
archived.size !== expected.size ||
|
||||
archived.sha256 !== expected.sha256
|
||||
) {
|
||||
throw new Error(
|
||||
`语音模型 ZIP 与当前模型目录不匹配:${expected.name}`
|
||||
@@ -544,7 +575,8 @@ export class SpeechModelManager {
|
||||
private beginOperation(
|
||||
modelId: string,
|
||||
kind: SpeechModelOperation['kind'],
|
||||
totalBytes: number | null
|
||||
totalBytes: number | null,
|
||||
downloadSource?: ModelDownloadSource
|
||||
): ActiveOperation {
|
||||
if (this.operations.has(modelId)) {
|
||||
throw new Error('该模型已有进行中的操作')
|
||||
@@ -557,7 +589,8 @@ export class SpeechModelManager {
|
||||
phase: 'preparing',
|
||||
currentFile: null,
|
||||
completedBytes: 0,
|
||||
totalBytes
|
||||
totalBytes,
|
||||
...(downloadSource ? { downloadSource } : {})
|
||||
}
|
||||
}
|
||||
this.operations.set(modelId, operation)
|
||||
@@ -605,55 +638,25 @@ export class SpeechModelManager {
|
||||
return directory
|
||||
}
|
||||
|
||||
private async fetchFollowingRedirects(
|
||||
initialUrl: string,
|
||||
signal: AbortSignal
|
||||
): Promise<Response> {
|
||||
let url = validateDownloadUrl(initialUrl)
|
||||
for (let redirectCount = 0; ; redirectCount += 1) {
|
||||
ensureNotAborted(signal)
|
||||
const response = await this.transport(url, {
|
||||
method: 'GET',
|
||||
redirect: 'manual',
|
||||
credentials: 'omit',
|
||||
cache: 'no-store',
|
||||
signal
|
||||
})
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
if (redirectCount >= MAX_REDIRECTS) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error('模型下载重定向次数过多')
|
||||
}
|
||||
const location = response.headers.get('location')
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
if (!location) {
|
||||
throw new Error('模型下载重定向缺少地址')
|
||||
}
|
||||
url = validateDownloadUrl(new URL(location, url).toString())
|
||||
continue
|
||||
}
|
||||
return response
|
||||
}
|
||||
}
|
||||
|
||||
private async downloadFile(
|
||||
file: SpeechModelFileSpec & {
|
||||
download: NonNullable<SpeechModelFileSpec['download']>
|
||||
},
|
||||
file: ResolvedModelArtifactFile<SpeechModelFileSpec['role']>,
|
||||
destination: string,
|
||||
operation: ActiveOperation,
|
||||
signal: AbortSignal
|
||||
): Promise<void> {
|
||||
if (
|
||||
file.download.size > this.maxFileBytes ||
|
||||
file.download.size <= 0
|
||||
file.size > this.maxFileBytes ||
|
||||
file.size <= 0
|
||||
) {
|
||||
throw new RangeError(`模型文件大小超出限制:${file.name}`)
|
||||
}
|
||||
const response = await this.fetchFollowingRedirects(
|
||||
file.download.url,
|
||||
signal
|
||||
)
|
||||
const response = await fetchModelDownloadResponse({
|
||||
transport: this.transport,
|
||||
initialUrl: file.target.url,
|
||||
redirectHosts: file.target.redirectHosts,
|
||||
signal,
|
||||
modelLabel: '模型'
|
||||
})
|
||||
if (!response.ok) {
|
||||
await response.body?.cancel().catch(() => undefined)
|
||||
throw new Error(`模型下载失败:HTTP ${response.status}`)
|
||||
@@ -666,7 +669,7 @@ export class SpeechModelManager {
|
||||
const parsedLength = Number(declaredLength)
|
||||
if (
|
||||
!Number.isSafeInteger(parsedLength) ||
|
||||
parsedLength !== file.download.size
|
||||
parsedLength !== file.size
|
||||
) {
|
||||
await response.body.cancel().catch(() => undefined)
|
||||
throw new Error(`模型文件大小不匹配:${file.name}`)
|
||||
@@ -687,7 +690,7 @@ export class SpeechModelManager {
|
||||
}
|
||||
written += result.value.byteLength
|
||||
if (
|
||||
written > file.download.size ||
|
||||
written > file.size ||
|
||||
written > this.maxFileBytes
|
||||
) {
|
||||
await reader.cancel()
|
||||
@@ -703,10 +706,10 @@ export class SpeechModelManager {
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
if (written !== file.download.size) {
|
||||
if (written !== file.size) {
|
||||
throw new Error(`模型文件大小不匹配:${file.name}`)
|
||||
}
|
||||
if (hash.digest('hex') !== file.download.sha256) {
|
||||
if (hash.digest('hex') !== file.sha256) {
|
||||
throw new Error(`模型文件校验失败:${file.name}`)
|
||||
}
|
||||
await rename(partialPath, destination)
|
||||
@@ -741,10 +744,9 @@ export class SpeechModelManager {
|
||||
throw new RangeError(`模型文件大小无效:${expectedFile.name}`)
|
||||
}
|
||||
if (
|
||||
expectedFile.download &&
|
||||
(sourceFileInfo.size !== expectedFile.download.size ||
|
||||
(await hashFile(sourceFile, signal)).sha256 !==
|
||||
expectedFile.download.sha256)
|
||||
sourceFileInfo.size !== expectedFile.size ||
|
||||
(await hashFile(sourceFile, signal)).sha256 !==
|
||||
expectedFile.sha256
|
||||
) {
|
||||
throw new Error(`本地模型文件校验失败:${expectedFile.name}`)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user