feat: add document OCR and offline model archives

This commit is contained in:
lofyer
2026-08-11 16:49:51 +08:00
parent 19a4469561
commit e0e7bc573c
46 changed files with 7831 additions and 85 deletions
@@ -373,6 +373,47 @@ describe('SpeechModelManager downloads', () => {
operations: []
})
})
it('round-trips a verified model through an offline ZIP archive', async () => {
const userData = await temporaryDirectory()
const modelBytes = new TextEncoder().encode('verified model bytes')
const tokenBytes = new TextEncoder().encode('verified tokens')
const catalog = downloadableCatalog(modelBytes, tokenBytes)
const manager = new SpeechModelManager({
userDataDirectory: userData,
catalog,
fetch: vi.fn<typeof fetch>(async (input) => {
const bytes = String(input).endsWith('model.onnx')
? modelBytes
: tokenBytes
return new Response(bytes, {
headers: { 'content-length': String(bytes.byteLength) }
})
})
})
const archive = join(userData, 'speech-model.zip')
await manager.install('download-test-model')
await manager.exportArchive('download-test-model', archive)
await manager.remove('download-test-model')
await expect(
manager.importArchive('download-test-model', archive)
).resolves.toMatchObject({
id: 'download-test-model',
source: 'local',
files: [
{
name: 'model.onnx',
sha256: sha256(modelBytes)
},
{
name: 'tokens.txt',
sha256: sha256(tokenBytes)
}
]
})
})
})
describe('SpeechModelManager local import', () => {
+143
View File
@@ -25,12 +25,18 @@ import {
type SpeechModelSnapshot
} from '../../shared/speech-model-contracts'
import { SPEECH_MODEL_CATALOG } from './speech-model-catalog'
import {
exportModelArchive,
extractModelArchive
} from '../model-archive'
const DEFAULT_MAX_FILE_BYTES = 2 * 1024 * 1024 * 1024
const MAX_REDIRECTS = 3
const MANIFEST_FILE_NAME = 'manifest.json'
const SELECTION_FILE_NAME = '.selection.json'
const PARTIAL_SUFFIX = '.partial'
const MAXIMUM_ARCHIVE_BYTES = 4 * 1024 * 1024 * 1024 - 1
const ARCHIVE_OVERHEAD_BYTES = 1024 * 1024
const selectionSchema = z
.object({
@@ -380,6 +386,143 @@ export class SpeechModelManager {
}
}
async exportArchive(
modelId: string,
destinationPath: string
): Promise<void> {
const entry = this.requireCatalogEntry(modelId)
await this.ensureRoot()
const installed = (await this.readInstalled()).find(
(model) => model.id === entry.id
)
if (!installed) {
throw new Error('只能导出已安装的语音模型')
}
const directory = this.modelDirectory(entry.id)
const files = []
for (const expected of entry.files) {
const recorded = installed.files.find(
(file) =>
file.name === expected.name && file.role === expected.role
)
if (
!recorded ||
recorded.size <= 0 ||
recorded.size > this.maxFileBytes ||
(expected.download &&
(recorded.size !== expected.download.size ||
recorded.sha256 !== expected.download.sha256))
) {
throw new Error(`语音模型文件不可导出:${expected.name}`)
}
files.push({
name: expected.name,
role: expected.role,
size: recorded.size,
sha256: recorded.sha256
})
}
await exportModelArchive({
destinationPath,
sourceDirectory: directory,
descriptor: {
kind: 'speech',
modelId: entry.id,
displayName: entry.displayName,
files
}
})
}
async importArchive(
modelId: string,
archivePath: string
): Promise<InstalledSpeechModel> {
const entry = this.requireCatalogEntry(modelId)
const expectedTotal = entry.files.reduce(
(total, file) =>
total + (file.download?.size ?? this.maxFileBytes),
0
)
const maximumTotalBytes = Math.min(
MAXIMUM_ARCHIVE_BYTES,
expectedTotal + ARCHIVE_OVERHEAD_BYTES
)
const operation = this.beginOperation(
entry.id,
'import',
expectedTotal
)
let stagingDirectory: string | undefined
try {
await this.ensureRoot()
await this.assertNotInstalled(entry.id)
stagingDirectory = await this.createStagingDirectory(entry.id)
operation.progress.phase = 'transferring'
const descriptor = await extractModelArchive({
archivePath,
destinationDirectory: stagingDirectory,
expectedKind: 'speech',
expectedModelId: entry.id,
expectedFiles: entry.files.map((file) => ({
name: file.name,
role: file.role
})),
maximumArchiveBytes: Math.min(
MAXIMUM_ARCHIVE_BYTES,
maximumTotalBytes + ARCHIVE_OVERHEAD_BYTES
),
maximumFileBytes: this.maxFileBytes,
maximumTotalBytes,
signal: operation.controller.signal,
onProgress: (completedBytes) => {
operation.progress.completedBytes = completedBytes
}
})
for (const expected of entry.files) {
const archived = descriptor.files.find(
(file) =>
file.name === expected.name &&
file.role === expected.role
)
if (
!archived ||
archived.size > this.maxFileBytes ||
(expected.download &&
(archived.size !== expected.download.size ||
archived.sha256 !== expected.download.sha256))
) {
throw new Error(
`语音模型 ZIP 与当前模型目录不匹配:${expected.name}`
)
}
}
operation.progress.phase = 'installing'
operation.progress.currentFile = null
const installed = installedSpeechModelSchema.parse({
id: entry.id,
displayName: entry.displayName,
source: 'local',
installedAt: new Date().toISOString(),
files: descriptor.files
})
await writeFile(
safeChild(stagingDirectory, MANIFEST_FILE_NAME),
`${JSON.stringify(installed, null, 2)}\n`,
{ encoding: 'utf8', flag: 'wx' }
)
ensureNotAborted(operation.controller.signal)
await rename(stagingDirectory, this.modelDirectory(entry.id))
stagingDirectory = undefined
return installed
} finally {
this.operations.delete(entry.id)
if (stagingDirectory) {
await rm(stagingDirectory, { recursive: true, force: true })
}
}
}
private async ensureRoot(): Promise<void> {
await mkdir(this.rootDirectory, { recursive: true })
}