perf: parse, chunk and hash documents in a worker thread and add startup and document-parse benchmarks

This commit is contained in:
mesalogo
2026-10-03 13:54:24 +08:00
parent 9d03217f99
commit 10c5162811
15 changed files with 1787 additions and 33 deletions
+3 -2
View File
@@ -1,7 +1,8 @@
import { lookup as dnsLookup } from 'node:dns/promises'
import { request as httpRequest } from 'node:http'
import { request as httpsRequest } from 'node:https'
import { parseDocument, type ParsedDocument } from './document-parser'
import type { ParsedDocument } from './document-parser'
import { parseDocumentOffMain } from '../document-parse-client'
type ResolvedAddress = {
address: string
@@ -224,7 +225,7 @@ export class UrlImporter {
.trim() || url.hostname
).slice(0, 240)
: url.pathname.split('/').filter(Boolean).at(-1) ?? url.hostname
const document = await parseDocument(
const document = await parseDocumentOffMain(
isHtml ? `${title}.html` : `${title}.txt`,
response.body
)