feat: enhance local knowledge retrieval

src/main/knowledge/retrieval-text.ts

@@ -0,0 +1,43 @@
+const hanPattern = /\p{Script=Han}/u
+const latinTokenPattern = /[\p{Letter}\p{Number}_.$/@-]+/gu
+export const maximumContextPrefixCharacters = 512
+
+export function contextualIndexText(
+  content: string,
+  contextPrefix?: unknown
+): string {
+  return `${
+    typeof contextPrefix === 'string'
+      ? contextPrefix.slice(0, maximumContextPrefixCharacters)
+      : ''
+  }${content}`
+}
+
+export function containsHanText(value: string): boolean {
+  return hanPattern.test(value)
+}
+
+export function knowledgeRetrievalTerms(
+  value: string,
+  maximumTerms = Number.POSITIVE_INFINITY
+): string[] {
+  const normalized = value.normalize('NFKC').trim().toLowerCase()
+  const tokens: string[] = [
+    ...(normalized.match(latinTokenPattern) ?? [])
+  ]
+  for (const run of normalized.match(/\p{Script=Han}+/gu) ?? []) {
+    const characters = [...run]
+    if (characters.length === 1) {
+      tokens.push(characters[0]!)
+      continue
+    }
+    for (let index = 0; index < characters.length - 1; index += 1) {
+      tokens.push(`${characters[index]}${characters[index + 1]}`)
+    }
+  }
+  return [...new Set(tokens)].slice(0, maximumTerms)
+}
+
+export function createCjkSearchText(value: string): string {
+  return knowledgeRetrievalTerms(value).join(' ')
+}