44 lines
1.2 KiB
TypeScript
44 lines
1.2 KiB
TypeScript
const hanPattern = /\p{Script=Han}/u
|
|
const latinTokenPattern = /[\p{Letter}\p{Number}_.$/@-]+/gu
|
|
export const maximumContextPrefixCharacters = 512
|
|
|
|
export function contextualIndexText(
|
|
content: string,
|
|
contextPrefix?: unknown
|
|
): string {
|
|
return `${
|
|
typeof contextPrefix === 'string'
|
|
? contextPrefix.slice(0, maximumContextPrefixCharacters)
|
|
: ''
|
|
}${content}`
|
|
}
|
|
|
|
export function containsHanText(value: string): boolean {
|
|
return hanPattern.test(value)
|
|
}
|
|
|
|
export function knowledgeRetrievalTerms(
|
|
value: string,
|
|
maximumTerms = Number.POSITIVE_INFINITY
|
|
): string[] {
|
|
const normalized = value.normalize('NFKC').trim().toLowerCase()
|
|
const tokens: string[] = [
|
|
...(normalized.match(latinTokenPattern) ?? [])
|
|
]
|
|
for (const run of normalized.match(/\p{Script=Han}+/gu) ?? []) {
|
|
const characters = [...run]
|
|
if (characters.length === 1) {
|
|
tokens.push(characters[0]!)
|
|
continue
|
|
}
|
|
for (let index = 0; index < characters.length - 1; index += 1) {
|
|
tokens.push(`${characters[index]}${characters[index + 1]}`)
|
|
}
|
|
}
|
|
return [...new Set(tokens)].slice(0, maximumTerms)
|
|
}
|
|
|
|
export function createCjkSearchText(value: string): string {
|
|
return knowledgeRetrievalTerms(value).join(' ')
|
|
}
|