feat: 检索页聚合搜索,修复中文关键词导致主进程崩溃

一次查询所有已启用数据源并按源分组展示,每组最多 5 条,
超出时提供「查看更多」跳转到该源的单源检索。

同时修复聚合并发暴露出的网络层问题:

- 主进程崩溃:Memory of the World 会把搜索关键词原样回写进
  ETag 响应头,中文关键词下 Electron net.fetch 在内部 emit
  回调里抛 ByteString TypeError,await 无法捕获,主进程直接
  崩溃且 Promise 永不 settle。现拦截该类异常并回退到 undici。
- OpenLibrary / LibGen 要求关键词至少 3 字符,否则返回 422。
  改为前置校验,直接返回空结果加说明,耗时从 14s 降到 3ms。
- 重试只针对可自愈错误(超时/5xx/限流),不再为 4xx 白等;
  重试时收紧超时,避免最坏耗时翻倍。镜像轮询类源关闭内层
  重试,换镜像交给 tryMirrors/raceMirrors。
- 网络错误文案改为可读提示,不再把整条 URL 抛给用户。
- Sci-Hub 非 DOI 关键词返回空而非报错,避免聚合结果刷屏。

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
lofyer
2026-07-26 11:59:40 +08:00
co-authored by factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent 1a1288ce18
commit de3e1d8a44
10 changed files with 484 additions and 72 deletions
+7 -4
View File
@@ -2,6 +2,9 @@ const { fetchJson, clampPage } = require('./http');
const BASE = 'https://gutendex.com/books';
// gutendex 偶发瞬时超时,fetchJson 默认已自动重试一次
const getJson = fetchJson;
function coverOf(formats) {
if (!formats) return '';
return formats['image/jpeg'] || formats['image/png'] || '';
@@ -49,7 +52,7 @@ module.exports = {
async list(page) {
page = clampPage(page);
const j = await fetchJson(`${BASE}?page=${page}`);
const j = await getJson(`${BASE}?page=${page}`);
const total = j.count || 0;
const maxPage = Math.max(1, Math.ceil(total / 32));
return { items: (j.results || []).map(toItem), maxPage, page };
@@ -57,14 +60,14 @@ module.exports = {
async search(keyword, page) {
page = clampPage(page);
const j = await fetchJson(`${BASE}?search=${encodeURIComponent(keyword)}&page=${page}`);
const j = await getJson(`${BASE}?search=${encodeURIComponent(keyword)}&page=${page}`);
const total = j.count || 0;
const maxPage = Math.max(1, Math.ceil(total / 32));
return { items: (j.results || []).map(toItem), maxPage, page };
},
async detail(postId) {
const b = await fetchJson(`${BASE}/${encodeURIComponent(postId)}`);
const b = await getJson(`${BASE}/${encodeURIComponent(postId)}`);
return {
postId: String(b.id),
title: b.title,
@@ -79,7 +82,7 @@ module.exports = {
},
async download(postId) {
const b = await fetchJson(`${BASE}/${encodeURIComponent(postId)}`);
const b = await getJson(`${BASE}/${encodeURIComponent(postId)}`);
const files = bookFiles(b.formats).map((f) => ({
name: `${String(b.title).replace(/[\\/:*?"<>|]/g, '_').slice(0, 60)}.${EXT[f.format] || 'bin'}`,
link: f.link,
+154 -15
View File
@@ -9,9 +9,8 @@ function setProxy(url) {
proxyUrl = String(url || '').trim();
dispatcher = null;
if (!proxyUrl) return;
// Electron 下由 session.setProxy 统一接管代理,不需要 undici dispatcher。
// 仅在纯 Node 环境(脚本/测试)才构建 ProxyAgent
if (process.versions.electron) return;
// Electron 下代理主要由 session.setProxy 接管;这里仍然构建 ProxyAgent,
// 供 net.fetch 不可用时的 undici 回退路径使用
try {
const { ProxyAgent } = require('undici');
dispatcher = new ProxyAgent({
@@ -25,18 +24,96 @@ function setProxy(url) {
function getProxy() { return proxyUrl; }
function fetchWithProxy(url, options = {}) {
// 在 Electron 主进程中优先使用 net.fetch(走 Chromium 网络栈,由 session.setProxy 控制代理)
if (process.versions.electron) {
try {
const { net } = require('electron');
return net.fetch(url, options);
} catch (e) { /* fallback */ }
// --- Electron net.fetch 响应头非 ASCII 崩溃的兜底 ---
// 部分站点(如 Memory of the World)会把搜索关键词原样回写进 ETag 等响应头。
// 关键词含中文时,Electron 的 net.fetch 在其内部 emit 回调里抛出
// "Cannot convert argument to a ByteString"。该异常无法被 await 捕获:
// 主进程直接崩溃,且对应的 fetch Promise 永远不会 settle。
//
// 兜底策略:拦截这一类 uncaughtException(其余异常原样交还默认处理),
// 记录 origin 后让仍在挂起中的同源请求转为可捕获的失败,再用 undici 重试。
// 加宽限期是因为异常与请求之间无法直接关联,宽限期内正常返回的请求不受影响。
const HEADER_BUG_GRACE = 1200;
// origin -> 该源已知会触发响应头崩溃(后续请求直接走 undici)
const headerBugOrigins = new Set();
// 正在进行中的 net.fetch: origin -> Set<abortFn>
const pendingByOrigin = new Map();
function originOf(url) {
try { return new URL(url).origin; } catch (e) { return ''; }
}
function isHeaderByteStringError(e) {
return !!e && (e instanceof TypeError || e.name === 'TypeError') && /ByteString/i.test(e.message || '');
}
let rethrowing = false;
function onUncaught(e) {
if (isHeaderByteStringError(e)) {
// 只有在宽限期后仍未返回的请求,才判定为被该异常卡死
const victims = [];
for (const set of pendingByOrigin.values()) victims.push(...set);
setTimeout(() => { for (const fail of victims) fail(); }, HEADER_BUG_GRACE);
return;
}
if (rethrowing) return;
rethrowing = true;
process.removeListener('uncaughtException', onUncaught);
setImmediate(() => { throw e; });
}
process.on('uncaughtException', onUncaught);
function undiciFetch(url, options) {
if (dispatcher) return fetch(url, { ...options, dispatcher });
return fetch(url, options);
}
function netFetch(url, options) {
const { net } = require('electron');
const origin = originOf(url);
return new Promise((resolve, reject) => {
let settled = false;
let set = pendingByOrigin.get(origin);
if (!set) { set = new Set(); pendingByOrigin.set(origin, set); }
const fail = () => {
if (settled) return;
settled = true;
cleanup();
headerBugOrigins.add(origin);
const err = new Error('响应头包含非 ASCII 字符,Electron 网络栈无法处理');
err.code = 'HEADER_BYTESTRING';
reject(err);
};
function cleanup() {
set.delete(fail);
if (!set.size) pendingByOrigin.delete(origin);
}
set.add(fail);
net.fetch(url, options).then(
(r) => { if (settled) return; settled = true; cleanup(); resolve(r); },
(e) => { if (settled) return; settled = true; cleanup(); reject(e); }
);
});
}
async function fetchWithProxy(url, options = {}) {
// Electron 主进程优先用 net.fetch(走 Chromium 网络栈,由 session.setProxy 控制代理)
if (process.versions.electron) {
// 已知有问题的源直接走 undici,避免每次都重新触发一遍崩溃
if (headerBugOrigins.has(originOf(url))) return undiciFetch(url, options);
try {
return await netFetch(url, options);
} catch (e) {
if (e && e.code === 'HEADER_BYTESTRING') return undiciFetch(url, options);
throw e;
}
}
return undiciFetch(url, options);
}
// 简易 cookie jar: Map<domain, Map<name, value>>
const cookieJar = new Map();
@@ -101,7 +178,12 @@ async function fetchRaw(url, options = {}) {
return res;
} catch (e) {
if (e && (e.name === 'AbortError' || /abort/i.test(e.message || ''))) {
throw new Error(`请求超时: ${url}`);
throw new Error('请求超时,站点无响应');
}
// undici / Chromium 的底层网络错误信息很不友好,统一换成可读文案
const msg = (e && e.message) || String(e);
if (/fetch failed|ERR_|ENOTFOUND|ECONNREFUSED|ECONNRESET|EAI_AGAIN|socket hang up/i.test(msg)) {
throw new Error('网络连接失败,请检查网络或代理设置');
}
throw e;
} finally {
@@ -109,18 +191,49 @@ async function fetchRaw(url, options = {}) {
}
}
async function fetchText(url, options = {}) {
const STATUS_HINT = {
403: '站点拒绝访问(403',
404: '资源不存在(404',
422: '查询条件不被接受(422',
429: '请求过于频繁,请稍后再试(429)',
500: '站点内部错误(500',
502: '站点网关错误(502',
503: '站点暂时不可用(503',
504: '站点网关超时(504'
};
async function fetchOnce(url, options) {
const res = await fetchRaw(url, options);
if (!res.ok) throw new Error(`请求失败: ${res.status} ${url}`);
if (!res.ok) throw new Error(STATUS_HINT[res.status] || `请求失败HTTP ${res.status}`);
return res.text();
}
// 默认对瞬时故障(超时/网络抖动/5xx/限流)自动重试一次。
// 镜像轮询类数据源(LibGen / Sci-Hub / Z-Library)应传 retries: 0
// 由上层的 tryMirrors / raceMirrors 负责换镜像,避免重试叠加放大耗时。
//
// 重试会把最坏耗时翻倍(超时 15s -> 30s),聚合搜索下体感很差。
// 因此重试时缩短单次超时:宁可放弃这次重试,也不要让整组结果一直转圈。
function fetchText(url, options = {}) {
const { retries, retryDelay, ...rest } = options;
const tries = (retries === undefined ? 1 : retries) + 1;
if (tries <= 1) return fetchOnce(url, rest);
const baseTimeout = rest.timeout === undefined ? DEFAULT_TIMEOUT : rest.timeout;
return withRetry(
(attempt) => fetchOnce(url, attempt === 0
? rest
: { ...rest, timeout: Math.max(3000, Math.round(baseTimeout / 2)) }),
{ tries, delay: retryDelay || 800 }
);
}
async function fetchJson(url, options = {}) {
const text = await fetchText(url, options);
try {
return JSON.parse(text);
} catch (e) {
throw new Error(`JSON 解析失败: ${url}`);
throw new Error('返回内容不是有效 JSON');
}
}
@@ -145,4 +258,30 @@ function clampPage(n, min) {
return n;
}
module.exports = { UA, fetchRaw, fetchText, fetchJson, decodeEntities, stripTags, clampPage, getCookies, setCookies, clearCookies, setProxy, getProxy, fetchWithProxy };
// 部分站点(OpenLibrary / LibGen)对过短的关键词直接返回 422。
// 与其把 HTTP 错误抛给用户,不如提前返回空结果并带上说明。
function tooShort(keyword, min = 3) {
const kw = String(keyword || '').trim();
if (kw.length >= min) return null;
return { items: [], maxPage: 1, page: 1, note: `关键词太短,该源要求至少 ${min} 个字符` };
}
// 可自愈的错误:超时、网络抖动、5xx、限流。4xx 属请求本身的问题,重试无意义。
function isRetryable(e) {
return /超时|网络连接失败|站点内部错误|站点网关|暂时不可用|过于频繁/.test((e && e.message) || '');
}
// 带退避的重试包装。仅在错误可自愈时重试,避免为 4xx 白等几秒。
// fn 会收到当前尝试序号(0 起),便于按次调整超时等参数。
async function withRetry(fn, { tries = 2, delay = 1000 } = {}) {
for (let i = 0; ; i++) {
try {
return await fn(i);
} catch (e) {
if (i >= tries - 1 || !isRetryable(e)) throw e;
await new Promise((r) => setTimeout(r, delay * (i + 1)));
}
}
}
module.exports = { UA, fetchRaw, fetchText, fetchJson, decodeEntities, stripTags, clampPage, tooShort, isRetryable, withRetry, getCookies, setCookies, clearCookies, setProxy, getProxy, fetchWithProxy };
+6 -4
View File
@@ -9,7 +9,7 @@
// 策略:优先用新版站点搜索(当前唯一可用);经典镜像作为兜底,
// 一旦恢复即可自动参与(raceMirrors 有 5 分钟冷却重试机制)。
const { fetchText, decodeEntities, clampPage } = require('./http');
const { fetchText, decodeEntities, clampPage, tooShort } = require('./http');
const { raceMirrors } = require('./mirror');
// 新版站点(当前可用)
@@ -166,7 +166,7 @@ async function webSearch(keyword, page) {
const kw = encodeURIComponent(keyword);
return raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
const url = `${base}/s/${kw}${page > 1 ? `?page=${page}` : ''}`;
const html = await fetchText(url, { timeout: TIMEOUT });
const html = await fetchText(url, { timeout: TIMEOUT, retries: 0 });
const items = parseWebResults(html, base);
if (!items.length && !/searchResultBox|resItemBox|Nothing found/i.test(html)) {
throw new Error('页面结构无法识别');
@@ -185,7 +185,7 @@ function buildDownloadLinks(md5) {
async function fetchBookPage(id) {
return raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
const html = await fetchText(`${base}/book/${id}`, { timeout: TIMEOUT });
const html = await fetchText(`${base}/book/${id}`, { timeout: TIMEOUT, retries: 0 });
return { html, base };
});
}
@@ -200,7 +200,7 @@ module.exports = {
// 新版站点有 /popular 榜单
try {
const r = await raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
const html = await fetchText(`${base}/popular`, { timeout: TIMEOUT });
const html = await fetchText(`${base}/popular`, { timeout: TIMEOUT, retries: 0 });
return { items: parseWebResults(html, base), base };
});
return { items: r.items, maxPage: 1, page: 1 };
@@ -213,6 +213,8 @@ module.exports = {
page = clampPage(page);
const q = String(keyword || '').trim();
if (!q) return { items: [], maxPage: 1, page };
const short = tooShort(q);
if (short) return { ...short, page };
let r;
try {
+5 -7
View File
@@ -1,14 +1,10 @@
const { fetchJson, clampPage } = require('./http');
const { fetchJson, clampPage, tooShort } = require('./http');
const BASE = 'https://openlibrary.org';
const PAGE_SIZE = 20;
async function fetchWithRetry(url, tries = 2) {
let last;
for (let i = 0; i < tries; i++) {
try { return await fetchJson(url); } catch (e) { last = e; await new Promise((r) => setTimeout(r, 1200)); }
}
throw last;
function fetchWithRetry(url) {
return fetchJson(url, { retryDelay: 1200 });
}
function coverOf(doc) {
@@ -44,6 +40,8 @@ module.exports = {
async search(keyword, page) {
page = clampPage(page);
const short = tooShort(keyword);
if (short) return { ...short, page };
const offset = (page - 1) * PAGE_SIZE;
const j = await fetchWithRetry(`${BASE}/search.json?q=${encodeURIComponent(keyword)}&limit=${PAGE_SIZE}&offset=${offset}&fields=${FIELDS}`);
const maxPage = Math.max(1, Math.ceil((j.numFound || 0) / PAGE_SIZE));
+4 -5
View File
@@ -66,7 +66,7 @@ function extractPdf(html, base) {
async function fetchSciHub(base, doi) {
const url = `${base}/${doi}`;
const html = await fetchText(url);
const html = await fetchText(url, { retries: 0 });
if (isChallenge(html)) {
throw new Error(`${base} 启用了人机验证`);
}
@@ -97,13 +97,12 @@ module.exports = {
return { items: [], maxPage: 1, page: 1 };
},
// 仅支持 DOI。非 DOI 关键词返回空而不是抛错,
// 这样在聚合搜索里不会被当成"失败的源"刷屏。
async search(keyword, page) {
page = clampPage(page);
const doi = normalizeDoi(keyword);
if (!doi) return { items: [], maxPage: 1, page: 1 };
if (!DOI_RE.test(doi)) {
throw new Error('Sci-Hub 仅支持 DOI 查询,例如 10.1038/nature12373');
}
if (!doi || !DOI_RE.test(doi)) return { items: [], maxPage: 1, page: 1, note: '仅支持 DOI 查询,例如 10.1038/nature12373' };
const r = await resolve(doi);
return {
items: [{
+3 -10
View File
@@ -4,16 +4,9 @@ const BASE = 'https://api.semanticscholar.org/graph/v1';
const PAGE_SIZE = 25;
const FIELDS = 'title,authors,year,abstract,openAccessPdf,externalIds,url,venue';
async function fetchWithRetry(url, tries = 3) {
let last;
for (let i = 0; i < tries; i++) {
try { return await fetchJson(url); } catch (e) {
last = e;
if (!/429/.test(e.message)) throw e;
await new Promise((r) => setTimeout(r, 2500 * (i + 1)));
}
}
throw last;
// 该 API 对匿名调用限流很严。只重试一次,避免在聚合搜索里长时间阻塞整组结果。
function fetchWithRetry(url) {
return fetchJson(url, { retryDelay: 1500 });
}
function toItem(p) {
+3 -1
View File
@@ -67,6 +67,7 @@ async function doLogin() {
const j = await fetchJson(apiUrl(m, '/eapi/user/login'), {
method: 'POST',
headers: FORM,
retries: 0,
body: form({ email: creds.email, password: creds.password })
});
if (!j || !j.success || !j.user) throw new Error(errMessage(j) || '登录失败');
@@ -88,10 +89,11 @@ async function callOn(mirror, path, params, session, method) {
j = await fetchJson(apiUrl(mirror, path), {
method: 'POST',
headers: FORM,
retries: 0,
body: form({ ...params, ...cred })
});
} else {
j = await fetchJson(apiUrl(mirror, path, { ...params, ...cred }));
j = await fetchJson(apiUrl(mirror, path, { ...params, ...cred }), { retries: 0 });
}
const msg = errMessage(j);
if (msg) {