feat: 检索页聚合搜索,修复中文关键词导致主进程崩溃
一次查询所有已启用数据源并按源分组展示,每组最多 5 条, 超出时提供「查看更多」跳转到该源的单源检索。 同时修复聚合并发暴露出的网络层问题: - 主进程崩溃:Memory of the World 会把搜索关键词原样回写进 ETag 响应头,中文关键词下 Electron net.fetch 在内部 emit 回调里抛 ByteString TypeError,await 无法捕获,主进程直接 崩溃且 Promise 永不 settle。现拦截该类异常并回退到 undici。 - OpenLibrary / LibGen 要求关键词至少 3 字符,否则返回 422。 改为前置校验,直接返回空结果加说明,耗时从 14s 降到 3ms。 - 重试只针对可自愈错误(超时/5xx/限流),不再为 4xx 白等; 重试时收紧超时,避免最坏耗时翻倍。镜像轮询类源关闭内层 重试,换镜像交给 tryMirrors/raceMirrors。 - 网络错误文案改为可读提示,不再把整条 URL 抛给用户。 - Sci-Hub 非 DOI 关键词返回空而非报错,避免聚合结果刷屏。 Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
parent
1a1288ce18
commit
de3e1d8a44
@@ -2,6 +2,9 @@ const { fetchJson, clampPage } = require('./http');
|
||||
|
||||
const BASE = 'https://gutendex.com/books';
|
||||
|
||||
// gutendex 偶发瞬时超时,fetchJson 默认已自动重试一次
|
||||
const getJson = fetchJson;
|
||||
|
||||
function coverOf(formats) {
|
||||
if (!formats) return '';
|
||||
return formats['image/jpeg'] || formats['image/png'] || '';
|
||||
@@ -49,7 +52,7 @@ module.exports = {
|
||||
|
||||
async list(page) {
|
||||
page = clampPage(page);
|
||||
const j = await fetchJson(`${BASE}?page=${page}`);
|
||||
const j = await getJson(`${BASE}?page=${page}`);
|
||||
const total = j.count || 0;
|
||||
const maxPage = Math.max(1, Math.ceil(total / 32));
|
||||
return { items: (j.results || []).map(toItem), maxPage, page };
|
||||
@@ -57,14 +60,14 @@ module.exports = {
|
||||
|
||||
async search(keyword, page) {
|
||||
page = clampPage(page);
|
||||
const j = await fetchJson(`${BASE}?search=${encodeURIComponent(keyword)}&page=${page}`);
|
||||
const j = await getJson(`${BASE}?search=${encodeURIComponent(keyword)}&page=${page}`);
|
||||
const total = j.count || 0;
|
||||
const maxPage = Math.max(1, Math.ceil(total / 32));
|
||||
return { items: (j.results || []).map(toItem), maxPage, page };
|
||||
},
|
||||
|
||||
async detail(postId) {
|
||||
const b = await fetchJson(`${BASE}/${encodeURIComponent(postId)}`);
|
||||
const b = await getJson(`${BASE}/${encodeURIComponent(postId)}`);
|
||||
return {
|
||||
postId: String(b.id),
|
||||
title: b.title,
|
||||
@@ -79,7 +82,7 @@ module.exports = {
|
||||
},
|
||||
|
||||
async download(postId) {
|
||||
const b = await fetchJson(`${BASE}/${encodeURIComponent(postId)}`);
|
||||
const b = await getJson(`${BASE}/${encodeURIComponent(postId)}`);
|
||||
const files = bookFiles(b.formats).map((f) => ({
|
||||
name: `${String(b.title).replace(/[\\/:*?"<>|]/g, '_').slice(0, 60)}.${EXT[f.format] || 'bin'}`,
|
||||
link: f.link,
|
||||
|
||||
+154
-15
@@ -9,9 +9,8 @@ function setProxy(url) {
|
||||
proxyUrl = String(url || '').trim();
|
||||
dispatcher = null;
|
||||
if (!proxyUrl) return;
|
||||
// Electron 下由 session.setProxy 统一接管代理,不需要 undici dispatcher。
|
||||
// 仅在纯 Node 环境(脚本/测试)才构建 ProxyAgent。
|
||||
if (process.versions.electron) return;
|
||||
// Electron 下代理主要由 session.setProxy 接管;这里仍然构建 ProxyAgent,
|
||||
// 供 net.fetch 不可用时的 undici 回退路径使用。
|
||||
try {
|
||||
const { ProxyAgent } = require('undici');
|
||||
dispatcher = new ProxyAgent({
|
||||
@@ -25,18 +24,96 @@ function setProxy(url) {
|
||||
|
||||
function getProxy() { return proxyUrl; }
|
||||
|
||||
function fetchWithProxy(url, options = {}) {
|
||||
// 在 Electron 主进程中优先使用 net.fetch(走 Chromium 网络栈,由 session.setProxy 控制代理)
|
||||
if (process.versions.electron) {
|
||||
try {
|
||||
const { net } = require('electron');
|
||||
return net.fetch(url, options);
|
||||
} catch (e) { /* fallback */ }
|
||||
// --- Electron net.fetch 响应头非 ASCII 崩溃的兜底 ---
|
||||
// 部分站点(如 Memory of the World)会把搜索关键词原样回写进 ETag 等响应头。
|
||||
// 关键词含中文时,Electron 的 net.fetch 在其内部 emit 回调里抛出
|
||||
// "Cannot convert argument to a ByteString"。该异常无法被 await 捕获:
|
||||
// 主进程直接崩溃,且对应的 fetch Promise 永远不会 settle。
|
||||
//
|
||||
// 兜底策略:拦截这一类 uncaughtException(其余异常原样交还默认处理),
|
||||
// 记录 origin 后让仍在挂起中的同源请求转为可捕获的失败,再用 undici 重试。
|
||||
// 加宽限期是因为异常与请求之间无法直接关联,宽限期内正常返回的请求不受影响。
|
||||
const HEADER_BUG_GRACE = 1200;
|
||||
|
||||
// origin -> 该源已知会触发响应头崩溃(后续请求直接走 undici)
|
||||
const headerBugOrigins = new Set();
|
||||
// 正在进行中的 net.fetch: origin -> Set<abortFn>
|
||||
const pendingByOrigin = new Map();
|
||||
|
||||
function originOf(url) {
|
||||
try { return new URL(url).origin; } catch (e) { return ''; }
|
||||
}
|
||||
|
||||
function isHeaderByteStringError(e) {
|
||||
return !!e && (e instanceof TypeError || e.name === 'TypeError') && /ByteString/i.test(e.message || '');
|
||||
}
|
||||
|
||||
let rethrowing = false;
|
||||
function onUncaught(e) {
|
||||
if (isHeaderByteStringError(e)) {
|
||||
// 只有在宽限期后仍未返回的请求,才判定为被该异常卡死
|
||||
const victims = [];
|
||||
for (const set of pendingByOrigin.values()) victims.push(...set);
|
||||
setTimeout(() => { for (const fail of victims) fail(); }, HEADER_BUG_GRACE);
|
||||
return;
|
||||
}
|
||||
if (rethrowing) return;
|
||||
rethrowing = true;
|
||||
process.removeListener('uncaughtException', onUncaught);
|
||||
setImmediate(() => { throw e; });
|
||||
}
|
||||
process.on('uncaughtException', onUncaught);
|
||||
|
||||
function undiciFetch(url, options) {
|
||||
if (dispatcher) return fetch(url, { ...options, dispatcher });
|
||||
return fetch(url, options);
|
||||
}
|
||||
|
||||
function netFetch(url, options) {
|
||||
const { net } = require('electron');
|
||||
const origin = originOf(url);
|
||||
return new Promise((resolve, reject) => {
|
||||
let settled = false;
|
||||
let set = pendingByOrigin.get(origin);
|
||||
if (!set) { set = new Set(); pendingByOrigin.set(origin, set); }
|
||||
|
||||
const fail = () => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
cleanup();
|
||||
headerBugOrigins.add(origin);
|
||||
const err = new Error('响应头包含非 ASCII 字符,Electron 网络栈无法处理');
|
||||
err.code = 'HEADER_BYTESTRING';
|
||||
reject(err);
|
||||
};
|
||||
function cleanup() {
|
||||
set.delete(fail);
|
||||
if (!set.size) pendingByOrigin.delete(origin);
|
||||
}
|
||||
set.add(fail);
|
||||
|
||||
net.fetch(url, options).then(
|
||||
(r) => { if (settled) return; settled = true; cleanup(); resolve(r); },
|
||||
(e) => { if (settled) return; settled = true; cleanup(); reject(e); }
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchWithProxy(url, options = {}) {
|
||||
// Electron 主进程优先用 net.fetch(走 Chromium 网络栈,由 session.setProxy 控制代理)
|
||||
if (process.versions.electron) {
|
||||
// 已知有问题的源直接走 undici,避免每次都重新触发一遍崩溃
|
||||
if (headerBugOrigins.has(originOf(url))) return undiciFetch(url, options);
|
||||
try {
|
||||
return await netFetch(url, options);
|
||||
} catch (e) {
|
||||
if (e && e.code === 'HEADER_BYTESTRING') return undiciFetch(url, options);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
return undiciFetch(url, options);
|
||||
}
|
||||
|
||||
// 简易 cookie jar: Map<domain, Map<name, value>>
|
||||
const cookieJar = new Map();
|
||||
|
||||
@@ -101,7 +178,12 @@ async function fetchRaw(url, options = {}) {
|
||||
return res;
|
||||
} catch (e) {
|
||||
if (e && (e.name === 'AbortError' || /abort/i.test(e.message || ''))) {
|
||||
throw new Error(`请求超时: ${url}`);
|
||||
throw new Error('请求超时,站点无响应');
|
||||
}
|
||||
// undici / Chromium 的底层网络错误信息很不友好,统一换成可读文案
|
||||
const msg = (e && e.message) || String(e);
|
||||
if (/fetch failed|ERR_|ENOTFOUND|ECONNREFUSED|ECONNRESET|EAI_AGAIN|socket hang up/i.test(msg)) {
|
||||
throw new Error('网络连接失败,请检查网络或代理设置');
|
||||
}
|
||||
throw e;
|
||||
} finally {
|
||||
@@ -109,18 +191,49 @@ async function fetchRaw(url, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchText(url, options = {}) {
|
||||
const STATUS_HINT = {
|
||||
403: '站点拒绝访问(403)',
|
||||
404: '资源不存在(404)',
|
||||
422: '查询条件不被接受(422)',
|
||||
429: '请求过于频繁,请稍后再试(429)',
|
||||
500: '站点内部错误(500)',
|
||||
502: '站点网关错误(502)',
|
||||
503: '站点暂时不可用(503)',
|
||||
504: '站点网关超时(504)'
|
||||
};
|
||||
|
||||
async function fetchOnce(url, options) {
|
||||
const res = await fetchRaw(url, options);
|
||||
if (!res.ok) throw new Error(`请求失败: ${res.status} ${url}`);
|
||||
if (!res.ok) throw new Error(STATUS_HINT[res.status] || `请求失败(HTTP ${res.status})`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
// 默认对瞬时故障(超时/网络抖动/5xx/限流)自动重试一次。
|
||||
// 镜像轮询类数据源(LibGen / Sci-Hub / Z-Library)应传 retries: 0,
|
||||
// 由上层的 tryMirrors / raceMirrors 负责换镜像,避免重试叠加放大耗时。
|
||||
//
|
||||
// 重试会把最坏耗时翻倍(超时 15s -> 30s),聚合搜索下体感很差。
|
||||
// 因此重试时缩短单次超时:宁可放弃这次重试,也不要让整组结果一直转圈。
|
||||
function fetchText(url, options = {}) {
|
||||
const { retries, retryDelay, ...rest } = options;
|
||||
const tries = (retries === undefined ? 1 : retries) + 1;
|
||||
if (tries <= 1) return fetchOnce(url, rest);
|
||||
|
||||
const baseTimeout = rest.timeout === undefined ? DEFAULT_TIMEOUT : rest.timeout;
|
||||
return withRetry(
|
||||
(attempt) => fetchOnce(url, attempt === 0
|
||||
? rest
|
||||
: { ...rest, timeout: Math.max(3000, Math.round(baseTimeout / 2)) }),
|
||||
{ tries, delay: retryDelay || 800 }
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchJson(url, options = {}) {
|
||||
const text = await fetchText(url, options);
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch (e) {
|
||||
throw new Error(`JSON 解析失败: ${url}`);
|
||||
throw new Error('返回内容不是有效 JSON');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -145,4 +258,30 @@ function clampPage(n, min) {
|
||||
return n;
|
||||
}
|
||||
|
||||
module.exports = { UA, fetchRaw, fetchText, fetchJson, decodeEntities, stripTags, clampPage, getCookies, setCookies, clearCookies, setProxy, getProxy, fetchWithProxy };
|
||||
// 部分站点(OpenLibrary / LibGen)对过短的关键词直接返回 422。
|
||||
// 与其把 HTTP 错误抛给用户,不如提前返回空结果并带上说明。
|
||||
function tooShort(keyword, min = 3) {
|
||||
const kw = String(keyword || '').trim();
|
||||
if (kw.length >= min) return null;
|
||||
return { items: [], maxPage: 1, page: 1, note: `关键词太短,该源要求至少 ${min} 个字符` };
|
||||
}
|
||||
|
||||
// 可自愈的错误:超时、网络抖动、5xx、限流。4xx 属请求本身的问题,重试无意义。
|
||||
function isRetryable(e) {
|
||||
return /超时|网络连接失败|站点内部错误|站点网关|暂时不可用|过于频繁/.test((e && e.message) || '');
|
||||
}
|
||||
|
||||
// 带退避的重试包装。仅在错误可自愈时重试,避免为 4xx 白等几秒。
|
||||
// fn 会收到当前尝试序号(0 起),便于按次调整超时等参数。
|
||||
async function withRetry(fn, { tries = 2, delay = 1000 } = {}) {
|
||||
for (let i = 0; ; i++) {
|
||||
try {
|
||||
return await fn(i);
|
||||
} catch (e) {
|
||||
if (i >= tries - 1 || !isRetryable(e)) throw e;
|
||||
await new Promise((r) => setTimeout(r, delay * (i + 1)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { UA, fetchRaw, fetchText, fetchJson, decodeEntities, stripTags, clampPage, tooShort, isRetryable, withRetry, getCookies, setCookies, clearCookies, setProxy, getProxy, fetchWithProxy };
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
// 策略:优先用新版站点搜索(当前唯一可用);经典镜像作为兜底,
|
||||
// 一旦恢复即可自动参与(raceMirrors 有 5 分钟冷却重试机制)。
|
||||
|
||||
const { fetchText, decodeEntities, clampPage } = require('./http');
|
||||
const { fetchText, decodeEntities, clampPage, tooShort } = require('./http');
|
||||
const { raceMirrors } = require('./mirror');
|
||||
|
||||
// 新版站点(当前可用)
|
||||
@@ -166,7 +166,7 @@ async function webSearch(keyword, page) {
|
||||
const kw = encodeURIComponent(keyword);
|
||||
return raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
|
||||
const url = `${base}/s/${kw}${page > 1 ? `?page=${page}` : ''}`;
|
||||
const html = await fetchText(url, { timeout: TIMEOUT });
|
||||
const html = await fetchText(url, { timeout: TIMEOUT, retries: 0 });
|
||||
const items = parseWebResults(html, base);
|
||||
if (!items.length && !/searchResultBox|resItemBox|Nothing found/i.test(html)) {
|
||||
throw new Error('页面结构无法识别');
|
||||
@@ -185,7 +185,7 @@ function buildDownloadLinks(md5) {
|
||||
|
||||
async function fetchBookPage(id) {
|
||||
return raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
|
||||
const html = await fetchText(`${base}/book/${id}`, { timeout: TIMEOUT });
|
||||
const html = await fetchText(`${base}/book/${id}`, { timeout: TIMEOUT, retries: 0 });
|
||||
return { html, base };
|
||||
});
|
||||
}
|
||||
@@ -200,7 +200,7 @@ module.exports = {
|
||||
// 新版站点有 /popular 榜单
|
||||
try {
|
||||
const r = await raceMirrors('libgen-web', WEB_MIRRORS, async (base) => {
|
||||
const html = await fetchText(`${base}/popular`, { timeout: TIMEOUT });
|
||||
const html = await fetchText(`${base}/popular`, { timeout: TIMEOUT, retries: 0 });
|
||||
return { items: parseWebResults(html, base), base };
|
||||
});
|
||||
return { items: r.items, maxPage: 1, page: 1 };
|
||||
@@ -213,6 +213,8 @@ module.exports = {
|
||||
page = clampPage(page);
|
||||
const q = String(keyword || '').trim();
|
||||
if (!q) return { items: [], maxPage: 1, page };
|
||||
const short = tooShort(q);
|
||||
if (short) return { ...short, page };
|
||||
|
||||
let r;
|
||||
try {
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
const { fetchJson, clampPage } = require('./http');
|
||||
const { fetchJson, clampPage, tooShort } = require('./http');
|
||||
|
||||
const BASE = 'https://openlibrary.org';
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
async function fetchWithRetry(url, tries = 2) {
|
||||
let last;
|
||||
for (let i = 0; i < tries; i++) {
|
||||
try { return await fetchJson(url); } catch (e) { last = e; await new Promise((r) => setTimeout(r, 1200)); }
|
||||
}
|
||||
throw last;
|
||||
function fetchWithRetry(url) {
|
||||
return fetchJson(url, { retryDelay: 1200 });
|
||||
}
|
||||
|
||||
function coverOf(doc) {
|
||||
@@ -44,6 +40,8 @@ module.exports = {
|
||||
|
||||
async search(keyword, page) {
|
||||
page = clampPage(page);
|
||||
const short = tooShort(keyword);
|
||||
if (short) return { ...short, page };
|
||||
const offset = (page - 1) * PAGE_SIZE;
|
||||
const j = await fetchWithRetry(`${BASE}/search.json?q=${encodeURIComponent(keyword)}&limit=${PAGE_SIZE}&offset=${offset}&fields=${FIELDS}`);
|
||||
const maxPage = Math.max(1, Math.ceil((j.numFound || 0) / PAGE_SIZE));
|
||||
|
||||
@@ -66,7 +66,7 @@ function extractPdf(html, base) {
|
||||
|
||||
async function fetchSciHub(base, doi) {
|
||||
const url = `${base}/${doi}`;
|
||||
const html = await fetchText(url);
|
||||
const html = await fetchText(url, { retries: 0 });
|
||||
if (isChallenge(html)) {
|
||||
throw new Error(`${base} 启用了人机验证`);
|
||||
}
|
||||
@@ -97,13 +97,12 @@ module.exports = {
|
||||
return { items: [], maxPage: 1, page: 1 };
|
||||
},
|
||||
|
||||
// 仅支持 DOI。非 DOI 关键词返回空而不是抛错,
|
||||
// 这样在聚合搜索里不会被当成"失败的源"刷屏。
|
||||
async search(keyword, page) {
|
||||
page = clampPage(page);
|
||||
const doi = normalizeDoi(keyword);
|
||||
if (!doi) return { items: [], maxPage: 1, page: 1 };
|
||||
if (!DOI_RE.test(doi)) {
|
||||
throw new Error('Sci-Hub 仅支持 DOI 查询,例如 10.1038/nature12373');
|
||||
}
|
||||
if (!doi || !DOI_RE.test(doi)) return { items: [], maxPage: 1, page: 1, note: '仅支持 DOI 查询,例如 10.1038/nature12373' };
|
||||
const r = await resolve(doi);
|
||||
return {
|
||||
items: [{
|
||||
|
||||
@@ -4,16 +4,9 @@ const BASE = 'https://api.semanticscholar.org/graph/v1';
|
||||
const PAGE_SIZE = 25;
|
||||
const FIELDS = 'title,authors,year,abstract,openAccessPdf,externalIds,url,venue';
|
||||
|
||||
async function fetchWithRetry(url, tries = 3) {
|
||||
let last;
|
||||
for (let i = 0; i < tries; i++) {
|
||||
try { return await fetchJson(url); } catch (e) {
|
||||
last = e;
|
||||
if (!/429/.test(e.message)) throw e;
|
||||
await new Promise((r) => setTimeout(r, 2500 * (i + 1)));
|
||||
}
|
||||
}
|
||||
throw last;
|
||||
// 该 API 对匿名调用限流很严。只重试一次,避免在聚合搜索里长时间阻塞整组结果。
|
||||
function fetchWithRetry(url) {
|
||||
return fetchJson(url, { retryDelay: 1500 });
|
||||
}
|
||||
|
||||
function toItem(p) {
|
||||
|
||||
+3
-1
@@ -67,6 +67,7 @@ async function doLogin() {
|
||||
const j = await fetchJson(apiUrl(m, '/eapi/user/login'), {
|
||||
method: 'POST',
|
||||
headers: FORM,
|
||||
retries: 0,
|
||||
body: form({ email: creds.email, password: creds.password })
|
||||
});
|
||||
if (!j || !j.success || !j.user) throw new Error(errMessage(j) || '登录失败');
|
||||
@@ -88,10 +89,11 @@ async function callOn(mirror, path, params, session, method) {
|
||||
j = await fetchJson(apiUrl(mirror, path), {
|
||||
method: 'POST',
|
||||
headers: FORM,
|
||||
retries: 0,
|
||||
body: form({ ...params, ...cred })
|
||||
});
|
||||
} else {
|
||||
j = await fetchJson(apiUrl(mirror, path, { ...params, ...cred }));
|
||||
j = await fetchJson(apiUrl(mirror, path, { ...params, ...cred }), { retries: 0 });
|
||||
}
|
||||
const msg = errMessage(j);
|
||||
if (msg) {
|
||||
|
||||
Reference in New Issue
Block a user