feat: 发布 v2.1.0 开放书源扩展
构建与发布 / 单测与集成测试 (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (arm64, linux, ubuntu-24.04-arm) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (arm64, macos, macos-15) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (x64, linux, ubuntu-24.04) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (x64, windows, windows-2025) (push) Has been cancelled
构建与发布 / 发布 GitHub Release (push) Has been cancelled
构建与发布 / 单测与集成测试 (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (arm64, linux, ubuntu-24.04-arm) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (arm64, macos, macos-15) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (x64, linux, ubuntu-24.04) (push) Has been cancelled
构建与发布 / 打包 ${{ matrix.platform }} ${{ matrix.arch }} (x64, windows, windows-2025) (push) Has been cancelled
构建与发布 / 发布 GitHub Release (push) Has been cancelled
This commit is contained in:
@@ -22,7 +22,7 @@ AI 助手,可把选中文本、当前页、全文或框选区域作为上下
|
||||
|
||||
## 功能
|
||||
|
||||
- **多源检索**:12 个数据源统一的搜索、详情、下载流程
|
||||
- **多源检索**:16 个数据源统一的搜索、详情、下载流程
|
||||
- **本地书库**:收藏条目、下载文件、封面缓存、阅读状态管理
|
||||
- **任务中心**:全局查看下载进度,离开详情页后继续下载,支持暂停、断点续传和删除未完成任务
|
||||
- **内置阅读器**:PDF、EPUB、无 DRM 的 MOBI/KF7/KF8 与 TXT/Markdown 阅读,支持进度、书签、选文和笔记
|
||||
@@ -50,6 +50,10 @@ DRM 保护的 MOBI/AZW/AZW3、KFX、Topaz 以及损坏或不兼容的文件不
|
||||
| arXiv 论文 | `arxiv` | 预印本,支持全文检索 |
|
||||
| Gutenberg 公版书 | `gutenberg` | 公共领域图书 |
|
||||
| Open Library 图书 | `openlibrary` | 图书元数据与借阅入口 |
|
||||
| OpenStax 开放教材 | `openstax` | 开放许可教材与 PDF 全文 |
|
||||
| 开放教材图书馆 | `opentextbook` | 开放教材目录与授权获取入口 |
|
||||
| 中文维基文库 | `wikisource-zh` | 中文经典与公共领域作品 |
|
||||
| 英文维基文库 | `wikisource-en` | 英文经典与公共领域作品 |
|
||||
| DOAJ 开放期刊 | `doaj` | 开放获取期刊论文 |
|
||||
| PMC 生物医学 | `pmc` | PubMed Central 全文 |
|
||||
| bioRxiv 预印本 | `biorxiv` | 仅浏览最新列表,不支持关键词搜索 |
|
||||
@@ -70,7 +74,7 @@ DRM 保护的 MOBI/AZW/AZW3、KFX、Topaz 以及损坏或不兼容的文件不
|
||||
2. 双击目录中的 `PeopleLib.exe`。
|
||||
3. 保留整个程序目录,不要只移动 exe。用户数据默认保存在程序同级的 `data/`。
|
||||
|
||||
当前版本为 **2.0.0**。可在「设置」中手动检查更新,也可启用启动时自动检查。检测到新版本后,应用会打开对应的 GitHub Release 下载页,更新前请退出旧版本并覆盖程序文件,`data/` 目录无需替换。
|
||||
当前版本为 **2.1.0**。可在「设置」中手动检查更新,也可启用启动时自动检查。检测到新版本后,应用会打开对应的 GitHub Release 下载页,更新前请退出旧版本并覆盖程序文件,`data/` 目录无需替换。
|
||||
|
||||
## 开发
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ const { pathToFileURL } = require('url');
|
||||
const { Readable, Transform } = require('stream');
|
||||
const { pipeline } = require('stream/promises');
|
||||
|
||||
const DL_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
|
||||
const RELEASES_API = 'https://api.github.com/repos/lofyer/peoplelib/releases/latest';
|
||||
const RELEASES_PAGE = 'https://github.com/lofyer/peoplelib/releases';
|
||||
|
||||
@@ -115,7 +114,7 @@ const aiClient = require('./src/reader/ai-client');
|
||||
const aiSessions = require('./src/reader/ai-sessions');
|
||||
const aiImages = require('./src/reader/ai-images');
|
||||
const { normalizeVisualContexts } = require('./src/reader/visual-context');
|
||||
const { setProxy, getProxy, fetchWithProxy } = require('./src/sources/http');
|
||||
const { UA: DL_UA, setProxy, getProxy, fetchWithProxy } = require('./src/sources/http');
|
||||
zlibAuth.init(userDataDir, safeStorage);
|
||||
semanticKey.init(userDataDir, safeStorage);
|
||||
settings.init(userDataDir);
|
||||
|
||||
Generated
+2
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "peoplelib",
|
||||
"version": "2.0.0",
|
||||
"version": "2.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "peoplelib",
|
||||
"version": "2.0.0",
|
||||
"version": "2.1.0",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"foliate-js": "1.0.1",
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "peoplelib",
|
||||
"version": "2.0.0",
|
||||
"description": "开放获取文献与图书客户端(arXiv / Gutenberg / Open Library / DOAJ / PMC / bioRxiv / Standard Ebooks / Semantic Scholar / LibGen / Z-Library)",
|
||||
"version": "2.1.0",
|
||||
"description": "多源开放文献、电子书与本地书库客户端",
|
||||
"main": "main.js",
|
||||
"author": "peoplelib",
|
||||
"license": "MIT",
|
||||
|
||||
@@ -57,6 +57,19 @@ app.whenReady().then(async () => {
|
||||
const configReadyMs = Date.now() - startedAt;
|
||||
check('慢速书库扫描不会阻塞窗口配置加载', configReadyMs < 1200, `${configReadyMs}ms`);
|
||||
|
||||
const sourceRows = await waitUntil(() => win.webContents.executeJavaScript(`(() => {
|
||||
const rows = Array.from(document.querySelectorAll('#sourceList input[data-id]')).map((input) => ({
|
||||
id: input.dataset.id,
|
||||
checked: input.checked,
|
||||
name: input.parentElement.querySelector('span').textContent
|
||||
}));
|
||||
return rows.length >= 16 ? rows : null;
|
||||
})()`));
|
||||
const expectedSources = ['openstax', 'opentextbook', 'wikisource-zh', 'wikisource-en'];
|
||||
check('新增开放书源出现在设置页且新安装默认启用',
|
||||
expectedSources.every((id) => sourceRows.some((row) => row.id === id && row.checked)),
|
||||
sourceRows.filter((row) => expectedSources.includes(row.id)).map((row) => `${row.id}:${row.name}`).join(', '));
|
||||
|
||||
await waitUntil(() => scanStartedAt > 0, 7000);
|
||||
check('启动维护在首屏完成后延迟执行', scanStartedAt - startedAt >= 1400,
|
||||
`${scanStartedAt - startedAt}ms`);
|
||||
|
||||
@@ -70,6 +70,12 @@ test('下载校验协议,拒绝 file:// 等非 http(s)', () => {
|
||||
assert.ok(/仅允许打开 HTTP 或 HTTPS 链接/.test(mainSrc), 'openExternal 缺协议校验');
|
||||
});
|
||||
|
||||
test('下载请求使用可识别且带项目地址的 User-Agent', () => {
|
||||
assert.match(mainSrc, /UA:\s*DL_UA[\s\S]+require\('\.\/src\/sources\/http'\)/);
|
||||
const httpSrc = fs.readFileSync(path.join(__dirname, '..', 'sources', 'http.js'), 'utf8');
|
||||
assert.match(httpSrc, /PeopleLib\/2\.1\.0 \(\+https:\/\/github\.com\/lofyer\/peoplelib\)/);
|
||||
});
|
||||
|
||||
test('移除书籍默认保留阅读资料,仅显式勾选时清理', () => {
|
||||
const start = mainSrc.indexOf("ipcMain.handle('library:remove'");
|
||||
const end = mainSrc.indexOf('// 下载文件', start);
|
||||
|
||||
+236
-1
@@ -7,7 +7,8 @@ const sources = require('../sources');
|
||||
|
||||
test('注册表:每个源都实现完整接口', () => {
|
||||
const list = sources.listSources();
|
||||
assert.ok(list.length >= 12);
|
||||
assert.ok(list.length >= 16);
|
||||
assert.strictEqual(new Set(list.map((source) => source.id)).size, list.length, '数据源 ID 不能重复');
|
||||
for (const s of list) {
|
||||
const m = sources.getSource(s.id);
|
||||
for (const fn of ['list', 'search', 'detail', 'download']) {
|
||||
@@ -17,6 +18,13 @@ test('注册表:每个源都实现完整接口', () => {
|
||||
}
|
||||
});
|
||||
|
||||
test('注册表:开放教材与中英文维基文库已启用', () => {
|
||||
const ids = sources.listSources().map((source) => source.id);
|
||||
for (const id of ['openstax', 'opentextbook', 'wikisource-zh', 'wikisource-en']) {
|
||||
assert.ok(ids.includes(id), `缺少新数据源: ${id}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('注册表:未知 id 抛错', () => {
|
||||
assert.throws(() => sources.getSource('nope'), /未知数据源/);
|
||||
});
|
||||
@@ -275,6 +283,233 @@ test('arxiv: 解析 atom feed 并取 pdf 链接', async () => {
|
||||
assert.strictEqual(d.files[0].link, 'https://arxiv.org/pdf/2201.00978v1');
|
||||
});
|
||||
|
||||
// --- OpenStax ---
|
||||
|
||||
test('openstax: 只展示 live 教材并支持本地关键词分页', async () => {
|
||||
const openstax = h.freshRequire('sources/openstax.js');
|
||||
const books = Array.from({ length: 21 }, (_, i) => ({
|
||||
id: i + 1,
|
||||
slug: `books/book-${i + 1}`,
|
||||
book_state: 'live',
|
||||
title: i === 20 ? 'Advanced Calculus' : `Biology ${i + 1}`,
|
||||
subjects: i === 20 ? ['Math'] : ['Science'],
|
||||
subject_categories: []
|
||||
}));
|
||||
books.push({ id: 99, slug: 'books/draft', book_state: 'draft', title: 'Draft Calculus' });
|
||||
h.setHandler(h.routes([['/apps/cms/api/books', { body: { books } }]]));
|
||||
|
||||
const page2 = await openstax.list(2);
|
||||
assert.strictEqual(page2.items.length, 1);
|
||||
assert.strictEqual(page2.maxPage, 2);
|
||||
const found = await openstax.search('advanced math', 1);
|
||||
assert.deepStrictEqual(found.items.map((item) => item.postId), ['21']);
|
||||
assert.strictEqual(found.items[0].url, 'https://openstax.org/details/books/book-21');
|
||||
});
|
||||
|
||||
test('openstax: 详情解析作者、许可和日期', async () => {
|
||||
const openstax = h.freshRequire('sources/openstax.js');
|
||||
h.setHandler(h.routes([['/apps/cms/api/v2/pages/76/', {
|
||||
body: {
|
||||
id: 76,
|
||||
meta: { slug: 'calculus-volume-3', html_url: 'https://openstax.org/details/books/calculus-volume-3' },
|
||||
title: 'Calculus Volume 3',
|
||||
publish_date: '2016-03-30',
|
||||
authors: [{ value: { name: 'Gilbert Strang' } }, { name: 'Second Author' }],
|
||||
book_subjects: { subject_name: 'Math' },
|
||||
book_categories: [{ subject_name: 'Calculus' }],
|
||||
description: '<p>Open <b>calculus</b> textbook.</p>',
|
||||
license_name: 'Creative Commons Attribution-NonCommercial-ShareAlike License',
|
||||
license_version: '4.0',
|
||||
digital_isbn_13: '978-1-947172-16-6'
|
||||
}
|
||||
}]]));
|
||||
|
||||
const detail = await openstax.detail('76');
|
||||
assert.deepStrictEqual(detail.authors, ['Gilbert Strang', 'Second Author']);
|
||||
assert.strictEqual(detail.date, '2016-03-30');
|
||||
assert.strictEqual(detail.brief, 'Open calculus textbook.');
|
||||
assert.ok(detail.tags.includes('主题:Math'));
|
||||
assert.ok(detail.tags.some((tag) => tag.includes('4.0')));
|
||||
});
|
||||
|
||||
test('openstax: PDF 去重且非法 id 不发请求', async () => {
|
||||
const openstax = h.freshRequire('sources/openstax.js');
|
||||
h.resetCalls();
|
||||
h.setHandler(h.routes([['/apps/cms/api/v2/pages/76/', {
|
||||
body: {
|
||||
id: 76,
|
||||
meta: { slug: 'calculus-volume-3' },
|
||||
title: 'Calculus: Volume 3',
|
||||
pdf_url: 'https://assets.openstax.org/calculus.pdf',
|
||||
high_resolution_pdf_url: 'https://assets.openstax.org/calculus.pdf',
|
||||
license_url: 'https://creativecommons.org/licenses/by-nc-sa/4.0/'
|
||||
}
|
||||
}]]));
|
||||
|
||||
const download = await openstax.download('76');
|
||||
assert.strictEqual(download.files.length, 1);
|
||||
assert.strictEqual(download.files[0].name, 'Calculus_ Volume 3.pdf');
|
||||
await assert.rejects(openstax.detail('../76'), /无效的 OpenStax ID/);
|
||||
assert.strictEqual(h.getCalls().length, 1);
|
||||
});
|
||||
|
||||
// --- Open Textbook Library ---
|
||||
|
||||
test('opentextbook: 搜索结果解析作者和服务端分页', async () => {
|
||||
const opentextbook = h.freshRequire('sources/opentextbook.js');
|
||||
h.setHandler(h.routes([['textbooks.json?q=calculus&page=2', {
|
||||
body: {
|
||||
data: [{
|
||||
id: 10,
|
||||
title: 'Calculus',
|
||||
copyright_year: 2023,
|
||||
contributors: [
|
||||
{ first_name: 'Gilbert', last_name: 'Strang' },
|
||||
{ corporate: true, title: 'Open Education Team' }
|
||||
],
|
||||
url: 'https://open.umn.edu/opentextbooks/textbooks/calculus'
|
||||
}],
|
||||
links: { total_pages: 10, total_count: 98 }
|
||||
}
|
||||
}]]));
|
||||
|
||||
const result = await opentextbook.search('calculus', 2);
|
||||
assert.strictEqual(result.page, 2);
|
||||
assert.strictEqual(result.maxPage, 10);
|
||||
assert.strictEqual(result.items[0].subtitle, 'Gilbert Strang, Open Education Team');
|
||||
assert.strictEqual(result.items[0].date, '2023');
|
||||
});
|
||||
|
||||
test('opentextbook: 详情展开 data 并保留单书许可', async () => {
|
||||
const opentextbook = h.freshRequire('sources/opentextbook.js');
|
||||
h.setHandler(h.routes([['textbooks/10.json', {
|
||||
body: {
|
||||
data: {
|
||||
id: 10,
|
||||
title: 'Calculus',
|
||||
edition_statement: 'Third Edition',
|
||||
copyright_year: 1991,
|
||||
license: 'Attribution-NonCommercial-ShareAlike',
|
||||
language: 'eng',
|
||||
description: '<p>Free <b>calculus</b> textbook.</p>',
|
||||
contributors: [{ first_name: 'Gilbert', last_name: 'Strang' }],
|
||||
subjects: [{ name: 'Mathematics' }],
|
||||
url: 'https://open.umn.edu/opentextbooks/textbooks/calculus'
|
||||
}
|
||||
}
|
||||
}]]));
|
||||
|
||||
const detail = await opentextbook.detail('10');
|
||||
assert.deepStrictEqual(detail.authors, ['Gilbert Strang']);
|
||||
assert.strictEqual(detail.brief, 'Free calculus textbook.');
|
||||
assert.ok(detail.tags.includes('版本:Third Edition'));
|
||||
assert.ok(detail.tags.includes('许可:Attribution-NonCommercial-ShareAlike'));
|
||||
});
|
||||
|
||||
test('opentextbook: 只有真实文件 URL 才进入下载列表', async () => {
|
||||
const opentextbook = h.freshRequire('sources/opentextbook.js');
|
||||
h.resetCalls();
|
||||
h.setHandler(h.routes([['textbooks/10.json', {
|
||||
body: {
|
||||
data: {
|
||||
id: 10,
|
||||
title: 'Calculus: Third Edition',
|
||||
url: 'https://open.umn.edu/opentextbooks/textbooks/calculus',
|
||||
formats: [
|
||||
{ type: 'PDF', url: 'https://ocw.mit.edu/courses/calculus/open-textbook/' },
|
||||
{ type: 'PDF', url: 'https://cdn.example/calculus.pdf?download=1' },
|
||||
{ type: 'EPUB', url: 'https://cdn.example/calculus.epub' },
|
||||
{ type: 'EPUB', url: 'https://cdn.example/calculus.epub' }
|
||||
]
|
||||
}
|
||||
}
|
||||
}]]));
|
||||
|
||||
const download = await opentextbook.download('10');
|
||||
assert.deepStrictEqual(download.files.map((file) => file.format), ['PDF', 'EPUB']);
|
||||
assert.strictEqual(download.links[1].name, 'PDF 获取页');
|
||||
assert.strictEqual(download.files[0].name, 'Calculus_ Third Edition.pdf');
|
||||
await assert.rejects(opentextbook.download('10/../../x'), /无效的开放教材 ID/);
|
||||
assert.strictEqual(h.getCalls().length, 1);
|
||||
});
|
||||
|
||||
// --- Wikisource ---
|
||||
|
||||
test('wikisource: 搜索使用整数偏移并携带可识别 User-Agent', async () => {
|
||||
const wikisource = h.freshRequire('sources/wikisource-zh.js');
|
||||
let request = null;
|
||||
h.setHandler((url, options) => {
|
||||
request = { url, options };
|
||||
return h.makeResponse({
|
||||
body: {
|
||||
query: {
|
||||
searchinfo: { totalhits: 24753 },
|
||||
search: [{ pageid: 6, title: '論語' }]
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const result = await wikisource.search('論語', 2);
|
||||
assert.strictEqual(result.items[0].postId, '6');
|
||||
assert.strictEqual(result.items[0].subtitle, '中文');
|
||||
assert.strictEqual(result.maxPage, 500, 'MediaWiki 搜索最多允许偏移到 10000 条');
|
||||
assert.ok(request.url.includes('sroffset=20'), request.url);
|
||||
assert.match(request.options.headers['User-Agent'], /PeopleLib\/2\.1\.0/);
|
||||
});
|
||||
|
||||
test('wikisource: 浏览按 continuation 令牌翻页', async () => {
|
||||
const wikisource = h.freshRequire('sources/wikisource-en.js');
|
||||
const urls = [];
|
||||
h.setHandler((url) => {
|
||||
urls.push(url);
|
||||
if (url.includes('apcontinue=')) {
|
||||
return h.makeResponse({ body: { query: { allpages: [{ pageid: 2, title: 'Second Book' }] } } });
|
||||
}
|
||||
return h.makeResponse({
|
||||
body: {
|
||||
continue: { apcontinue: 'Second Book', continue: '-||' },
|
||||
query: { allpages: [{ pageid: 1, title: 'First Book' }] }
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
const first = await wikisource.list(1);
|
||||
const second = await wikisource.list(2);
|
||||
assert.strictEqual(first.maxPage, 2);
|
||||
assert.strictEqual(second.items[0].postId, '2');
|
||||
assert.ok(urls[1].includes('apcontinue=Second+Book'), urls[1]);
|
||||
});
|
||||
|
||||
test('wikisource: 详情和导出链接由服务端标题生成', async () => {
|
||||
const wikisource = h.freshRequire('sources/wikisource-zh.js');
|
||||
h.resetCalls();
|
||||
h.setHandler(h.routes([['pageids=6', {
|
||||
body: {
|
||||
query: {
|
||||
pages: [{
|
||||
pageid: 6,
|
||||
title: '論語/學而第一',
|
||||
extract: '<p>學而時習之。</p>',
|
||||
fullurl: 'https://zh.wikisource.org/wiki/%E8%AB%96%E8%AA%9E',
|
||||
thumbnail: { source: 'https://upload.wikimedia.org/cover.jpg' }
|
||||
}]
|
||||
}
|
||||
}
|
||||
}]]));
|
||||
|
||||
const detail = await wikisource.detail('6');
|
||||
assert.strictEqual(detail.brief, '學而時習之。');
|
||||
assert.strictEqual(detail.cover, 'https://upload.wikimedia.org/cover.jpg');
|
||||
const download = await wikisource.download('6');
|
||||
assert.deepStrictEqual(download.files.map((file) => file.format), ['EPUB', 'PDF']);
|
||||
assert.ok(download.files[0].link.includes('lang=zh'));
|
||||
assert.ok(download.files[0].link.includes('page=%E8%AB%96%E8%AA%9E%2F%E5%AD%B8%E8%80%8C%E7%AC%AC%E4%B8%80'));
|
||||
assert.strictEqual(download.files[0].name, '論語_學而第一.epub');
|
||||
await assert.rejects(wikisource.detail('../6'), /无效的中文维基文库 ID/);
|
||||
assert.strictEqual(h.getCalls().length, 2);
|
||||
});
|
||||
|
||||
// --- Z-Library ---
|
||||
|
||||
test('zlib: postId 缺 hash 时详情仍可用', async () => {
|
||||
|
||||
@@ -14,9 +14,7 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
const crypto = require('crypto');
|
||||
const atomic = require('../atomic-file');
|
||||
const { fetchWithProxy } = require('../sources/http');
|
||||
|
||||
const DL_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
|
||||
const { UA: DL_UA, fetchWithProxy } = require('../sources/http');
|
||||
|
||||
const SCHEMA_VERSION = 4;
|
||||
const MAX_TAGS = 50;
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
|
||||
const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36 PeopleLib/2.1.0 (+https://github.com/lofyer/peoplelib)';
|
||||
const { fetch: undiciFetch, ProxyAgent } = require('undici');
|
||||
|
||||
// 代理配置:默认直连(空字符串)。一旦设置,所有 HTTP 请求统一走该代理。
|
||||
|
||||
+22
-1
@@ -10,8 +10,29 @@ const libgen = require('./libgen');
|
||||
const zlib = require('./zlib');
|
||||
const scihub = require('./scihub');
|
||||
const motw = require('./motw');
|
||||
const openstax = require('./openstax');
|
||||
const opentextbook = require('./opentextbook');
|
||||
const wikisourceZh = require('./wikisource-zh');
|
||||
const wikisourceEn = require('./wikisource-en');
|
||||
|
||||
const sources = [arxiv, gutenberg, openlibrary, doaj, pmc, biorxiv, standardebooks, semanticscholar, libgen, zlib, scihub, motw];
|
||||
const sources = [
|
||||
arxiv,
|
||||
gutenberg,
|
||||
openlibrary,
|
||||
openstax,
|
||||
opentextbook,
|
||||
wikisourceZh,
|
||||
wikisourceEn,
|
||||
doaj,
|
||||
pmc,
|
||||
biorxiv,
|
||||
standardebooks,
|
||||
semanticscholar,
|
||||
libgen,
|
||||
zlib,
|
||||
scihub,
|
||||
motw
|
||||
];
|
||||
const byId = new Map(sources.map((s) => [s.id, s]));
|
||||
|
||||
function listSources() {
|
||||
|
||||
@@ -0,0 +1,180 @@
|
||||
const { fetchJson, clampPage, stripTags } = require('./http');
|
||||
|
||||
const BASE = 'https://openstax.org';
|
||||
const CATALOG_URL = `${BASE}/apps/cms/api/books`;
|
||||
const DETAIL_BASE = `${BASE}/apps/cms/api/v2/pages`;
|
||||
const PAGE_SIZE = 20;
|
||||
const CATALOG_TTL = 30 * 60 * 1000;
|
||||
|
||||
let catalogPromise = null;
|
||||
let catalogAt = 0;
|
||||
|
||||
function getJson(url) {
|
||||
return fetchJson(url);
|
||||
}
|
||||
|
||||
async function loadCatalog() {
|
||||
const now = Date.now();
|
||||
if (catalogPromise && now - catalogAt < CATALOG_TTL) return catalogPromise;
|
||||
const promise = getJson(CATALOG_URL).then((j) => {
|
||||
if (!j || !Array.isArray(j.books)) throw new Error('OpenStax 返回了无法识别的书目');
|
||||
return j.books.filter((b) => b && b.id && b.book_state === 'live');
|
||||
});
|
||||
catalogPromise = promise;
|
||||
catalogAt = now;
|
||||
try {
|
||||
return await promise;
|
||||
} catch (e) {
|
||||
if (catalogPromise === promise) {
|
||||
catalogPromise = null;
|
||||
catalogAt = 0;
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
function validateId(postId) {
|
||||
const id = String(postId || '');
|
||||
if (!/^\d+$/.test(id)) throw new Error('无效的 OpenStax ID');
|
||||
return id;
|
||||
}
|
||||
|
||||
function pageUrl(book) {
|
||||
const slug = String(book && book.slug || '').replace(/^\/+/, '');
|
||||
return slug ? `${BASE}/details/${slug}` : `${BASE}/subjects`;
|
||||
}
|
||||
|
||||
function toItem(book) {
|
||||
return {
|
||||
postId: String(book.id),
|
||||
title: book.title || '(无标题)',
|
||||
cover: book.cover_url || '',
|
||||
date: '',
|
||||
url: pageUrl(book),
|
||||
subtitle: (book.subjects || []).slice(0, 3).join(' · ')
|
||||
};
|
||||
}
|
||||
|
||||
function pack(books, page) {
|
||||
const start = (page - 1) * PAGE_SIZE;
|
||||
return {
|
||||
items: books.slice(start, start + PAGE_SIZE).map(toItem),
|
||||
maxPage: Math.max(1, Math.ceil(books.length / PAGE_SIZE)),
|
||||
page
|
||||
};
|
||||
}
|
||||
|
||||
function matches(book, keyword) {
|
||||
const tokens = String(keyword || '').trim().toLocaleLowerCase().split(/\s+/).filter(Boolean);
|
||||
if (!tokens.length) return true;
|
||||
const text = [
|
||||
book.title,
|
||||
...(book.subjects || []),
|
||||
...(book.subject_categories || [])
|
||||
].filter(Boolean).join(' ').toLocaleLowerCase();
|
||||
return tokens.every((token) => text.includes(token));
|
||||
}
|
||||
|
||||
function authorsOf(book) {
|
||||
return (book.authors || []).map((author) => {
|
||||
if (typeof author === 'string') return author;
|
||||
return author && author.value && author.value.name
|
||||
? author.value.name
|
||||
: (author && author.name) || '';
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
function subjectNames(value) {
|
||||
const items = Array.isArray(value) ? value : (value ? [value] : []);
|
||||
return items.map((item) => (
|
||||
typeof item === 'string' ? item : (item && (item.subject_name || item.name)) || ''
|
||||
)).filter(Boolean);
|
||||
}
|
||||
|
||||
async function bookDetail(postId) {
|
||||
const id = validateId(postId);
|
||||
const book = await getJson(`${DETAIL_BASE}/${id}/`);
|
||||
if (!book || !book.id) throw new Error('未找到该 OpenStax 教材');
|
||||
return book;
|
||||
}
|
||||
|
||||
function detailUrl(book) {
|
||||
return (book.meta && book.meta.html_url) || `${BASE}/details/books/${book.meta && book.meta.slug || book.id}`;
|
||||
}
|
||||
|
||||
function licenseLabel(book) {
|
||||
return [book.license_name, book.license_version].filter(Boolean).join(' ');
|
||||
}
|
||||
|
||||
function safeName(title) {
|
||||
return String(title || 'OpenStax 教材').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
id: 'openstax',
|
||||
name: 'OpenStax 开放教材',
|
||||
supportsSearch: true,
|
||||
|
||||
async list(page) {
|
||||
page = clampPage(page);
|
||||
return pack(await loadCatalog(), page);
|
||||
},
|
||||
|
||||
async search(keyword, page) {
|
||||
page = clampPage(page);
|
||||
const books = (await loadCatalog()).filter((book) => matches(book, keyword));
|
||||
return pack(books, page);
|
||||
},
|
||||
|
||||
async detail(postId) {
|
||||
const book = await bookDetail(postId);
|
||||
const subjects = [
|
||||
...subjectNames(book.book_subjects),
|
||||
...subjectNames(book.book_categories)
|
||||
];
|
||||
return {
|
||||
postId: String(book.id),
|
||||
title: book.title || '(无标题)',
|
||||
cover: book.cover_url || '',
|
||||
authors: authorsOf(book),
|
||||
date: String(book.publish_date || '').slice(0, 10),
|
||||
tags: [
|
||||
...subjects.slice(0, 4).map((subject) => `主题:${subject}`),
|
||||
licenseLabel(book) ? `许可:${licenseLabel(book)}` : '',
|
||||
book.digital_isbn_13 ? `ISBN:${book.digital_isbn_13}` : ''
|
||||
].filter(Boolean),
|
||||
brief: stripTags(book.description || ''),
|
||||
url: detailUrl(book),
|
||||
links: [
|
||||
{ name: 'OpenStax 页', url: detailUrl(book) },
|
||||
...(book.webview_link || book.webview_rex_link
|
||||
? [{ name: '在线阅读', url: book.webview_link || book.webview_rex_link }]
|
||||
: [])
|
||||
]
|
||||
};
|
||||
},
|
||||
|
||||
async download(postId) {
|
||||
const book = await bookDetail(postId);
|
||||
const files = [];
|
||||
const seen = new Set();
|
||||
for (const [url, label] of [
|
||||
[book.pdf_url, 'PDF'],
|
||||
[book.high_resolution_pdf_url, '高清 PDF']
|
||||
]) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
files.push({
|
||||
name: `${safeName(book.title)}${label === '高清 PDF' ? '-高清' : ''}.pdf`,
|
||||
link: url,
|
||||
format: 'PDF'
|
||||
});
|
||||
}
|
||||
const links = [{ name: 'OpenStax 页', url: detailUrl(book) }];
|
||||
if (book.webview_link || book.webview_rex_link) {
|
||||
links.push({ name: '在线阅读', url: book.webview_link || book.webview_rex_link });
|
||||
}
|
||||
if (book.license_url) links.push({ name: '许可说明', url: book.license_url });
|
||||
return { files, links };
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,147 @@
|
||||
const { fetchJson, clampPage, stripTags } = require('./http');
|
||||
|
||||
const BASE = 'https://open.umn.edu/opentextbooks';
|
||||
|
||||
function getJson(url) {
|
||||
return fetchJson(url);
|
||||
}
|
||||
|
||||
function validateId(postId) {
|
||||
const id = String(postId || '');
|
||||
if (!/^\d+$/.test(id)) throw new Error('无效的开放教材 ID');
|
||||
return id;
|
||||
}
|
||||
|
||||
function unwrapBook(value) {
|
||||
return value && value.data && !Array.isArray(value.data) ? value.data : value;
|
||||
}
|
||||
|
||||
function contributorsOf(book) {
|
||||
return (book.contributors || []).map((person) => {
|
||||
if (!person) return '';
|
||||
if (person.corporate) return person.title || person.name || '';
|
||||
return [person.first_name, person.middle_name, person.last_name].filter(Boolean).join(' ');
|
||||
}).filter(Boolean);
|
||||
}
|
||||
|
||||
function pageUrl(book) {
|
||||
return book.url || `${BASE}/textbooks/${book.id}`;
|
||||
}
|
||||
|
||||
function toItem(book) {
|
||||
return {
|
||||
postId: String(book.id),
|
||||
title: book.title || '(无标题)',
|
||||
cover: book.cover_url || book.cover || '',
|
||||
date: book.copyright_year ? String(book.copyright_year) : '',
|
||||
url: pageUrl(book),
|
||||
subtitle: contributorsOf(book).slice(0, 3).join(', ')
|
||||
};
|
||||
}
|
||||
|
||||
function pack(response, page) {
|
||||
const links = response && response.links || {};
|
||||
return {
|
||||
items: (response && response.data || []).map(toItem),
|
||||
maxPage: Math.max(1, Number(links.total_pages) || page),
|
||||
page
|
||||
};
|
||||
}
|
||||
|
||||
async function bookDetail(postId) {
|
||||
const id = validateId(postId);
|
||||
const book = unwrapBook(await getJson(`${BASE}/textbooks/${id}.json`));
|
||||
if (!book || !book.id) throw new Error('未找到该开放教材');
|
||||
return book;
|
||||
}
|
||||
|
||||
function formatLabel(value) {
|
||||
return String(value || '资源').trim().toUpperCase();
|
||||
}
|
||||
|
||||
function extensionOf(type) {
|
||||
const value = formatLabel(type);
|
||||
if (value.includes('EPUB')) return 'epub';
|
||||
if (value.includes('PDF')) return 'pdf';
|
||||
if (value.includes('MOBI') || value.includes('KINDLE')) return 'mobi';
|
||||
return '';
|
||||
}
|
||||
|
||||
function isDirectFile(format) {
|
||||
if (!format || !format.url) return false;
|
||||
const ext = extensionOf(format.type);
|
||||
if (!ext) return false;
|
||||
try {
|
||||
const url = new URL(format.url);
|
||||
return new RegExp(`\\.${ext}(?:$|[?#])`, 'i').test(url.pathname + url.search + url.hash);
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function safeName(title) {
|
||||
return String(title || '开放教材').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
id: 'opentextbook',
|
||||
name: '开放教材图书馆',
|
||||
supportsSearch: true,
|
||||
|
||||
async list(page) {
|
||||
page = clampPage(page);
|
||||
return pack(await getJson(`${BASE}/textbooks.json?page=${page}`), page);
|
||||
},
|
||||
|
||||
async search(keyword, page) {
|
||||
page = clampPage(page);
|
||||
const query = String(keyword || '').trim();
|
||||
const suffix = query ? `?q=${encodeURIComponent(query)}&page=${page}` : `?page=${page}`;
|
||||
return pack(await getJson(`${BASE}/textbooks.json${suffix}`), page);
|
||||
},
|
||||
|
||||
async detail(postId) {
|
||||
const book = await bookDetail(postId);
|
||||
return {
|
||||
postId: String(book.id),
|
||||
title: book.title || '(无标题)',
|
||||
cover: book.cover_url || book.cover || '',
|
||||
authors: contributorsOf(book),
|
||||
date: book.copyright_year ? String(book.copyright_year) : '',
|
||||
tags: [
|
||||
book.edition_statement ? `版本:${book.edition_statement}` : '',
|
||||
book.license ? `许可:${book.license}` : '',
|
||||
book.language ? `语言:${book.language}` : '',
|
||||
...(book.subjects || []).slice(0, 4).map((subject) => (
|
||||
subject && subject.name ? `主题:${subject.name}` : ''
|
||||
))
|
||||
].filter(Boolean),
|
||||
brief: stripTags(book.description || ''),
|
||||
url: pageUrl(book),
|
||||
links: [{ name: '开放教材图书馆页', url: pageUrl(book) }]
|
||||
};
|
||||
},
|
||||
|
||||
async download(postId) {
|
||||
const book = await bookDetail(postId);
|
||||
const files = [];
|
||||
const links = [{ name: '开放教材图书馆页', url: pageUrl(book) }];
|
||||
const seen = new Set();
|
||||
for (const format of book.formats || []) {
|
||||
if (!format || !format.url || seen.has(format.url)) continue;
|
||||
seen.add(format.url);
|
||||
const label = formatLabel(format.type);
|
||||
const ext = extensionOf(label);
|
||||
if (isDirectFile(format)) {
|
||||
files.push({
|
||||
name: `${safeName(book.title)}.${ext}`,
|
||||
link: format.url,
|
||||
format: label
|
||||
});
|
||||
} else {
|
||||
links.push({ name: `${label} 获取页`, url: format.url });
|
||||
}
|
||||
}
|
||||
return { files, links };
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,6 @@
|
||||
module.exports = require('./wikisource').create({
|
||||
id: 'wikisource-en',
|
||||
name: '英文维基文库',
|
||||
lang: 'en',
|
||||
label: '英文'
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
module.exports = require('./wikisource').create({
|
||||
id: 'wikisource-zh',
|
||||
name: '中文维基文库',
|
||||
lang: 'zh',
|
||||
label: '中文'
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
const { fetchJson, clampPage, stripTags } = require('./http');
|
||||
|
||||
const PAGE_SIZE = 20;
|
||||
|
||||
function create({ id, name, lang, label }) {
|
||||
const base = `https://${lang}.wikisource.org`;
|
||||
const pageTokens = new Map([[1, '']]);
|
||||
|
||||
function apiUrl(params) {
|
||||
const query = new URLSearchParams({
|
||||
...params,
|
||||
format: 'json',
|
||||
formatversion: '2'
|
||||
});
|
||||
return `${base}/w/api.php?${query}`;
|
||||
}
|
||||
|
||||
function getJson(params) {
|
||||
return fetchJson(apiUrl(params));
|
||||
}
|
||||
|
||||
function validateId(postId) {
|
||||
const idValue = String(postId || '');
|
||||
if (!/^\d+$/.test(idValue)) throw new Error(`无效的${label}维基文库 ID`);
|
||||
return idValue;
|
||||
}
|
||||
|
||||
function toItem(page) {
|
||||
return {
|
||||
postId: String(page.pageid),
|
||||
title: page.title || '(无标题)',
|
||||
cover: page.thumbnail && page.thumbnail.source || '',
|
||||
date: '',
|
||||
url: page.fullurl || `${base}/?curid=${page.pageid}`,
|
||||
subtitle: label
|
||||
};
|
||||
}
|
||||
|
||||
async function fetchListPage(page) {
|
||||
let nearest = 1;
|
||||
for (const known of pageTokens.keys()) {
|
||||
if (known <= page && known > nearest) nearest = known;
|
||||
}
|
||||
let token = pageTokens.get(nearest) || '';
|
||||
let result = null;
|
||||
for (let current = nearest; current <= page; current++) {
|
||||
const params = {
|
||||
action: 'query',
|
||||
list: 'allpages',
|
||||
apnamespace: '0',
|
||||
apfilterredir: 'nonredirects',
|
||||
aplimit: String(PAGE_SIZE)
|
||||
};
|
||||
if (token) params.apcontinue = token;
|
||||
result = await getJson(params);
|
||||
const next = result && result.continue && result.continue.apcontinue;
|
||||
if (next) pageTokens.set(current + 1, next);
|
||||
if (current === page || !next) break;
|
||||
token = next;
|
||||
}
|
||||
return result || { query: { allpages: [] } };
|
||||
}
|
||||
|
||||
async function pageDetail(postId) {
|
||||
const pageId = validateId(postId);
|
||||
const response = await getJson({
|
||||
action: 'query',
|
||||
prop: 'extracts|pageimages|info',
|
||||
pageids: pageId,
|
||||
exintro: '1',
|
||||
explaintext: '1',
|
||||
piprop: 'thumbnail',
|
||||
pithumbsize: '300',
|
||||
inprop: 'url'
|
||||
});
|
||||
const page = response && response.query && response.query.pages && response.query.pages[0];
|
||||
if (!page || page.missing) throw new Error(`未找到该${label}维基文库页面`);
|
||||
return page;
|
||||
}
|
||||
|
||||
function fileName(title) {
|
||||
return String(title || '维基文库作品').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
|
||||
}
|
||||
|
||||
function exportUrl(title, format) {
|
||||
const query = new URLSearchParams({ lang, page: title, format });
|
||||
return `https://ws-export.wmcloud.org/?${query}`;
|
||||
}
|
||||
|
||||
async function list(page) {
|
||||
page = clampPage(page);
|
||||
const response = await fetchListPage(page);
|
||||
const items = response && response.query && response.query.allpages || [];
|
||||
return {
|
||||
items: items.map(toItem),
|
||||
maxPage: response && response.continue ? page + 1 : page,
|
||||
page
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
id,
|
||||
name,
|
||||
supportsSearch: true,
|
||||
|
||||
list,
|
||||
|
||||
async search(keyword, page) {
|
||||
page = Math.min(clampPage(page), 500);
|
||||
const query = String(keyword || '').trim();
|
||||
if (!query) return list(page);
|
||||
const offset = (page - 1) * PAGE_SIZE;
|
||||
const response = await getJson({
|
||||
action: 'query',
|
||||
list: 'search',
|
||||
srsearch: query,
|
||||
srnamespace: '0',
|
||||
srlimit: String(PAGE_SIZE),
|
||||
sroffset: String(offset),
|
||||
srprop: 'size|wordcount|timestamp|snippet'
|
||||
});
|
||||
const search = response && response.query && response.query.search || [];
|
||||
const total = response && response.query && response.query.searchinfo
|
||||
? Number(response.query.searchinfo.totalhits) || 0
|
||||
: search.length;
|
||||
return {
|
||||
items: search.map(toItem),
|
||||
maxPage: Math.max(1, Math.ceil(Math.min(total, 10000) / PAGE_SIZE)),
|
||||
page
|
||||
};
|
||||
},
|
||||
|
||||
async detail(postId) {
|
||||
const page = await pageDetail(postId);
|
||||
return {
|
||||
postId: String(page.pageid),
|
||||
title: page.title || '(无标题)',
|
||||
cover: page.thumbnail && page.thumbnail.source || '',
|
||||
authors: [],
|
||||
date: '',
|
||||
tags: [`语言:${label}`, '许可:以原始页面标注为准'],
|
||||
brief: stripTags(page.extract || ''),
|
||||
url: page.fullurl || `${base}/?curid=${page.pageid}`,
|
||||
links: [{ name: `${label}维基文库页`, url: page.fullurl || `${base}/?curid=${page.pageid}` }]
|
||||
};
|
||||
},
|
||||
|
||||
async download(postId) {
|
||||
const page = await pageDetail(postId);
|
||||
const title = page.title || `wikisource-${page.pageid}`;
|
||||
const safeTitle = fileName(title);
|
||||
return {
|
||||
files: [
|
||||
{ name: `${safeTitle}.epub`, link: exportUrl(title, 'epub'), format: 'EPUB' },
|
||||
{ name: `${safeTitle}.pdf`, link: exportUrl(title, 'pdf'), format: 'PDF' }
|
||||
],
|
||||
links: [{
|
||||
name: `${label}维基文库页`,
|
||||
url: page.fullurl || `${base}/?curid=${page.pageid}`
|
||||
}]
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { create };
|
||||
Reference in New Issue
Block a user