feat: 发布 v2.1.0 开放书源扩展

src/sources/http.js

@@ -1,4 +1,4 @@
-const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36';
+const UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0 Safari/537.36 PeopleLib/2.1.0 (+https://github.com/lofyer/peoplelib)';
 const { fetch: undiciFetch, ProxyAgent } = require('undici');
 
 // 代理配置：默认直连（空字符串）。一旦设置，所有 HTTP 请求统一走该代理。

src/sources/index.js

@@ -10,8 +10,29 @@ const libgen = require('./libgen');
 const zlib = require('./zlib');
 const scihub = require('./scihub');
 const motw = require('./motw');
+const openstax = require('./openstax');
+const opentextbook = require('./opentextbook');
+const wikisourceZh = require('./wikisource-zh');
+const wikisourceEn = require('./wikisource-en');
 
-const sources = [arxiv, gutenberg, openlibrary, doaj, pmc, biorxiv, standardebooks, semanticscholar, libgen, zlib, scihub, motw];
+const sources = [
+  arxiv,
+  gutenberg,
+  openlibrary,
+  openstax,
+  opentextbook,
+  wikisourceZh,
+  wikisourceEn,
+  doaj,
+  pmc,
+  biorxiv,
+  standardebooks,
+  semanticscholar,
+  libgen,
+  zlib,
+  scihub,
+  motw
+];
 const byId = new Map(sources.map((s) => [s.id, s]));
 
 function listSources() {

src/sources/openstax.js

@@ -0,0 +1,180 @@
+const { fetchJson, clampPage, stripTags } = require('./http');
+
+const BASE = 'https://openstax.org';
+const CATALOG_URL = `${BASE}/apps/cms/api/books`;
+const DETAIL_BASE = `${BASE}/apps/cms/api/v2/pages`;
+const PAGE_SIZE = 20;
+const CATALOG_TTL = 30 * 60 * 1000;
+
+let catalogPromise = null;
+let catalogAt = 0;
+
+function getJson(url) {
+  return fetchJson(url);
+}
+
+async function loadCatalog() {
+  const now = Date.now();
+  if (catalogPromise && now - catalogAt < CATALOG_TTL) return catalogPromise;
+  const promise = getJson(CATALOG_URL).then((j) => {
+    if (!j || !Array.isArray(j.books)) throw new Error('OpenStax 返回了无法识别的书目');
+    return j.books.filter((b) => b && b.id && b.book_state === 'live');
+  });
+  catalogPromise = promise;
+  catalogAt = now;
+  try {
+    return await promise;
+  } catch (e) {
+    if (catalogPromise === promise) {
+      catalogPromise = null;
+      catalogAt = 0;
+    }
+    throw e;
+  }
+}
+
+function validateId(postId) {
+  const id = String(postId || '');
+  if (!/^\d+$/.test(id)) throw new Error('无效的 OpenStax ID');
+  return id;
+}
+
+function pageUrl(book) {
+  const slug = String(book && book.slug || '').replace(/^\/+/, '');
+  return slug ? `${BASE}/details/${slug}` : `${BASE}/subjects`;
+}
+
+function toItem(book) {
+  return {
+    postId: String(book.id),
+    title: book.title || '(无标题)',
+    cover: book.cover_url || '',
+    date: '',
+    url: pageUrl(book),
+    subtitle: (book.subjects || []).slice(0, 3).join(' · ')
+  };
+}
+
+function pack(books, page) {
+  const start = (page - 1) * PAGE_SIZE;
+  return {
+    items: books.slice(start, start + PAGE_SIZE).map(toItem),
+    maxPage: Math.max(1, Math.ceil(books.length / PAGE_SIZE)),
+    page
+  };
+}
+
+function matches(book, keyword) {
+  const tokens = String(keyword || '').trim().toLocaleLowerCase().split(/\s+/).filter(Boolean);
+  if (!tokens.length) return true;
+  const text = [
+    book.title,
+    ...(book.subjects || []),
+    ...(book.subject_categories || [])
+  ].filter(Boolean).join(' ').toLocaleLowerCase();
+  return tokens.every((token) => text.includes(token));
+}
+
+function authorsOf(book) {
+  return (book.authors || []).map((author) => {
+    if (typeof author === 'string') return author;
+    return author && author.value && author.value.name
+      ? author.value.name
+      : (author && author.name) || '';
+  }).filter(Boolean);
+}
+
+function subjectNames(value) {
+  const items = Array.isArray(value) ? value : (value ? [value] : []);
+  return items.map((item) => (
+    typeof item === 'string' ? item : (item && (item.subject_name || item.name)) || ''
+  )).filter(Boolean);
+}
+
+async function bookDetail(postId) {
+  const id = validateId(postId);
+  const book = await getJson(`${DETAIL_BASE}/${id}/`);
+  if (!book || !book.id) throw new Error('未找到该 OpenStax 教材');
+  return book;
+}
+
+function detailUrl(book) {
+  return (book.meta && book.meta.html_url) || `${BASE}/details/books/${book.meta && book.meta.slug || book.id}`;
+}
+
+function licenseLabel(book) {
+  return [book.license_name, book.license_version].filter(Boolean).join(' ');
+}
+
+function safeName(title) {
+  return String(title || 'OpenStax 教材').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
+}
+
+module.exports = {
+  id: 'openstax',
+  name: 'OpenStax 开放教材',
+  supportsSearch: true,
+
+  async list(page) {
+    page = clampPage(page);
+    return pack(await loadCatalog(), page);
+  },
+
+  async search(keyword, page) {
+    page = clampPage(page);
+    const books = (await loadCatalog()).filter((book) => matches(book, keyword));
+    return pack(books, page);
+  },
+
+  async detail(postId) {
+    const book = await bookDetail(postId);
+    const subjects = [
+      ...subjectNames(book.book_subjects),
+      ...subjectNames(book.book_categories)
+    ];
+    return {
+      postId: String(book.id),
+      title: book.title || '(无标题)',
+      cover: book.cover_url || '',
+      authors: authorsOf(book),
+      date: String(book.publish_date || '').slice(0, 10),
+      tags: [
+        ...subjects.slice(0, 4).map((subject) => `主题：${subject}`),
+        licenseLabel(book) ? `许可：${licenseLabel(book)}` : '',
+        book.digital_isbn_13 ? `ISBN：${book.digital_isbn_13}` : ''
+      ].filter(Boolean),
+      brief: stripTags(book.description || ''),
+      url: detailUrl(book),
+      links: [
+        { name: 'OpenStax 页', url: detailUrl(book) },
+        ...(book.webview_link || book.webview_rex_link
+          ? [{ name: '在线阅读', url: book.webview_link || book.webview_rex_link }]
+          : [])
+      ]
+    };
+  },
+
+  async download(postId) {
+    const book = await bookDetail(postId);
+    const files = [];
+    const seen = new Set();
+    for (const [url, label] of [
+      [book.pdf_url, 'PDF'],
+      [book.high_resolution_pdf_url, '高清 PDF']
+    ]) {
+      if (!url || seen.has(url)) continue;
+      seen.add(url);
+      files.push({
+        name: `${safeName(book.title)}${label === '高清 PDF' ? '-高清' : ''}.pdf`,
+        link: url,
+        format: 'PDF'
+      });
+    }
+    const links = [{ name: 'OpenStax 页', url: detailUrl(book) }];
+    if (book.webview_link || book.webview_rex_link) {
+      links.push({ name: '在线阅读', url: book.webview_link || book.webview_rex_link });
+    }
+    if (book.license_url) links.push({ name: '许可说明', url: book.license_url });
+    return { files, links };
+  }
+};

src/sources/opentextbook.js

@@ -0,0 +1,147 @@
+const { fetchJson, clampPage, stripTags } = require('./http');
+
+const BASE = 'https://open.umn.edu/opentextbooks';
+
+function getJson(url) {
+  return fetchJson(url);
+}
+
+function validateId(postId) {
+  const id = String(postId || '');
+  if (!/^\d+$/.test(id)) throw new Error('无效的开放教材 ID');
+  return id;
+}
+
+function unwrapBook(value) {
+  return value && value.data && !Array.isArray(value.data) ? value.data : value;
+}
+
+function contributorsOf(book) {
+  return (book.contributors || []).map((person) => {
+    if (!person) return '';
+    if (person.corporate) return person.title || person.name || '';
+    return [person.first_name, person.middle_name, person.last_name].filter(Boolean).join(' ');
+  }).filter(Boolean);
+}
+
+function pageUrl(book) {
+  return book.url || `${BASE}/textbooks/${book.id}`;
+}
+
+function toItem(book) {
+  return {
+    postId: String(book.id),
+    title: book.title || '(无标题)',
+    cover: book.cover_url || book.cover || '',
+    date: book.copyright_year ? String(book.copyright_year) : '',
+    url: pageUrl(book),
+    subtitle: contributorsOf(book).slice(0, 3).join(', ')
+  };
+}
+
+function pack(response, page) {
+  const links = response && response.links || {};
+  return {
+    items: (response && response.data || []).map(toItem),
+    maxPage: Math.max(1, Number(links.total_pages) || page),
+    page
+  };
+}
+
+async function bookDetail(postId) {
+  const id = validateId(postId);
+  const book = unwrapBook(await getJson(`${BASE}/textbooks/${id}.json`));
+  if (!book || !book.id) throw new Error('未找到该开放教材');
+  return book;
+}
+
+function formatLabel(value) {
+  return String(value || '资源').trim().toUpperCase();
+}
+
+function extensionOf(type) {
+  const value = formatLabel(type);
+  if (value.includes('EPUB')) return 'epub';
+  if (value.includes('PDF')) return 'pdf';
+  if (value.includes('MOBI') || value.includes('KINDLE')) return 'mobi';
+  return '';
+}
+
+function isDirectFile(format) {
+  if (!format || !format.url) return false;
+  const ext = extensionOf(format.type);
+  if (!ext) return false;
+  try {
+    const url = new URL(format.url);
+    return new RegExp(`\\.${ext}(?:$|[?#])`, 'i').test(url.pathname + url.search + url.hash);
+  } catch (e) {
+    return false;
+  }
+}
+
+function safeName(title) {
+  return String(title || '开放教材').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
+}
+
+module.exports = {
+  id: 'opentextbook',
+  name: '开放教材图书馆',
+  supportsSearch: true,
+
+  async list(page) {
+    page = clampPage(page);
+    return pack(await getJson(`${BASE}/textbooks.json?page=${page}`), page);
+  },
+
+  async search(keyword, page) {
+    page = clampPage(page);
+    const query = String(keyword || '').trim();
+    const suffix = query ? `?q=${encodeURIComponent(query)}&page=${page}` : `?page=${page}`;
+    return pack(await getJson(`${BASE}/textbooks.json${suffix}`), page);
+  },
+
+  async detail(postId) {
+    const book = await bookDetail(postId);
+    return {
+      postId: String(book.id),
+      title: book.title || '(无标题)',
+      cover: book.cover_url || book.cover || '',
+      authors: contributorsOf(book),
+      date: book.copyright_year ? String(book.copyright_year) : '',
+      tags: [
+        book.edition_statement ? `版本：${book.edition_statement}` : '',
+        book.license ? `许可：${book.license}` : '',
+        book.language ? `语言：${book.language}` : '',
+        ...(book.subjects || []).slice(0, 4).map((subject) => (
+          subject && subject.name ? `主题：${subject.name}` : ''
+        ))
+      ].filter(Boolean),
+      brief: stripTags(book.description || ''),
+      url: pageUrl(book),
+      links: [{ name: '开放教材图书馆页', url: pageUrl(book) }]
+    };
+  },
+
+  async download(postId) {
+    const book = await bookDetail(postId);
+    const files = [];
+    const links = [{ name: '开放教材图书馆页', url: pageUrl(book) }];
+    const seen = new Set();
+    for (const format of book.formats || []) {
+      if (!format || !format.url || seen.has(format.url)) continue;
+      seen.add(format.url);
+      const label = formatLabel(format.type);
+      const ext = extensionOf(label);
+      if (isDirectFile(format)) {
+        files.push({
+          name: `${safeName(book.title)}.${ext}`,
+          link: format.url,
+          format: label
+        });
+      } else {
+        links.push({ name: `${label} 获取页`, url: format.url });
+      }
+    }
+    return { files, links };
+  }
+};

src/sources/wikisource-en.js

@@ -0,0 +1,6 @@
+module.exports = require('./wikisource').create({
+  id: 'wikisource-en',
+  name: '英文维基文库',
+  lang: 'en',
+  label: '英文'
+});

src/sources/wikisource-zh.js

@@ -0,0 +1,6 @@
+module.exports = require('./wikisource').create({
+  id: 'wikisource-zh',
+  name: '中文维基文库',
+  lang: 'zh',
+  label: '中文'
+});

src/sources/wikisource.js

@@ -0,0 +1,166 @@
+const { fetchJson, clampPage, stripTags } = require('./http');
+
+const PAGE_SIZE = 20;
+
+function create({ id, name, lang, label }) {
+  const base = `https://${lang}.wikisource.org`;
+  const pageTokens = new Map([[1, '']]);
+
+  function apiUrl(params) {
+    const query = new URLSearchParams({
+      ...params,
+      format: 'json',
+      formatversion: '2'
+    });
+    return `${base}/w/api.php?${query}`;
+  }
+
+  function getJson(params) {
+    return fetchJson(apiUrl(params));
+  }
+
+  function validateId(postId) {
+    const idValue = String(postId || '');
+    if (!/^\d+$/.test(idValue)) throw new Error(`无效的${label}维基文库 ID`);
+    return idValue;
+  }
+
+  function toItem(page) {
+    return {
+      postId: String(page.pageid),
+      title: page.title || '(无标题)',
+      cover: page.thumbnail && page.thumbnail.source || '',
+      date: '',
+      url: page.fullurl || `${base}/?curid=${page.pageid}`,
+      subtitle: label
+    };
+  }
+
+  async function fetchListPage(page) {
+    let nearest = 1;
+    for (const known of pageTokens.keys()) {
+      if (known <= page && known > nearest) nearest = known;
+    }
+    let token = pageTokens.get(nearest) || '';
+    let result = null;
+    for (let current = nearest; current <= page; current++) {
+      const params = {
+        action: 'query',
+        list: 'allpages',
+        apnamespace: '0',
+        apfilterredir: 'nonredirects',
+        aplimit: String(PAGE_SIZE)
+      };
+      if (token) params.apcontinue = token;
+      result = await getJson(params);
+      const next = result && result.continue && result.continue.apcontinue;
+      if (next) pageTokens.set(current + 1, next);
+      if (current === page || !next) break;
+      token = next;
+    }
+    return result || { query: { allpages: [] } };
+  }
+
+  async function pageDetail(postId) {
+    const pageId = validateId(postId);
+    const response = await getJson({
+      action: 'query',
+      prop: 'extracts|pageimages|info',
+      pageids: pageId,
+      exintro: '1',
+      explaintext: '1',
+      piprop: 'thumbnail',
+      pithumbsize: '300',
+      inprop: 'url'
+    });
+    const page = response && response.query && response.query.pages && response.query.pages[0];
+    if (!page || page.missing) throw new Error(`未找到该${label}维基文库页面`);
+    return page;
+  }
+
+  function fileName(title) {
+    return String(title || '维基文库作品').replace(/[\\/:*?"<>|]/g, '_').slice(0, 80);
+  }
+
+  function exportUrl(title, format) {
+    const query = new URLSearchParams({ lang, page: title, format });
+    return `https://ws-export.wmcloud.org/?${query}`;
+  }
+
+  async function list(page) {
+    page = clampPage(page);
+    const response = await fetchListPage(page);
+    const items = response && response.query && response.query.allpages || [];
+    return {
+      items: items.map(toItem),
+      maxPage: response && response.continue ? page + 1 : page,
+      page
+    };
+  }
+
+  return {
+    id,
+    name,
+    supportsSearch: true,
+
+    list,
+
+    async search(keyword, page) {
+      page = Math.min(clampPage(page), 500);
+      const query = String(keyword || '').trim();
+      if (!query) return list(page);
+      const offset = (page - 1) * PAGE_SIZE;
+      const response = await getJson({
+        action: 'query',
+        list: 'search',
+        srsearch: query,
+        srnamespace: '0',
+        srlimit: String(PAGE_SIZE),
+        sroffset: String(offset),
+        srprop: 'size|wordcount|timestamp|snippet'
+      });
+      const search = response && response.query && response.query.search || [];
+      const total = response && response.query && response.query.searchinfo
+        ? Number(response.query.searchinfo.totalhits) || 0
+        : search.length;
+      return {
+        items: search.map(toItem),
+        maxPage: Math.max(1, Math.ceil(Math.min(total, 10000) / PAGE_SIZE)),
+        page
+      };
+    },
+
+    async detail(postId) {
+      const page = await pageDetail(postId);
+      return {
+        postId: String(page.pageid),
+        title: page.title || '(无标题)',
+        cover: page.thumbnail && page.thumbnail.source || '',
+        authors: [],
+        date: '',
+        tags: [`语言：${label}`, '许可：以原始页面标注为准'],
+        brief: stripTags(page.extract || ''),
+        url: page.fullurl || `${base}/?curid=${page.pageid}`,
+        links: [{ name: `${label}维基文库页`, url: page.fullurl || `${base}/?curid=${page.pageid}` }]
+      };
+    },
+
+    async download(postId) {
+      const page = await pageDetail(postId);
+      const title = page.title || `wikisource-${page.pageid}`;
+      const safeTitle = fileName(title);
+      return {
+        files: [
+          { name: `${safeTitle}.epub`, link: exportUrl(title, 'epub'), format: 'EPUB' },
+          { name: `${safeTitle}.pdf`, link: exportUrl(title, 'pdf'), format: 'PDF' }
+        ],
+        links: [{
+          name: `${label}维基文库页`,
+          url: page.fullurl || `${base}/?curid=${page.pageid}`
+        }]
+      };
+    }
+  };
+}
+
+module.exports = { create };