From 07afe18fdb2a02d40af8096e6c3435fc7e5e81c6 Mon Sep 17 00:00:00 2001 From: HCLonely Date: Mon, 23 May 2022 18:02:50 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=BD=BF=E7=94=A8bgm?= =?UTF-8?q?=E6=BA=90=E6=97=B6=E7=95=AA=E5=89=A7=E6=98=BE=E7=A4=BA=E9=87=8D?= =?UTF-8?q?=E5=A4=8D=E7=9A=84BUG;=E4=BF=AE=E5=A4=8D=E4=BD=BF=E7=94=A8bgm?= =?UTF-8?q?=E6=BA=90=E6=97=B6=E8=8E=B7=E5=8F=96=E7=95=AA=E5=89=A7=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E4=B8=8D=E5=85=A8=E7=9A=84BUG;=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E4=BD=BF=E7=94=A8bgm=E6=BA=90=E6=97=B6=E7=95=AA=E5=89=A7?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E8=8E=B7=E5=8F=96=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 1 - lib/get-bgm-data.js | 323 +++++++++++++++++++++++----------------- package.json | 2 +- src/lib/get-bgm-data.js | 56 ++++--- 4 files changed, 220 insertions(+), 162 deletions(-) diff --git a/README.md b/README.md index a89a7aa..2b16043 100644 --- a/README.md +++ b/README.md @@ -88,7 +88,6 @@ cinema: # 追剧设置 登录[Bangumi](https://bangumi.tv/)后打开控制台(`Ctrl`+`Shift`+`J`)，输入`CHOBITS_UID`回车，下面会输出`id` - ## 示例 ![示例图片](https://github.com/HCLonely/hexo-bilibili-bangumi/raw/master/example.png) diff --git a/lib/get-bgm-data.js b/lib/get-bgm-data.js index a9b778e..ffe8f3d 100644 --- a/lib/get-bgm-data.js +++ b/lib/get-bgm-data.js @@ -29,8 +29,10 @@ var tunnel = require('tunnel'); var bangumiData = require('bangumi-data'); var getItemsId = /*#__PURE__*/function () { - var _ref = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee(vmid, status, showProgress) { - var items, bar, response, $, pageNum, _loop, i; + var _ref = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee(vmid, status, showProgress, sourceDir, proxy) { + var _response$request$pat; + + var items, bar, response, username, $, pageNum, _loop, i; return _regenerator["default"].wrap(function _callee$(_context2) { while (1) { @@ -42,21 +44,44 @@ var getItemsId = /*#__PURE__*/function () { case 3: response = _context2.sent; + username = (_response$request$pat = response.request.path.match(/anime\/list\/(.*?)\//)) === null || _response$request$pat === void 0 ? void 0 : _response$request$pat[1]; + + if (username) { + _context2.next = 7; + break; + } + + return _context2.abrupt("return", console.error('Failed to get "username"!')); + case 7: if (!(response !== null && response !== void 0 && response.data)) { - _context2.next = 19; + _context2.next = 28; break; } $ = cheerio.load(response.data); - pageNum = $('#multipage').find('a').length; - items = $('#browserItemList>li').map(function (index, el) { + pageNum = Math.max.apply(Math, (0, _toConsumableArray2["default"])($('#multipage').find('a').map(function (index, el) { + var _$$attr, _$$attr$match; + + return parseInt(((_$$attr = $(el).attr('href')) === null || _$$attr === void 0 ? void 0 : (_$$attr$match = _$$attr.match(/\?page=([\d]+)/)) === null || _$$attr$match === void 0 ? void 0 : _$$attr$match[1]) || '0', 10); + }).get())) || $('#multipage').find('a').length; + _context2.t0 = items.push; + _context2.t1 = items; + _context2.t2 = _toConsumableArray2["default"]; + _context2.next = 15; + return getBangumiData($('#browserItemList>li').map(function (index, el) { return { id: $(el).attr('id').replace('item_', ''), cover: $(el).find('img').attr('src'), name: $(el).find('h3>a').text() }; - }).get(); + }).get(), sourceDir, proxy); + + case 15: + _context2.t3 = _context2.sent; + _context2.t4 = (0, _context2.t2)(_context2.t3); + + _context2.t0.apply.call(_context2.t0, _context2.t1, _context2.t4); if (showProgress) { // eslint-disable-next-line no-nested-ternary @@ -68,14 +93,13 @@ var getItemsId = /*#__PURE__*/function () { } if (!(pageNum < 2)) { - _context2.next = 12; + _context2.next = 21; break; } - log.info('正在获取番剧详细数据，请耐心等待...'); return _context2.abrupt("return", items); - case 12: + case 21: _loop = /*#__PURE__*/_regenerator["default"].mark(function _loop(i) { var response, $; return _regenerator["default"].wrap(function _loop$(_context) { @@ -84,22 +108,32 @@ var getItemsId = /*#__PURE__*/function () { case 0: if (showProgress) bar.tick(); _context.next = 3; - return axios.get("https://bangumi.tv/anime/list/".concat(vmid, "/").concat(status, "?page=").concat(i), { + return axios.get("https://bangumi.tv/anime/list/".concat(username, "/").concat(status, "?page=").concat(i), { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.69' }); case 3: response = _context.sent; $ = cheerio.load(response.data); - items = [].concat((0, _toConsumableArray2["default"])(items), (0, _toConsumableArray2["default"])($('#browserItemList>li').map(function (index, el) { + _context.t0 = items.push; + _context.t1 = items; + _context.t2 = _toConsumableArray2["default"]; + _context.next = 10; + return getBangumiData($('#browserItemList>li').map(function (index, el) { return { id: $(el).attr('id').replace('item_', ''), cover: $(el).find('img').attr('src'), name: $(el).find('h3>a').text() }; - }).get())); + }).get(), sourceDir, proxy); + + case 10: + _context.t3 = _context.sent; + _context.t4 = (0, _context.t2)(_context.t3); - case 6: + _context.t0.apply.call(_context.t0, _context.t1, _context.t4); + + case 13: case "end": return _context.stop(); } @@ -108,24 +142,23 @@ var getItemsId = /*#__PURE__*/function () { }); i = 2; - case 14: + case 23: if (!(i <= pageNum)) { - _context2.next = 19; + _context2.next = 28; break; } - return _context2.delegateYield(_loop(i), "t0", 16); + return _context2.delegateYield(_loop(i), "t5", 25); - case 16: + case 25: i++; - _context2.next = 14; + _context2.next = 23; break; - case 19: - log.info('正在获取番剧详细数据，耗时与追番数量成正比，请耐心等待...'); + case 28: return _context2.abrupt("return", items); - case 21: + case 29: case "end": return _context2.stop(); } @@ -133,7 +166,7 @@ var getItemsId = /*#__PURE__*/function () { }, _callee); })); - return function getItemsId(_x, _x2, _x3) { + return function getItemsId(_x, _x2, _x3, _x4, _x5) { return _ref.apply(this, arguments); }; }(); @@ -155,68 +188,106 @@ var TYPE = { }; var getBangumiData = /*#__PURE__*/function () { - var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(items, sourceDir, proxy) { - return _regenerator["default"].wrap(function _callee2$(_context3) { + var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(items, sourceDir, proxy) { + return _regenerator["default"].wrap(function _callee3$(_context4) { while (1) { - switch (_context3.prev = _context3.next) { + switch (_context4.prev = _context4.next) { case 0: - _context3.next = 2; - return Promise.all(items.map(function (item) { - var cachePath = path.join(sourceDir, '/_data/Bangumi-Subject-Cache'); - var subjectPath = path.join(cachePath, "".concat(item.id, ".json")); - - if (!fs.existsSync(cachePath)) { - fs.mkdirsSync(cachePath); - } + _context4.next = 2; + return Promise.allSettled(items.map( /*#__PURE__*/function () { + var _ref3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(item) { + var cachePath, subjectPath, options; + return _regenerator["default"].wrap(function _callee2$(_context3) { + while (1) { + switch (_context3.prev = _context3.next) { + case 0: + cachePath = path.join(sourceDir, '/_data/Bangumi-Subject-Cache'); + subjectPath = path.join(cachePath, "".concat(item.id, ".json")); + + if (!fs.existsSync(cachePath)) { + fs.mkdirsSync(cachePath); + } + + if (!fs.existsSync(subjectPath)) { + _context3.next = 5; + break; + } + + return _context3.abrupt("return", { + config: { + itemData: item + }, + data: fs.readFileSync(subjectPath).toString(), + status: 200 + }); + + case 5: + options = { + itemData: item, + responseType: 'json', + validateStatus: function validateStatus(status) { + return status >= 200 && status < 300 || status === 403; + }, + proxy: false, + timeout: 30 * 1000 + }; - if (fs.existsSync(subjectPath)) { - return { - config: { - itemData: item - }, - data: fs.readFileSync(subjectPath).toString(), - status: 200 - }; - } + if (proxy !== null && proxy !== void 0 && proxy.host && proxy !== null && proxy !== void 0 && proxy.port) { + options.httpsAgent = tunnel.httpsOverHttp({ + proxy: proxy, + options: { + rejectUnauthorized: false + } + }); + } + + _context3.next = 9; + return axios.get("https://cdn.jsdelivr.net/gh/czy0729/Bangumi-Subject@master/data/".concat(parseInt(parseInt(item.id, 10) / 100, 10), "/").concat(item.id, ".json"), options).then(function (response) { + return response; + })["catch"](function (error) { + if (error.response) { + console.error('Error', error.response.status); + } else { + console.error('Error', error.stack); + } + + return { + config: { + itemData: item + }, + error: error + }; + }); + + case 9: + return _context3.abrupt("return", _context3.sent); + + case 10: + case "end": + return _context3.stop(); + } + } + }, _callee2); + })); - var options = { - itemData: item, - responseType: 'json', - validateStatus: function validateStatus(status) { - return status >= 200 && status < 300 || status === 403; - }, - proxy: false + return function (_x9) { + return _ref3.apply(this, arguments); }; + }())); - if (proxy !== null && proxy !== void 0 && proxy.host && proxy !== null && proxy !== void 0 && proxy.port) { - options.httpsAgent = tunnel.httpsOverHttp({ - proxy: proxy, - options: { - rejectUnauthorized: false - } - }); - } + case 2: + return _context4.abrupt("return", _context4.sent.map(function (_ref4) { + var value = _ref4.value, + reason = _ref4.reason; - return axios.get("https://cdn.jsdelivr.net/gh/czy0729/Bangumi-Subject@master/data/".concat(parseInt(parseInt(item.id, 10) / 100, 10), "/").concat(item.id, ".json"), options).then(function (response) { - return response; - })["catch"](function (error) { - if (error.response) { - console.error('Error', error.response.status); - } else { - console.error('Error', error.stack); - } + var _ref5 = value || reason, + config = _ref5.config, + data = _ref5.data, + status = _ref5.status; - return error; - }); - })); - - case 2: - return _context3.abrupt("return", _context3.sent.map(function (_ref3) { - var config = _ref3.config, - data = _ref3.data, - status = _ref3.status; + var bangumiData = data; - if (status === 403 || !data) { + if (reason || status === 403 || !data) { return { id: config.itemData.id, title: jp2cnName(config.itemData.name), @@ -234,29 +305,29 @@ var getBangumiData = /*#__PURE__*/function () { if (typeof data === 'string') { try { // eslint-disable-next-line no-param-reassign - data = JSON.parse(data.replace(/(? { - let items = []; +const getItemsId = async (vmid, status, showProgress, sourceDir, proxy) => { + const items = []; let bar; const response = await axios.get(`https://bangumi.tv/anime/list/${vmid}/${status}?page=1`); + const username = response.request.path.match(/anime\/list\/(.*?)\//)?.[1]; + if (!username) { + return console.error('Failed to get "username"!'); + } if (response?.data) { const $ = cheerio.load(response.data); - const pageNum = $('#multipage').find('a').length; - items = $('#browserItemList>li').map((index, el) => ({ + const pageNum = Math.max(...$('#multipage').find('a') + .map((index, el) => parseInt($(el).attr('href') + ?.match(/\?page=([\d]+)/)?.[1] || '0', 10)) + .get()) || $('#multipage').find('a').length; + items.push(...await getBangumiData($('#browserItemList>li').map((index, el) => ({ id: $(el).attr('id') .replace('item_', ''), cover: $(el).find('img') @@ -26,7 +33,7 @@ const getItemsId = async (vmid, status, showProgress) => { name: $(el).find('h3>a') .text() })) - .get(); + .get(), sourceDir, proxy)); if (showProgress) { // eslint-disable-next-line no-nested-ternary @@ -35,18 +42,17 @@ const getItemsId = async (vmid, status, showProgress) => { bar.tick(); } if (pageNum < 2) { - log.info('正在获取番剧详细数据，请耐心等待...'); return items; } // eslint-disable-next-line no-plusplus for (let i = 2; i <= pageNum; i++) { if (showProgress) bar.tick(); - const response = await axios.get(`https://bangumi.tv/anime/list/${vmid}/${status}?page=${i}`, { + const response = await axios.get(`https://bangumi.tv/anime/list/${username}/${status}?page=${i}`, { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.69' }); const $ = cheerio.load(response.data); - items = [...items, ...$('#browserItemList>li').map((index, el) => ({ + items.push(...await getBangumiData($('#browserItemList>li').map((index, el) => ({ id: $(el).attr('id') .replace('item_', ''), cover: $(el).find('img') @@ -54,10 +60,10 @@ const getItemsId = async (vmid, status, showProgress) => { name: $(el).find('h3>a') .text() })) - .get()]; + .get(), sourceDir, proxy)); } } - log.info('正在获取番剧详细数据，耗时与追番数量成正比，请耐心等待...'); + // log.info('正在获取番剧详细数据，耗时与追番数量成正比，请耐心等待...'); return items; }; @@ -70,9 +76,9 @@ const TYPE = { 4: '游戏', 6: '三次元' }; -const getBangumiData = async (items, sourceDir, proxy) => (await Promise.all( +const getBangumiData = async (items, sourceDir, proxy) => (await Promise.allSettled( items.map( - (item) => { + async (item) => { const cachePath = path.join(sourceDir, '/_data/Bangumi-Subject-Cache'); const subjectPath = path.join(cachePath, `${item.id}.json`); if (!fs.existsSync(cachePath)) { @@ -87,7 +93,8 @@ const getBangumiData = async (items, sourceDir, proxy) => (await Promise.all( validateStatus(status) { return (status >= 200 && status < 300) || status === 403; }, - proxy: false + proxy: false, + timeout: 30 * 1000 }; if (proxy?.host && proxy?.port) { options.httpsAgent = tunnel.httpsOverHttp({ @@ -97,7 +104,7 @@ const getBangumiData = async (items, sourceDir, proxy) => (await Promise.all( } }); } - return axios.get(`https://cdn.jsdelivr.net/gh/czy0729/Bangumi-Subject@master/data/${ + return await axios.get(`https://cdn.jsdelivr.net/gh/czy0729/Bangumi-Subject@master/data/${ parseInt(parseInt(item.id, 10) / 100, 10)}/${item.id}.json`, options) .then((response) => response) .catch((error) => { @@ -106,12 +113,17 @@ const getBangumiData = async (items, sourceDir, proxy) => (await Promise.all( } else { console.error('Error', error.stack); } - return error; + return { + config: { itemData: item }, + error + }; }); } ) -)).map(({ config, data, status }) => { - if (status === 403 || !data) { +)).map(({ value, reason }) => { + const { config, data, status } = value || reason; + let bangumiData = data; + if (reason || status === 403 || !data) { return { id: config.itemData.id, title: jp2cnName(config.itemData.name), @@ -128,13 +140,13 @@ const getBangumiData = async (items, sourceDir, proxy) => (await Promise.all( if (typeof data === 'string') { try { // eslint-disable-next-line no-param-reassign - data = JSON.parse(data.replace(/(? (await Promise.all( module.exports.getBgmData = async function getBgmData(vmid, showProgress, sourceDir, proxy) { log.info('Getting bangumis, please wait...'); const startTime = new Date().getTime(); - const wantWatch = await getBangumiData(await getItemsId(vmid, 'wish', showProgress), sourceDir, proxy); - const watching = await getBangumiData(await getItemsId(vmid, 'do', showProgress), sourceDir, proxy); - const watched = await getBangumiData(await getItemsId(vmid, 'collect', showProgress), sourceDir, proxy); + const wantWatch = await getItemsId(vmid, 'wish', showProgress, sourceDir, proxy); + const watching = await getItemsId(vmid, 'do', showProgress, sourceDir, proxy); + const watched = await getItemsId(vmid, 'collect', showProgress, sourceDir, proxy); const endTime = new Date().getTime(); log.info(`${wantWatch.length + watching.length + watched.length} bangumis have been loaded in ${endTime - startTime} ms`); const bangumis = { wantWatch, watching, watched };