From a881676cdf273cd86e48c69e0f7cc79e65586f65 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Sat, 6 Nov 2021 02:44:21 -0700 Subject: [PATCH 01/12] Convert to typescript --- sitemapper.d.ts | 5 ++ src/assets/{sitemapper.js => sitemapper.ts} | 78 +++++++++++++-------- 2 files changed, 52 insertions(+), 31 deletions(-) rename src/assets/{sitemapper.js => sitemapper.ts} (87%) diff --git a/sitemapper.d.ts b/sitemapper.d.ts index c60f4e5..8fbfcda 100644 --- a/sitemapper.d.ts +++ b/sitemapper.d.ts @@ -13,9 +13,14 @@ export interface SitemapperErrorData { export interface SitemapperOptions { concurrency?: number; debug?: boolean; + gzip?: boolean; + headers?: Headers; lastmod?: number; + method?: string; rejectUnauthorized?: boolean; requestHeaders?: {[name: string]: string}; + resolveWithFullResponse?: boolean; + responseType?: string; retries?: number; timeout?: number; url?: string; diff --git a/src/assets/sitemapper.js b/src/assets/sitemapper.ts similarity index 87% rename from src/assets/sitemapper.js rename to src/assets/sitemapper.ts index f74c808..15fd206 100644 --- a/src/assets/sitemapper.js +++ b/src/assets/sitemapper.ts @@ -6,16 +6,32 @@ * @author Sean Burke <@seantomburke> */ -import { parseStringPromise } from "xml2js"; -import got from "got"; -import zlib from "zlib"; -import pLimit from "p-limit"; -import isGzip from "is-gzip"; +import { parseStringPromise } from 'xml2js'; +import got, { Headers, OptionsOfTextResponseBody } from 'got'; +import zlib from 'zlib'; +import pLimit from 'p-limit'; +import isGzip from 'is-gzip'; +import Url from 'url'; +import path from 'path'; +import { SitemapperOptions, SitemapperResponse} from '../../sitemapper'; +import { ErrorCallback } from 'typescript'; +import { Response } from 'got'; +import { Buffer } from 'buffer'; /** * @typedef {Object} Sitemapper */ export default class Sitemapper { + public url: string; + public timeout: number; + public timeoutTable: Object; + public requestHeaders: any; + public debug: boolean; + public retries: number; + public rejectUnauthorized: boolean; + public concurrency: number; + public lastmod: number; + /** * Construct the Sitemapper class * @@ -34,14 +50,14 @@ export default class Sitemapper { * lastmod: 1630693759 * }); */ - constructor(options) { - const settings = options || { requestHeaders: {} }; - this.url = settings.url; + constructor(options: SitemapperOptions) { + const settings: SitemapperOptions = options || { requestHeaders: {}}; + this.url = settings.url || ''; this.timeout = settings.timeout || 15000; this.timeoutTable = {}; this.lastmod = settings.lastmod || 0; this.requestHeaders = settings.requestHeaders; - this.debug = settings.debug; + this.debug = settings.debug || false; this.concurrency = settings.concurrency || 10; this.retries = settings.retries || 0; this.rejectUnauthorized = @@ -57,10 +73,10 @@ export default class Sitemapper { * @example sitemapper.fetch('example.xml') * .then((sites) => console.log(sites)); */ - async fetch(url = this.url) { + async fetch(url: string = this.url) { // initialize empty variables - let results = { - url: "", + let results: any = { + url: '', sites: [], errors: [], }; @@ -73,7 +89,7 @@ export default class Sitemapper { try { // crawl the URL results = await this.crawl(url); - } catch (e) { + } catch (e: any) { // show errors that may occur if (this.debug) { console.error(e); @@ -103,7 +119,7 @@ export default class Sitemapper { * @param {Timeout} duration * @example sitemapper.timeout = 15000; // 15 seconds */ - static set timeout(duration) { + static set timeout(duration: Number) { this.timeout = duration; } @@ -133,7 +149,7 @@ export default class Sitemapper { * @param {string} url - url for making requests. Should be a link to a sitemaps.xml * @example sitemapper.url = 'https://wp.seantburke.com/sitemap.xml' */ - static set url(url) { + static set url(url: string) { this.url = url; } @@ -142,7 +158,7 @@ export default class Sitemapper { * @returns {string} * @example console.log(sitemapper.url) */ - static get url() { + static get url(): string { return this.url; } @@ -151,7 +167,7 @@ export default class Sitemapper { * @param {Boolean} option - set whether to show debug logs in output. * @example sitemapper.debug = true; */ - static set debug(option) { + static set debug(option: boolean) { this.debug = option; } @@ -160,21 +176,21 @@ export default class Sitemapper { * @returns {Boolean} * @example console.log(sitemapper.debug) */ - static get debug() { + static get debug(): boolean { return this.debug; } /** - * Requests the URL and uses parseStringPromise to parse through and find the data + * Requests the URL and uses parsestringPromise to parse through and find the data * * @private * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) * @returns {Promise} */ - async parse(url = this.url) { + async parse(url: string = this.url): Promise<{error, data}> { // setup the response options for the got request - const requestOptions = { - method: "GET", + const requestOptions: any = { + method: 'GET', resolveWithFullResponse: true, gzip: true, responseType: "buffer", @@ -192,7 +208,7 @@ export default class Sitemapper { this.initializeTimeout(url, requester); // get the response from the requester promise - const response = await requester; + const response: any = await requester; // if the response does not have a successful status code then clear the timeout for this url. if (!response || response.statusCode !== 200) { @@ -200,7 +216,7 @@ export default class Sitemapper { return { error: response.error, data: response }; } - let responseBody; + let responseBody: Buffer; if (isGzip(response.rawBody)) { responseBody = await this.decompressResponseBody(response.body); @@ -246,7 +262,7 @@ export default class Sitemapper { * @param {string} url - url to use as a hash in the timeoutTable * @param {Promise} requester - the promise that creates the web request to the url */ - initializeTimeout(url, requester) { + initializeTimeout(url: string, requester: { cancel: Function }) { // this will throw a CancelError which will be handled in the parent that calls this method. this.timeoutTable[url] = setTimeout(() => requester.cancel(), this.timeout); } @@ -260,9 +276,9 @@ export default class Sitemapper { * @param {integer} retryIndex - Number of retry attempts fro this URL (e.g. 0 for 1st attempt, 1 for second attempty etc.) * @returns {Promise} */ - async crawl(url, retryIndex = 0) { + async crawl(url: string, retryIndex: number = 0): Promise { try { - const { error, data } = await this.parse(url); + const { error, data } : { error: Error, data: {name: string, sitemapindex: { sitemap: Array<{loc: string[], lastmod: number }>}, urlset: { url: Array<{ loc: string[], lastmod: number }>}, } } = await this.parse(url); // The promise resolved, remove the timeout clearTimeout(this.timeoutTable[url]); @@ -305,7 +321,7 @@ export default class Sitemapper { } // filter out any urls that are older than the lastmod const sites = data.urlset.url - .filter((site) => { + .filter((site: ({ loc: Array, lastmod: number })) => { if (this.lastmod === 0) return true; if (site.lastmod === undefined) return false; const modified = new Date(site.lastmod[0]).getTime(); @@ -414,7 +430,7 @@ export default class Sitemapper { * @param {Buffer} body - body of the gzipped file * @returns {Boolean} */ - decompressResponseBody(body) { + decompressResponseBody(body: Buffer): Promise { return new Promise((resolve, reject) => { const buffer = Buffer.from(body); zlib.gunzip(buffer, (err, result) => { @@ -450,7 +466,7 @@ export default class Sitemapper { * * @typedef {Object} ParseData * - * @property {Error} error that either comes from `parseStringPromise` or `got` or custom error + * @property {Error} error that either comes from `parsestringPromise` or `got` or custom error * @property {Object} data * @property {string} data.url - URL of sitemap * @property {Array} data.urlset - Array of returned URLs @@ -502,7 +518,7 @@ export default class Sitemapper { /** * An array of urls * - * @typedef {String[]} SitesArray + * @typedef {string[]} SitesArray * @example [ * 'https://www.google.com', * 'https://www.linkedin.com' From 26b2ade12ed1cc56ebdb4b384f59f13045b872d0 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Sat, 6 Nov 2021 03:17:13 -0700 Subject: [PATCH 02/12] TypeScript Working --- lib/assets/sitemapper.d.ts | 201 +++++++++++++++++++++++++++++++++++++ lib/assets/sitemapper.js | 4 +- lib/examples/google.js | 2 - lib/examples/index.js | 2 - package.json | 6 +- src/assets/sitemapper.ts | 22 +++- src/tests/test.js | 4 +- src/tests/test.ts.ts | 6 +- 8 files changed, 229 insertions(+), 18 deletions(-) create mode 100644 lib/assets/sitemapper.d.ts delete mode 100644 lib/examples/google.js delete mode 100644 lib/examples/index.js diff --git a/lib/assets/sitemapper.d.ts b/lib/assets/sitemapper.d.ts new file mode 100644 index 0000000..bad9111 --- /dev/null +++ b/lib/assets/sitemapper.d.ts @@ -0,0 +1,201 @@ +/** + * Sitemap Parser + * + * Copyright (c) 2020 Sean Thomas Burke + * Licensed under the MIT license. + * @author Sean Burke <@seantomburke> + */ +/// +import { SitemapperOptions } from '../../sitemapper'; +/** + * @typedef {Object} Sitemapper + */ +export default class Sitemapper { + url: string; + timeout: number; + timeoutTable: Object; + requestHeaders: any; + debug: boolean; + /** + * Construct the Sitemapper class + * + * @params {Object} options to set + * @params {string} [options.url] - the Sitemap url (e.g https://wp.seantburke.com/sitemap.xml) + * @params {Timeout} [options.timeout] - @see {timeout} + * + * @example let sitemap = new Sitemapper({ + * url: 'https://wp.seantburke.com/sitemap.xml', + * timeout: 15000 + * }); + */ + constructor(options?: SitemapperOptions); + /** + * Gets the sites from a sitemap.xml with a given URL + * + * @public + * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @returns {Promise} + * @example sitemapper.fetch('example.xml') + * .then((sites) => console.log(sites)); + */ + fetch(url?: string): Promise<{ + url: string; + sites: string[]; + }>; + /** + * Get the timeout + * + * @example console.log(sitemapper.timeout); + * @returns {Timeout} + */ + static get timeout(): Number; + /** + * Set the timeout + * + * @public + * @param {Timeout} duration + * @example sitemapper.timeout = 15000; // 15 seconds + */ + static set timeout(duration: Number); + /** + * + * @param {string} url - url for making requests. Should be a link to a sitemaps.xml + * @example sitemapper.url = 'https://wp.seantburke.com/sitemap.xml' + */ + static set url(url: string); + /** + * Get the url to parse + * @returns {string} + * @example console.log(sitemapper.url) + */ + static get url(): string; + /** + * Setter for the debug state + * @param {Boolean} option - set whether to show debug logs in output. + * @example sitemapper.debug = true; + */ + static set debug(option: boolean); + /** + * Getter for the debug state + * @returns {Boolean} + * @example console.log(sitemapper.debug) + */ + static get debug(): boolean; + /** + * Requests the URL and uses parsestringPromise to parse through and find the data + * + * @private + * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @returns {Promise} + */ + parse(url?: string): Promise<{ + error: any; + data: any; + }>; + /** + * Timeouts are necessary for large xml trees. This will cancel the call if the request is taking + * too long, but will still allow the promises to resolve. + * + * @private + * @param {string} url - url to use as a hash in the timeoutTable + * @param {Promise} requester - the promise that creates the web request to the url + */ + initializeTimeout(url: string, requester: { + cancel: Function; + }): void; + /** + * Recursive function that will go through a sitemaps tree and get all the sites + * + * @private + * @recursive + * @param {string} url - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @returns {Promise | Promise} + */ + crawl(url: string): Promise>; + /** + * Gets the sites from a sitemap.xml with a given URL + * + * @deprecated + * @param {string} url - url to query + * @param {getSitesCallback} callback - callback for sites and error + * @callback + */ + getSites(url: string, callback: any): Promise; + /** + * Check to see if the url is a gzipped url + * + * @param {string} url - url to query + * @returns {Boolean} + */ + isGzip(url: string): boolean; + /** + * Decompress the gzipped response body using zlib.gunzip + * + * @param {Buffer} body - body of the gzipped file + * @returns {Boolean} + */ + decompressResponseBody(body: Buffer): Promise; +} +/** + * Callback for the getSites method + * + * @callback getSitesCallback + * @param {Object} error - error from callback + * @param {Array} sites - an Array of sitemaps + */ +/** + * Timeout in milliseconds + * + * @typedef {Number} Timeout + * the number of milliseconds before all requests timeout. The promises will still resolve so + * you'll still receive parts of the request, but maybe not all urls + * default is 15000 which is 15 seconds + */ +/** + * Resolve handler type for the promise in this.parse() + * + * @typedef {Object} ParseData + * + * @property {Error} error that either comes from `parsestringPromise` or `got` or custom error + * @property {Object} data + * @property {string} data.url - URL of sitemap + * @property {Array} data.urlset - Array of returned URLs + * @property {string} data.urlset.url - single Url + * @property {Object} data.sitemapindex - index of sitemap + * @property {string} data.sitemapindex.sitemap - Sitemap + * @example { + * error: "There was an error!" + * data: { + * url: 'https://linkedin.com', + * urlset: [{ + * url: 'https://www.linkedin.com/project1' + * },[{ + * url: 'https://www.linkedin.com/project2' + * }] + * } + * } + */ +/** + * Resolve handler type for the promise in this.parse() + * + * @typedef {Object} SitesData + * + * @property {string} url - the original url used to query the data + * @property {SitesArray} sites + * @example { + * url: 'https://linkedin.com/sitemap.xml', + * sites: [ + * 'https://linkedin.com/project1', + * 'https://linkedin.com/project2' + * ] + * } + */ +/** + * An array of urls + * + * @typedef {string[]} SitesArray + * @example [ + * 'https://www.google.com', + * 'https://www.linkedin.com' + * ] + */ diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index 95619fe..74798a4 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -1,2 +1,2 @@ -"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_pLimit=_interopRequireDefault(require("p-limit")),_isGzip=_interopRequireDefault(require("is-gzip"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.lastmod=b.lastmod||0,this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0,this.rejectUnauthorized=!1!==b.rejectUnauthorized}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1{if(0===c.lastmod)return!0;if(void 0===a.lastmod)return!1;var b=new Date(a.lastmod[0]).getTime();return b>=c.lastmod}).map(a=>a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0===a.errors.length).reduce((a,b)=>{var{sites:c}=b;return[...a,...c]},[]),j=h.filter(a=>0!==a.errors.length).reduce((a,b)=>{var{errors:c}=b;return[...a,...c]},[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; -//# sourceMappingURL=sitemapper.js.map \ No newline at end of file +"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_pLimit=_interopRequireDefault(require("p-limit")),_isGzip=_interopRequireDefault(require("is-gzip"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0,this.rejectUnauthorized=b.rejectUnauthorized||!0}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0===a.errors.length).reduce((a,b)=>{var{sites:c}=b;return[...a,...c]},[]),j=h.filter(a=>0!==a.errors.length).reduce((a,b)=>{var{errors:c}=b;return[...a,...c]},[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; +//# sourceMappingURL=sitemapper.js.map diff --git a/lib/examples/google.js b/lib/examples/google.js deleted file mode 100644 index ea2caa7..0000000 --- a/lib/examples/google.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";var _sitemapper=_interopRequireDefault(require("../assets/sitemapper.js"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}var Google=new _sitemapper.default({url:"https://www.google.com/work/sitemap.xml",debug:!1,timeout:15e3});Google.fetch().then(a=>console.log(a.sites)).catch(a=>console.log(a)); -//# sourceMappingURL=google.js.map \ No newline at end of file diff --git a/lib/examples/index.js b/lib/examples/index.js deleted file mode 100644 index 64b6198..0000000 --- a/lib/examples/index.js +++ /dev/null @@ -1,2 +0,0 @@ -"use strict";var _sitemapper=_interopRequireDefault(require("../assets/sitemapper"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}var exampleURL="https://www.walmart.com/sitemap_topic.xml",sitemapper=new _sitemapper.default({url:"https://www.walmart.com/sitemap_topic.xml",debug:!1,timeout:1e4,concurrency:10,retries:0});_asyncToGenerator(function*(){try{var a=yield sitemapper.fetch();console.log(a)}catch(a){console.error(a)}})(); -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/package.json b/package.json index d2d1230..a8c4598 100644 --- a/package.json +++ b/package.json @@ -18,8 +18,7 @@ }, "license": "MIT", "files": [ - "lib", - "sitemapper.d.ts" + "lib" ], "main": "./lib/assets/sitemapper.js", "types": "./sitemapper.d.ts", @@ -33,12 +32,13 @@ "url": "http://www.seantburke.com" }, "scripts": { - "compile": "babel src -d lib -s && tsc --project ./src/tests/", + "compile": "tsc -d --sourceMap --outDir lib -t esnext ./src/**/*.ts && babel lib -d lib -s", "build": "npm run clean && npm run compile", "start": "npm run build && node lib/examples/index.js", "test": "npm run build && mocha ./lib/tests/*.js && npm run lint", "lint": "eslint src", "clean": "rm -rf lib", + "tsc": "tsc", "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md" }, "maintainers": [ diff --git a/src/assets/sitemapper.ts b/src/assets/sitemapper.ts index 15fd206..88d5d95 100644 --- a/src/assets/sitemapper.ts +++ b/src/assets/sitemapper.ts @@ -7,15 +7,17 @@ */ import { parseStringPromise } from 'xml2js'; -import got, { Headers, OptionsOfTextResponseBody } from 'got'; +// @ts-ignore +import got from 'got'; +// @ts-ignore import zlib from 'zlib'; import pLimit from 'p-limit'; import isGzip from 'is-gzip'; +// @ts-ignore import Url from 'url'; +// @ts-ignore import path from 'path'; import { SitemapperOptions, SitemapperResponse} from '../../sitemapper'; -import { ErrorCallback } from 'typescript'; -import { Response } from 'got'; import { Buffer } from 'buffer'; /** @@ -50,7 +52,7 @@ export default class Sitemapper { * lastmod: 1630693759 * }); */ - constructor(options: SitemapperOptions) { + constructor(options?: SitemapperOptions) { const settings: SitemapperOptions = options || { requestHeaders: {}}; this.url = settings.url || ''; this.timeout = settings.timeout || 15000; @@ -424,6 +426,18 @@ export default class Sitemapper { return callback(err, sites); } + /** + * Check to see if the url is a gzipped url + * + * @param {string} url - url to query + * @returns {Boolean} + */ + isGzip(url: string) { + const parsed = Url.parse(url); + const ext = path.extname(parsed.path || ''); + return ext === '.gz'; + } + /** * Decompress the gzipped response body using zlib.gunzip * diff --git a/src/tests/test.js b/src/tests/test.js index 0a6a893..424ec79 100644 --- a/src/tests/test.js +++ b/src/tests/test.js @@ -51,8 +51,8 @@ describe('Sitemapper', function () { }); it('should set url', () => { - sitemapper.url = 1000; - sitemapper.url.should.equal(1000); + sitemapper.url = 'https://wp.seantburke.com/sitemap.xml'; + sitemapper.url.should.equal('https://wp.seantburke.com/sitemap.xml'); }); }); diff --git a/src/tests/test.ts.ts b/src/tests/test.ts.ts index ee0576c..7ad61d7 100644 --- a/src/tests/test.ts.ts +++ b/src/tests/test.ts.ts @@ -1,7 +1,7 @@ import 'async'; import 'assert'; import 'should'; -import isUrl = require('is-url'); +const isUrl = require('is-url'); // @ts-ignore import Sitemapper from '../../lib/assets/sitemapper.js'; @@ -52,8 +52,8 @@ describe('Sitemapper', function () { }); it('should set url', () => { - sitemapper.url = 1000; - sitemapper.url.should.equal(1000); + sitemapper.url = 'https://wp.seantburke.com/sitemap.xml'; + sitemapper.url.should.equal('https://wp.seantburke.com/sitemap.xml'); }); }); From 4916d37e6c07c604692d1d0f2c0a27c83dffba2f Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 21:47:51 -0700 Subject: [PATCH 03/12] tsconfig.js --- src/tests/tsconfig.json | 24 --------- tsconfig.json | 109 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 24 deletions(-) delete mode 100644 src/tests/tsconfig.json create mode 100644 tsconfig.json diff --git a/src/tests/tsconfig.json b/src/tests/tsconfig.json deleted file mode 100644 index 80ab693..0000000 --- a/src/tests/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "compilerOptions": { - "allowJs": false, - "alwaysStrict": true, - "declaration": false, - "module": "commonjs", - "moduleResolution": "node", - "lib": [ - "dom", - "esnext" - ], - "newLine": "lf", - "target": "es2015", - "baseUrl": "./", - "outDir": "../../lib/tests/", - "preserveConstEnums": true, - "removeComments": false, - "strict": true, - "noImplicitAny": false - }, - "include": [ - "./*.ts" - ] -} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..6f00a06 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,109 @@ +{ + "compilerOptions": { + /* Visit https://aka.ms/tsconfig to read more about this file */ + + /* Projects */ + // "incremental": true, /* Save .tsbuildinfo files to allow for incremental compilation of projects. */ + // "composite": true, /* Enable constraints that allow a TypeScript project to be used with project references. */ + // "tsBuildInfoFile": "./.tsbuildinfo", /* Specify the path to .tsbuildinfo incremental compilation file. */ + // "disableSourceOfProjectReferenceRedirect": true, /* Disable preferring source files instead of declaration files when referencing composite projects. */ + // "disableSolutionSearching": true, /* Opt a project out of multi-project reference checking when editing. */ + // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ + + /* Language and Environment */ + "target": "es2015", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "lib": [ + "dom", + "esnext" + ], /* Specify a set of bundled library declaration files that describe the target runtime environment. */ + // "jsx": "preserve", /* Specify what JSX code is generated. */ + // "experimentalDecorators": true, /* Enable experimental support for TC39 stage 2 draft decorators. */ + // "emitDecoratorMetadata": true, /* Emit design-type metadata for decorated declarations in source files. */ + // "jsxFactory": "", /* Specify the JSX factory function used when targeting React JSX emit, e.g. 'React.createElement' or 'h'. */ + // "jsxFragmentFactory": "", /* Specify the JSX Fragment reference used for fragments when targeting React JSX emit e.g. 'React.Fragment' or 'Fragment'. */ + // "jsxImportSource": "", /* Specify module specifier used to import the JSX factory functions when using 'jsx: react-jsx*'. */ + // "reactNamespace": "", /* Specify the object invoked for 'createElement'. This only applies when targeting 'react' JSX emit. */ + // "noLib": true, /* Disable including any library files, including the default lib.d.ts. */ + // "useDefineForClassFields": true, /* Emit ECMAScript-standard-compliant class fields. */ + // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ + + /* Modules */ + "module": "commonjs", /* Specify what module code is generated. */ + // "rootDir": "./", /* Specify the root folder within your source files. */ + "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ + "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ + // "paths": {}, /* Specify a set of entries that re-map imports to additional lookup locations. */ + // "rootDirs": [], /* Allow multiple folders to be treated as one when resolving modules. */ + // "typeRoots": [], /* Specify multiple folders that act like './node_modules/@types'. */ + // "types": [], /* Specify type package names to be included without being referenced in a source file. */ + // "allowUmdGlobalAccess": true, /* Allow accessing UMD globals from modules. */ + // "moduleSuffixes": [], /* List of file name suffixes to search when resolving a module. */ + // "resolveJsonModule": true, /* Enable importing .json files. */ + // "noResolve": true, /* Disallow 'import's, 'require's or ''s from expanding the number of files TypeScript should add to a project. */ + + /* JavaScript Support */ + "allowJs": true, /* Allow JavaScript files to be a part of your program. Use the 'checkJS' option to get errors from these files. */ + // "checkJs": true, /* Enable error reporting in type-checked JavaScript files. */ + // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ + + /* Emit */ + "declaration": false, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + // "declarationMap": true, /* Create sourcemaps for d.ts files. */ + // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ + // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ + "outDir": "./lib/tests", /* Specify an output folder for all emitted files. */ + "removeComments": false, /* Disable emitting comments. */ + // "noEmit": true, /* Disable emitting files from a compilation. */ + // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ + // "importsNotUsedAsValues": "remove", /* Specify emit/checking behavior for imports that are only used for types. */ + // "downlevelIteration": true, /* Emit more compliant, but verbose and less performant JavaScript for iteration. */ + // "sourceRoot": "", /* Specify the root path for debuggers to find the reference source code. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Include sourcemap files inside the emitted JavaScript. */ + // "inlineSources": true, /* Include source code in the sourcemaps inside the emitted JavaScript. */ + // "emitBOM": true, /* Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. */ + "newLine": "lf", /* Set the newline character for emitting files. */ + // "stripInternal": true, /* Disable emitting declarations that have '@internal' in their JSDoc comments. */ + // "noEmitHelpers": true, /* Disable generating custom helper functions like '__extends' in compiled output. */ + // "noEmitOnError": true, /* Disable emitting files if any type checking errors are reported. */ + "preserveConstEnums": true, /* Disable erasing 'const enum' declarations in generated code. */ + // "declarationDir": "./", /* Specify the output directory for generated declaration files. */ + // "preserveValueImports": true, /* Preserve unused imported values in the JavaScript output that would otherwise be removed. */ + + /* Interop Constraints */ + // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ + // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ + // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ + "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ + + /* Type Checking */ + "strict": true, /* Enable all strict type-checking options. */ + "noImplicitAny": true, /* Enable error reporting for expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* When type checking, take into account 'null' and 'undefined'. */ + // "strictFunctionTypes": true, /* When assigning functions, check to ensure parameters and the return values are subtype-compatible. */ + // "strictBindCallApply": true, /* Check that the arguments for 'bind', 'call', and 'apply' methods match the original function. */ + // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ + // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ + // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ + "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ + // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ + // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ + // "noImplicitReturns": true, /* Enable error reporting for codepaths that do not explicitly return in a function. */ + // "noFallthroughCasesInSwitch": true, /* Enable error reporting for fallthrough cases in switch statements. */ + // "noUncheckedIndexedAccess": true, /* Add 'undefined' to a type when accessed using an index. */ + // "noImplicitOverride": true, /* Ensure overriding members in derived classes are marked with an override modifier. */ + // "noPropertyAccessFromIndexSignature": true, /* Enforces using indexed accessors for keys declared using an indexed type. */ + // "allowUnusedLabels": true, /* Disable error reporting for unused labels. */ + // "allowUnreachableCode": true, /* Disable error reporting for unreachable code. */ + + /* Completeness */ + // "skipDefaultLibCheck": true, /* Skip type checking .d.ts files that are included with TypeScript. */ + "skipLibCheck": true /* Skip type checking all .d.ts files. */ + }, + "include": [ + "./*.ts" + ] +} From f5b93425afa3be795edc0b72c28c3efbf398d7a5 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 22:26:31 -0700 Subject: [PATCH 04/12] Updating with ESM --- .eslintrc.js | 2 +- lib/assets/sitemapper.d.ts | 63 +- lib/assets/sitemapper.js | 463 ++++++++++- package-lock.json | 1227 +++++++++++------------------ package.json | 17 +- src/assets/sitemapper.ts | 27 +- src/tests/test.es5.js | 180 ----- src/tests/{test.ts.ts => test.ts} | 28 +- tsconfig.json | 17 +- 9 files changed, 1013 insertions(+), 1011 deletions(-) delete mode 100644 src/tests/test.es5.js rename src/tests/{test.ts.ts => test.ts} (90%) diff --git a/.eslintrc.js b/.eslintrc.js index ced0800..1cd1bf2 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,4 +1,4 @@ -module.exports = { +export default { extends: 'eslint:recommended', parserOptions: { ecmaVersion: 8, diff --git a/lib/assets/sitemapper.d.ts b/lib/assets/sitemapper.d.ts index bad9111..e2a4dae 100644 --- a/lib/assets/sitemapper.d.ts +++ b/lib/assets/sitemapper.d.ts @@ -16,12 +16,19 @@ export default class Sitemapper { timeoutTable: Object; requestHeaders: any; debug: boolean; + retries: number; + rejectUnauthorized: boolean; + concurrency: number; /** * Construct the Sitemapper class * * @params {Object} options to set * @params {string} [options.url] - the Sitemap url (e.g https://wp.seantburke.com/sitemap.xml) * @params {Timeout} [options.timeout] - @see {timeout} + * @params {boolean} [options.debug] - Enables/Disables additional logging + * @params {integer} [options.concurrency] - The number of concurrent sitemaps to crawl (e.g. 2 will crawl no more than 2 sitemaps at the same time) + * @params {integer} [options.retries] - The maximum number of retries to attempt when crawling fails (e.g. 1 for 1 retry, 2 attempts in total) + * @params {boolean} [options.rejectUnauthorized] - If true (default), it will throw on invalid certificates, such as expired or self-signed ones. * * @example let sitemap = new Sitemapper({ * url: 'https://wp.seantburke.com/sitemap.xml', @@ -40,7 +47,8 @@ export default class Sitemapper { */ fetch(url?: string): Promise<{ url: string; - sites: string[]; + sites: any; + errors: any; }>; /** * Get the timeout @@ -109,9 +117,10 @@ export default class Sitemapper { * @private * @recursive * @param {string} url - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) - * @returns {Promise | Promise} + * @param {integer} retryIndex - Number of retry attempts fro this URL (e.g. 0 for 1st attempt, 1 for second attempty etc.) + * @returns {Promise} */ - crawl(url: string): Promise>; + crawl(url: string, retryIndex?: number): Promise; /** * Gets the sites from a sitemap.xml with a given URL * @@ -120,7 +129,7 @@ export default class Sitemapper { * @param {getSitesCallback} callback - callback for sites and error * @callback */ - getSites(url: string, callback: any): Promise; + getSites(url: string | undefined, callback: Function): Promise; /** * Check to see if the url is a gzipped url * @@ -164,7 +173,7 @@ export default class Sitemapper { * @property {Object} data.sitemapindex - index of sitemap * @property {string} data.sitemapindex.sitemap - Sitemap * @example { - * error: "There was an error!" + * error: 'There was an error!' * data: { * url: 'https://linkedin.com', * urlset: [{ @@ -182,11 +191,24 @@ export default class Sitemapper { * * @property {string} url - the original url used to query the data * @property {SitesArray} sites + * @property {ErrorDataArray} errors * @example { * url: 'https://linkedin.com/sitemap.xml', * sites: [ * 'https://linkedin.com/project1', * 'https://linkedin.com/project2' + * ], + * errors: [ + * { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * }, + * { + * type: 'HTTPError', + * url: 'https://www.walmart.com/sitemap_tp2.xml', + * retries: 0 + * }, * ] * } */ @@ -199,3 +221,34 @@ export default class Sitemapper { * 'https://www.linkedin.com' * ] */ +/** + * An array of Error data objects + * + * @typedef {ErrorData[]} ErrorDataArray + * @example [ + * { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * }, + * { + * type: 'HTTPError', + * url: 'https://www.walmart.com/sitemap_tp2.xml', + * retries: 0 + * }, + * ] + */ +/** + * An object containing details about the errors which occurred during the crawl + * + * @typedef {Object} ErrorData + * + * @property {string} type - The error type which was returned + * @property {string} url - The sitemap URL which returned the error + * @property {Number} errors - The total number of retries attempted after receiving the first error + * @example { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * } + */ diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index 74798a4..65406d0 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -1,2 +1,461 @@ -"use strict";Object.defineProperty(exports,"__esModule",{value:!0}),exports.default=void 0;var _xml2js=require("xml2js"),_got=_interopRequireDefault(require("got")),_zlib=_interopRequireDefault(require("zlib")),_pLimit=_interopRequireDefault(require("p-limit")),_isGzip=_interopRequireDefault(require("is-gzip"));function _interopRequireDefault(a){return a&&a.__esModule?a:{default:a}}function asyncGeneratorStep(a,b,c,d,e,f,g){try{var h=a[f](g),i=h.value}catch(a){return void c(a)}h.done?b(i):Promise.resolve(i).then(d,e)}function _asyncToGenerator(a){return function(){var b=this,c=arguments;return new Promise(function(d,e){function f(a){asyncGeneratorStep(h,d,e,f,g,"next",a)}function g(a){asyncGeneratorStep(h,d,e,f,g,"throw",a)}var h=a.apply(b,c);f(void 0)})}}class Sitemapper{constructor(a){var b=a||{requestHeaders:{}};this.url=b.url,this.timeout=b.timeout||15e3,this.timeoutTable={},this.requestHeaders=b.requestHeaders,this.debug=b.debug,this.concurrency=b.concurrency||10,this.retries=b.retries||0,this.rejectUnauthorized=b.rejectUnauthorized||!0}fetch(){var a=arguments,b=this;return _asyncToGenerator(function*(){var c=0b.cancel(),this.timeout)}crawl(a){var b=arguments,c=this;return _asyncToGenerator(function*(){var d=1a.loc&&a.loc[0]);return{sites:m,errors:[]}}if(l&&l.sitemapindex){c.debug&&console.debug("Additional sitemap found during \"crawl('".concat(a,"')\""));var e=l.sitemapindex.sitemap.map(a=>a.loc&&a.loc[0]),f=(0,_pLimit.default)(c.concurrency),g=e.map(a=>f(()=>c.crawl(a))),h=yield Promise.all(g),i=h.filter(a=>0===a.errors.length).reduce((a,b)=>{var{sites:c}=b;return[...a,...c]},[]),j=h.filter(a=>0!==a.errors.length).reduce((a,b)=>{var{errors:c}=b;return[...a,...c]},[]);return{sites:i,errors:j}}return d{var d=Buffer.from(a);_zlib.default.gunzip(d,(a,d)=>{a?c(a):b(d)})})}}exports.default=Sitemapper,module.exports=exports.default,module.exports.default=exports.default; -//# sourceMappingURL=sitemapper.js.map +/** + * Sitemap Parser + * + * Copyright (c) 2020 Sean Thomas Burke + * Licensed under the MIT license. + * @author Sean Burke <@seantomburke> + */ +import { parseStringPromise } from 'xml2js'; +// @ts-ignore +import got from 'got'; +// @ts-ignore +import zlib from 'zlib'; +import pLimit from 'p-limit'; +import isGzip from 'is-gzip'; +// @ts-ignore +import Url from 'url'; +// @ts-ignore +import path from 'path'; +import { Buffer } from 'buffer'; +/** + * @typedef {Object} Sitemapper + */ +export default class Sitemapper { + /** + * Construct the Sitemapper class + * + * @params {Object} options to set + * @params {string} [options.url] - the Sitemap url (e.g https://wp.seantburke.com/sitemap.xml) + * @params {Timeout} [options.timeout] - @see {timeout} + * @params {boolean} [options.debug] - Enables/Disables additional logging + * @params {integer} [options.concurrency] - The number of concurrent sitemaps to crawl (e.g. 2 will crawl no more than 2 sitemaps at the same time) + * @params {integer} [options.retries] - The maximum number of retries to attempt when crawling fails (e.g. 1 for 1 retry, 2 attempts in total) + * @params {boolean} [options.rejectUnauthorized] - If true (default), it will throw on invalid certificates, such as expired or self-signed ones. + * + * @example let sitemap = new Sitemapper({ + * url: 'https://wp.seantburke.com/sitemap.xml', + * timeout: 15000 + * }); + */ + constructor(options) { + const settings = options || { requestHeaders: {} }; + this.url = settings.url || ''; + this.timeout = settings.timeout || 15000; + this.timeoutTable = {}; + this.requestHeaders = settings.requestHeaders; + this.debug = settings.debug || false; + this.concurrency = settings.concurrency || 10; + this.retries = settings.retries || 0; + this.rejectUnauthorized = settings.rejectUnauthorized || true; + } + /** + * Gets the sites from a sitemap.xml with a given URL + * + * @public + * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @returns {Promise} + * @example sitemapper.fetch('example.xml') + * .then((sites) => console.log(sites)); + */ + async fetch(url = this.url) { + // initialize empty variables + let results = { + url: '', + sites: [], + errors: [], + }; + // attempt to set the variables with the crawl + try { + // crawl the URL + results = await this.crawl(url); + } + catch (e) { + // show errors that may occur + if (this.debug) { + console.error(e); + } + } + return { + url, + sites: results.sites || [], + errors: results.errors || [], + }; + } + /** + * Get the timeout + * + * @example console.log(sitemapper.timeout); + * @returns {Timeout} + */ + static get timeout() { + return this.timeout; + } + /** + * Set the timeout + * + * @public + * @param {Timeout} duration + * @example sitemapper.timeout = 15000; // 15 seconds + */ + static set timeout(duration) { + this.timeout = duration; + } + /** + * + * @param {string} url - url for making requests. Should be a link to a sitemaps.xml + * @example sitemapper.url = 'https://wp.seantburke.com/sitemap.xml' + */ + static set url(url) { + this.url = url; + } + /** + * Get the url to parse + * @returns {string} + * @example console.log(sitemapper.url) + */ + static get url() { + return this.url; + } + /** + * Setter for the debug state + * @param {Boolean} option - set whether to show debug logs in output. + * @example sitemapper.debug = true; + */ + static set debug(option) { + this.debug = option; + } + /** + * Getter for the debug state + * @returns {Boolean} + * @example console.log(sitemapper.debug) + */ + static get debug() { + return this.debug; + } + /** + * Requests the URL and uses parsestringPromise to parse through and find the data + * + * @private + * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @returns {Promise} + */ + async parse(url = this.url) { + // setup the response options for the got request + const requestOptions = { + method: 'GET', + resolveWithFullResponse: true, + gzip: true, + responseType: 'buffer', + headers: this.requestHeaders, + https: { + rejectUnauthorized: this.rejectUnauthorized, + } + }; + try { + // create a request Promise with the url and request options + const requester = got.get(url, requestOptions); + // initialize the timeout method based on the URL, and pass the request object. + this.initializeTimeout(url, requester); + // get the response from the requester promise + const response = await requester; + // if the response does not have a successful status code then clear the timeout for this url. + if (!response || response.statusCode !== 200) { + clearTimeout(this.timeoutTable[url]); + return { error: response.error, data: response }; + } + let responseBody; + if (isGzip(response.rawBody)) { + responseBody = await this.decompressResponseBody(response.body); + } + else { + responseBody = response.body; + } + // otherwise parse the XML that was returned. + const data = await parseStringPromise(responseBody); + // return the results + return { error: null, data }; + } + catch (error) { + // If the request was canceled notify the user of the timeout + if (error.name === 'CancelError') { + return { + error: `Request timed out after ${this.timeout} milliseconds for url: '${url}'`, + data: error + }; + } + // Otherwise notify of another error + return { + error: `Error occurred: ${error.name}`, + data: error + }; + } + } + /** + * Timeouts are necessary for large xml trees. This will cancel the call if the request is taking + * too long, but will still allow the promises to resolve. + * + * @private + * @param {string} url - url to use as a hash in the timeoutTable + * @param {Promise} requester - the promise that creates the web request to the url + */ + initializeTimeout(url, requester) { + // this will throw a CancelError which will be handled in the parent that calls this method. + this.timeoutTable[url] = setTimeout(() => requester.cancel(), this.timeout); + } + /** + * Recursive function that will go through a sitemaps tree and get all the sites + * + * @private + * @recursive + * @param {string} url - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) + * @param {integer} retryIndex - Number of retry attempts fro this URL (e.g. 0 for 1st attempt, 1 for second attempty etc.) + * @returns {Promise} + */ + async crawl(url, retryIndex = 0) { + try { + const { error, data } = await this.parse(url); + // The promise resolved, remove the timeout + clearTimeout(this.timeoutTable[url]); + if (error) { + // Handle errors during sitemap parsing / request + // Retry on error until you reach the retry limit set in the settings + if (retryIndex < this.retries) { + if (this.debug) { + console.log(`(Retry attempt: ${retryIndex + 1} / ${this.retries}) ${url} due to ${data.name} on previous request`); + } + return this.crawl(url, retryIndex + 1); + } + if (this.debug) { + console.error(`Error occurred during "crawl('${url}')":\n\r Error: ${error}`); + } + // Fail and log error + return { + sites: [], + errors: [{ + type: data.name, + url, + retries: retryIndex, + }] + }; + } + else if (data && data.urlset && data.urlset.url) { + // Handle URLs found inside the sitemap + if (this.debug) { + console.debug(`Urlset found during "crawl('${url}')"`); + } + const sites = data.urlset.url.map(site => site.loc && site.loc[0]); + return { + sites, + errors: [] + }; + } + else if (data && data.sitemapindex) { + // Handle child sitemaps found inside the active sitemap + if (this.debug) { + console.debug(`Additional sitemap found during "crawl('${url}')"`); + } + // Map each child url into a promise to create an array of promises + const sitemap = data.sitemapindex.sitemap.map(map => map.loc && map.loc[0]); + // Parse all child urls within the concurrency limit in the settings + const limit = pLimit(this.concurrency); + const promiseArray = sitemap.map(site => limit(() => this.crawl(site))); + // Make sure all the promises resolve then filter and reduce the array + const results = await Promise.all(promiseArray); + const sites = results + .filter(result => (result.errors.length === 0)) + .reduce((prev, { sites }) => [...prev, ...sites], []); + const errors = results + .filter(result => (result.errors.length !== 0)) + .reduce((prev, { errors }) => [...prev, ...errors], []); + return { + sites, + errors, + }; + } + // Retry on error until you reach the retry limit set in the settings + if (retryIndex < this.retries) { + if (this.debug) { + console.log(`(Retry attempt: ${retryIndex + 1} / ${this.retries}) ${url} due to ${data.name} on previous request`); + } + return this.crawl(url, retryIndex + 1); + } + if (this.debug) { + console.error(`Unknown state during "crawl('${url})'":`, error, data); + } + // Fail and log error + return { + sites: [], + errors: [{ + url, + type: data.name || 'UnknownStateError', + retries: retryIndex + }] + }; + } + catch (e) { + if (this.debug) { + this.debug && console.error(e); + } + } + } + /** + * Gets the sites from a sitemap.xml with a given URL + * + * @deprecated + * @param {string} url - url to query + * @param {getSitesCallback} callback - callback for sites and error + * @callback + */ + async getSites(url = this.url, callback) { + console.warn(// eslint-disable-line no-console + '\r\nWarning:', 'function .getSites() is deprecated, please use the function .fetch()\r\n'); + let err = {}; + let sites = []; + try { + const response = await this.fetch(url); + sites = response.sites; + } + catch (error) { + err = error; + } + return callback(err, sites); + } + /** + * Check to see if the url is a gzipped url + * + * @param {string} url - url to query + * @returns {Boolean} + */ + isGzip(url) { + const parsed = Url.parse(url); + const ext = path.extname(parsed.path || ''); + return ext === '.gz'; + } + /** + * Decompress the gzipped response body using zlib.gunzip + * + * @param {Buffer} body - body of the gzipped file + * @returns {Boolean} + */ + decompressResponseBody(body) { + return new Promise((resolve, reject) => { + const buffer = Buffer.from(body); + zlib.gunzip(buffer, (err, result) => { + if (err) { + reject(err); + } + else { + resolve(result); + } + }); + }); + } +} +/** + * Callback for the getSites method + * + * @callback getSitesCallback + * @param {Object} error - error from callback + * @param {Array} sites - an Array of sitemaps + */ +/** + * Timeout in milliseconds + * + * @typedef {Number} Timeout + * the number of milliseconds before all requests timeout. The promises will still resolve so + * you'll still receive parts of the request, but maybe not all urls + * default is 15000 which is 15 seconds + */ +/** + * Resolve handler type for the promise in this.parse() + * + * @typedef {Object} ParseData + * + * @property {Error} error that either comes from `parsestringPromise` or `got` or custom error + * @property {Object} data + * @property {string} data.url - URL of sitemap + * @property {Array} data.urlset - Array of returned URLs + * @property {string} data.urlset.url - single Url + * @property {Object} data.sitemapindex - index of sitemap + * @property {string} data.sitemapindex.sitemap - Sitemap + * @example { + * error: 'There was an error!' + * data: { + * url: 'https://linkedin.com', + * urlset: [{ + * url: 'https://www.linkedin.com/project1' + * },[{ + * url: 'https://www.linkedin.com/project2' + * }] + * } + * } + */ +/** + * Resolve handler type for the promise in this.parse() + * + * @typedef {Object} SitesData + * + * @property {string} url - the original url used to query the data + * @property {SitesArray} sites + * @property {ErrorDataArray} errors + * @example { + * url: 'https://linkedin.com/sitemap.xml', + * sites: [ + * 'https://linkedin.com/project1', + * 'https://linkedin.com/project2' + * ], + * errors: [ + * { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * }, + * { + * type: 'HTTPError', + * url: 'https://www.walmart.com/sitemap_tp2.xml', + * retries: 0 + * }, + * ] + * } + */ +/** + * An array of urls + * + * @typedef {string[]} SitesArray + * @example [ + * 'https://www.google.com', + * 'https://www.linkedin.com' + * ] + */ +/** + * An array of Error data objects + * + * @typedef {ErrorData[]} ErrorDataArray + * @example [ + * { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * }, + * { + * type: 'HTTPError', + * url: 'https://www.walmart.com/sitemap_tp2.xml', + * retries: 0 + * }, + * ] + */ +/** + * An object containing details about the errors which occurred during the crawl + * + * @typedef {Object} ErrorData + * + * @property {string} type - The error type which was returned + * @property {string} url - The sitemap URL which returned the error + * @property {Number} errors - The total number of retries attempted after receiving the first error + * @example { + * type: 'CancelError', + * url: 'https://www.walmart.com/sitemap_tp1.xml', + * retries: 0 + * } + */ +//# sourceMappingURL=sitemapper.js.map \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 66bdbed..8aa207c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5,12 +5,13 @@ "requires": true, "packages": { "": { + "name": "sitemapper", "version": "3.2.5", "license": "MIT", "dependencies": { "got": "^11.8.0", "is-gzip": "2.0.0", - "p-limit": "^3.1.0", + "p-limit": "^4.0.0", "xml2js": "^0.4.23" }, "devDependencies": { @@ -20,6 +21,7 @@ "@babel/runtime": "^7.12.5", "@types/async": "^3.2.4", "@types/got": "^9.6.11", + "@types/is-gzip": "^2.0.0", "@types/is-url": "^1.2.28", "@types/mocha": "^8.0.4", "@types/xml2js": "^0.4.7", @@ -27,15 +29,15 @@ "babel-plugin-add-module-exports": "^1.0.4", "babel-preset-minify": "^0.5.1", "documentation": "^13.1.0", - "eslint": "^7.14.0", + "eslint": "^8.20.0", "is-url": "^1.2.4", - "mocha": "^8.2.1", + "mocha": "^10.0.0", "should": "^13.2.3", "ts-node": "^9.0.0", "typescript": "^4.1.2" }, "engines": { - "node": ">= 10.0.0" + "node": ">=14.16" } }, "node_modules/@ampproject/remapping": { @@ -1656,25 +1658,31 @@ } }, "node_modules/@eslint/eslintrc": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", - "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", + "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", "dev": true, "dependencies": { "ajv": "^6.12.4", - "debug": "^4.1.1", - "espree": "^7.3.0", - "globals": "^13.9.0", - "ignore": "^4.0.6", + "debug": "^4.3.2", + "espree": "^9.3.2", + "globals": "^13.15.0", + "ignore": "^5.2.0", "import-fresh": "^3.2.1", - "js-yaml": "^3.13.1", - "minimatch": "^3.0.4", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@eslint/eslintrc/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "node_modules/@eslint/eslintrc/node_modules/globals": { "version": "13.17.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", @@ -1690,6 +1698,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@eslint/eslintrc/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/@eslint/eslintrc/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -1715,12 +1735,12 @@ } }, "node_modules/@humanwhocodes/config-array": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", - "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "version": "0.9.5", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", + "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==", "dev": true, "dependencies": { - "@humanwhocodes/object-schema": "^1.2.0", + "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", "minimatch": "^3.0.4" }, @@ -1852,6 +1872,12 @@ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" }, + "node_modules/@types/is-gzip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/is-gzip/-/is-gzip-2.0.0.tgz", + "integrity": "sha512-jG6MJGI45YAPE+3cjtBKUymbTIcMWSVEjfDS70okgTMjfjvC2GP1FuD9htugr36g9MFTT3KOjZDVoYrgeGJ8mg==", + "dev": true + }, "node_modules/@types/is-url": { "version": "1.2.30", "resolved": "https://registry.npmjs.org/@types/is-url/-/is-url-1.2.30.tgz", @@ -2076,15 +2102,6 @@ "url": "https://github.com/sponsors/epoberezkin" } }, - "node_modules/ansi-colors": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", - "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", - "dev": true, - "engines": { - "node": ">=6" - } - }, "node_modules/ansi-html": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz", @@ -2218,15 +2235,6 @@ "node": ">=0.10.0" } }, - "node_modules/astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -4246,18 +4254,6 @@ "once": "^1.4.0" } }, - "node_modules/enquirer": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", - "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", - "dev": true, - "dependencies": { - "ansi-colors": "^4.1.1" - }, - "engines": { - "node": ">=8.6" - } - }, "node_modules/error": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/error/-/error-7.2.1.tgz", @@ -4295,49 +4291,44 @@ } }, "node_modules/eslint": { - "version": "7.32.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", - "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.20.0.tgz", + "integrity": "sha512-d4ixhz5SKCa1D6SCPrivP7yYVi7nyD6A4vs6HIAul9ujBzcEmZVM3/0NN/yu5nKhmO1wjp5xQ46iRfmDGlOviA==", "dev": true, "dependencies": { - "@babel/code-frame": "7.12.11", - "@eslint/eslintrc": "^0.4.3", - "@humanwhocodes/config-array": "^0.5.0", + "@eslint/eslintrc": "^1.3.0", + "@humanwhocodes/config-array": "^0.9.2", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", - "debug": "^4.0.1", + "debug": "^4.3.2", "doctrine": "^3.0.0", - "enquirer": "^2.3.5", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^5.1.1", - "eslint-utils": "^2.1.0", - "eslint-visitor-keys": "^2.0.0", - "espree": "^7.3.1", + "eslint-scope": "^7.1.1", + "eslint-utils": "^3.0.0", + "eslint-visitor-keys": "^3.3.0", + "espree": "^9.3.2", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "functional-red-black-tree": "^1.0.1", - "glob-parent": "^5.1.2", - "globals": "^13.6.0", - "ignore": "^4.0.6", + "glob-parent": "^6.0.1", + "globals": "^13.15.0", + "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", - "js-yaml": "^3.13.1", + "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "progress": "^2.0.0", - "regexpp": "^3.1.0", - "semver": "^7.2.1", - "strip-ansi": "^6.0.0", + "regexpp": "^3.2.0", + "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", - "table": "^6.0.9", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" }, @@ -4345,50 +4336,44 @@ "eslint": "bin/eslint.js" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" }, "funding": { "url": "https://opencollective.com/eslint" } }, "node_modules/eslint-scope": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", + "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", "dev": true, "dependencies": { "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" + "estraverse": "^5.2.0" }, "engines": { - "node": ">=8.0.0" + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/eslint-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", - "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", + "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", "dev": true, "dependencies": { - "eslint-visitor-keys": "^1.1.0" + "eslint-visitor-keys": "^2.0.0" }, "engines": { - "node": ">=6" + "node": "^10.0.0 || ^12.0.0 || >= 14.0.0" }, "funding": { "url": "https://github.com/sponsors/mysticatea" + }, + "peerDependencies": { + "eslint": ">=5" } }, "node_modules/eslint-utils/node_modules/eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/eslint-visitor-keys": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", @@ -4397,13 +4382,13 @@ "node": ">=10" } }, - "node_modules/eslint/node_modules/@babel/code-frame": { - "version": "7.12.11", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", - "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "node_modules/eslint-visitor-keys": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", + "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", "dev": true, - "dependencies": { - "@babel/highlight": "^7.10.4" + "engines": { + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, "node_modules/eslint/node_modules/ansi-styles": { @@ -4421,6 +4406,12 @@ "url": "https://github.com/chalk/ansi-styles?sponsor=1" } }, + "node_modules/eslint/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "node_modules/eslint/node_modules/chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -4467,6 +4458,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/eslint/node_modules/glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "dependencies": { + "is-glob": "^4.0.3" + }, + "engines": { + "node": ">=10.13.0" + } + }, "node_modules/eslint/node_modules/globals": { "version": "13.17.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", @@ -4491,19 +4494,16 @@ "node": ">=8" } }, - "node_modules/eslint/node_modules/semver": { - "version": "7.3.7", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "node_modules/eslint/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "dependencies": { - "lru-cache": "^6.0.0" + "argparse": "^2.0.1" }, "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" + "js-yaml": "bin/js-yaml.js" } }, "node_modules/eslint/node_modules/strip-json-comments": { @@ -4543,26 +4543,29 @@ } }, "node_modules/espree": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", - "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", + "version": "9.3.2", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.2.tgz", + "integrity": "sha512-D211tC7ZwouTIuY5x9XnS0E9sWNChB7IYKX/Xp5eQj3nFXhqmiUDB9q27y76oFl8jTg3pXcQx/bpxMfs3CIZbA==", "dev": true, "dependencies": { - "acorn": "^7.4.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^1.3.0" + "acorn": "^8.7.1", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" }, "engines": { - "node": "^10.12.0 || >=12.0.0" + "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, - "node_modules/espree/node_modules/eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "node_modules/espree/node_modules/acorn": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "dev": true, + "bin": { + "acorn": "bin/acorn" + }, "engines": { - "node": ">=4" + "node": ">=0.4.0" } }, "node_modules/esprima": { @@ -4590,15 +4593,6 @@ "node": ">=0.10" } }, - "node_modules/esquery/node_modules/estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/esrecurse": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz", @@ -4611,7 +4605,7 @@ "node": ">=4.0" } }, - "node_modules/esrecurse/node_modules/estraverse": { + "node_modules/estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", @@ -4620,15 +4614,6 @@ "node": ">=4.0" } }, - "node_modules/estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", - "dev": true, - "engines": { - "node": ">=4.0" - } - }, "node_modules/estree-walker": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", @@ -5499,15 +5484,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "dev": true }, - "node_modules/growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true, - "engines": { - "node": ">=4.x" - } - }, "node_modules/handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -5741,9 +5717,9 @@ } }, "node_modules/ignore": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", - "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", "dev": true, "engines": { "node": ">= 4" @@ -6117,6 +6093,18 @@ "node": ">=0.10.0" } }, + "node_modules/is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -6411,22 +6399,20 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "node_modules/lodash.truncate": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", - "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", - "dev": true - }, "node_modules/log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "dev": true, "dependencies": { - "chalk": "^4.0.0" + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" }, "engines": { "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" } }, "node_modules/log-symbols/node_modules/ansi-styles": { @@ -7394,43 +7380,40 @@ } }, "node_modules/mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "dependencies": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" }, "bin": { "_mocha": "bin/_mocha", - "mocha": "bin/mocha" + "mocha": "bin/mocha.js" }, "engines": { - "node": ">= 10.12.0" + "node": ">= 14.0.0" }, "funding": { "type": "opencollective", @@ -7467,27 +7450,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "node_modules/mocha/node_modules/chokidar": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", - "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", - "dev": true, - "dependencies": { - "anymatch": "~3.1.1", - "braces": "~3.0.2", - "glob-parent": "~5.1.0", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.5.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "optionalDependencies": { - "fsevents": "~2.3.1" - } - }, "node_modules/mocha/node_modules/cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -7517,29 +7479,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "node_modules/mocha/node_modules/debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "dev": true, - "dependencies": { - "ms": "2.1.2" - }, - "engines": { - "node": ">=6.0" - }, - "peerDependenciesMeta": { - "supports-color": { - "optional": true - } - } - }, - "node_modules/mocha/node_modules/debug/node_modules/ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - }, "node_modules/mocha/node_modules/diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -7562,9 +7501,9 @@ } }, "node_modules/mocha/node_modules/glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "dependencies": { "fs.realpath": "^1.0.0", @@ -7581,6 +7520,18 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mocha/node_modules/glob/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, "node_modules/mocha/node_modules/has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", @@ -7591,9 +7542,9 @@ } }, "node_modules/mocha/node_modules/js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "dependencies": { "argparse": "^2.0.1" @@ -7603,15 +7554,24 @@ } }, "node_modules/mocha/node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", "dev": true, "dependencies": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" }, "engines": { - "node": "*" + "node": ">=10" + } + }, + "node_modules/mocha/node_modules/minimatch/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" } }, "node_modules/mocha/node_modules/ms": { @@ -7620,18 +7580,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, - "node_modules/mocha/node_modules/readdirp": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", - "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, "node_modules/mocha/node_modules/strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -7789,9 +7737,9 @@ "dev": true }, "node_modules/nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", "dev": true, "bin": { "nanoid": "bin/nanoid.cjs" @@ -8096,14 +8044,14 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" }, "engines": { - "node": ">=10" + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -8124,6 +8072,33 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-1.0.0.tgz", @@ -8408,15 +8383,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, - "node_modules/progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true, - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/property-information": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", @@ -8968,15 +8934,6 @@ "node": ">=0.10.0" } }, - "node_modules/require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -9104,9 +9061,9 @@ } }, "node_modules/serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", "dev": true, "dependencies": { "randombytes": "^2.1.0" @@ -9252,56 +9209,6 @@ "node": ">=6" } }, - "node_modules/slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/chalk/slice-ansi?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "dependencies": { - "color-convert": "^2.0.1" - }, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/chalk/ansi-styles?sponsor=1" - } - }, - "node_modules/slice-ansi/node_modules/color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "dependencies": { - "color-name": "~1.1.4" - }, - "engines": { - "node": ">=7.0.0" - } - }, - "node_modules/slice-ansi/node_modules/color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - }, "node_modules/snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -10100,44 +10007,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/table": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/table/-/table-6.8.0.tgz", - "integrity": "sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA==", - "dev": true, - "dependencies": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/table/node_modules/ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "dependencies": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/epoberezkin" - } - }, - "node_modules/table/node_modules/json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - }, "node_modules/text-extensions": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", @@ -10899,94 +10768,42 @@ "dependencies": { "http-parser-js": ">=0.5.1", "safe-buffer": ">=5.1.0", - "websocket-extensions": ">=0.1.1" - }, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/websocket-extensions": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", - "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/which": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", - "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", - "dev": true, - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "node-which": "bin/node-which" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/which-module": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", - "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", - "dev": true - }, - "node_modules/wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, - "dependencies": { - "string-width": "^1.0.2 || 2" - } - }, - "node_modules/wide-align/node_modules/ansi-regex": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", - "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/wide-align/node_modules/is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true, + "websocket-extensions": ">=0.1.1" + }, "engines": { - "node": ">=4" + "node": ">=0.8.0" } }, - "node_modules/wide-align/node_modules/string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", + "node_modules/websocket-extensions": { + "version": "0.1.4", + "resolved": "https://registry.npmjs.org/websocket-extensions/-/websocket-extensions-0.1.4.tgz", + "integrity": "sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg==", "dev": true, - "dependencies": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - }, "engines": { - "node": ">=4" + "node": ">=0.8.0" } }, - "node_modules/wide-align/node_modules/strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", + "node_modules/which": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz", + "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==", "dev": true, "dependencies": { - "ansi-regex": "^3.0.0" + "isexe": "^2.0.0" + }, + "bin": { + "node-which": "bin/node-which" }, "engines": { - "node": ">=4" + "node": ">= 8" } }, + "node_modules/which-module": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/which-module/-/which-module-2.0.0.tgz", + "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", + "dev": true + }, "node_modules/word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -11003,9 +10820,9 @@ "dev": true }, "node_modules/workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", "dev": true }, "node_modules/wrap-ansi": { @@ -11264,11 +11081,11 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -12407,22 +12224,28 @@ } }, "@eslint/eslintrc": { - "version": "0.4.3", - "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.4.3.tgz", - "integrity": "sha512-J6KFFz5QCYUJq3pf0mjEcCJVERbzv71PUIDczuh9JkwGEzced6CO5ADLHB1rbf/+oPBtoPfMYNOpGDzCANlbXw==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.3.0.tgz", + "integrity": "sha512-UWW0TMTmk2d7hLcWD1/e2g5HDM/HQ3csaLSqXCfqwh4uNDuNqlaKWXmEsL4Cs41Z0KnILNvwbHAah3C2yt06kw==", "dev": true, "requires": { "ajv": "^6.12.4", - "debug": "^4.1.1", - "espree": "^7.3.0", - "globals": "^13.9.0", - "ignore": "^4.0.6", + "debug": "^4.3.2", + "espree": "^9.3.2", + "globals": "^13.15.0", + "ignore": "^5.2.0", "import-fresh": "^3.2.1", - "js-yaml": "^3.13.1", - "minimatch": "^3.0.4", + "js-yaml": "^4.1.0", + "minimatch": "^3.1.2", "strip-json-comments": "^3.1.1" }, "dependencies": { + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "globals": { "version": "13.17.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", @@ -12432,6 +12255,15 @@ "type-fest": "^0.20.2" } }, + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "requires": { + "argparse": "^2.0.1" + } + }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -12447,12 +12279,12 @@ } }, "@humanwhocodes/config-array": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.5.0.tgz", - "integrity": "sha512-FagtKFz74XrTl7y6HCzQpwDfXP0yhxe9lHLD1UZxjvZIcbyRz8zTFF/yYNfSfzU414eDwZ1SrO0Qvtyf+wFMQg==", + "version": "0.9.5", + "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.9.5.tgz", + "integrity": "sha512-ObyMyWxZiCu/yTisA7uzx81s40xR2fD5Cg/2Kq7G02ajkNubJf6BopgDTmDyc3U7sXpNKM8cYOw7s7Tyr+DnCw==", "dev": true, "requires": { - "@humanwhocodes/object-schema": "^1.2.0", + "@humanwhocodes/object-schema": "^1.2.1", "debug": "^4.1.1", "minimatch": "^3.0.4" } @@ -12560,6 +12392,12 @@ "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.1.tgz", "integrity": "sha512-SZs7ekbP8CN0txVG2xVRH6EgKmEm31BOxA07vkFaETzZz1xh+cbt8BcI0slpymvwhx5dlFnQG2rTlPVQn+iRPQ==" }, + "@types/is-gzip": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@types/is-gzip/-/is-gzip-2.0.0.tgz", + "integrity": "sha512-jG6MJGI45YAPE+3cjtBKUymbTIcMWSVEjfDS70okgTMjfjvC2GP1FuD9htugr36g9MFTT3KOjZDVoYrgeGJ8mg==", + "dev": true + }, "@types/is-url": { "version": "1.2.30", "resolved": "https://registry.npmjs.org/@types/is-url/-/is-url-1.2.30.tgz", @@ -12769,12 +12607,6 @@ "uri-js": "^4.2.2" } }, - "ansi-colors": { - "version": "4.1.3", - "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", - "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", - "dev": true - }, "ansi-html": { "version": "0.0.7", "resolved": "https://registry.npmjs.org/ansi-html/-/ansi-html-0.0.7.tgz", @@ -12872,12 +12704,6 @@ "integrity": "sha512-Q+JC7Whu8HhmTdBph/Tq59IoRtoy6KAm5zzPv00WdujX82lbAL8K7WVjne7vdCsAmbF4AYaDOPyO3k0kl8qIrw==", "dev": true }, - "astral-regex": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz", - "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==", - "dev": true - }, "async": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/async/-/async-3.2.4.tgz", @@ -14506,15 +14332,6 @@ "once": "^1.4.0" } }, - "enquirer": { - "version": "2.3.6", - "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.6.tgz", - "integrity": "sha512-yjNnPr315/FjS4zIsUxYguYUPP2e1NK4d7E7ZOLiyYCcbFBiTMyID+2wvm2w6+pZ/odMA7cRkjhsPbltwBOrLg==", - "dev": true, - "requires": { - "ansi-colors": "^4.1.1" - } - }, "error": { "version": "7.2.1", "resolved": "https://registry.npmjs.org/error/-/error-7.2.1.tgz", @@ -14546,62 +14363,48 @@ "dev": true }, "eslint": { - "version": "7.32.0", - "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", - "integrity": "sha512-VHZ8gX+EDfz+97jGcgyGCyRia/dPOd6Xh9yPv8Bl1+SoaIwD+a/vlrOmGRUyOYu7MwUhc7CxqeaDZU13S4+EpA==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/eslint/-/eslint-8.20.0.tgz", + "integrity": "sha512-d4ixhz5SKCa1D6SCPrivP7yYVi7nyD6A4vs6HIAul9ujBzcEmZVM3/0NN/yu5nKhmO1wjp5xQ46iRfmDGlOviA==", "dev": true, "requires": { - "@babel/code-frame": "7.12.11", - "@eslint/eslintrc": "^0.4.3", - "@humanwhocodes/config-array": "^0.5.0", + "@eslint/eslintrc": "^1.3.0", + "@humanwhocodes/config-array": "^0.9.2", "ajv": "^6.10.0", "chalk": "^4.0.0", "cross-spawn": "^7.0.2", - "debug": "^4.0.1", + "debug": "^4.3.2", "doctrine": "^3.0.0", - "enquirer": "^2.3.5", "escape-string-regexp": "^4.0.0", - "eslint-scope": "^5.1.1", - "eslint-utils": "^2.1.0", - "eslint-visitor-keys": "^2.0.0", - "espree": "^7.3.1", + "eslint-scope": "^7.1.1", + "eslint-utils": "^3.0.0", + "eslint-visitor-keys": "^3.3.0", + "espree": "^9.3.2", "esquery": "^1.4.0", "esutils": "^2.0.2", "fast-deep-equal": "^3.1.3", "file-entry-cache": "^6.0.1", "functional-red-black-tree": "^1.0.1", - "glob-parent": "^5.1.2", - "globals": "^13.6.0", - "ignore": "^4.0.6", + "glob-parent": "^6.0.1", + "globals": "^13.15.0", + "ignore": "^5.2.0", "import-fresh": "^3.0.0", "imurmurhash": "^0.1.4", "is-glob": "^4.0.0", - "js-yaml": "^3.13.1", + "js-yaml": "^4.1.0", "json-stable-stringify-without-jsonify": "^1.0.1", "levn": "^0.4.1", "lodash.merge": "^4.6.2", - "minimatch": "^3.0.4", + "minimatch": "^3.1.2", "natural-compare": "^1.4.0", "optionator": "^0.9.1", - "progress": "^2.0.0", - "regexpp": "^3.1.0", - "semver": "^7.2.1", - "strip-ansi": "^6.0.0", + "regexpp": "^3.2.0", + "strip-ansi": "^6.0.1", "strip-json-comments": "^3.1.0", - "table": "^6.0.9", "text-table": "^0.2.0", "v8-compile-cache": "^2.0.3" }, "dependencies": { - "@babel/code-frame": { - "version": "7.12.11", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", - "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", - "dev": true, - "requires": { - "@babel/highlight": "^7.10.4" - } - }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", @@ -14611,6 +14414,12 @@ "color-convert": "^2.0.1" } }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, "chalk": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", @@ -14642,6 +14451,15 @@ "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", "dev": true }, + "glob-parent": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", + "integrity": "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A==", + "dev": true, + "requires": { + "is-glob": "^4.0.3" + } + }, "globals": { "version": "13.17.0", "resolved": "https://registry.npmjs.org/globals/-/globals-13.17.0.tgz", @@ -14657,13 +14475,13 @@ "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", "dev": true }, - "semver": { - "version": "7.3.7", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.7.tgz", - "integrity": "sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==", + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "requires": { - "lru-cache": "^6.0.0" + "argparse": "^2.0.1" } }, "strip-json-comments": { @@ -14690,53 +14508,53 @@ } }, "eslint-scope": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz", - "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==", + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", + "integrity": "sha512-QKQM/UXpIiHcLqJ5AOyIW7XZmzjkzQXYE54n1++wb0u9V/abW3l9uQnxX8Z5Xd18xyKIMTUAyQ0k1e8pz6LUrw==", "dev": true, "requires": { "esrecurse": "^4.3.0", - "estraverse": "^4.1.1" + "estraverse": "^5.2.0" } }, "eslint-utils": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-2.1.0.tgz", - "integrity": "sha512-w94dQYoauyvlDc43XnGB8lU3Zt713vNChgt4EWwhXAP2XkBvndfxF0AgIqKOOasjPIPzj9JqgwkwbCYD0/V3Zg==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/eslint-utils/-/eslint-utils-3.0.0.tgz", + "integrity": "sha512-uuQC43IGctw68pJA1RgbQS8/NP7rch6Cwd4j3ZBtgo4/8Flj4eGE7ZYSZRN3iq5pVUv6GPdW5Z1RFleo84uLDA==", "dev": true, "requires": { - "eslint-visitor-keys": "^1.1.0" + "eslint-visitor-keys": "^2.0.0" }, "dependencies": { "eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", + "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", "dev": true } } }, "eslint-visitor-keys": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.1.0.tgz", - "integrity": "sha512-0rSmRBzXgDzIsD6mGdJgevzgezI534Cer5L/vyMX0kHzT/jiB43jRhd9YUlMGYLQy2zprNmoT8qasCGtY+QaKw==", + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz", + "integrity": "sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA==", "dev": true }, "espree": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.1.tgz", - "integrity": "sha512-v3JCNCE64umkFpmkFGqzVKsOT0tN1Zr+ueqLZfpV1Ob8e+CEgPWa+OxCoGH3tnhimMKIaBm4m/vaRpJ/krRz2g==", + "version": "9.3.2", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.3.2.tgz", + "integrity": "sha512-D211tC7ZwouTIuY5x9XnS0E9sWNChB7IYKX/Xp5eQj3nFXhqmiUDB9q27y76oFl8jTg3pXcQx/bpxMfs3CIZbA==", "dev": true, "requires": { - "acorn": "^7.4.0", - "acorn-jsx": "^5.3.1", - "eslint-visitor-keys": "^1.3.0" + "acorn": "^8.7.1", + "acorn-jsx": "^5.3.2", + "eslint-visitor-keys": "^3.3.0" }, "dependencies": { - "eslint-visitor-keys": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz", - "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==", + "acorn": { + "version": "8.8.0", + "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", + "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "dev": true } } @@ -14754,14 +14572,6 @@ "dev": true, "requires": { "estraverse": "^5.1.0" - }, - "dependencies": { - "estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true - } } }, "esrecurse": { @@ -14771,20 +14581,12 @@ "dev": true, "requires": { "estraverse": "^5.2.0" - }, - "dependencies": { - "estraverse": { - "version": "5.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true - } } }, "estraverse": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-4.3.0.tgz", - "integrity": "sha512-39nnKffWz8xN1BU/2c79n9nB9HDzo0niYUqx6xyqUnyoAnQyyWpOTdZEeiCch8BBu515t4wp9ZmgVfVhn9EBpw==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", "dev": true }, "estree-walker": { @@ -15477,12 +15279,6 @@ "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==", "dev": true }, - "growl": { - "version": "1.10.5", - "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", - "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", - "dev": true - }, "handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -15654,9 +15450,9 @@ } }, "ignore": { - "version": "4.0.6", - "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", - "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.0.tgz", + "integrity": "sha512-CmxgYGiEPCLhfLnpPp1MoRmifwEIOgjcHXxOBjv7mY96c+eWScsOP9c112ZyLdWHi0FxHjI+4uVhKYp/gcdRmQ==", "dev": true }, "import-fresh": { @@ -15924,6 +15720,12 @@ "unc-path-regex": "^0.1.2" } }, + "is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true + }, "is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -16163,19 +15965,14 @@ "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, - "lodash.truncate": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/lodash.truncate/-/lodash.truncate-4.4.2.tgz", - "integrity": "sha512-jttmRe7bRse52OsWIMDLaXxWqRAmtIUccAQ3garviCqJjafXOfNMO0yMfNpdD6zbGaTU0P5Nz7e7gAT6cKmJRw==", - "dev": true - }, "log-symbols": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", - "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", "dev": true, "requires": { - "chalk": "^4.0.0" + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" }, "dependencies": { "ansi-styles": { @@ -16903,33 +16700,30 @@ } }, "mocha": { - "version": "8.4.0", - "resolved": "https://registry.npmjs.org/mocha/-/mocha-8.4.0.tgz", - "integrity": "sha512-hJaO0mwDXmZS4ghXsvPVriOhsxQ7ofcpQdm8dE+jISUOKopitvnXFQmpRR7jd2K6VBG6E26gU3IAbXXGIbu4sQ==", + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.0.0.tgz", + "integrity": "sha512-0Wl+elVUD43Y0BqPZBzZt8Tnkw9CMUdNYnUsTfOM1vuhJVZL+kiesFYsqwBkEEuEixaiPe5ZQdqDgX2jddhmoA==", "dev": true, "requires": { "@ungap/promise-all-settled": "1.1.2", "ansi-colors": "4.1.1", "browser-stdout": "1.3.1", - "chokidar": "3.5.1", - "debug": "4.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", "diff": "5.0.0", "escape-string-regexp": "4.0.0", "find-up": "5.0.0", - "glob": "7.1.6", - "growl": "1.10.5", + "glob": "7.2.0", "he": "1.2.0", - "js-yaml": "4.0.0", - "log-symbols": "4.0.0", - "minimatch": "3.0.4", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", "ms": "2.1.3", - "nanoid": "3.1.20", - "serialize-javascript": "5.0.1", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", "strip-json-comments": "3.1.1", "supports-color": "8.1.1", - "which": "2.0.2", - "wide-align": "1.1.3", - "workerpool": "6.1.0", + "workerpool": "6.2.1", "yargs": "16.2.0", "yargs-parser": "20.2.4", "yargs-unparser": "2.0.0" @@ -16956,22 +16750,6 @@ "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", "dev": true }, - "chokidar": { - "version": "3.5.1", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.5.1.tgz", - "integrity": "sha512-9+s+Od+W0VJJzawDma/gvBNQqkTiqYTWLuZoyAsivsI4AaWTCzHG06/TMjsf1cYe9Cb97UCEhjz7HvnPk2p/tw==", - "dev": true, - "requires": { - "anymatch": "~3.1.1", - "braces": "~3.0.2", - "fsevents": "~2.3.1", - "glob-parent": "~5.1.0", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.5.0" - } - }, "cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", @@ -16998,23 +16776,6 @@ "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", "dev": true }, - "debug": { - "version": "4.3.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", - "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", - "dev": true, - "requires": { - "ms": "2.1.2" - }, - "dependencies": { - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", - "dev": true - } - } - }, "diff": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/diff/-/diff-5.0.0.tgz", @@ -17028,9 +16789,9 @@ "dev": true }, "glob": { - "version": "7.1.6", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.6.tgz", - "integrity": "sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==", + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", "dev": true, "requires": { "fs.realpath": "^1.0.0", @@ -17039,6 +16800,17 @@ "minimatch": "^3.0.4", "once": "^1.3.0", "path-is-absolute": "^1.0.0" + }, + "dependencies": { + "minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "requires": { + "brace-expansion": "^1.1.7" + } + } } }, "has-flag": { @@ -17048,21 +16820,32 @@ "dev": true }, "js-yaml": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.0.0.tgz", - "integrity": "sha512-pqon0s+4ScYUvX30wxQi3PogGFAlUyH0awepWvwkj4jD4v+ova3RiYw8bmA6x2rDrEaj8i/oWKoRxpVNW+Re8Q==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", "dev": true, "requires": { "argparse": "^2.0.1" } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", "dev": true, "requires": { - "brace-expansion": "^1.1.7" + "brace-expansion": "^2.0.1" + }, + "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + } } }, "ms": { @@ -17071,15 +16854,6 @@ "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "dev": true }, - "readdirp": { - "version": "3.5.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.5.0.tgz", - "integrity": "sha512-cMhu7c/8rdhkHXWsY+osBhfSy0JikwpHK/5+imo+LpeasTF8ouErHrlYkwT0++njiyuDvc7OFY5T3ukvZ8qmFQ==", - "dev": true, - "requires": { - "picomatch": "^2.2.1" - } - }, "strip-json-comments": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-3.1.1.tgz", @@ -17205,9 +16979,9 @@ "dev": true }, "nanoid": { - "version": "3.1.20", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.1.20.tgz", - "integrity": "sha512-a1cQNyczgKbLX9jwbS/+d7W8fX/RfgYR7lVWwWOGIPNgK2m0MWvrGF6/m4kk6U3QcFMnZf3RIhL0v2Jgh/0Uxw==", + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.3.tgz", + "integrity": "sha512-p1sjXuopFs0xg+fPASzQ28agW1oHD7xDsd9Xkf3T15H3c/cifrFHVwrh74PdoklAPi+i7MdRsE47vm2r6JoB+w==", "dev": true }, "nanomatch": { @@ -17446,11 +17220,11 @@ "integrity": "sha512-BZOr3nRQHOntUjTrH8+Lh54smKHoHyur8We1V8DSMVrl5A2malOOwuJRnKRDjSnkoeBh4at6BwEnb5I7Jl31wg==" }, "p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-4.0.0.tgz", + "integrity": "sha512-5b0R4txpzjPWVw/cXXUResoD4hb6U/x9BH08L7nw+GN1sezDzPdxeRvpc9c433fZhBan/wusjbCsqwqm4EIBIQ==", "requires": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" } }, "p-locate": { @@ -17460,6 +17234,23 @@ "dev": true, "requires": { "p-limit": "^3.0.2" + }, + "dependencies": { + "p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "requires": { + "yocto-queue": "^0.1.0" + } + }, + "yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true + } } }, "p-try": { @@ -17677,12 +17468,6 @@ "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==", "dev": true }, - "progress": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", - "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", - "dev": true - }, "property-information": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/property-information/-/property-information-5.6.0.tgz", @@ -18112,12 +17897,6 @@ "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", "dev": true }, - "require-from-string": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true - }, "require-main-filename": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/require-main-filename/-/require-main-filename-2.0.0.tgz", @@ -18217,9 +17996,9 @@ "dev": true }, "serialize-javascript": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-5.0.1.tgz", - "integrity": "sha512-SaaNal9imEO737H2c05Og0/8LUXG7EnsZyMa8MzkmuHoELfT6txuj0cMqRj6zfPKnmQ1yasR4PCJc8x+M4JSPA==", + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", + "integrity": "sha512-Qr3TosvguFt8ePWqsvRfrKyQXIiW+nGbYpy8XK24NQHE83caxWt+mIymTT19DGFbNWNLfEwsrkSmN64lVWB9ag==", "dev": true, "requires": { "randombytes": "^2.1.0" @@ -18346,43 +18125,6 @@ "integrity": "sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==", "dev": true }, - "slice-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz", - "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==", - "dev": true, - "requires": { - "ansi-styles": "^4.0.0", - "astral-regex": "^2.0.0", - "is-fullwidth-code-point": "^3.0.0" - }, - "dependencies": { - "ansi-styles": { - "version": "4.3.0", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", - "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, - "requires": { - "color-convert": "^2.0.1" - } - }, - "color-convert": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", - "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, - "requires": { - "color-name": "~1.1.4" - } - }, - "color-name": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true - } - } - }, "snapdragon": { "version": "0.8.2", "resolved": "https://registry.npmjs.org/snapdragon/-/snapdragon-0.8.2.tgz", @@ -19045,39 +18787,6 @@ "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==", "dev": true }, - "table": { - "version": "6.8.0", - "resolved": "https://registry.npmjs.org/table/-/table-6.8.0.tgz", - "integrity": "sha512-s/fitrbVeEyHKFa7mFdkuQMWlH1Wgw/yEXMt5xACT4ZpzWFluehAxRtUUQKPuWhaLAWhFcVx6w3oC8VKaUfPGA==", - "dev": true, - "requires": { - "ajv": "^8.0.1", - "lodash.truncate": "^4.4.2", - "slice-ansi": "^4.0.0", - "string-width": "^4.2.3", - "strip-ansi": "^6.0.1" - }, - "dependencies": { - "ajv": { - "version": "8.11.0", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", - "integrity": "sha512-wGgprdCvMalC0BztXvitD2hC04YffAvtsUn93JbGXYLAtCUO4xd17mCCZQxUOItiBwZvJScWo8NIvQMQ71rdpg==", - "dev": true, - "requires": { - "fast-deep-equal": "^3.1.1", - "json-schema-traverse": "^1.0.0", - "require-from-string": "^2.0.2", - "uri-js": "^4.2.2" - } - }, - "json-schema-traverse": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true - } - } - }, "text-extensions": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/text-extensions/-/text-extensions-1.9.0.tgz", @@ -19687,48 +19396,6 @@ "integrity": "sha512-B+enWhmw6cjfVC7kS8Pj9pCrKSc5txArRyaYGe088shv/FGWH+0Rjx/xPgtsWfsUtS27FkP697E4DDhgrgoc0Q==", "dev": true }, - "wide-align": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/wide-align/-/wide-align-1.1.3.tgz", - "integrity": "sha512-QGkOQc8XL6Bt5PwnsExKBPuMKBxnGxWWW3fU55Xt4feHozMUhdUMaBCk290qpm/wG5u/RSKzwdAC4i51YigihA==", - "dev": true, - "requires": { - "string-width": "^1.0.2 || 2" - }, - "dependencies": { - "ansi-regex": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.1.tgz", - "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", - "dev": true - }, - "is-fullwidth-code-point": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz", - "integrity": "sha512-VHskAKYM8RfSFXwee5t5cbN5PZeq1Wrh6qd5bkyiXIf6UQcN6w/A0eXM9r6t8d+GYOh+o6ZhiEnb88LN/Y8m2w==", - "dev": true - }, - "string-width": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz", - "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==", - "dev": true, - "requires": { - "is-fullwidth-code-point": "^2.0.0", - "strip-ansi": "^4.0.0" - } - }, - "strip-ansi": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz", - "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "dev": true, - "requires": { - "ansi-regex": "^3.0.0" - } - } - } - }, "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", @@ -19742,9 +19409,9 @@ "dev": true }, "workerpool": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.1.0.tgz", - "integrity": "sha512-toV7q9rWNYha963Pl/qyeZ6wG+3nnsyvolaNUS8+R5Wtw6qJPTxIlOP1ZSvcGhEJw+l3HMMmtiNo9Gl61G4GVg==", + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/workerpool/-/workerpool-6.2.1.tgz", + "integrity": "sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==", "dev": true }, "wrap-ansi": { @@ -19940,9 +19607,9 @@ "dev": true }, "yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.0.0.tgz", + "integrity": "sha512-9bnSc/HEW2uRy67wc+T8UwauLuPJVn28jb+GtJY16iiKWyvmYJRXVT4UamsAEGQfPohgr2q4Tq0sQbQlxTfi1g==" }, "zwitch": { "version": "1.0.5", diff --git a/package.json b/package.json index a8c4598..483e2d2 100644 --- a/package.json +++ b/package.json @@ -20,8 +20,9 @@ "files": [ "lib" ], - "main": "./lib/assets/sitemapper.js", + "exports": "./lib/assets/sitemapper.js", "types": "./sitemapper.d.ts", + "type": "module", "repository": { "type": "git", "url": "git://github.com/seantomburke/sitemapper.git" @@ -32,13 +33,14 @@ "url": "http://www.seantburke.com" }, "scripts": { - "compile": "tsc -d --sourceMap --outDir lib -t esnext ./src/**/*.ts && babel lib -d lib -s", + "compile:tsc": "tsc --project tsconfig.json", + "compile:babel": "babel lib -d lib -s", + "compile": "npm run tsc", "build": "npm run clean && npm run compile", "start": "npm run build && node lib/examples/index.js", "test": "npm run build && mocha ./lib/tests/*.js && npm run lint", "lint": "eslint src", "clean": "rm -rf lib", - "tsc": "tsc", "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md" }, "maintainers": [ @@ -53,7 +55,7 @@ "test": "./test" }, "engines": { - "node": ">= 10.0.0" + "node": ">=14.16" }, "devDependencies": { "@babel/cli": "^7.12.8", @@ -62,6 +64,7 @@ "@babel/runtime": "^7.12.5", "@types/async": "^3.2.4", "@types/got": "^9.6.11", + "@types/is-gzip": "^2.0.0", "@types/is-url": "^1.2.28", "@types/mocha": "^8.0.4", "@types/xml2js": "^0.4.7", @@ -69,9 +72,9 @@ "babel-plugin-add-module-exports": "^1.0.4", "babel-preset-minify": "^0.5.1", "documentation": "^13.1.0", - "eslint": "^7.14.0", + "eslint": "^8.20.0", "is-url": "^1.2.4", - "mocha": "^8.2.1", + "mocha": "^10.0.0", "should": "^13.2.3", "ts-node": "^9.0.0", "typescript": "^4.1.2" @@ -79,7 +82,7 @@ "dependencies": { "got": "^11.8.0", "is-gzip": "2.0.0", - "p-limit": "^3.1.0", + "p-limit": "^4.0.0", "xml2js": "^0.4.23" } } diff --git a/src/assets/sitemapper.ts b/src/assets/sitemapper.ts index 88d5d95..e77e5ec 100644 --- a/src/assets/sitemapper.ts +++ b/src/assets/sitemapper.ts @@ -12,6 +12,7 @@ import got from 'got'; // @ts-ignore import zlib from 'zlib'; import pLimit from 'p-limit'; +// @ts-ignore import isGzip from 'is-gzip'; // @ts-ignore import Url from 'url'; @@ -142,7 +143,7 @@ export default class Sitemapper { * @param {Number} timestamp * @example sitemapper.lastmod = 1630694181; // Unix timestamp */ - static set lastmod(timestamp) { + static set lastmod(timestamp: number) { this.lastmod = timestamp; } @@ -189,7 +190,7 @@ export default class Sitemapper { * @param {string} [url] - the Sitemaps url (e.g https://wp.seantburke.com/sitemap.xml) * @returns {Promise} */ - async parse(url: string = this.url): Promise<{error, data}> { + async parse(url: string = this.url): Promise<{error: any, data: any}> { // setup the response options for the got request const requestOptions: any = { method: 'GET', @@ -214,7 +215,7 @@ export default class Sitemapper { // if the response does not have a successful status code then clear the timeout for this url. if (!response || response.statusCode !== 200) { - clearTimeout(this.timeoutTable[url]); + clearTimeout((this.timeoutTable as any)[url]); return { error: response.error, data: response }; } @@ -231,7 +232,7 @@ export default class Sitemapper { // return the results return { error: null, data }; - } catch (error) { + } catch (error: any) { // If the request was canceled notify the user of the timeout if (error.name === "CancelError") { return { @@ -266,7 +267,7 @@ export default class Sitemapper { */ initializeTimeout(url: string, requester: { cancel: Function }) { // this will throw a CancelError which will be handled in the parent that calls this method. - this.timeoutTable[url] = setTimeout(() => requester.cancel(), this.timeout); + (this.timeoutTable as any)[url] = setTimeout(() => requester.cancel(), this.timeout); } /** @@ -282,7 +283,7 @@ export default class Sitemapper { try { const { error, data } : { error: Error, data: {name: string, sitemapindex: { sitemap: Array<{loc: string[], lastmod: number }>}, urlset: { url: Array<{ loc: string[], lastmod: number }>}, } } = await this.parse(url); // The promise resolved, remove the timeout - clearTimeout(this.timeoutTable[url]); + clearTimeout((this.timeoutTable as any)[url]); if (error) { // Handle errors during sitemap parsing / request @@ -326,7 +327,7 @@ export default class Sitemapper { .filter((site: ({ loc: Array, lastmod: number })) => { if (this.lastmod === 0) return true; if (site.lastmod === undefined) return false; - const modified = new Date(site.lastmod[0]).getTime(); + const modified = new Date((site.lastmod as any)[0]).getTime(); return modified >= this.lastmod; }) @@ -407,12 +408,10 @@ export default class Sitemapper { * @param {string} url - url to query * @param {getSitesCallback} callback - callback for sites and error * @callback - */ - async getSites(url = this.url, callback) { - console.warn( - // eslint-disable-line no-console - "\r\nWarning:", - "function .getSites() is deprecated, please use the function .fetch()\r\n" + */ + async getSites(url = this.url, callback: Function) { + console.warn( // eslint-disable-line no-console + '\r\nWarning:', 'function .getSites() is deprecated, please use the function .fetch()\r\n' ); let err = {}; @@ -420,7 +419,7 @@ export default class Sitemapper { try { const response = await this.fetch(url); sites = response.sites; - } catch (error) { + } catch (error: any) { err = error; } return callback(err, sites); diff --git a/src/tests/test.es5.js b/src/tests/test.es5.js deleted file mode 100644 index e5e535b..0000000 --- a/src/tests/test.es5.js +++ /dev/null @@ -1,180 +0,0 @@ -require('async'); -require('assert'); -require('should'); -const isUrl = require('is-url'); - -const Sitemapper = require('../../lib/assets/sitemapper.js'); -var sitemapper; - -describe('Sitemapper', function () { - - beforeEach(() => { - sitemapper = new Sitemapper(); - }); - - describe('Sitemapper Class', function () { - - it('should have initializeTimeout method', () => { - sitemapper.initializeTimeout.should.be.Function; - }); - - it('should have crawl method', () => { - sitemapper.crawl.should.be.Function; - }); - - it('should have parse method', () => { - sitemapper.parse.should.be.Function; - }); - - it('should have fetch method', () => { - sitemapper.fetch.should.be.Function; - }); - - it('should construct with a url', () => { - sitemapper = new Sitemapper({ - url: 'google.com', - }); - sitemapper.url.should.equal('google.com'); - }); - - it('should construct with a timeout', () => { - sitemapper = new Sitemapper({ - timeout: 1000, - }); - sitemapper.timeout.should.equal(1000); - }); - - it('should set timeout', () => { - sitemapper.timeout = 1000; - sitemapper.timeout.should.equal(1000); - }); - - it('should set url', () => { - sitemapper.url = 1000; - sitemapper.url.should.equal(1000); - }); - }); - - describe('fetch Method resolves sites to array', function () { - it('https://wp.seantburke.com/sitemap.xml sitemaps should be an array', function (done) { - this.timeout(30000); - const url = 'https://wp.seantburke.com/sitemap.xml'; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - data.url.should.equal(url); - data.sites.length.should.be.above(2); - isUrl(data.sites[0]).should.be.true; - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - - it('gibberish.gibberish should fail silently with an empty array', function (done) { - this.timeout(30000); - const url = 'http://gibberish.gibberish'; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - data.errors.should.be.Array; - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - - it('https://www.google.com/work/sitemap.xml sitemaps should be an array', function (done) { - this.timeout(30000); - const url = 'https://www.google.com/work/sitemap.xml'; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - data.url.should.equal(url); - data.sites.length.should.be.above(2); - isUrl(data.sites[0]).should.be.true; - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - - it('https://www.golinks.io/sitemap.xml sitemaps should be an array', function (done) { - this.timeout(30000); - const url = 'https://www.golinks.io/sitemap.xml'; - sitemapper.timeout = 5000; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - data.url.should.equal(url); - data.sites.length.should.be.above(2); - isUrl(data.sites[0]).should.be.true; - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - - it('https://www.golinks.io/sitemap.xml sitemaps should return an empty array when timing out', function (done) { - this.timeout(30000); - const url = 'https://www.golinks.io/sitemap.xml'; - sitemapper.timeout = 1; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - }); - - describe('gzipped sitemaps', function () { - beforeEach(() => { - sitemapper = new Sitemapper({ - requestHeaders: { - 'Accept-Encoding': 'gzip,deflate,sdch', - } - }); - }); - - it('https://www.banggood.com/sitemap/category.xml.gz gzip should be a non-empty array', function (done) { - this.timeout(30000); - const url = 'https://www.banggood.com/sitemap/category.xml.gz'; - sitemapper.timeout = 10000; - sitemapper.fetch(url) - .then(data => { - data.sites.should.be.Array; - data.errors.should.be.Array; - data.sites.length.should.be.greaterThan(0); - done(); - }) - .catch(error => { - console.error('Test failed'); - done(error); - }); - }); - }); - - describe('getSites method', function () { - it('getSites should be backwards compatible', function (done) { - this.timeout(30000); - const url = 'https://wp.seantburke.com/sitemap.xml'; - sitemapper.getSites(url, (err, sites) => { - sites.should.be.Array; - isUrl(sites[0]).should.be.true; - done(); - }); - }); - }); -}); diff --git a/src/tests/test.ts.ts b/src/tests/test.ts similarity index 90% rename from src/tests/test.ts.ts rename to src/tests/test.ts index 7ad61d7..945262b 100644 --- a/src/tests/test.ts.ts +++ b/src/tests/test.ts @@ -1,7 +1,7 @@ import 'async'; import 'assert'; import 'should'; -const isUrl = require('is-url'); +import isUrl from 'is-url'; // @ts-ignore import Sitemapper from '../../lib/assets/sitemapper.js'; @@ -62,14 +62,14 @@ describe('Sitemapper', function () { this.timeout(30000); const url = 'https://wp.seantburke.com/sitemap.xml'; sitemapper.fetch(url) - .then(data => { + .then((data: any) => { data.sites.should.be.Array; data.url.should.equal(url); data.sites.length.should.be.above(2); isUrl(data.sites[0]).should.be.true; done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -79,12 +79,12 @@ describe('Sitemapper', function () { this.timeout(30000); const url = 'http://gibberish.gibberish'; sitemapper.fetch(url) - .then(data => { + .then((data :any) => { data.sites.should.be.Array; data.errors.should.be.Array; done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -94,14 +94,14 @@ describe('Sitemapper', function () { this.timeout(30000); const url = 'https://www.google.com/work/sitemap.xml'; sitemapper.fetch(url) - .then(data => { + .then((data: any) => { data.sites.should.be.Array; data.url.should.equal(url); data.sites.length.should.be.above(2); isUrl(data.sites[0]).should.be.true; done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -112,14 +112,14 @@ describe('Sitemapper', function () { const url = 'https://www.golinks.io/sitemap.xml'; sitemapper.timeout = 5000; sitemapper.fetch(url) - .then(data => { + .then((data: any) => { data.sites.should.be.Array; data.url.should.equal(url); data.sites.length.should.be.above(2); isUrl(data.sites[0]).should.be.true; done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -131,11 +131,11 @@ describe('Sitemapper', function () { sitemapper.timeout = 1; sitemapper.fetch(url) - .then(data => { + .then((data: any) => { data.sites.should.be.Array; done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -156,13 +156,13 @@ describe('Sitemapper', function () { const url = 'https://www.banggood.com/sitemap/category.xml.gz'; sitemapper.timeout = 10000; sitemapper.fetch(url) - .then(data => { + .then((data: any) => { data.sites.should.be.Array; data.errors.should.be.Array; data.sites.length.should.be.greaterThan(0); done(); }) - .catch(error => { + .catch((error: any) => { console.error('Test failed'); done(error); }); @@ -173,7 +173,7 @@ describe('Sitemapper', function () { it('getSites should be backwards compatible', function (done) { this.timeout(30000); const url = 'https://wp.seantburke.com/sitemap.xml'; - sitemapper.getSites(url, (err, sites) => { + sitemapper.getSites(url, (err: any, sites: Array) => { sites.should.be.Array; isUrl(sites[0]).should.be.true; done(); diff --git a/tsconfig.json b/tsconfig.json index 6f00a06..762c711 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,7 +11,7 @@ // "disableReferencedProjectLoad": true, /* Reduce the number of projects loaded automatically by TypeScript. */ /* Language and Environment */ - "target": "es2015", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ + "target": "esnext", /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */ "lib": [ "dom", "esnext" @@ -28,7 +28,7 @@ // "moduleDetection": "auto", /* Control what method is used to detect module-format JS files. */ /* Modules */ - "module": "commonjs", /* Specify what module code is generated. */ + "module": "esnext", /* Specify what module code is generated. */ // "rootDir": "./", /* Specify the root folder within your source files. */ "moduleResolution": "node", /* Specify how TypeScript looks up a file from a given module specifier. */ "baseUrl": "./", /* Specify the base directory to resolve non-relative module names. */ @@ -47,12 +47,12 @@ // "maxNodeModuleJsDepth": 1, /* Specify the maximum folder depth used for checking JavaScript files from 'node_modules'. Only applicable with 'allowJs'. */ /* Emit */ - "declaration": false, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ + "declaration": true, /* Generate .d.ts files from TypeScript and JavaScript files in your project. */ // "declarationMap": true, /* Create sourcemaps for d.ts files. */ // "emitDeclarationOnly": true, /* Only output d.ts files and not JavaScript files. */ - // "sourceMap": true, /* Create source map files for emitted JavaScript files. */ + "sourceMap": true, /* Create source map files for emitted JavaScript files. */ // "outFile": "./", /* Specify a file that bundles all outputs into one JavaScript file. If 'declaration' is true, also designates a file that bundles all .d.ts output. */ - "outDir": "./lib/tests", /* Specify an output folder for all emitted files. */ + "outDir": "./lib", /* Specify an output folder for all emitted files. */ "removeComments": false, /* Disable emitting comments. */ // "noEmit": true, /* Disable emitting files from a compilation. */ // "importHelpers": true, /* Allow importing helper functions from tslib once per project, instead of including them per-file. */ @@ -73,7 +73,7 @@ /* Interop Constraints */ // "isolatedModules": true, /* Ensure that each file can be safely transpiled without relying on other imports. */ - // "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ + "allowSyntheticDefaultImports": true, /* Allow 'import x from y' when a module doesn't have a default export. */ "esModuleInterop": true, /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */ // "preserveSymlinks": true, /* Disable resolving symlinks to their realpath. This correlates to the same flag in node. */ "forceConsistentCasingInFileNames": true, /* Ensure that casing is correct in imports. */ @@ -87,7 +87,7 @@ // "strictPropertyInitialization": true, /* Check for class properties that are declared but not set in the constructor. */ // "noImplicitThis": true, /* Enable error reporting when 'this' is given the type 'any'. */ // "useUnknownInCatchVariables": true, /* Default catch clause variables as 'unknown' instead of 'any'. */ - "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ + // "alwaysStrict": true, /* Ensure 'use strict' is always emitted. */ // "noUnusedLocals": true, /* Enable error reporting when local variables aren't read. */ // "noUnusedParameters": true, /* Raise an error when a function parameter isn't read. */ // "exactOptionalPropertyTypes": true, /* Interpret optional property types as written, rather than adding 'undefined'. */ @@ -104,6 +104,7 @@ "skipLibCheck": true /* Skip type checking all .d.ts files. */ }, "include": [ - "./*.ts" + "./*.ts", + "./src/**/*.ts" ] } From e40f2fe50856e00c217b2c8ce4cf614f7902e931 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 22:46:27 -0700 Subject: [PATCH 05/12] Rename eslintrc --- .eslintrc.js => .eslintrc.cjs | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .eslintrc.js => .eslintrc.cjs (100%) diff --git a/.eslintrc.js b/.eslintrc.cjs similarity index 100% rename from .eslintrc.js rename to .eslintrc.cjs From 3e2056b243f3d22336d31af42731ff7855c94567 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 22:46:55 -0700 Subject: [PATCH 06/12] Updating with TypeScript --- .eslintrc.cjs | 11 ++++-- example.es6.js | 41 -------------------- example.js | 77 +++++++++++++++----------------------- lib/assets/sitemapper.d.ts | 1 + lib/assets/sitemapper.js | 12 ++++-- package-lock.json | 28 ++++++++++++++ package.json | 5 ++- src/assets/sitemapper.ts | 4 -- src/tests/test.ts | 1 + 9 files changed, 79 insertions(+), 101 deletions(-) delete mode 100644 example.es6.js diff --git a/.eslintrc.cjs b/.eslintrc.cjs index 1cd1bf2..5f7b240 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -1,8 +1,11 @@ -export default { - extends: 'eslint:recommended', +// @ts-check +const { defineConfig } = require('eslint-define-config'); + +module.exports = defineConfig({ + extends: "eslint:recommended", parserOptions: { ecmaVersion: 8, - sourceType: 'module', + sourceType: "module", ecmaFeatures: {}, }, env: { @@ -10,4 +13,4 @@ export default { mocha: true, es6: true, }, -}; +}); diff --git a/example.es6.js b/example.es6.js deleted file mode 100644 index 5fd9e24..0000000 --- a/example.es6.js +++ /dev/null @@ -1,41 +0,0 @@ -import Sitemapper from 'sitemapper'; - -(async () => { - const sitemapper = new Sitemapper(); - - const Google = new Sitemapper({ - url: 'https://www.google.com/work/sitemap.xml', - debug: false, - timeout: 15000, // 15 seconds - }); - - try { - const data = await Google.fetch(); - console.log(data.sites); - } catch(error) { - console.log(error); - } - - sitemapper.timeout = 5000; - - try { - const { url, sites } = await sitemapper.fetch('https://wp.seantburke.com/sitemap.xml'); - console.log(`url:${url}`, 'sites:', sites); - } catch(error) { - console.log(error) - } - - try { - const { url, sites } = await sitemapper.fetch('http://www.cnn.com/sitemaps/sitemap-index.xml'); - console.log(`url:${url}`, 'sites:', sites); - } catch(error) { - console.log(error) - } - - try { - const { url, sites } = await sitemapper.fetch('http://www.stubhub.com/new-sitemap/us/sitemap-US-en-index.xml'); - console.log(`url:${url}`, 'sites:', sites); - } catch(error) { - console.log(error) - } -})(); diff --git a/example.js b/example.js index 80437ae..5fd9e24 100644 --- a/example.js +++ b/example.js @@ -1,56 +1,41 @@ -var Sitemapper = require('sitemapper'); +import Sitemapper from 'sitemapper'; -// Instantiate an instance with options -var Google = new Sitemapper({ - url: 'https://www.google.com/work/sitemap.xml', - debug: false, - timeout: 15000 //15 seconds -}); +(async () => { + const sitemapper = new Sitemapper(); -// Then fetch -Google.fetch() - .then(function (data) { - console.log(data); - }) - .catch(function (error) { - console.log(error); - }); - -// Instantiate an instance with no options -var sitemapper = new Sitemapper(); -sitemapper.timeout = 5000; - -sitemapper.fetch('https://wp.seantburke.com/sitemap.xml') - .then(function (data) { - console.log(data); - }) - .catch(function (error) { - console.log(error); + const Google = new Sitemapper({ + url: 'https://www.google.com/work/sitemap.xml', + debug: false, + timeout: 15000, // 15 seconds }); -sitemapper.fetch('http://www.cnn.com/sitemaps/sitemap-index.xml') - .then(function (data) { - console.log('sites:', data.sites, 'url', data.url); - }) - .catch(function (error) { + try { + const data = await Google.fetch(); + console.log(data.sites); + } catch(error) { console.log(error); - }); + } -sitemapper.fetch('http://www.stubhub.com/new-sitemap/us/sitemap-US-en-index.xml') - .then(function (data) { - console.log('sites:', data.sites, 'url', data.url); - }) - .catch(function (error) { - console.log(error); - }); + sitemapper.timeout = 5000; -// Version 1.0.0 example which has been deprecated. -sitemapper.getSites('https://wp.seantburke.com/sitemap.xml', function (err, sites) { - if (!err) { - console.log(sites); + try { + const { url, sites } = await sitemapper.fetch('https://wp.seantburke.com/sitemap.xml'); + console.log(`url:${url}`, 'sites:', sites); + } catch(error) { + console.log(error) } - else { - console.log(err); + + try { + const { url, sites } = await sitemapper.fetch('http://www.cnn.com/sitemaps/sitemap-index.xml'); + console.log(`url:${url}`, 'sites:', sites); + } catch(error) { + console.log(error) } -}); + try { + const { url, sites } = await sitemapper.fetch('http://www.stubhub.com/new-sitemap/us/sitemap-US-en-index.xml'); + console.log(`url:${url}`, 'sites:', sites); + } catch(error) { + console.log(error) + } +})(); diff --git a/lib/assets/sitemapper.d.ts b/lib/assets/sitemapper.d.ts index e2a4dae..c7be7b4 100644 --- a/lib/assets/sitemapper.d.ts +++ b/lib/assets/sitemapper.d.ts @@ -7,6 +7,7 @@ */ /// import { SitemapperOptions } from '../../sitemapper'; +import { Buffer } from 'buffer'; /** * @typedef {Object} Sitemapper */ diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index 65406d0..b91c589 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -6,21 +6,25 @@ * @author Sean Burke <@seantomburke> */ import { parseStringPromise } from 'xml2js'; -// @ts-ignore import got from 'got'; -// @ts-ignore import zlib from 'zlib'; import pLimit from 'p-limit'; import isGzip from 'is-gzip'; -// @ts-ignore import Url from 'url'; -// @ts-ignore import path from 'path'; import { Buffer } from 'buffer'; /** * @typedef {Object} Sitemapper */ export default class Sitemapper { + url; + timeout; + timeoutTable; + requestHeaders; + debug; + retries; + rejectUnauthorized; + concurrency; /** * Construct the Sitemapper class * diff --git a/package-lock.json b/package-lock.json index 8aa207c..49eee43 100644 --- a/package-lock.json +++ b/package-lock.json @@ -30,6 +30,7 @@ "babel-preset-minify": "^0.5.1", "documentation": "^13.1.0", "eslint": "^8.20.0", + "eslint-define-config": "^1.5.1", "is-url": "^1.2.4", "mocha": "^10.0.0", "should": "^13.2.3", @@ -4342,6 +4343,27 @@ "url": "https://opencollective.com/eslint" } }, + "node_modules/eslint-define-config": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/eslint-define-config/-/eslint-define-config-1.5.1.tgz", + "integrity": "sha512-6gxrmN7aKGffaO8dCtMMKyo3IxbWymMQ248p4lf8GbaFRcLsqOXHFdUhhM0Hcy1NudvnpwHcfbDf7Nh9pIm1TA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/Shinigami92" + }, + { + "type": "paypal", + "url": "https://www.paypal.com/donate/?hosted_button_id=L7GY729FBKTZY" + } + ], + "engines": { + "node": ">= 14.6.0", + "npm": ">= 6.0.0", + "pnpm": ">= 7.0.0" + } + }, "node_modules/eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", @@ -14507,6 +14529,12 @@ } } }, + "eslint-define-config": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/eslint-define-config/-/eslint-define-config-1.5.1.tgz", + "integrity": "sha512-6gxrmN7aKGffaO8dCtMMKyo3IxbWymMQ248p4lf8GbaFRcLsqOXHFdUhhM0Hcy1NudvnpwHcfbDf7Nh9pIm1TA==", + "dev": true + }, "eslint-scope": { "version": "7.1.1", "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-7.1.1.tgz", diff --git a/package.json b/package.json index 483e2d2..ee9519f 100644 --- a/package.json +++ b/package.json @@ -35,11 +35,11 @@ "scripts": { "compile:tsc": "tsc --project tsconfig.json", "compile:babel": "babel lib -d lib -s", - "compile": "npm run tsc", + "compile": "npm run compile:tsc", "build": "npm run clean && npm run compile", "start": "npm run build && node lib/examples/index.js", "test": "npm run build && mocha ./lib/tests/*.js && npm run lint", - "lint": "eslint src", + "lint": "eslint ./src", "clean": "rm -rf lib", "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md" }, @@ -73,6 +73,7 @@ "babel-preset-minify": "^0.5.1", "documentation": "^13.1.0", "eslint": "^8.20.0", + "eslint-define-config": "^1.5.1", "is-url": "^1.2.4", "mocha": "^10.0.0", "should": "^13.2.3", diff --git a/src/assets/sitemapper.ts b/src/assets/sitemapper.ts index e77e5ec..8401473 100644 --- a/src/assets/sitemapper.ts +++ b/src/assets/sitemapper.ts @@ -7,16 +7,12 @@ */ import { parseStringPromise } from 'xml2js'; -// @ts-ignore import got from 'got'; -// @ts-ignore import zlib from 'zlib'; import pLimit from 'p-limit'; // @ts-ignore import isGzip from 'is-gzip'; -// @ts-ignore import Url from 'url'; -// @ts-ignore import path from 'path'; import { SitemapperOptions, SitemapperResponse} from '../../sitemapper'; import { Buffer } from 'buffer'; diff --git a/src/tests/test.ts b/src/tests/test.ts index 945262b..efa863d 100644 --- a/src/tests/test.ts +++ b/src/tests/test.ts @@ -1,6 +1,7 @@ import 'async'; import 'assert'; import 'should'; +// @ts-ignore import isUrl from 'is-url'; // @ts-ignore From 3a24c82caaaf026b658ee97c299019e9ca138ae6 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 22:50:32 -0700 Subject: [PATCH 07/12] Bump version to 4.0.0 --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ee9519f..bddeffd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sitemapper", - "version": "3.2.5", + "version": "4.0.0", "description": "Parser for XML Sitemaps to be used with Robots.txt and web crawlers", "keywords": [ "parse", @@ -18,7 +18,8 @@ }, "license": "MIT", "files": [ - "lib" + "lib", + "sitemapper.d.ts" ], "exports": "./lib/assets/sitemapper.js", "types": "./sitemapper.d.ts", From 7362a7a24b998a61dc2ffbff13232ce27eb55bf8 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 22:59:54 -0700 Subject: [PATCH 08/12] node versions minimum 14 --- .github/workflows/npm-publish.yml | 4 ++-- .github/workflows/test.yml | 2 +- .github/workflows/version-bump.yml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index f6d2a0b..4f0eb47 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 12 + node-version: 14 - run: npm ci - run: npm test @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 12 + node-version: 14 registry-url: https://registry.npmjs.org/ - run: npm ci - run: npm publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fe5475e..e708ee2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [10.x, 12.x, 14.x] + node-version: [14.x, 16.x, 18.x] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index 59e55ee..a53464f 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -18,7 +18,7 @@ jobs: nodeVersion: ${{ steps.bump_version.outputs.version }} strategy: matrix: - node-version: [10.x] + node-version: [14.x] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} @@ -38,7 +38,7 @@ jobs: needs: build strategy: matrix: - node-version: [10.x] + node-version: [14.x] steps: - name: draft release id: draft_release From 5bd8b450944b9b89d679abe593602832de1a5a05 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 23:14:07 -0700 Subject: [PATCH 09/12] verison 14.16 --- .github/workflows/npm-publish.yml | 4 ++-- .github/workflows/test.yml | 2 +- .github/workflows/version-bump.yml | 4 ++-- README.md | 12 ++++++++++++ 4 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 4f0eb47..d5e1ec2 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 14.16 - run: npm ci - run: npm test @@ -25,7 +25,7 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: - node-version: 14 + node-version: 14.16 registry-url: https://registry.npmjs.org/ - run: npm ci - run: npm publish diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e708ee2..fe3ec04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x, 18.x] + node-version: [14.16, 16.x, 18.x] steps: - uses: actions/checkout@v2 diff --git a/.github/workflows/version-bump.yml b/.github/workflows/version-bump.yml index a53464f..29f26a7 100644 --- a/.github/workflows/version-bump.yml +++ b/.github/workflows/version-bump.yml @@ -18,7 +18,7 @@ jobs: nodeVersion: ${{ steps.bump_version.outputs.version }} strategy: matrix: - node-version: [14.x] + node-version: [14.16] steps: - uses: actions/checkout@v2 - name: Use Node.js ${{ matrix.node-version }} @@ -38,7 +38,7 @@ jobs: needs: build strategy: matrix: - node-version: [14.x] + node-version: [14.16] steps: - name: draft release id: draft_release diff --git a/README.md b/README.md index e728b02..d6c3d4d 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,18 @@ ![Test](https://github.com/seantomburke/sitemapper/workflows/test/badge.svg?branch=master&event=push) Parse through a sitemaps xml to get all the urls for your crawler. + +## Version 4 + +Version 4.0.0 introduces breaking changes and supports ECMAScript Modules (ESM) +If upgrading to 4.0.0 you will not be able to use require() to import the dependency. +You must use import() to import the dependencies. +You will also need to upgrade to Node version >=14.16 + +## Version 3 + + + ## Version 2 ### Installation From cbc1f0a2de42e5756d3c31174c39f9c8f8425e13 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Thu, 28 Jul 2022 23:26:37 -0700 Subject: [PATCH 10/12] Updating Read Me --- README.md | 120 ++++++++++++++++++++++-------------------------------- 1 file changed, 49 insertions(+), 71 deletions(-) diff --git a/README.md b/README.md index d6c3d4d..f4e2438 100644 --- a/README.md +++ b/README.md @@ -15,61 +15,27 @@ Parse through a sitemaps xml to get all the urls for your crawler. ## Version 4 -Version 4.0.0 introduces breaking changes and supports ECMAScript Modules (ESM) +> Version 4.0.0 introduces breaking changes and supports ECMAScript Modules (ESM) If upgrading to 4.0.0 you will not be able to use require() to import the dependency. You must use import() to import the dependencies. -You will also need to upgrade to Node version >=14.16 - -## Version 3 - - - -## Version 2 +You will also need to upgrade to `Node >= 14.16` ### Installation ```bash -npm install sitemapper --save +npm install sitemapper@latest --save ``` ### Simple Example ```javascript -const Sitemapper = require('sitemapper'); +import Sitemapper from 'sitemapper'; const sitemap = new Sitemapper(); -sitemap.fetch('https://wp.seantburke.com/sitemap.xml').then(function(sites) { - console.log(sites); -}); - -``` -### Examples in ES6 -```javascript -import Sitemapper from 'sitemapper'; - (async () => { - const Google = new Sitemapper({ - url: 'https://www.google.com/work/sitemap.xml', - timeout: 15000, // 15 seconds - }); - - try { - const { sites } = await Google.fetch(); + const { sites } = await sitemap.fetch('https://wp.seantburke.com/sitemap.xml'); console.log(sites); - } catch (error) { - console.log(error); - } })(); - -// or - -const sitemapper = new Sitemapper(); -sitemapper.timeout = 5000; - -sitemapper.fetch('https://wp.seantburke.com/sitemap.xml') - .then(({ url, sites }) => console.log(`url:${url}`, 'sites:', sites)) - .catch(error => console.log(error)); ``` - # Options You can add options on the initial Sitemapper object when instantiating it. @@ -113,56 +79,68 @@ const sitemapper = new Sitemapper({ ``` -### Examples in ES5 +### Examples in ESM ```javascript -var Sitemapper = require('sitemapper'); - -var Google = new Sitemapper({ - url: 'https://www.google.com/work/sitemap.xml', - timeout: 15000 //15 seconds -}); +import Sitemapper from 'sitemapper'; -Google.fetch() - .then(function (data) { - console.log(data); - }) - .catch(function (error) { - console.log(error); +(async () => { + const Google = new Sitemapper({ + url: 'https://www.google.com/work/sitemap.xml', + timeout: 15000, // 15 seconds }); + try { + const { sites } = await Google.fetch(); + console.log(sites); + } catch (error) { + console.log(error); + } +})(); // or - -var sitemapper = new Sitemapper(); - +const sitemapper = new Sitemapper(); sitemapper.timeout = 5000; -sitemapper.fetch('https://wp.seantburke.com/sitemap.xml') - .then(function (data) { - console.log(data); - }) - .catch(function (error) { - console.log(error); - }); +sitemapper.fetch('https://wp.seantburke.com/sitemap.xml') + .then(({ url, sites }) => console.log(`url:${url}`, 'sites:', sites)) + .catch(error => console.log(error)); ``` -## Version 1 +## Version 3 + +> Works for `Node 10.X` to `Node 12.X` + +### Installation ```bash -npm install sitemapper@1.1.1 --save +npm install sitemapper@3.2 --save ``` ### Simple Example +```javascript +const Sitemapper = require('sitemapper'); + +const sitemap = new Sitemapper(); +sitemap.fetch('https://wp.seantburke.com/sitemap.xml').then(function(sites) { + console.log(sites); +}); + +### Example in JS ```javascript var Sitemapper = require('sitemapper'); -var sitemapper = new Sitemapper(); - -sitemapper.getSites('https://wp.seantburke.com/sitemap.xml', function(err, sites) { - if (!err) { - console.log(sites); - } +var Google = new Sitemapper({ + url: 'https://www.google.com/work/sitemap.xml', + timeout: 15000 //15 seconds }); -``` + +Google.fetch() + .then(function (data) { + console.log(data); + }) + .catch(function (error) { + console.log(error); + }); + From a1de9e1646ac01afcc6fb3932899cbb805e94899 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Fri, 29 Jul 2022 00:35:08 -0700 Subject: [PATCH 11/12] Adding lastmod stuff --- lib/assets/sitemapper.d.ts | 20 ++++++++- lib/assets/sitemapper.js | 89 ++++++++++++++++++++++++++++++-------- 2 files changed, 89 insertions(+), 20 deletions(-) diff --git a/lib/assets/sitemapper.d.ts b/lib/assets/sitemapper.d.ts index c7be7b4..249a10d 100644 --- a/lib/assets/sitemapper.d.ts +++ b/lib/assets/sitemapper.d.ts @@ -20,6 +20,7 @@ export default class Sitemapper { retries: number; rejectUnauthorized: boolean; concurrency: number; + lastmod: number; /** * Construct the Sitemapper class * @@ -30,10 +31,12 @@ export default class Sitemapper { * @params {integer} [options.concurrency] - The number of concurrent sitemaps to crawl (e.g. 2 will crawl no more than 2 sitemaps at the same time) * @params {integer} [options.retries] - The maximum number of retries to attempt when crawling fails (e.g. 1 for 1 retry, 2 attempts in total) * @params {boolean} [options.rejectUnauthorized] - If true (default), it will throw on invalid certificates, such as expired or self-signed ones. + * @params {lastmod} [options.lastmod] - the minimum lastmod value for urls * * @example let sitemap = new Sitemapper({ * url: 'https://wp.seantburke.com/sitemap.xml', - * timeout: 15000 + * timeout: 15000, + * lastmod: 1630693759 * }); */ constructor(options?: SitemapperOptions); @@ -66,6 +69,21 @@ export default class Sitemapper { * @example sitemapper.timeout = 15000; // 15 seconds */ static set timeout(duration: Number); + /** + * Get the lastmod minimum value + * + * @example console.log(sitemapper.lastmod); + * @returns {Number} + */ + static get lastmod(): number; + /** + * Set the lastmod minimum value + * + * @public + * @param {Number} timestamp + * @example sitemapper.lastmod = 1630694181; // Unix timestamp + */ + static set lastmod(timestamp: number); /** * * @param {string} url - url for making requests. Should be a link to a sitemaps.xml diff --git a/lib/assets/sitemapper.js b/lib/assets/sitemapper.js index b91c589..7db9fea 100644 --- a/lib/assets/sitemapper.js +++ b/lib/assets/sitemapper.js @@ -9,6 +9,7 @@ import { parseStringPromise } from 'xml2js'; import got from 'got'; import zlib from 'zlib'; import pLimit from 'p-limit'; +// @ts-ignore import isGzip from 'is-gzip'; import Url from 'url'; import path from 'path'; @@ -25,6 +26,7 @@ export default class Sitemapper { retries; rejectUnauthorized; concurrency; + lastmod; /** * Construct the Sitemapper class * @@ -35,10 +37,12 @@ export default class Sitemapper { * @params {integer} [options.concurrency] - The number of concurrent sitemaps to crawl (e.g. 2 will crawl no more than 2 sitemaps at the same time) * @params {integer} [options.retries] - The maximum number of retries to attempt when crawling fails (e.g. 1 for 1 retry, 2 attempts in total) * @params {boolean} [options.rejectUnauthorized] - If true (default), it will throw on invalid certificates, such as expired or self-signed ones. + * @params {lastmod} [options.lastmod] - the minimum lastmod value for urls * * @example let sitemap = new Sitemapper({ * url: 'https://wp.seantburke.com/sitemap.xml', - * timeout: 15000 + * timeout: 15000, + * lastmod: 1630693759 * }); */ constructor(options) { @@ -46,11 +50,13 @@ export default class Sitemapper { this.url = settings.url || ''; this.timeout = settings.timeout || 15000; this.timeoutTable = {}; + this.lastmod = settings.lastmod || 0; this.requestHeaders = settings.requestHeaders; this.debug = settings.debug || false; this.concurrency = settings.concurrency || 10; this.retries = settings.retries || 0; - this.rejectUnauthorized = settings.rejectUnauthorized || true; + this.rejectUnauthorized = + settings.rejectUnauthorized === false ? false : true; } /** * Gets the sites from a sitemap.xml with a given URL @@ -69,6 +75,9 @@ export default class Sitemapper { errors: [], }; // attempt to set the variables with the crawl + if (this.debug) { + console.debug(`Using minimum lastmod value of ${this.lastmod}`); + } try { // crawl the URL results = await this.crawl(url); @@ -104,6 +113,25 @@ export default class Sitemapper { static set timeout(duration) { this.timeout = duration; } + /** + * Get the lastmod minimum value + * + * @example console.log(sitemapper.lastmod); + * @returns {Number} + */ + static get lastmod() { + return this.lastmod; + } + /** + * Set the lastmod minimum value + * + * @public + * @param {Number} timestamp + * @example sitemapper.lastmod = 1630694181; // Unix timestamp + */ + static set lastmod(timestamp) { + this.lastmod = timestamp; + } /** * * @param {string} url - url for making requests. Should be a link to a sitemaps.xml @@ -149,11 +177,11 @@ export default class Sitemapper { method: 'GET', resolveWithFullResponse: true, gzip: true, - responseType: 'buffer', + responseType: "buffer", headers: this.requestHeaders, https: { rejectUnauthorized: this.rejectUnauthorized, - } + }, }; try { // create a request Promise with the url and request options @@ -181,16 +209,23 @@ export default class Sitemapper { } catch (error) { // If the request was canceled notify the user of the timeout - if (error.name === 'CancelError') { + if (error.name === "CancelError") { return { error: `Request timed out after ${this.timeout} milliseconds for url: '${url}'`, - data: error + data: error, + }; + } + // If an HTTPError include error http code + if (error.name === "HTTPError") { + return { + error: `HTTP Error occurred: ${error.message}`, + data: error, }; } // Otherwise notify of another error return { error: `Error occurred: ${error.name}`, - data: error + data: error, }; } } @@ -235,11 +270,14 @@ export default class Sitemapper { // Fail and log error return { sites: [], - errors: [{ + errors: [ + { type: data.name, + message: error, url, retries: retryIndex, - }] + }, + ], }; } else if (data && data.urlset && data.urlset.url) { @@ -247,10 +285,20 @@ export default class Sitemapper { if (this.debug) { console.debug(`Urlset found during "crawl('${url}')"`); } - const sites = data.urlset.url.map(site => site.loc && site.loc[0]); + // filter out any urls that are older than the lastmod + const sites = data.urlset.url + .filter((site) => { + if (this.lastmod === 0) + return true; + if (site.lastmod === undefined) + return false; + const modified = new Date(site.lastmod[0]).getTime(); + return modified >= this.lastmod; + }) + .map((site) => site.loc && site.loc[0]); return { sites, - errors: [] + errors: [], }; } else if (data && data.sitemapindex) { @@ -259,17 +307,17 @@ export default class Sitemapper { console.debug(`Additional sitemap found during "crawl('${url}')"`); } // Map each child url into a promise to create an array of promises - const sitemap = data.sitemapindex.sitemap.map(map => map.loc && map.loc[0]); + const sitemap = data.sitemapindex.sitemap.map((map) => map.loc && map.loc[0]); // Parse all child urls within the concurrency limit in the settings const limit = pLimit(this.concurrency); - const promiseArray = sitemap.map(site => limit(() => this.crawl(site))); + const promiseArray = sitemap.map((site) => limit(() => this.crawl(site))); // Make sure all the promises resolve then filter and reduce the array const results = await Promise.all(promiseArray); const sites = results - .filter(result => (result.errors.length === 0)) + .filter((result) => result.errors.length === 0) .reduce((prev, { sites }) => [...prev, ...sites], []); const errors = results - .filter(result => (result.errors.length !== 0)) + .filter((result) => result.errors.length !== 0) .reduce((prev, { errors }) => [...prev, ...errors], []); return { sites, @@ -289,11 +337,14 @@ export default class Sitemapper { // Fail and log error return { sites: [], - errors: [{ + errors: [ + { url, - type: data.name || 'UnknownStateError', - retries: retryIndex - }] + type: data.name || "UnknownStateError", + message: "An unknown error occurred.", + retries: retryIndex, + }, + ], }; } catch (e) { From 262d9f3d26c527edf1bc01f5e2f34c9b7c4834d5 Mon Sep 17 00:00:00 2001 From: Sean Thomas Burke Date: Fri, 29 Jul 2022 00:48:32 -0700 Subject: [PATCH 12/12] Adding Type --- sitemapper.d.ts | 2 +- src/assets/sitemapper.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sitemapper.d.ts b/sitemapper.d.ts index 8fbfcda..005b952 100644 --- a/sitemapper.d.ts +++ b/sitemapper.d.ts @@ -1,6 +1,6 @@ export interface SitemapperResponse { url: string; - sites: string[]; + sites: Array; errors: SitemapperErrorData[]; } diff --git a/src/assets/sitemapper.ts b/src/assets/sitemapper.ts index 8401473..8de6682 100644 --- a/src/assets/sitemapper.ts +++ b/src/assets/sitemapper.ts @@ -74,7 +74,7 @@ export default class Sitemapper { */ async fetch(url: string = this.url) { // initialize empty variables - let results: any = { + let results: SitemapperResponse = { url: '', sites: [], errors: [], @@ -351,7 +351,7 @@ export default class Sitemapper { // Make sure all the promises resolve then filter and reduce the array const results = await Promise.all(promiseArray); const sites = results - .filter((result) => result.errors.length === 0) + .filter((result: { errors: Array }) => result.errors.length === 0) .reduce((prev, { sites }) => [...prev, ...sites], []); const errors = results .filter((result) => result.errors.length !== 0) @@ -411,7 +411,7 @@ export default class Sitemapper { ); let err = {}; - let sites = []; + let sites: string[] = []; try { const response = await this.fetch(url); sites = response.sites;