Filepath and raw XML Text files are now supported by the fetch method

.gitignore

@@ -6,3 +6,4 @@ npm-debug.log
 tmp
 lib/tests/
 lib/**/*.map
+yarn.lock

README.md

@@ -1,3 +1,5 @@
+## NPM : https://www.npmjs.com/package/@myagizmaktav/sitemapper
+
 ## Sitemap-parser
 [![Code Scanning](https://github.com/seantomburke/sitemapper/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/seantomburke/sitemapper/actions/workflows/codeql-analysis.yml)
 [![NPM Publish](https://github.com/seantomburke/sitemapper/actions/workflows/npm-publish.yml/badge.svg)](https://github.com/seantomburke/sitemapper/actions/workflows/npm-publish.yml)
@@ -35,7 +37,24 @@ sitemap.fetch('https://wp.seantburke.com/sitemap.xml').then(function(sites) {
   console.log(sites);
 });
 
+// OR 
+
+sitemap.fetch("D:\\githubprojects\\src\\data\\sites-xmls\\Sitemap.xml").then(function(sites) {
+  console.log(sites);
+}
+);
+
+// OR 
+
+sitemap.fetch('<?xml version="1.0" encoding="utf-8" standalone="yes" ?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://wp.seantburke.com/?p=231</loc></url></urlset>').then(function(sites) {
+  console.log(sites);
+}
+);
+
+
+
 ```
+
 ### Examples in ES6
 ```javascript
 import Sitemapper from 'sitemapper';

package.json

@@ -1,5 +1,5 @@
 {
-  "name": "sitemapper",
+  "name": "@myagizmaktav/sitemapper",
   "version": "3.2.8",
   "description": "Parser for XML Sitemaps to be used with Robots.txt and web crawlers",
   "keywords": [
@@ -11,10 +11,10 @@
     "crawlers",
     "webcrawler"
   ],
-  "homepage": "http://github.com/seantomburke/sitemapper",
+  "homepage": "https://github.com/myagizmaktav/sitemapper",
   "tonicExampleFilename": "example.js",
   "bugs": {
-    "url": "http://github.com/seantomburke/sitemapper/issues"
+    "url": "https://github.com/myagizmaktav/sitemapper/issues"
   },
   "license": "MIT",
   "files": [
@@ -25,13 +25,9 @@
   "types": "./sitemapper.d.ts",
   "repository": {
     "type": "git",
-    "url": "git://github.com/seantomburke/sitemapper.git"
-  },
-  "author": {
-    "name": "Sean Thomas Burke",
-    "email": "[email protected]",
-    "url": "http://www.seantburke.com"
+    "url": "git+https://github.com/myagizmaktav/sitemapper.git"
   },
+  "author": "Mehmet Yağız Maktav <[email protected]>",
   "scripts": {
     "compile": "babel src -d lib -s && tsc --project ./src/tests/",
     "build": "npm run clean && npm run compile",
@@ -42,15 +38,11 @@
     "docs": "documentation build ./src/assets/sitemapper.js -f md > docs.md"
   },
   "maintainers": [
-    {
-      "name": "Sean Thomas Burke",
-      "email": "[email protected]",
-      "url": "http://www.seantburke.com"
-    }
+    "Sean Thomas Burke <[email protected]> (http://www.seantburke.com)",
+    "Mehmet Yağız Maktav <[email protected]>"
   ],
   "directories": {
-    "lib": "./lib",
-    "test": "./test"
+    "lib": "lib"
   },
   "engines": {
     "node": ">= 10.0.0"

src/assets/sitemapper.js

@@ -11,6 +11,7 @@ import got from "got";
 import zlib from "zlib";
 import pLimit from "p-limit";
 import isGzip from "is-gzip";
+import fs from "fs";
 
 /**
  * @typedef {Object} Sitemapper
@@ -186,33 +187,63 @@ export default class Sitemapper {
         rejectUnauthorized: this.rejectUnauthorized,
       },
     };
-
+    const isUrlRegex =
+      // eslint-disable-next-line no-useless-escape
+      /^https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)/;
     try {
-      // create a request Promise with the url and request options
-      const requester = got.get(url, requestOptions);
+      let data;
 
-      // initialize the timeout method based on the URL, and pass the request object.
-      this.initializeTimeout(url, requester);
+      // if is this url
+      if (isUrlRegex.test(url)) {
+        // create a request Promise with the url and request options
+        const requester = got.get(url, requestOptions);
 
-      // get the response from the requester promise
-      const response = await requester;
+        // initialize the timeout method based on the URL, and pass the request object.
+        this.initializeTimeout(url, requester);
 
-      // if the response does not have a successful status code then clear the timeout for this url.
-      if (!response || response.statusCode !== 200) {
-        clearTimeout(this.timeoutTable[url]);
-        return { error: response.error, data: response };
-      }
+        // get the response from the requester promise
+        let response = await requester;
+        // console.log(String(response.body));
+
+        // if the response does not have a successful status code then clear the timeout for this url.
+        if (!response || response.statusCode !== 200) {
+          clearTimeout(this.timeoutTable[url]);
+          return { error: response.error, data: response };
+        }
+        let responseBody;
+
+        if (isGzip(response.rawBody)) {
+          responseBody = await this.decompressResponseBody(response.body);
+        } else {
+          responseBody = response.body;
+        }
 
-      let responseBody;
+        data = await parseStringPromise(responseBody);
+      }
+      // if is this raw XmlText
+      else if (url.includes("xmlns") && url.includes("sitemaps")) {
+        let responseBody;
+        if (isGzip(url)) {
+          responseBody = await this.decompressResponseBody(url);
+        } else {
+          responseBody = await url;
+        }
 
-      if (isGzip(response.rawBody)) {
-        responseBody = await this.decompressResponseBody(response.body);
-      } else {
-        responseBody = response.body;
+        data = await parseStringPromise(responseBody);
       }
+      // if is this file path
+      else if (url.includes(":\\")) {
+        const rawBody = await fs.readFileSync(url, "utf-8");
+        let responseBody;
+        if (isGzip(rawBody)) {
+          responseBody = await this.decompressResponseBody(rawBody);
+        } else {
+          responseBody = await rawBody;
+        }
 
+        data = await parseStringPromise(responseBody);
+      } else throw new Error("Invalid url");
       // otherwise parse the XML that was returned.
-      const data = await parseStringPromise(responseBody);
 
       // return the results
       return { error: null, data };

src/tests/test.js

@@ -262,4 +262,29 @@ describe('Sitemapper', function () {
       });
     });
   });
+
+  describe("fetch readsRawData", function () {
+    it("fetch should be read raw data", function (done) {
+      this.timeout(30000);
+      const url =
+        '<?xml version="1.0" encoding="utf-8" standalone="yes" ?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><url><loc>https://wp.seantburke.com/?p=234</loc></url><url><loc>https://wp.seantburke.com/?p=231</loc></url></urlset>';
+      sitemapper.getSites(url, (err, sites) => {
+        sites.should.be.Array;
+        isUrl(sites[0]).should.be.true;
+        done();
+      });
+    });
+  });
+  describe("fetch readsFilePath", function () {
+    it("fetch should be read filePath", function (done) {
+      this.timeout(30000);
+      const url =
+        "D:\\githubprojects\\ProjectSeriesBackend\\src\\data\\sites-xmls\\Sitemap.xml";
+      sitemapper.getSites(url, (err, sites) => {
+        sites.should.be.Array;
+        isUrl(sites[0]).should.be.true;
+        done();
+      });
+    });
+  });
 });