chore(api): add norobo crawl handling

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@a11ywatch/core",
-  "version": "0.5.46",
+  "version": "0.5.47",
   "description": "a11ywatch central api",
   "main": "./server.js",
   "scripts": {
@@ -12,7 +12,7 @@
     "pub": "tsc && cp package.json dist && cp package-lock.json dist && cp README.md dist && cp LICENSE dist && cd dist && npm publish"
   },
   "dependencies": {
-    "@a11ywatch/protos": "^0.3.1",
+    "@a11ywatch/protos": "^0.3.3",
     "@a11ywatch/website-source-builder": "^0.0.32",
     "@fastify/cookie": "^6.0.0",
     "@fastify/cors": "^7.0.0",

src/core/actions/accessibility/watcher_crawl.ts

@@ -6,7 +6,7 @@ interface CrawlParams {
   url?: string;
   scan?: boolean; // determine scan or crawl method
   userId?: number;
-  robots?: boolean; // respect robots txt file
+  robots?: boolean; // respect robots txt file defaults to true
   subdomains?: boolean;
   tld?: boolean;
 }
@@ -31,7 +31,7 @@ export const watcherCrawl = async ({
   const crawlParams = {
     url: String(initUrl(url, true)),
     id: userId,
-    norobots: !robots,
+    robots,
     subdomains: subdomains,
     tld: tld,
   };

src/core/streams/crawl-config.ts

@@ -2,7 +2,14 @@ import { SUPER_MODE } from "../../config/config";
 import { getWebsite } from "../controllers/websites";
 
 // get the website crawl configuration
-export const getCrawlConfig = async ({ id, role, url, tld, subdomains }) => {
+export const getCrawlConfig = async ({
+  id,
+  role,
+  url,
+  tld,
+  subdomains,
+  robots = true,
+}) => {
   let subdomainsEnabled = subdomains;
   let tldEnabled = tld;
 
@@ -29,5 +36,6 @@ export const getCrawlConfig = async ({ id, role, url, tld, subdomains }) => {
     userId: id,
     subdomains: subdomainsEnabled,
     tld: tldEnabled,
+    robots,
   };
 };

src/core/streams/crawl.ts

@@ -44,6 +44,7 @@ export const crawlStream = async (
       role: userNext.role,
       subdomains: body.subdomains,
       tld: body.tld,
+      robots: body.robots,
     });
 
     res.raw.write("[");

src/core/utils/crawl-stream-slim.ts

@@ -14,7 +14,7 @@ export const crawlHttpStreamSlim = (
   client?: string,
   onlyData?: boolean // remove issues and other data from stream
 ): Promise<boolean> => {
-  const { url, userId, subdomains, tld } = props;
+  const { url, userId, subdomains, tld, robots } = props;
 
   setImmediate(async () => {
     await watcherCrawl({
@@ -23,6 +23,7 @@ export const crawlHttpStreamSlim = (
       subdomains: !!subdomains,
       tld: !!tld,
       scan: true,
+      robots,
     });
   });
 

src/core/utils/crawl-stream.ts

@@ -10,6 +10,7 @@ export type CrawlProps = {
   userId?: number;
   subdomains?: boolean;
   tld?: boolean;
+  robots?: boolean; // prevent robots txt
 };
 
 // crawl website and wait for finished emit event to continue @return Website[] use for testing.
@@ -18,7 +19,7 @@ export const crawlHttpStream = (
   res: FastifyContext["reply"],
   client?: string
 ): Promise<boolean> => {
-  const { url, userId, subdomains, tld } = props;
+  const { url, userId, subdomains, tld, robots = true } = props;
 
   setImmediate(async () => {
     await watcherCrawl({
@@ -27,6 +28,7 @@ export const crawlHttpStream = (
       subdomains: !!subdomains,
       tld: !!tld,
       scan: true,
+      robots,
     });
   });
 

src/proto/calls/core-crawl.ts

@@ -33,10 +33,12 @@ export const coreCrawl = async (call: ServerCallStreaming) => {
 
 // crawl website slim and wait for finished emit event to continue @return Website[].
 export const crawlStreaming = (
-  props: CrawlProps,
+  props: CrawlProps & {
+    norobo?: boolean;
+  },
   call: ServerCallStreaming
 ): Promise<boolean> => {
-  const { url, userId, subdomains, tld } = props;
+  const { url, userId, subdomains, tld, norobo } = props;
 
   setImmediate(async () => {
     await watcherCrawl({
@@ -45,6 +47,7 @@ export const crawlStreaming = (
       subdomains: !!subdomains,
       tld: !!tld,
       scan: true,
+      robots: !norobo,
     });
   });