perf(crawl): add async streaming parse and shared arc

a11ywatch · Mar 2, 2023 · 60ad80c · 60ad80c
1 parent d3baa03
commit 60ad80c
Show file tree

Hide file tree

Showing 9 changed files with 319 additions and 201 deletions.
diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "website_crawler"
-version = "0.7.70"
+version = "0.7.71"
 authors = ["Jeff Mendez <[email protected]>"]
 edition = "2018"
 description = "gRPC tokio based web crawler"

diff --git a/src/lib.rs b/src/lib.rs
@@ -1,4 +1,3 @@
-#![forbid(unsafe_code)]
 extern crate sitemap;
 extern crate tokio;
 extern crate tonic;

diff --git a/src/packages/mod.rs b/src/packages/mod.rs
@@ -1,3 +1,3 @@
-pub mod spider;
+pub mod robotparser;
 pub mod scraper;
-pub mod robotparser;
+pub mod spider;