enhancement: send an event that scraping has started

devflowinc · Nov 21, 2024 · e4f8e14 · e4f8e14
1 parent f9e0f5b
commit e4f8e14
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 0 deletions.
diff --git a/server/src/bin/crawl-worker.rs b/server/src/bin/crawl-worker.rs
@@ -830,6 +830,19 @@ async fn scrape_worker(
             }
         }
 
+        event_queue
+            .send(ClickHouseEvent::WorkerEvent(
+                WorkerEvent::from_details(
+                    crawl_request.dataset_id,
+                    models::EventType::CrawlStarted {
+                        scrape_id: crawl_request.scrape_id,
+                        crawl_options: crawl_request.clone().crawl_options,
+                    },
+                )
+                .into(),
+            ))
+            .await;
+
         match crawl(crawl_request.clone(), pool.clone(), redis_pool.clone()).await {
             Ok(scrape_report) => {
                 log::info!("Scrape job completed: {:?}", scrape_report);

diff --git a/server/src/data/models.rs b/server/src/data/models.rs
@@ -1781,6 +1781,11 @@ pub enum EventType {
     GroupChunksUpdated { group_id: uuid::Uuid },
     #[display(fmt = "group_chunks_action_failed")]
     GroupChunksActionFailed { group_id: uuid::Uuid, error: String },
+    #[display(fmt = "crawl_started")]
+    CrawlStarted {
+        scrape_id: uuid::Uuid,
+        crawl_options: CrawlOptions,
+    },
     #[display(fmt = "crawl_completed")]
     CrawlCompleted {
         scrape_id: uuid::Uuid,
@@ -1810,6 +1815,7 @@ impl EventType {
             EventTypeRequest::GroupChunksUpdated,
             EventTypeRequest::GroupChunksActionFailed,
             EventTypeRequest::CrawlCompleted,
+            EventTypeRequest::CrawlStarted,
             EventTypeRequest::CrawlFailed,
         ]
     }
@@ -6003,6 +6009,8 @@ pub enum EventTypeRequest {
     CrawlCompleted,
     #[display(fmt = "crawl_failed")]
     CrawlFailed,
+    #[display(fmt = "crawl_started")]
+    CrawlStarted,
 }
 
 #[derive(Debug, Clone, Deserialize, Serialize)]