Removed unnecessary results channel

Unbewohnte · Feb 14, 2023 · c2ec207 · c2ec207
1 parent 812fd2a
commit c2ec207
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 24 deletions.
diff --git a/src/dashboard/dashboard.go b/src/dashboard/dashboard.go
@@ -43,7 +43,7 @@ type PageData struct {
 	Stats worker.Statistics
 }
 
-func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statistics) *Dashboard {
+func NewDashboard(port uint16, webConf *config.Conf, pool *worker.Pool) *Dashboard {
 	mux := http.NewServeMux()
 	res, err := fs.Sub(resFS, "res")
 	if err != nil {
@@ -63,7 +63,7 @@ func NewDashboard(port uint16, webConf *config.Conf, statistics *worker.Statisti
 	})
 
 	mux.HandleFunc("/stats", func(w http.ResponseWriter, req *http.Request) {
-		jsonStats, err := json.MarshalIndent(statistics, "", " ")
+		jsonStats, err := json.MarshalIndent(pool.Stats, "", " ")
 		if err != nil {
 			http.Error(w, "Failed to marshal statistics", http.StatusInternalServerError)
 			logger.Error("Failed to marshal stats to send to the dashboard: %s", err)

diff --git a/src/main.go b/src/main.go
@@ -156,17 +156,6 @@ func main() {
 	}
 	logger.Info("Successfully opened configuration file")
 
-	// Prepare global statistics variable
-	statistics := worker.Statistics{}
-
-	// open dashboard if needed
-	var board *dashboard.Dashboard = nil
-	if conf.Dashboard.UseDashboard {
-		board = dashboard.NewDashboard(conf.Dashboard.Port, conf, &statistics)
-		go board.Launch()
-		logger.Info("Launched dashboard at http://localhost:%d", conf.Dashboard.Port)
-	}
-
 	// sanitize and correct inputs
 	if len(conf.InitialPages) == 0 {
 		logger.Error("No initial page URLs have been set")
@@ -344,9 +333,6 @@ func main() {
 		logger.SetOutput(nil)
 	}
 
-	jobs := make(chan web.Job, conf.Workers*5)
-	results := make(chan web.Result, conf.Workers*5)
-
 	// create visit queue file if not turned off
 	var visitQueueFile *os.File = nil
 	if !conf.InMemoryVisitQueue {
@@ -363,6 +349,7 @@ func main() {
 	}
 
 	// create initial jobs
+	initialJobs := make(chan web.Job, conf.Workers*5)
 	if !conf.InMemoryVisitQueue {
 		for _, initialPage := range conf.InitialPages {
 			var newJob web.Job = web.Job{
@@ -379,16 +366,19 @@ func main() {
 		visitQueueFile.Seek(0, io.SeekStart)
 	} else {
 		for _, initialPage := range conf.InitialPages {
-			jobs <- web.Job{
+			initialJobs <- web.Job{
 				URL:    initialPage,
 				Search: conf.Search,
 				Depth:  conf.Depth,
 			}
 		}
 	}
 
+	// Prepare global statistics variable
+	statistics := worker.Statistics{}
+
 	// form a worker pool
-	workerPool := worker.NewWorkerPool(jobs, results, conf.Workers, &worker.WorkerConf{
+	workerPool := worker.NewWorkerPool(initialJobs, conf.Workers, &worker.WorkerConf{
 		Search:             &conf.Search,
 		Requests:           &conf.Requests,
 		Save:               &conf.Save,
@@ -403,6 +393,14 @@ func main() {
 	}, &statistics)
 	logger.Info("Created a worker pool with %d workers", conf.Workers)
 
+	// open dashboard if needed
+	var board *dashboard.Dashboard = nil
+	if conf.Dashboard.UseDashboard {
+		board = dashboard.NewDashboard(conf.Dashboard.Port, conf, workerPool)
+		go board.Launch()
+		logger.Info("Launched dashboard at http://localhost:%d", conf.Dashboard.Port)
+	}
+
 	// launch concurrent scraping !
 	workerPool.Work()
 	logger.Info("Started scraping...")
@@ -436,7 +434,4 @@ func main() {
 
 	// stop workers
 	workerPool.Stop()
-
-	// close results channel
-	close(results)
 }
diff --git a/src/worker/pool.go b/src/worker/pool.go
@@ -48,7 +48,7 @@ type Pool struct {
 }
 
 // Create a new worker pool
-func NewWorkerPool(jobs chan web.Job, results chan web.Result, workerCount uint, workerConf *WorkerConf, stats *Statistics) *Pool {
+func NewWorkerPool(initialJobs chan web.Job, workerCount uint, workerConf *WorkerConf, stats *Statistics) *Pool {
 	var newPool Pool = Pool{
 		workersCount: workerCount,
 		workers:      nil,
@@ -61,7 +61,7 @@ func NewWorkerPool(jobs chan web.Job, results chan web.Result, workerCount uint,
 
 	var i uint
 	for i = 0; i < workerCount; i++ {
-		newWorker := NewWorker(jobs, results, workerConf, &newPool.visited, newPool.Stats)
+		newWorker := NewWorker(initialJobs, workerConf, &newPool.visited, newPool.Stats)
 		newPool.workers = append(newPool.workers, &newWorker)
 	}
 

diff --git a/src/worker/worker.go b/src/worker/worker.go
@@ -62,7 +62,7 @@ type Worker struct {
 }
 
 // Create a new worker
-func NewWorker(jobs chan web.Job, results chan web.Result, conf *WorkerConf, visited *visited, stats *Statistics) Worker {
+func NewWorker(jobs chan web.Job, conf *WorkerConf, visited *visited, stats *Statistics) Worker {
 	return Worker{
 		Jobs:    jobs,
 		Conf:    conf,