From 04de1c0a7fc23ba454c1351839e7e0288323c716 Mon Sep 17 00:00:00 2001
From: Aditya Thebe <contact@adityathebe.com>
Date: Wed, 3 Jul 2024 21:30:12 +0545
Subject: [PATCH 1/2] feat: persist scrape results locally

---
 scrapers/runscrapers.go | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/scrapers/runscrapers.go b/scrapers/runscrapers.go
index 406fccf7..34b6fd29 100644
--- a/scrapers/runscrapers.go
+++ b/scrapers/runscrapers.go
@@ -1,9 +1,11 @@
 package scrapers
 
 import (
+	"encoding/gob"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 
 	"github.com/flanksource/commons/logger"
@@ -18,6 +20,11 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
+func init() {
+	gob.Register(map[string]any{})
+	gob.Register([]any{})
+}
+
 // RunK8ObjScraper extracts & saves the given kubernetes objects.
 func RunK8ObjScraper(ctx api.ScrapeContext, config v1.Kubernetes, objs []*unstructured.Unstructured) ([]v1.ScrapeResult, error) {
 	var results v1.ScrapeResults
@@ -44,6 +51,23 @@ func RunK8IncrementalScraper(ctx api.ScrapeContext, config v1.Kubernetes, events
 
 // Run ...
 func Run(ctx api.ScrapeContext) ([]v1.ScrapeResult, error) {
+	var localResultPath string
+	if base, ok := os.LookupEnv("SCRAPE_RESULT_DIR"); ok {
+		localResultPath = fmt.Sprintf("%s/%s.gob", base, ctx.ScrapeConfig().Name)
+		f, err := os.Open(localResultPath)
+		if err != nil && !errors.Is(err, os.ErrNotExist) {
+			return nil, fmt.Errorf("failed to open local result path: %w", err)
+		} else if err == nil {
+			defer f.Close()
+
+			var results []v1.ScrapeResult
+			if err := gob.NewDecoder(f).Decode(&results); err != nil {
+				return nil, fmt.Errorf("failed to decode local result path: %w", err)
+			}
+			return results, nil
+		}
+	}
+
 	var results v1.ScrapeResults
 	for _, scraper := range All {
 		if !scraper.CanScrape(ctx.ScrapeConfig().Spec) {
@@ -69,6 +93,17 @@ func Run(ctx api.ScrapeContext) ([]v1.ScrapeResult, error) {
 		}
 	}
 
+	if localResultPath != "" {
+		writeFile, err := os.Create(localResultPath)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create local result file: %w", err)
+		}
+		defer writeFile.Close()
+		if err := gob.NewEncoder(writeFile).Encode(results); err != nil {
+			return nil, fmt.Errorf("failed to encode local result path: %w", err)
+		}
+	}
+
 	ctx.Logger.V(1).Infof("Completed scraping with %d results.", len(results))
 	return results, nil
 }

From a637e20732309c1223883f6356f8ab02c0ebe1bc Mon Sep 17 00:00:00 2001
From: Moshe Immerman <moshe@flanksource.com>
Date: Mon, 29 Jul 2024 09:24:49 +0300
Subject: [PATCH 2/2] Update scrapers/runscrapers.go

---
 scrapers/runscrapers.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapers/runscrapers.go b/scrapers/runscrapers.go
index f614fee2..d6e1a79f 100644
--- a/scrapers/runscrapers.go
+++ b/scrapers/runscrapers.go
@@ -52,6 +52,7 @@ func RunK8IncrementalScraper(ctx api.ScrapeContext, config v1.Kubernetes, events
 func Run(ctx api.ScrapeContext) ([]v1.ScrapeResult, error) {
 	var localResultPath string
 	if base, ok := os.LookupEnv("SCRAPE_RESULT_DIR"); ok {
+                 logger.Errorf("Returning existing scrape results from: $SCRAPE_RESULT_DIR")
 		localResultPath = fmt.Sprintf("%s/%s.gob", base, ctx.ScrapeConfig().Name)
 		f, err := os.Open(localResultPath)
 		if err != nil && !errors.Is(err, os.ErrNotExist) {