start logging everything in the solver

go.mod

@@ -87,6 +87,7 @@ require (
 	github.com/rivo/uniseg v0.4.4 // indirect
 	github.com/sergi/go-diff v1.1.0 // indirect
 	github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible // indirect
+	github.com/simonfrey/jsonl v0.0.0-20230718192858-195f01ae6957
 	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

go.sum

@@ -323,6 +323,8 @@ github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNX
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible h1:Bn1aCHHRnjv4Bl16T8rcaFjYSrGrIZvpiGO6P3Q4GpU=
 github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/simonfrey/jsonl v0.0.0-20230718192858-195f01ae6957 h1:x4Ok+mXir8ZApXlFuAf2JayR1ZhiVgqBBCQ8msaNJkk=
+github.com/simonfrey/jsonl v0.0.0-20230718192858-195f01ae6957/go.mod h1:G4w16caPmc6at7u4fmkj/8OAoOnM9mkmJr2fvL0vhaw=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/skeema/knownhosts v1.2.0 h1:h9r9cf0+u7wSE+M183ZtMGgOJKiL96brpaz5ekfJCpM=
 github.com/skeema/knownhosts v1.2.0/go.mod h1:g4fPeYpque7P0xefxtGzV81ihjC8sX2IqpAoNkjxbMo=

pkg/solver/store/memory/store.go

@@ -2,10 +2,12 @@ package store
 
 import (
 	"fmt"
+	"os"
 	"sync"
 
 	"github.com/bacalhau-project/lilypad/pkg/data"
 	"github.com/bacalhau-project/lilypad/pkg/solver/store"
+	"github.com/simonfrey/jsonl"
 )
 
 type SolverStoreMemory struct {
@@ -15,47 +17,66 @@ type SolverStoreMemory struct {
 	resultMap        map[string]*data.Result
 	matchDecisionMap map[string]*data.MatchDecision
 	mutex            sync.RWMutex
+	logWriters       map[string]jsonl.Writer
 }
 
 func getMatchID(resourceOffer string, jobOffer string) string {
 	return fmt.Sprintf("%s-%s", resourceOffer, jobOffer)
 }
 
 func NewSolverStoreMemory() (*SolverStoreMemory, error) {
+	logWriters := make(map[string]jsonl.Writer)
+
+	kinds := []string{"job_offers", "resource_offers", "deals", "decisions", "results"}
+	for k := range kinds {
+		logfile, err := os.OpenFile(fmt.Sprintf("/var/tmp/lilypad_%s.jsonl", kinds[k]), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return nil, err
+		}
+		logWriters[kinds[k]] = jsonl.NewWriter(logfile)
+	}
+
 	return &SolverStoreMemory{
 		jobOfferMap:      map[string]*data.JobOfferContainer{},
 		resourceOfferMap: map[string]*data.ResourceOfferContainer{},
 		dealMap:          map[string]*data.DealContainer{},
 		resultMap:        map[string]*data.Result{},
 		matchDecisionMap: map[string]*data.MatchDecision{},
+		logWriters:       logWriters,
 	}, nil
 }
 
 func (s *SolverStoreMemory) AddJobOffer(jobOffer data.JobOfferContainer) (*data.JobOfferContainer, error) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	s.jobOfferMap[jobOffer.ID] = &jobOffer
+
+	s.logWriters["job_offers"].Write(jobOffer)
 	return &jobOffer, nil
 }
 
 func (s *SolverStoreMemory) AddResourceOffer(resourceOffer data.ResourceOfferContainer) (*data.ResourceOfferContainer, error) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	s.resourceOfferMap[resourceOffer.ID] = &resourceOffer
+
+	s.logWriters["resource_offers"].Write(resourceOffer)
 	return &resourceOffer, nil
 }
 
 func (s *SolverStoreMemory) AddDeal(deal data.DealContainer) (*data.DealContainer, error) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	s.dealMap[deal.ID] = &deal
+	s.logWriters["deals"].Write(deal)
 	return &deal, nil
 }
 
 func (s *SolverStoreMemory) AddResult(result data.Result) (*data.Result, error) {
 	s.mutex.Lock()
 	defer s.mutex.Unlock()
 	s.resultMap[result.DealID] = &result
+	s.logWriters["results"].Write(result)
 	return &result, nil
 }
 
@@ -74,6 +95,7 @@ func (s *SolverStoreMemory) AddMatchDecision(resourceOffer string, jobOffer stri
 		Result:        result,
 	}
 	s.matchDecisionMap[id] = decision
+	s.logWriters["decisions"].Write(decision)
 	return decision, nil
 }