feat: read nodes and links

go.mod

@@ -5,7 +5,11 @@ go 1.21
 require github.com/rs/zerolog v1.28.0
 
 require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
 	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,14 +1,23 @@
 github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY=
 github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/silo/default_backend.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+type BackendInMemory struct {
+	links []DataLink
+	nodes []DataNode
+}
+
+func NewBackendInMemory() *BackendInMemory {
+	return &BackendInMemory{
+		links: []DataLink{},
+		nodes: []DataNode{},
+	}
+}
+
+func (b *BackendInMemory) StoreLink(link DataLink) error {
+	b.links = append(b.links, link)
+
+	return nil
+}
+
+func (b *BackendInMemory) StoreNode(node DataNode) error {
+	b.nodes = append(b.nodes, node)
+
+	return nil
+}
+
+func (b *BackendInMemory) ReadLinks() DataLinkReader { //nolint:ireturn
+	return NewDataLinkReaderInMemory(b.links)
+}
+
+func (b *BackendInMemory) ReadNodes() DataNodeReader { //nolint:ireturn
+	return NewDataNodeReaderInMemory(b.nodes)
+}

pkg/silo/default_datalinkreader.go

@@ -0,0 +1,46 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+import "io"
+
+type DataLinkReaderInMemory struct {
+	rows  []DataLink
+	index int
+}
+
+func NewDataLinkReaderInMemory(rows []DataLink) *DataLinkReaderInMemory {
+	return &DataLinkReaderInMemory{
+		rows:  rows,
+		index: -1,
+	}
+}
+
+func (r *DataLinkReaderInMemory) ReadDataLink() (DataLink, error) {
+	r.index++
+
+	if r.index >= len(r.rows) {
+		return DataLink{}, io.EOF
+	}
+
+	return r.rows[r.index], nil
+}
+
+func (r *DataLinkReaderInMemory) Close() error {
+	return nil
+}

pkg/silo/default_datanodereader.go

@@ -0,0 +1,46 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+import "io"
+
+type DataNodeReaderInMemory struct {
+	rows  []DataNode
+	index int
+}
+
+func NewDataNodeReaderInMemory(rows []DataNode) *DataNodeReaderInMemory {
+	return &DataNodeReaderInMemory{
+		rows:  rows,
+		index: -1,
+	}
+}
+
+func (r *DataNodeReaderInMemory) ReadDataNode() (DataNode, error) {
+	r.index++
+
+	if r.index >= len(r.rows) {
+		return DataNode{}, io.EOF
+	}
+
+	return r.rows[r.index], nil
+}
+
+func (r *DataNodeReaderInMemory) Close() error {
+	return nil
+}

pkg/silo/default_datarowreader.go

@@ -0,0 +1,48 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+import (
+	"io"
+)
+
+type DataRowReaderInMemory struct {
+	rows  []DataRow
+	index int
+}
+
+func NewDataRowReaderInMemory(rows []DataRow) *DataRowReaderInMemory {
+	return &DataRowReaderInMemory{
+		rows:  rows,
+		index: -1,
+	}
+}
+
+func (r *DataRowReaderInMemory) ReadDataRow() (DataRow, error) {
+	r.index++
+
+	if r.index >= len(r.rows) {
+		return nil, io.EOF
+	}
+
+	return r.rows[r.index], nil
+}
+
+func (r *DataRowReaderInMemory) Close() error {
+	return nil
+}

pkg/silo/driven.go

@@ -0,0 +1,40 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+type DataRowReader interface {
+	ReadDataRow() (DataRow, error)
+	Close() error
+}
+
+type DataLinkReader interface {
+	ReadDataLink() (DataLink, error)
+	Close() error
+}
+
+type DataNodeReader interface {
+	ReadDataNode() (DataNode, error)
+	Close() error
+}
+
+type Backend interface {
+	StoreLink(link DataLink) error
+	StoreNode(node DataNode) error
+	ReadLinks() DataLinkReader
+	ReadNodes() DataNodeReader
+}

pkg/silo/driver.go

@@ -0,0 +1,125 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+import (
+	"errors"
+	"fmt"
+	"io"
+)
+
+type Driver struct {
+	backend Backend
+}
+
+func NewDriver(backend Backend) *Driver {
+	return &Driver{
+		backend: backend,
+	}
+}
+
+func (d *Driver) Scan(input DataRowReader) error {
+	defer input.Close()
+
+	for {
+		datarow, err := input.ReadDataRow()
+		if err != nil && !errors.Is(err, io.EOF) {
+			return fmt.Errorf("%w: %w", ErrReadingNextInput, err)
+		}
+
+		if errors.Is(err, io.EOF) || datarow == nil {
+			break
+		}
+
+		nodes, links := scan(datarow)
+
+		for _, node := range nodes {
+			if err := d.backend.StoreNode(node); err != nil {
+				return fmt.Errorf("%w: %w", ErrPersistingData, err)
+			}
+		}
+
+		for _, link := range links {
+			if err := d.backend.StoreLink(link); err != nil {
+				return fmt.Errorf("%w: %w", ErrPersistingData, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func scan(datarow DataRow) ([]DataNode, []DataLink) {
+	nodes := []DataNode{}
+	links := []DataLink{}
+
+	for key, value := range datarow {
+		if value != nil {
+			nodes = append(nodes, DataNode{Key: key, Data: value})
+		}
+	}
+
+	// find all pairs in nodes
+	for i := 0; i < len(nodes); i++ {
+		for j := i + 1; j < len(nodes); j++ {
+			links = append(links, DataLink{E1: nodes[i], E2: nodes[j]})
+		}
+	}
+
+	return nodes, links
+}
+
+func (d *Driver) ReadAllNodes() ([]DataNode, error) {
+	nodes := []DataNode{}
+
+	reader := d.backend.ReadNodes()
+	for {
+		node, err := reader.ReadDataNode()
+		if err != nil && !errors.Is(err, io.EOF) {
+			return nodes, fmt.Errorf("%w: %w", ErrReadingPersistedData, err)
+		}
+
+		if errors.Is(err, io.EOF) {
+			break
+		}
+
+		nodes = append(nodes, node)
+	}
+
+	return nodes, nil
+}
+
+func (d *Driver) ReadAllLinks() ([]DataLink, error) {
+	links := []DataLink{}
+
+	reader := d.backend.ReadLinks()
+	for {
+		link, err := reader.ReadDataLink()
+		if err != nil && !errors.Is(err, io.EOF) {
+			return links, fmt.Errorf("%w: %w", ErrReadingPersistedData, err)
+		}
+
+		if errors.Is(err, io.EOF) {
+			break
+		}
+
+		links = append(links, link)
+	}
+
+	return links, nil
+}