From 92c3721472dc74d633c7eec27be1e14f70c56523 Mon Sep 17 00:00:00 2001 From: Adrien Aury <44274230+adrienaury@users.noreply.github.com> Date: Thu, 21 Mar 2024 16:34:55 +0000 Subject: [PATCH] feat: read nodes and links --- go.mod | 4 + go.sum | 9 +++ pkg/silo/default_backend.go | 50 ++++++++++++ pkg/silo/default_datalinkreader.go | 46 +++++++++++ pkg/silo/default_datanodereader.go | 46 +++++++++++ pkg/silo/default_datarowreader.go | 48 +++++++++++ pkg/silo/driven.go | 40 +++++++++ pkg/silo/driver.go | 125 +++++++++++++++++++++++++++++ pkg/silo/driver_test.go | 50 ++++++++++++ pkg/silo/errors.go | 26 ++++++ pkg/silo/model.go | 30 +++++++ 11 files changed, 474 insertions(+) create mode 100644 pkg/silo/default_backend.go create mode 100644 pkg/silo/default_datalinkreader.go create mode 100644 pkg/silo/default_datanodereader.go create mode 100644 pkg/silo/default_datarowreader.go create mode 100644 pkg/silo/driven.go create mode 100644 pkg/silo/driver.go create mode 100644 pkg/silo/driver_test.go create mode 100644 pkg/silo/errors.go create mode 100644 pkg/silo/model.go diff --git a/go.mod b/go.mod index 136ce2e..619a0fe 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,11 @@ go 1.21 require github.com/rs/zerolog v1.28.0 require ( + github.com/davecgh/go-spew v1.1.1 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index c5e5441..63d483a 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,23 @@ github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/silo/default_backend.go b/pkg/silo/default_backend.go new file mode 100644 index 0000000..77900cb --- /dev/null +++ b/pkg/silo/default_backend.go @@ -0,0 +1,50 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +type BackendInMemory struct { + links []DataLink + nodes []DataNode +} + +func NewBackendInMemory() *BackendInMemory { + return &BackendInMemory{ + links: []DataLink{}, + nodes: []DataNode{}, + } +} + +func (b *BackendInMemory) StoreLink(link DataLink) error { + b.links = append(b.links, link) + + return nil +} + +func (b *BackendInMemory) StoreNode(node DataNode) error { + b.nodes = append(b.nodes, node) + + return nil +} + +func (b *BackendInMemory) ReadLinks() DataLinkReader { //nolint:ireturn + return NewDataLinkReaderInMemory(b.links) +} + +func (b *BackendInMemory) ReadNodes() DataNodeReader { //nolint:ireturn + return NewDataNodeReaderInMemory(b.nodes) +} diff --git a/pkg/silo/default_datalinkreader.go b/pkg/silo/default_datalinkreader.go new file mode 100644 index 0000000..c527f29 --- /dev/null +++ b/pkg/silo/default_datalinkreader.go @@ -0,0 +1,46 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import "io" + +type DataLinkReaderInMemory struct { + rows []DataLink + index int +} + +func NewDataLinkReaderInMemory(rows []DataLink) *DataLinkReaderInMemory { + return &DataLinkReaderInMemory{ + rows: rows, + index: -1, + } +} + +func (r *DataLinkReaderInMemory) ReadDataLink() (DataLink, error) { + r.index++ + + if r.index >= len(r.rows) { + return DataLink{}, io.EOF + } + + return r.rows[r.index], nil +} + +func (r *DataLinkReaderInMemory) Close() error { + return nil +} diff --git a/pkg/silo/default_datanodereader.go b/pkg/silo/default_datanodereader.go new file mode 100644 index 0000000..1022f42 --- /dev/null +++ b/pkg/silo/default_datanodereader.go @@ -0,0 +1,46 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import "io" + +type DataNodeReaderInMemory struct { + rows []DataNode + index int +} + +func NewDataNodeReaderInMemory(rows []DataNode) *DataNodeReaderInMemory { + return &DataNodeReaderInMemory{ + rows: rows, + index: -1, + } +} + +func (r *DataNodeReaderInMemory) ReadDataNode() (DataNode, error) { + r.index++ + + if r.index >= len(r.rows) { + return DataNode{}, io.EOF + } + + return r.rows[r.index], nil +} + +func (r *DataNodeReaderInMemory) Close() error { + return nil +} diff --git a/pkg/silo/default_datarowreader.go b/pkg/silo/default_datarowreader.go new file mode 100644 index 0000000..ce1ad96 --- /dev/null +++ b/pkg/silo/default_datarowreader.go @@ -0,0 +1,48 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import ( + "io" +) + +type DataRowReaderInMemory struct { + rows []DataRow + index int +} + +func NewDataRowReaderInMemory(rows []DataRow) *DataRowReaderInMemory { + return &DataRowReaderInMemory{ + rows: rows, + index: -1, + } +} + +func (r *DataRowReaderInMemory) ReadDataRow() (DataRow, error) { + r.index++ + + if r.index >= len(r.rows) { + return nil, io.EOF + } + + return r.rows[r.index], nil +} + +func (r *DataRowReaderInMemory) Close() error { + return nil +} diff --git a/pkg/silo/driven.go b/pkg/silo/driven.go new file mode 100644 index 0000000..fdb975f --- /dev/null +++ b/pkg/silo/driven.go @@ -0,0 +1,40 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +type DataRowReader interface { + ReadDataRow() (DataRow, error) + Close() error +} + +type DataLinkReader interface { + ReadDataLink() (DataLink, error) + Close() error +} + +type DataNodeReader interface { + ReadDataNode() (DataNode, error) + Close() error +} + +type Backend interface { + StoreLink(link DataLink) error + StoreNode(node DataNode) error + ReadLinks() DataLinkReader + ReadNodes() DataNodeReader +} diff --git a/pkg/silo/driver.go b/pkg/silo/driver.go new file mode 100644 index 0000000..b2ca828 --- /dev/null +++ b/pkg/silo/driver.go @@ -0,0 +1,125 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import ( + "errors" + "fmt" + "io" +) + +type Driver struct { + backend Backend +} + +func NewDriver(backend Backend) *Driver { + return &Driver{ + backend: backend, + } +} + +func (d *Driver) Scan(input DataRowReader) error { + defer input.Close() + + for { + datarow, err := input.ReadDataRow() + if err != nil && !errors.Is(err, io.EOF) { + return fmt.Errorf("%w: %w", ErrReadingNextInput, err) + } + + if errors.Is(err, io.EOF) || datarow == nil { + break + } + + nodes, links := scan(datarow) + + for _, node := range nodes { + if err := d.backend.StoreNode(node); err != nil { + return fmt.Errorf("%w: %w", ErrPersistingData, err) + } + } + + for _, link := range links { + if err := d.backend.StoreLink(link); err != nil { + return fmt.Errorf("%w: %w", ErrPersistingData, err) + } + } + } + + return nil +} + +func scan(datarow DataRow) ([]DataNode, []DataLink) { + nodes := []DataNode{} + links := []DataLink{} + + for key, value := range datarow { + if value != nil { + nodes = append(nodes, DataNode{Key: key, Data: value}) + } + } + + // find all pairs in nodes + for i := 0; i < len(nodes); i++ { + for j := i + 1; j < len(nodes); j++ { + links = append(links, DataLink{E1: nodes[i], E2: nodes[j]}) + } + } + + return nodes, links +} + +func (d *Driver) ReadAllNodes() ([]DataNode, error) { + nodes := []DataNode{} + + reader := d.backend.ReadNodes() + for { + node, err := reader.ReadDataNode() + if err != nil && !errors.Is(err, io.EOF) { + return nodes, fmt.Errorf("%w: %w", ErrReadingPersistedData, err) + } + + if errors.Is(err, io.EOF) { + break + } + + nodes = append(nodes, node) + } + + return nodes, nil +} + +func (d *Driver) ReadAllLinks() ([]DataLink, error) { + links := []DataLink{} + + reader := d.backend.ReadLinks() + for { + link, err := reader.ReadDataLink() + if err != nil && !errors.Is(err, io.EOF) { + return links, fmt.Errorf("%w: %w", ErrReadingPersistedData, err) + } + + if errors.Is(err, io.EOF) { + break + } + + links = append(links, link) + } + + return links, nil +} diff --git a/pkg/silo/driver_test.go b/pkg/silo/driver_test.go new file mode 100644 index 0000000..e25dce3 --- /dev/null +++ b/pkg/silo/driver_test.go @@ -0,0 +1,50 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo_test + +import ( + "testing" + + "github.com/cgi-fr/silo/pkg/silo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNominal(t *testing.T) { + t.Parallel() + + rows := []silo.DataRow{ + {"ID1": 1, "ID2": "1", "ID3": 1.0, "ID4": "00001"}, + {"ID1": 2, "ID2": "2", "ID3": 2.0, "ID4": "00002"}, + } + input := silo.NewDataRowReaderInMemory(rows) + + backend := silo.NewBackendInMemory() + driver := silo.NewDriver(backend) + + err := driver.Scan(input) + require.NoError(t, err) + + nodes, err := driver.ReadAllNodes() + require.NoError(t, err) + assert.Len(t, nodes, 8) + + links, err := driver.ReadAllLinks() + require.NoError(t, err) + assert.Len(t, links, 12) +} diff --git a/pkg/silo/errors.go b/pkg/silo/errors.go new file mode 100644 index 0000000..cf477c6 --- /dev/null +++ b/pkg/silo/errors.go @@ -0,0 +1,26 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +import "errors" + +var ( + ErrReadingNextInput = errors.New("error while reading next input") + ErrPersistingData = errors.New("error while persisting data") + ErrReadingPersistedData = errors.New("error while reading persisted data") +) diff --git a/pkg/silo/model.go b/pkg/silo/model.go new file mode 100644 index 0000000..4158cb0 --- /dev/null +++ b/pkg/silo/model.go @@ -0,0 +1,30 @@ +// Copyright (C) 2024 CGI France +// +// This file is part of SILO. +// +// SILO is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// SILO is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with SILO. If not, see . + +package silo + +type DataRow map[string]any + +type DataNode struct { + Key string + Data any +} + +type DataLink struct { + E1 DataNode + E2 DataNode +}