feat(scan): alias and filters

CHANGELOG.md

@@ -14,6 +14,12 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.2.0]
+
+- `Added` flag `--only` (short `-o`) to only scan a specific list of fields
+- `Added` flag `--alias` (short `-a`) to rename fields on the fly
+- `Fixed` self reference link are no longer counted in the links counter while scanning
+
 ## [0.1.0]
 
 - `Added` initial version

internal/app/cli/scan.go

@@ -28,21 +28,27 @@ import (
 )
 
 func NewScanCommand(parent string, stderr *os.File, stdout *os.File, stdin *os.File) *cobra.Command {
-	var passthrough bool
+	var (
+		passthrough bool
+		only        []string
+		aliases     map[string]string
+	)
 
 	cmd := &cobra.Command{ //nolint:exhaustruct
 		Use:     "scan path",
 		Short:   "Ingest data from stdin and update silo database stored in given path",
 		Example: "  lino pull database --table client | " + parent + " scan clients",
 		Args:    cobra.ExactArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := scan(cmd, args[0], passthrough); err != nil {
+			if err := scan(cmd, args[0], passthrough, only, aliases); err != nil {
 				log.Fatal().Err(err).Int("return", 1).Msg("end SILO")
 			}
 		},
 	}
 
 	cmd.Flags().BoolVarP(&passthrough, "passthrough", "p", false, "pass stdin to stdout")
+	cmd.Flags().StringSliceVarP(&only, "only", "o", []string{}, "only scan these columns, exclude all others")
+	cmd.Flags().StringToStringVarP(&aliases, "alias", "a", map[string]string{}, "use given aliases for each columns")
 
 	cmd.SetOut(stdout)
 	cmd.SetErr(stderr)
@@ -51,15 +57,15 @@ func NewScanCommand(parent string, stderr *os.File, stdout *os.File, stdin *os.F
 	return cmd
 }
 
-func scan(cmd *cobra.Command, path string, passthrough bool) error {
+func scan(cmd *cobra.Command, path string, passthrough bool, only []string, aliases map[string]string) error {
 	backend, err := infra.NewBackend(path)
 	if err != nil {
 		return fmt.Errorf("%w", err)
 	}
 
 	defer backend.Close()
 
-	driver := silo.NewDriver(backend, nil)
+	driver := silo.NewDriver(backend, nil, silo.WithKeys(only), silo.WithAliases(aliases))
 
 	var reader silo.DataRowReader
 

pkg/silo/config.go

@@ -0,0 +1,50 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+import "errors"
+
+type Config struct {
+	Include map[string]bool
+	Aliases map[string]string
+}
+
+func DefaultConfig() *Config {
+	config := Config{
+		Include: map[string]bool{},
+		Aliases: map[string]string{},
+	}
+
+	return &config
+}
+
+func (cfg *Config) validate() error {
+	var errs []error
+
+	for key := range cfg.Aliases {
+		if _, ok := cfg.Include[key]; !ok && len(cfg.Include) > 0 {
+			errs = append(errs, &ConfigScanAliasIsNotIncludedError{alias: key})
+		}
+	}
+
+	if len(errs) != 0 {
+		return errors.Join(errs...)
+	}
+
+	return nil
+}

pkg/silo/driver.go

@@ -26,14 +26,33 @@ import (
 )
 
 type Driver struct {
+	*Config
 	backend Backend
 	writer  DumpWriter
 }
 
-func NewDriver(backend Backend, writer DumpWriter) *Driver {
+func NewDriver(backend Backend, writer DumpWriter, options ...Option) *Driver {
+	errs := []error{}
+	config := DefaultConfig()
+
+	for _, option := range options {
+		if err := option.apply(config); err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if err := config.validate(); err != nil {
+		errs = append(errs, err)
+	}
+
+	if len(errs) > 0 {
+		panic(errs)
+	}
+
 	return &Driver{
 		backend: backend,
 		writer:  writer,
+		Config:  config,
 	}
 }
 
@@ -102,7 +121,7 @@ func (d *Driver) Scan(input DataRowReader, observers ...ScanObserver) error {
 			break
 		}
 
-		links := Scan(datarow)
+		links := d.scan(datarow)
 
 		log.Info().Int("links", len(links)).Interface("row", datarow).Msg("datarow scanned")
 
@@ -120,12 +139,14 @@ func (d *Driver) ingest(datarow DataRow, links []DataLink, observers ...ScanObse
 			return fmt.Errorf("%w: %w", ErrPersistingData, err)
 		}
 
-		if err := d.backend.Store(link.E2, link.E1); err != nil {
-			return fmt.Errorf("%w: %w", ErrPersistingData, err)
-		}
+		if link.E1 != link.E2 {
+			if err := d.backend.Store(link.E2, link.E1); err != nil {
+				return fmt.Errorf("%w: %w", ErrPersistingData, err)
+			}
 
-		for _, observer := range observers {
-			observer.IngestedLink(link)
+			for _, observer := range observers {
+				observer.IngestedLink(link)
+			}
 		}
 	}
 
@@ -136,12 +157,16 @@ func (d *Driver) ingest(datarow DataRow, links []DataLink, observers ...ScanObse
 	return nil
 }
 
-func Scan(datarow DataRow) []DataLink {
+func (cfg *Config) scan(datarow DataRow) []DataLink {
 	nodes := []DataNode{}
 	links := []DataLink{}
 
 	for key, value := range datarow {
-		if value != nil {
+		if _, included := cfg.Include[key]; value != nil && (included || len(cfg.Include) == 0) {
+			if alias, exist := cfg.Aliases[key]; exist {
+				key = alias
+			}
+
 			nodes = append(nodes, DataNode{Key: key, Data: value})
 		}
 	}

pkg/silo/errors.go

@@ -17,10 +17,21 @@
 
 package silo
 
-import "errors"
+import (
+	"errors"
+	"fmt"
+)
 
 var (
 	ErrReadingNextInput     = errors.New("error while reading next input")
 	ErrPersistingData       = errors.New("error while persisting data")
 	ErrReadingPersistedData = errors.New("error while reading persisted data")
 )
+
+type ConfigScanAliasIsNotIncludedError struct {
+	alias string
+}
+
+func (e *ConfigScanAliasIsNotIncludedError) Error() string {
+	return fmt.Sprintf("configuration error : alias [%s] is not included", e.alias)
+}

pkg/silo/options.go

@@ -0,0 +1,76 @@
+// Copyright (C) 2024 CGI France
+//
+// This file is part of SILO.
+//
+// SILO is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// SILO is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with SILO.  If not, see <http://www.gnu.org/licenses/>.
+
+package silo
+
+type Option interface {
+	applier
+}
+
+type option func(*Config) error
+
+func (f option) apply(cfg *Config) error {
+	return f(cfg)
+}
+
+type applier interface {
+	apply(cfg *Config) error
+}
+
+func Alias(key, alias string) Option { //nolint:ireturn
+	applier := func(cfg *Config) error {
+		cfg.Aliases[key] = alias
+
+		return nil
+	}
+
+	return option(applier)
+}
+
+func Include(key string) Option { //nolint:ireturn
+	applier := func(cfg *Config) error {
+		cfg.Include[key] = true
+
+		return nil
+	}
+
+	return option(applier)
+}
+
+func WithAliases(aliases map[string]string) Option { //nolint:ireturn
+	applier := func(cfg *Config) error {
+		for key, alias := range aliases {
+			cfg.Aliases[key] = alias
+		}
+
+		return nil
+	}
+
+	return option(applier)
+}
+
+func WithKeys(keys []string) Option { //nolint:ireturn
+	applier := func(cfg *Config) error {
+		for _, key := range keys {
+			cfg.Include[key] = true
+		}
+
+		return nil
+	}
+
+	return option(applier)
+}