Merge pull request #238 from FleekHQ/develop

Release to master

app/app.go

@@ -7,6 +7,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/FleekHQ/space-daemon/core/search/sqlite"
+
 	"github.com/FleekHQ/space-daemon/core"
 	"github.com/FleekHQ/space-daemon/grpc"
 
@@ -119,9 +121,13 @@ func (a *App) Start(ctx context.Context) error {
 
 	hubAuth := hub.New(appStore, kc, a.cfg)
 
+	// setup files search engine
+	searchEngine := sqlite.NewSearchEngine(sqlite.WithDBPath(a.cfg.GetString(config.SpaceStorePath, "")))
+	a.Run("SqliteSearchEngine", searchEngine)
+
 	// setup textile client
 	uc := textile.CreateUserClient(a.cfg.GetString(config.TextileHubTarget, ""))
-	textileClient := textile.NewClient(appStore, kc, hubAuth, uc, nil)
+	textileClient := textile.NewClient(appStore, kc, hubAuth, uc, nil, searchEngine)
 	err = a.RunAsync("TextileClient", textileClient, func() error {
 		return textileClient.Start(ctx, a.cfg)
 	})

config/map_config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"os"
+	"os/user"
+	"path/filepath"
 
 	"github.com/FleekHQ/space-daemon/core/env"
 )
@@ -17,8 +19,10 @@ func NewMap(envVal env.SpaceEnv, flags *Flags) Config {
 	configInt := make(map[string]int)
 	configBool := make(map[string]bool)
 
+	usr, _ := user.Current()
+
 	// default values
-	configStr[SpaceStorePath] = "~/.fleek-space"
+	configStr[SpaceStorePath] = filepath.Join(usr.HomeDir, ".fleek-space")
 	configStr[MountFuseDrive] = "false"
 	configStr[FuseDriveName] = "Space"
 	configInt[SpaceServerPort] = 9999

core/events/events.go

@@ -1,8 +1,6 @@
 package events
 
-import (
-	"os"
-)
+import "github.com/FleekHQ/space-daemon/core/space/domain"
 
 // These file defines events that daemon can propagate through all layers
 
@@ -25,18 +23,18 @@ const (
 )
 
 type FileEvent struct {
-	Path   string
-	Bucket string
-	Info   os.FileInfo
+	Info   domain.FileInfo
 	Type   FileEventType
+	Bucket string
+	DbID   string
 }
 
-func NewFileEvent(path, bucket string, eventType FileEventType, info os.FileInfo) FileEvent {
+func NewFileEvent(info domain.FileInfo, eventType FileEventType, bucket, dbID string) FileEvent {
 	return FileEvent{
-		Path:   path,
-		Bucket: bucket,
-		Type:   eventType,
 		Info:   info,
+		Type:   eventType,
+		Bucket: bucket,
+		DbID:   dbID,
 	}
 }
 

core/search/engines.go

@@ -0,0 +1,14 @@
+package search
+
+import (
+	"context"
+)
+
+// Represents Search Engines for File and Folders
+// Can be used for indexing and querying of File/Folders
+type FilesSearchEngine interface {
+	Start() error
+	InsertFileData(ctx context.Context, data *InsertIndexRecord) (*IndexRecord, error)
+	DeleteFileData(ctx context.Context, data *DeleteIndexRecord) error
+	QueryFileData(ctx context.Context, query string, limit int) ([]*IndexRecord, error)
+}

core/search/model.go

@@ -0,0 +1,28 @@
+package search
+
+type IndexRecord struct {
+	Id            string
+	ItemName      string
+	ItemExtension string
+	ItemPath      string
+	ItemType      string
+	// Metadata here
+	BucketSlug string
+	DbId       string
+}
+
+type InsertIndexRecord struct {
+	ItemName      string
+	ItemExtension string
+	ItemPath      string
+	ItemType      string
+	BucketSlug    string
+	DbId          string
+}
+
+type DeleteIndexRecord struct {
+	ItemName   string
+	ItemPath   string
+	BucketSlug string
+	DbId       string // DbId is only required for shared content
+}

core/search/sqlite/model.go

@@ -0,0 +1,13 @@
+package sqlite
+
+import "gorm.io/gorm"
+
+type SearchIndexRecord struct {
+	gorm.Model
+	ItemName      string `gorm:"index:idx_name_path_bucket,unique"`
+	ItemExtension string `gorm:"size:10"`
+	ItemPath      string `gorm:"index:idx_name_path_bucket,unique"`
+	ItemType      string
+	BucketSlug    string `gorm:"index:idx_name_path_bucket,unique"`
+	DbId          string `gorm:"index"`
+}

core/search/sqlite/options.go

@@ -0,0 +1,15 @@
+package sqlite
+
+import "gorm.io/gorm/logger"
+
+func WithDBPath(path string) Option {
+	return func(o *sqliteSearchOption) {
+		o.dbPath = path
+	}
+}
+
+func WithLogLevel(level logger.LogLevel) Option {
+	return func(o *sqliteSearchOption) {
+		o.logLevel = level
+	}
+}

core/search/sqlite/sqlite.go

@@ -0,0 +1,142 @@
+package sqlite
+
+import (
+	"context"
+	"os/user"
+	"path/filepath"
+	"strconv"
+	"strings"
+
+	"gorm.io/gorm/logger"
+
+	"github.com/FleekHQ/space-daemon/core/search"
+
+	"github.com/pkg/errors"
+	"gorm.io/driver/sqlite"
+	"gorm.io/gorm"
+)
+
+const DbFileName = "filesIndex.db"
+
+type sqliteSearchOption struct {
+	dbPath   string
+	logLevel logger.LogLevel
+}
+
+type Option func(o *sqliteSearchOption)
+
+// sqliteFilesSearchEngine is a files search engine that is backed by sqlite
+type sqliteFilesSearchEngine struct {
+	db   *gorm.DB
+	opts sqliteSearchOption
+}
+
+// Creates a new SQLite backed search engine for files and folders
+func NewSearchEngine(opts ...Option) *sqliteFilesSearchEngine {
+	usr, _ := user.Current()
+
+	searchOptions := sqliteSearchOption{
+		dbPath: filepath.Join(usr.HomeDir, ".fleek-space"),
+	}
+
+	for _, opt := range opts {
+		opt(&searchOptions)
+	}
+
+	return &sqliteFilesSearchEngine{
+		db:   nil,
+		opts: searchOptions,
+	}
+}
+
+func (s *sqliteFilesSearchEngine) Start() error {
+	dsn := filepath.Join(s.opts.dbPath, DbFileName)
+
+	if db, err := gorm.Open(sqlite.Open(dsn), &gorm.Config{
+		Logger: logger.Default.LogMode(s.opts.logLevel),
+	}); err != nil {
+		return errors.Wrap(err, "failed to open database")
+	} else {
+		s.db = db
+	}
+
+	return s.db.AutoMigrate(&SearchIndexRecord{})
+}
+
+func (s *sqliteFilesSearchEngine) InsertFileData(ctx context.Context, data *search.InsertIndexRecord) (*search.IndexRecord, error) {
+	record := SearchIndexRecord{
+		ItemName:      data.ItemName,
+		ItemExtension: data.ItemExtension,
+		ItemPath:      data.ItemPath,
+		ItemType:      data.ItemPath,
+		BucketSlug:    data.BucketSlug,
+		DbId:          data.DbId,
+	}
+	result := s.db.Create(&record)
+
+	if result.Error != nil {
+		if strings.Contains(result.Error.Error(), "UNIQUE constraint failed") {
+			return nil, errors.New("a similar file has already been inserted")
+		}
+		return nil, result.Error
+	}
+
+	return modelToIndexRecord(&record), nil
+}
+
+func (s *sqliteFilesSearchEngine) DeleteFileData(ctx context.Context, data *search.DeleteIndexRecord) error {
+	stmt := s.db.Where(
+		"item_name = ? AND item_path = ? AND bucket_slug = ?",
+		data.ItemName,
+		data.ItemPath,
+		data.BucketSlug,
+	)
+	if data.DbId != "" {
+		stmt = stmt.Where("dbId = ?", data.DbId)
+	}
+
+	result := stmt.Delete(&SearchIndexRecord{})
+
+	return result.Error
+}
+
+func (s *sqliteFilesSearchEngine) QueryFileData(ctx context.Context, query string, limit int) ([]*search.IndexRecord, error) {
+	var records []*SearchIndexRecord
+	result := s.db.Where(
+		"item_name LIKE ? OR item_extension = ?",
+		"%"+query+"%",
+		query,
+	).Limit(limit).Find(&records)
+
+	if result.Error != nil {
+		return nil, result.Error
+	}
+
+	searchResults := make([]*search.IndexRecord, len(records))
+	for i, record := range records {
+		searchResults[i] = modelToIndexRecord(record)
+	}
+
+	return searchResults, nil
+}
+
+func (s *sqliteFilesSearchEngine) Shutdown() error {
+	db, err := s.db.DB()
+	if err != nil {
+		return err
+	}
+
+	return db.Close()
+}
+
+func modelToIndexRecord(model *SearchIndexRecord) *search.IndexRecord {
+	return &search.IndexRecord{
+		Id:            strconv.Itoa(int(model.ID)),
+		ItemName:      model.ItemName,
+		ItemExtension: model.ItemExtension,
+		ItemPath:      model.ItemPath,
+		ItemType:      model.ItemType,
+		BucketSlug:    model.BucketSlug,
+		DbId:          model.DbId,
+	}
+}