diff --git a/.scripts/packages.list b/.scripts/packages.list index 53314018..60e19087 100644 --- a/.scripts/packages.list +++ b/.scripts/packages.list @@ -1,5 +1,6 @@ * + ansi * + cache/memory +* + cache/fs * + color * + cron * + csv diff --git a/CHANGELOG.md b/CHANGELOG.md index 6829e239..23202e60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,8 @@ ### [13.3.0](https://kaos.sh/ek/13.3.0) +- `[cache/fs]` Added cache with file system storage +- `[cache]` In-memory cache moved to `cache/memory` - `[sliceutil]` Added method `Join` ### [13.2.1](https://kaos.sh/ek/13.2.1) diff --git a/cache/fs/examples_test.go b/cache/fs/examples_test.go new file mode 100644 index 00000000..44fafa6d --- /dev/null +++ b/cache/fs/examples_test.go @@ -0,0 +1,128 @@ +package fs + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2024 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "fmt" + "time" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +func ExampleNew() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + fmt.Println(cache.Get("test")) +} + +func ExampleCache_Set() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + cache.Set("test", "ABCD", 15*time.Minute) + + fmt.Println(cache.Get("test")) +} + +func ExampleCache_Has() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + fmt.Println(cache.Has("test")) +} + +func ExampleCache_Get() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + fmt.Println(cache.Get("test")) +} + +func ExampleCache_Size() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + fmt.Println(cache.Size()) +} + +func ExampleCache_Expired() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + fmt.Println(cache.Expired()) +} + +func ExampleCache_GetWithExpiration() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + + item, exp := cache.GetWithExpiration("test") + + fmt.Println(item, exp.String()) +} + +func ExampleCache_Delete() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + cache.Delete("test") + + fmt.Println(cache.Get("test")) +} + +func ExampleCache_Flush() { + cache, _ := New(Config{ + Dir: "/path/to/cache", + DefaultExpiration: time.Minute, + CleanupInterval: time.Minute, + }) + + cache.Set("test", "ABCD") + cache.Flush() + + fmt.Println(cache.Get("test")) +} diff --git a/cache/fs/fs.go b/cache/fs/fs.go new file mode 100644 index 00000000..a2ac2e8c --- /dev/null +++ b/cache/fs/fs.go @@ -0,0 +1,291 @@ +//go:build !windows +// +build !windows + +// Package fs provides cache with file system storage +package fs + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2024 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "crypto/sha1" + "encoding/gob" + "fmt" + "hash" + "os" + "path" + "time" + + "github.com/essentialkaos/ek/v13/cache" + "github.com/essentialkaos/ek/v13/fsutil" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// MIN_EXPIRATION is minimal expiration duration +const MIN_EXPIRATION = time.Second + +// MIN_CLEANUP_INTERVAL is minimal cleanup interval +const MIN_CLEANUP_INTERVAL = time.Second + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Cache is fs cache instance +type Cache struct { + dir string + hasher hash.Hash + expiration time.Duration + isJanitorWorks bool +} + +// Config is cache configuration +type Config struct { + Dir string + DefaultExpiration time.Duration + CleanupInterval time.Duration +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// cacheItem is cache item +type cacheItem struct { + Data any +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// validate storage interface +var _ cache.Cache = (*Cache)(nil) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// New creates new cache instance +func New(config Config) (*Cache, error) { + err := config.Validate() + + if err != nil { + return nil, fmt.Errorf("Invalid configuration: %w", err) + } + + c := &Cache{ + dir: config.Dir, + expiration: config.DefaultExpiration, + hasher: sha1.New(), + } + + if config.CleanupInterval != 0 { + c.isJanitorWorks = true + go c.janitor(config.CleanupInterval) + } + + return c, nil +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Has returns true if cache contains data for given key +func (c *Cache) Has(key string) bool { + if c == nil || key == "" { + return false + } + + return fsutil.IsExist(c.getItemPath(key, false)) +} + +// Size returns number of items in cache +func (c *Cache) Size() int { + if c == nil { + return 0 + } + + return len(fsutil.List(c.dir, true)) +} + +// Expired returns number of expired items in cache +func (c *Cache) Expired() int { + if c == nil { + return 0 + } + + expired := 0 + now := time.Now() + items := fsutil.List(c.dir, true) + + fsutil.ListToAbsolute(c.dir, items) + + for _, item := range items { + mtime, _ := fsutil.GetMTime(item) + + if mtime.Before(now) { + expired++ + } + } + + return expired +} + +// Set adds or updates item in cache +func (c *Cache) Set(key string, data any, expiration ...time.Duration) bool { + if c == nil || data == nil || key == "" { + return false + } + + tmpFile := c.getItemPath(key, true) + fd, err := os.OpenFile(tmpFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0600) + + if err != nil { + return false + } + + err = gob.NewEncoder(fd).Encode(&cacheItem{data}) + + fd.Close() + + if err != nil { + os.Remove(tmpFile) + return false + } + + itemFile := c.getItemPath(key, false) + + err = os.Rename(tmpFile, itemFile) + + if err != nil { + os.Remove(tmpFile) + return false + } + + expr := c.expiration + + if len(expiration) > 0 { + expr = expiration[0] + } + + return os.Chtimes(itemFile, time.Time{}, time.Now().Add(expr)) == err +} + +// GetWithExpiration returns item from cache +func (c *Cache) Get(key string) any { + if c == nil || key == "" { + return nil + } + + fd, err := os.Open(c.getItemPath(key, false)) + + if err != nil { + return nil + } + + item := &cacheItem{} + err = gob.NewDecoder(fd).Decode(item) + + if err != nil { + return nil + } + + return item.Data +} + +// GetWithExpiration returns item expiration date +func (c *Cache) GetExpiration(key string) time.Time { + if c == nil || key == "" { + return time.Time{} + } + + mt, _ := fsutil.GetMTime(c.getItemPath(key, false)) + + return mt +} + +// GetWithExpiration returns item from cache and expiration date or nil +func (c *Cache) GetWithExpiration(key string) (any, time.Time) { + if c == nil || key == "" { + return nil, time.Time{} + } + + return c.Get(key), c.GetExpiration(key) +} + +// Delete removes item from cache +func (c *Cache) Delete(key string) bool { + if c == nil { + return false + } + + return os.Remove(c.getItemPath(key, false)) == nil +} + +// Flush removes all data from cache +func (c *Cache) Flush() bool { + if c == nil { + return false + } + + items := fsutil.List(c.dir, true) + fsutil.ListToAbsolute(c.dir, items) + + for _, item := range items { + os.Remove(item) + } + + return true +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// Validate validates cache configuration +func (c Config) Validate() error { + if c.DefaultExpiration < MIN_EXPIRATION { + return fmt.Errorf("Expiration is too short (< 1s)") + } + + if c.CleanupInterval != 0 && c.CleanupInterval < MIN_CLEANUP_INTERVAL { + return fmt.Errorf("Cleanup interval is too short (< 1s)") + } + + err := fsutil.ValidatePerms("DRWX", c.Dir) + + if err != nil { + return fmt.Errorf("Can't use given directory for cache: %w", err) + } + + return nil +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// getItemPath returns path to cache item +func (c *Cache) getItemPath(key string, temporary bool) string { + if temporary { + return path.Join(c.dir, "."+c.hashKey(key)) + } + + return path.Join(c.dir, c.hashKey(key)) +} + +// hashKey generates SHA-1 hash for given key +func (c *Cache) hashKey(key string) string { + return fmt.Sprintf("%64x", c.hasher.Sum([]byte(key))) +} + +// janitor is cache cleanup job +func (c *Cache) janitor(interval time.Duration) { + for range time.NewTicker(interval).C { + now := time.Now() + + items := fsutil.List(c.dir, true) + fsutil.ListToAbsolute(c.dir, items) + + for _, item := range items { + mtime, _ := fsutil.GetMTime(item) + + if mtime.Before(now) { + os.Remove(item) + } + } + } +} diff --git a/cache/fs/fs_stubs.go b/cache/fs/fs_stubs.go new file mode 100644 index 00000000..0bee6100 --- /dev/null +++ b/cache/fs/fs_stubs.go @@ -0,0 +1,90 @@ +//go:build windows +// +build windows + +// Package fs provides cache with file system storage +package fs + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2024 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "time" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +// ❗ MIN_EXPIRATION is minimal expiration duration +const MIN_EXPIRATION = time.Second + +// ❗ MIN_CLEANUP_INTERVAL is minimal cleanup interval +const MIN_CLEANUP_INTERVAL = time.Second + +// ////////////////////////////////////////////////////////////////////////////////// // + +// ❗ Cache is fs cache instance +type Cache struct{} + +// ❗ Config is cache configuration +type Config struct { + Dir string + DefaultExpiration time.Duration + CleanupInterval time.Duration +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// ❗ New creates new cache instance +func New(config Config) (*Cache, error) { + panic("UNSUPPORTED") +} + +// ////////////////////////////////////////////////////////////////////////////////// // + +// ❗ Has returns true if cache contains data for given key +func (c *Cache) Has(key string) bool { + panic("UNSUPPORTED") +} + +// ❗ Size returns number of items in cache +func (c *Cache) Size() int { + panic("UNSUPPORTED") +} + +// ❗ Expired returns number of expired items in cache +func (c *Cache) Expired() int { + panic("UNSUPPORTED") +} + +// ❗ Set adds or updates item in cache +func (c *Cache) Set(key string, data any, expiration ...time.Duration) bool { + panic("UNSUPPORTED") +} + +// ❗ GetWithExpiration returns item from cache +func (c *Cache) Get(key string) any { + panic("UNSUPPORTED") +} + +// ❗ GetWithExpiration returns item expiration date +func (c *Cache) GetExpiration(key string) time.Time { + panic("UNSUPPORTED") +} + +// ❗ GetWithExpiration returns item from cache and expiration date or nil +func (c *Cache) GetWithExpiration(key string) (any, time.Time) { + panic("UNSUPPORTED") +} + +// ❗ Delete removes item from cache +func (c *Cache) Delete(key string) bool { + panic("UNSUPPORTED") +} + +// ❗ Flush removes all data from cache +func (c *Cache) Flush() bool { + panic("UNSUPPORTED") +} diff --git a/cache/fs/fs_test.go b/cache/fs/fs_test.go new file mode 100644 index 00000000..faba74c3 --- /dev/null +++ b/cache/fs/fs_test.go @@ -0,0 +1,115 @@ +package fs + +// ////////////////////////////////////////////////////////////////////////////////// // +// // +// Copyright (c) 2024 ESSENTIAL KAOS // +// Apache License, Version 2.0 // +// // +// ////////////////////////////////////////////////////////////////////////////////// // + +import ( + "testing" + "time" + + . "github.com/essentialkaos/check" +) + +// ////////////////////////////////////////////////////////////////////////////////// // + +type CacheSuite struct{} + +// ////////////////////////////////////////////////////////////////////////////////// // + +func Test(t *testing.T) { TestingT(t) } + +// ////////////////////////////////////////////////////////////////////////////////// // + +var _ = Suite(&CacheSuite{}) + +// ////////////////////////////////////////////////////////////////////////////////// // + +func (s *CacheSuite) TestCache(c *C) { + cache, err := New(Config{ + DefaultExpiration: time.Second, + CleanupInterval: time.Second, + Dir: c.MkDir(), + }) + + c.Assert(err, IsNil) + c.Assert(cache, NotNil) + + cache.Set("1", "TEST") + cache.Set("2", "TEST") + cache.Set("3", "TEST", time.Minute) + + c.Assert(cache.Size(), Equals, 3) + + c.Assert(cache.Get("1"), Equals, "TEST") + c.Assert(cache.Get("2"), Equals, "TEST") + + c.Assert(cache.Has("2"), Equals, true) + c.Assert(cache.Has("4"), Equals, false) + + item, exp := cache.GetWithExpiration("1") + + c.Assert(item, Equals, "TEST") + c.Assert(exp.IsZero(), Not(Equals), true) + + cache.Delete("1") + + c.Assert(cache.Get("1"), Equals, nil) + c.Assert(cache.GetExpiration("3").IsZero(), Not(Equals), true) + + cache2, err := New(Config{ + DefaultExpiration: time.Second, + Dir: c.MkDir(), + }) + + cache2.Set("1", "TEST") + cache2.Set("2", "TEST", time.Minute) + + time.Sleep(2 * time.Second) + + item, _ = cache.GetWithExpiration("2") + + c.Assert(cache.Get("2"), Equals, nil) + c.Assert(item, Equals, nil) + + c.Assert(cache.Expired(), Equals, 0) + c.Assert(cache2.Expired(), Equals, 1) + + c.Assert(cache.Flush(), Equals, true) + +} + +func (s *CacheSuite) TestNil(c *C) { + var cache *Cache + + c.Assert(func() { cache.Set("1", "TEST") }, NotPanics) + c.Assert(func() { cache.Delete("1") }, NotPanics) + c.Assert(func() { cache.Flush() }, NotPanics) + + c.Assert(cache.Size(), Equals, 0) + c.Assert(cache.Expired(), Equals, 0) + c.Assert(cache.Get("1"), Equals, nil) + c.Assert(cache.Has("1"), Equals, false) + c.Assert(cache.GetExpiration("1").IsZero(), Equals, true) + + item, exp := cache.GetWithExpiration("1") + c.Assert(item, Equals, nil) + c.Assert(exp.IsZero(), Equals, true) +} + +func (s *CacheSuite) TestConfig(c *C) { + _, err := New(Config{DefaultExpiration: 1}) + + c.Assert(err.Error(), Equals, "Invalid configuration: Expiration is too short (< 1s)") + + _, err = New(Config{DefaultExpiration: time.Minute, CleanupInterval: 1}) + + c.Assert(err.Error(), Equals, "Invalid configuration: Cleanup interval is too short (< 1s)") + + _, err = New(Config{DefaultExpiration: time.Minute, CleanupInterval: time.Minute, Dir: "_unknown_"}) + + c.Assert(err.Error(), Equals, "Invalid configuration: Can't use given directory for cache: Directory _unknown_ doesn't exist or not accessible") +}