Merge pull request #1342 from luk3skyw4lker/feat/add-clickhouse-storage

feat: Clickhouse Storage Driver

.github/dependabot.yml

@@ -16,3 +16,9 @@ updates:
  - "🤖 Dependencies"
  schedule:
  interval: "daily"
+ - package-ecosystem: "gomod"
+ directory: "/clickhouse/" # Location of package manifests
+ labels:
+ - "🤖 Dependencies"
+ schedule:
+ interval: "daily"

.github/release-drafter-clickhouse.yml

@@ -0,0 +1,50 @@
+name-template: 'ClickHouse - v$RESOLVED_VERSION'
+tag-template: 'clickhouse/v$RESOLVED_VERSION'
+tag-prefix: clickhouse/v
+include-paths:
+ - clickhouse
+categories:
+ - title: '❗ Breaking Changes'
+ labels:
+ - '❗ BreakingChange'
+ - title: '🚀 New'
+ labels:
+ - '✏️ Feature'
+ - title: '🧹 Updates'
+ labels:
+ - '🧹 Updates'
+ - '🤖 Dependencies'
+ - title: '🐛 Fixes'
+ labels:
+ - '☢️ Bug'
+ - title: '📚 Documentation'
+ labels:
+ - '📒 Documentation'
+change-template: '- $TITLE (#$NUMBER)'
+change-title-escapes: '\<*_&' # You can add # and @ to disable mentions, and add ` to disable code blocks.
+exclude-contributors:
+ - dependabot
+ - dependabot[bot]
+version-resolver:
+ major:
+ labels:
+ - 'major'
+ - '❗ BreakingChange'
+ minor:
+ labels:
+ - 'minor'
+ - '✏️ Feature'
+ patch:
+ labels:
+ - 'patch'
+ - '📒 Documentation'
+ - '☢️ Bug'
+ - '🤖 Dependencies'
+ - '🧹 Updates'
+ default: patch
+template: |
+ $CHANGES
+
+ **Full Changelog**: https://github.com/$OWNER/$REPOSITORY/compare/$PREVIOUS_TAG...clickhouse/v$RESOLVED_VERSION
+
+ Thank you $CONTRIBUTORS for making this update possible.

.github/workflows/benchmark.yml

@@ -134,6 +134,11 @@ jobs:
  docker run --name scylladb -p 9042:9042 -p 19042:19042 -p 9160:9160 -p 7000:7000 -p 7001:7001 -p 7199:7199 -p 9180:9180 -d scylladb/scylla:latest --broadcast-address 127.0.0.1 --listen-address 0.0.0.0 --broadcast-rpc-address 127.0.0.1
  sleep 15 # Wait for ScyllaDb to initialize
 
+ - name: Startup Clickhouse
+ run: |
+ docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server
+ sleep 10 # Wait for Clickhouse to initialize
+
  - name: Setup Redis
  uses: shogo82148/actions-setup-redis@v1
  with:

.github/workflows/release-drafter-clickhouse.yml

@@ -0,0 +1,19 @@
+name: Release Drafter Clickhouse
+on:
+ push:
+ # branches to consider in the event; optional, defaults to all
+ branches:
+ - master
+ - main
+ paths:
+ - 'clickhouse/**'
+jobs:
+ draft_release_clickhouse:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - uses: release-drafter/release-drafter@v6
+ with:
+ config-name: release-drafter-clickhouse.yml
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-clickhouse.yml

@@ -0,0 +1,32 @@
+on:
+ push:
+ branches:
+ - master
+ - main
+ paths:
+ - 'clickhouse/**'
+ pull_request:
+ paths:
+ - 'clickhouse/**'
+name: 'Tests Clickhouse'
+jobs:
+ Tests:
+ runs-on: ubuntu-latest
+ strategy:
+ matrix:
+ go-version:
+ - 1.21.x
+ - 1.22.x
+ steps:
+ - name: Fetch Repository
+ uses: actions/checkout@v4
+ - name: Startup Clickhouse
+ run: |
+ docker run -d -p 9001:9000 --name clickhouse --ulimit nofile=262144:262144 clickhouse/clickhouse-server
+ sleep 30
+ - name: Install Go
+ uses: actions/setup-go@v5
+ with:
+ go-version: '${{ matrix.go-version }}'
+ - name: Run Test
+ run: cd ./clickhouse && go clean -testcache && go test ./... -v -race

.github/workflows/test-cloudflarekv.yml

@@ -3,10 +3,13 @@ name: Tests CloudflareKV
 on:
  push:
  branches:
+ - master
  - main
+ paths:
+ - 'cloudflarekv/**'
  pull_request:
- branches:
- - main
+ paths:
+ - 'cloudflarekv/**'
 
 jobs:
  Tests:
@@ -16,7 +19,6 @@ jobs:
  go-version:
  - 1.21.x
  - 1.22.x
-
  steps:
  - name: Checkout Repository
  uses: actions/checkout@v4

README.md

@@ -74,3 +74,4 @@ type Storage interface {
 - [S3](./s3/README.md) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Tests+S3%22"> <img src="https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-s3.yml?branch=main&label=%F0%9F%A7%AA%20&style=flat&color=75C46B" /> </a>
 - [ScyllaDB](./scylladb/README.md) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Tests+scylladb%22"> <img src="https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-scylladb.yml?branch=main&label=%F0%9F%A7%AA%20&style=flat&color=75C46B" /> </a>
 - [SQLite3](./sqlite3/README.md) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Tests+Sqlite3%22"> <img src="https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-sqlite3.yml?branch=main&label=%F0%9F%A7%AA%20&style=flat&color=75C46B" /> </a>
+- [ClickHouse](./clickhouse/README.md) <a href="https://github.com/gofiber/storage/actions?query=workflow%3A%22Tests+Clickhouse%22"> <img src="https://img.shields.io/github/actions/workflow/status/gofiber/storage/test-clickhouse.yml?branch=main&label=%F0%9F%A7%AA%20&style=flat&color=75C46B" /> </a>

clickhouse/README.md

@@ -0,0 +1,117 @@
+# Clickhouse
+
+A Clickhouse storage driver using [https://github.com/ClickHouse/clickhouse-go](https://github.com/ClickHouse/clickhouse-go).
+
+### Table of Contents
+
+- [Signatures](#signatures)
+- [Installation](#installation)
+- [Examples](#examples)
+- [Config](#config)
+- [Default Config](#default-config)
+
+### Signatures
+
+```go
+func New(config ...Config) (*Storage, error)
+func (s *Storage) Get(key string) ([]byte, error)
+func (s *Storage) Set(key string, val []byte, exp time.Duration) error
+func (s *Storage) Delete(key string) error
+func (s *Storage) Reset() error
+func (s *Storage) Close() error
+func (s *Storage) Conn() *Session
+```
+
+### Installation
+
+Clickhouse is supported on the latest two versions of Go:
+
+Install the clickhouse implementation:
+```bash
+go get github.com/gofiber/storage/clickhouse
+```
+
+Before running or testing this implementation, you must ensure a Clickhouse cluster is available.
+For local development, we recommend using the Clickhouse Docker image; it contains everything
+necessary for the client to operate correctly.
+
+To start Clickhouse using Docker, issue the following:
+
+```bash
+docker run -d -p 9000:9000 --name some-clickhouse-server --ulimit nofile=262144:262144 clickhouse/clickhouse-server
+```
+
+After running this command you're ready to start using the storage and connecting to the database.
+
+### Examples
+
+You can use the following options to create a clickhouse storage driver:
+```go
+import "github.com/gofiber/storage/clickhouse"
+
+// Initialize default config, to connect to localhost:9000 using the memory engine and with a clean table.
+store, err := clickhouse.New(clickhouse.Config{
+ Host: "localhost",
+ Port: 9000,
+ Clean: true,
+})
+
+// Initialize custom config to connect to a different host/port and use custom engine and with clean table.
+store, err := clickhouse.New(clickhouse.Config{
+ Host: "some-ip-address",
+ Port: 9000,
+ Engine: clickhouse.MergeTree,
+ Clean: true,
+})
+
+// Initialize to connect with TLS enabled with your own tls.Config and with clean table.
+tlsConfig := config := &tls.Config{...}
+
+store, err := clickhouse.New(clickhouse.Config{
+ Host: "some-ip-address",
+ Port: 9000,
+ Clean: true,
+ TLSConfig: tlsConfig,
+})
+```
+
+### Config
+
+```go
+// Config defines configuration options for Clickhouse connection.
+type Config struct {
+ // The host of the database. Ex: 127.0.0.1
+ Host string
+ // The port where the database is supposed to listen to. Ex: 9000
+ Port int
+ // The database that the connection should authenticate from
+ Database string
+ // The username to be used in the authentication
+ Username string
+ // The password to be used in the authentication
+ Password string
+ // The name of the table that will store the data
+ Table string
+ // The engine that should be used in the table
+ Engine string
+ // Should start a clean table, default false
+ Clean bool
+ // TLS configuration, default nil
+ TLSConfig *tls.Config
+ // Should the connection be in debug mode, default false
+ Debug bool
+ // The function to use with the debug config, default print function. It only works when debug is true
+ Debugf func(format string, v ...any)
+}
+```
+
+### Default Config
+
+```go
+var DefaultConfig = Config{
+ Host: "localhost",
+ Port: 9000,
+ Engine: "Memory",
+ Clean: false,
+}
+```

clickhouse/clickhouse.go

@@ -0,0 +1,126 @@
+package clickhouse
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+ "time"
+
+ driver "github.com/ClickHouse/clickhouse-go/v2"
+)
+
+type Storage struct {
+ session driver.Conn
+ context context.Context
+ table string
+}
+
+// New returns a new [*Storage] given a [Config].
+func New(configuration Config) (*Storage, error) {
+ cfg, engine, err := defaultConfig(configuration)
+ if err != nil {
+ return nil, err
+ }
+
+ conn, err := driver.Open(&cfg)
+ if err != nil {
+ return nil, err
+ }
+
+ ctx := context.Background()
+
+ queryWithEngine := fmt.Sprintf(createTableString, engine)
+ if err := conn.Exec(ctx, queryWithEngine, driver.Named("table", configuration.Table)); err != nil {
+ return nil, err
+ }
+
+ if configuration.Clean {
+ if err := conn.Exec(ctx, resetDataString, driver.Named("table", configuration.Table)); err != nil {
+ return nil, err
+ }
+ }
+
+ if err := conn.Ping(ctx); err != nil {
+ return nil, err
+ }
+
+ return &Storage{
+ session: conn,
+ context: ctx,
+ table: configuration.Table,
+ }, nil
+}
+
+func (s *Storage) Set(key string, value []byte, expiration time.Duration) error {
+ if len(key) <= 0 || len(value) <= 0 {
+ return nil
+ }
+
+ exp := time.Time{}
+ if expiration != 0 {
+ exp = time.Now().Add(expiration).UTC()
+ }
+
+ return s.
+ session.
+ Exec(
+ s.context,
+ insertDataString,
+ driver.Named("table", s.table),
+ driver.Named("key", key),
+ driver.Named("value", string(value)),
+ driver.Named("expiration", exp.Format("2006-01-02 15:04:05")),
+ )
+}
+
+func (s *Storage) Get(key string) ([]byte, error) {
+ if len(key) == 0 {
+ return []byte{}, nil
+ }
+
+ var result schema
+
+ row := s.session.QueryRow(
+ s.context,
+ selectDataString,
+ driver.Named("table", s.table),
+ driver.Named("key", key),
+ )
+ if row.Err() != nil {
+ return []byte{}, row.Err()
+ }
+
+ if err := row.ScanStruct(&result); err != nil {
+ if errors.Is(err, sql.ErrNoRows) {
+ return []byte{}, nil
+ }
+
+ return []byte{}, err
+ }
+
+ // The result.Expiration.IsZero() was returning a false value even when the time was
+ // set to be the zero value of the time.Time struct (Jan 1st 1970, 00:00:00 UTC)
+ // so we had to change the comparison
+ if !time.Unix(0, 0).Equal(result.Expiration) && result.Expiration.Before(time.Now().UTC()) {
+ return []byte{}, nil
+ }
+
+ return []byte(result.Value), nil
+}
+
+func (s *Storage) Delete(key string) error {
+ if len(key) == 0 {
+ return nil
+ }
+
+ return s.session.Exec(s.context, deleteDataString, driver.Named("table", s.table), driver.Named("key", key))
+}
+
+func (s *Storage) Reset() error {
+ return s.session.Exec(s.context, resetDataString, driver.Named("table", s.table))
+}
+
+func (s *Storage) Close() error {
+ return s.session.Close()
+}