Skip to content

Commit

Permalink
feat: add support for postgresql
Browse files Browse the repository at this point in the history
This allows users to connect to a postgresql backend for our ent
library. The ent client creation was broken into it's own function so we
can use the enttest library to pass in the test client for unit testing
in the future.
  • Loading branch information
mikhailswift committed Sep 26, 2023
1 parent 658b3f6 commit 56a98e9
Show file tree
Hide file tree
Showing 7 changed files with 111 additions and 34 deletions.
9 changes: 7 additions & 2 deletions cmd/archivista/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/testifysec/archivista"
"github.com/testifysec/archivista/internal/config"
"github.com/testifysec/archivista/internal/metadatastorage/mysqlstore"
"github.com/testifysec/archivista/internal/metadatastorage/sqlstore"
"github.com/testifysec/archivista/internal/objectstorage/blobstore"
"github.com/testifysec/archivista/internal/objectstorage/filestore"
"github.com/testifysec/archivista/internal/server"
Expand Down Expand Up @@ -86,7 +86,12 @@ func main() {
logrus.Fatalf("error initializing storage clients: %+v", err)
}

mysqlStore, mysqlStoreCh, err := mysqlstore.New(ctx, cfg.SQLStoreConnectionString)
entClient, err := sqlstore.NewEntClient(cfg.SQLStoreBackend, cfg.SQLStoreConnectionString)
if err != nil {
logrus.Fatalf("could not create ent client: %+v", err)
}

mysqlStore, mysqlStoreCh, err := sqlstore.New(ctx, entClient)
if err != nil {
logrus.Fatalf("error initializing mysql client: %+v", err)
}
Expand Down
26 changes: 26 additions & 0 deletions compose.psql.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Copyright 2023 The Archivista Contributors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

services:
psql:
image: postgres:16
restart: always
environment:
POSTGRES_USER: testify
POSTGRES_PASSWORD: example

archivista:
environment:
ARCHIVISTA_SQL_STORE_BACKEND: PSQL
ARCHIVISTA_SQL_STORE_CONNECTION_STRING: postgresql://testify:example@psql?sslmode=disable
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-multierror v1.1.1
github.com/kelseyhightower/envconfig v1.4.0
github.com/lib/pq v1.10.7
github.com/minio/minio-go v6.0.14+incompatible
github.com/sirupsen/logrus v1.9.0
github.com/spf13/cobra v1.5.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc=
github.com/matryer/moq v0.2.7/go.mod h1:kITsx543GOENm48TUAQyJ9+SAvFSr7iGQXPoth/VUBk=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
Expand Down
1 change: 1 addition & 0 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ type Config struct {
SPIFFEAddress string `default:"unix:///tmp/spire-agent/public/api.sock" desc:"SPIFFE server address" split_words:"true"`
SPIFFETrustedServerId string `default:"" desc:"Trusted SPIFFE server ID; defaults to any" split_words:"true"`
SQLStoreConnectionString string `default:"root:example@tcp(db)/testify" desc:"SQL store connection string" split_words:"true"`
SQLStoreBackend string `default:"MYSQL" desc:"SQL backend to use. Options are MYSQL, PSQL" split_words:"true"`

StorageBackend string `default:"" desc:"Backend to use for attestation storage. Options are FILE, BLOB, or empty string for disabled." split_words:"true"`
FileServeOn string `default:"" desc:"What address to serve files on. Only valid when using FILE storage backend." split_words:"true"`
Expand Down
72 changes: 72 additions & 0 deletions internal/metadatastorage/sqlstore/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
// Copyright 2023 The Archivista Contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sqlstore

import (
"fmt"
"strings"
"time"

"ariga.io/sqlcomment"
"entgo.io/ent/dialect"
"entgo.io/ent/dialect/sql"
"github.com/go-sql-driver/mysql"
"github.com/testifysec/archivista/ent"

_ "github.com/lib/pq"
)

// NewEntClient creates an ent client for use in the sqlmetadata store.
// Valid backends are MYSQL and PSQL.
func NewEntClient(sqlBackend string, connectionString string) (*ent.Client, error) {
var entDialect string
switch strings.ToUpper(sqlBackend) {
case "MYSQL":
dbConfig, err := mysql.ParseDSN(connectionString)
if err != nil {
return nil, fmt.Errorf("could not parse mysql connection string: %w", err)
}

// this tells the go-sql-driver to parse times from mysql to go's time.Time
// see https://github.com/go-sql-driver/mysql#timetime-support for details
dbConfig.ParseTime = true
entDialect = dialect.MySQL
connectionString = dbConfig.FormatDSN()
case "PSQL":
entDialect = dialect.Postgres
default:
return nil, fmt.Errorf("unknown sql backend: %s", sqlBackend)
}

drv, err := sql.Open(entDialect, connectionString)
if err != nil {
return nil, fmt.Errorf("could not open sql connection: %w", err)
}

db := drv.DB()
db.SetMaxIdleConns(10)
db.SetMaxOpenConns(100)
db.SetConnMaxLifetime(3 * time.Minute)
sqlcommentDrv := sqlcomment.NewDriver(drv,
sqlcomment.WithDriverVerTag(),
sqlcomment.WithTags(sqlcomment.Tags{
sqlcomment.KeyApplication: "archivista",
sqlcomment.KeyFramework: "net/http",
}),
)

client := ent.NewClient(ent.Driver(sqlcommentDrv))
return client, nil
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package mysqlstore
package sqlstore

import (
"context"
Expand All @@ -22,10 +22,7 @@ import (
"fmt"
"time"

"ariga.io/sqlcomment"
"entgo.io/ent/dialect/sql"
"github.com/digitorus/timestamp"
"github.com/go-sql-driver/mysql"
"github.com/sirupsen/logrus"
"github.com/testifysec/archivista/ent"
"github.com/testifysec/archivista/internal/metadatastorage"
Expand All @@ -47,34 +44,7 @@ type Store struct {
client *ent.Client
}

func New(ctx context.Context, connectionstring string) (*Store, <-chan error, error) {
dbConfig, err := mysql.ParseDSN(connectionstring)
if err != nil {
return nil, nil, err
}

dbConfig.ParseTime = true
connectionstring = dbConfig.FormatDSN()
drv, err := sql.Open("mysql", connectionstring)
if err != nil {
return nil, nil, err
}
sqlcommentDrv := sqlcomment.NewDriver(drv,
sqlcomment.WithDriverVerTag(),
sqlcomment.WithTags(sqlcomment.Tags{
sqlcomment.KeyApplication: "archivista",
sqlcomment.KeyFramework: "net/http",
}),
)

// TODO make sure these take affect in sqlcommentDrv
db := drv.DB()
db.SetMaxIdleConns(10)
db.SetMaxOpenConns(100)
db.SetConnMaxLifetime(3 * time.Minute)

client := ent.NewClient(ent.Driver(sqlcommentDrv))

func New(ctx context.Context, client *ent.Client) (*Store, <-chan error, error) {
errCh := make(chan error)

go func() {
Expand Down

0 comments on commit 56a98e9

Please sign in to comment.