diff --git a/cmd/archivista/main.go b/cmd/archivista/main.go index 0c13cd98..e6aed61b 100644 --- a/cmd/archivista/main.go +++ b/cmd/archivista/main.go @@ -39,7 +39,7 @@ import ( "github.com/sirupsen/logrus" "github.com/testifysec/archivista" "github.com/testifysec/archivista/internal/config" - "github.com/testifysec/archivista/internal/metadatastorage/mysqlstore" + "github.com/testifysec/archivista/internal/metadatastorage/sqlstore" "github.com/testifysec/archivista/internal/objectstorage/blobstore" "github.com/testifysec/archivista/internal/objectstorage/filestore" "github.com/testifysec/archivista/internal/server" @@ -86,7 +86,12 @@ func main() { logrus.Fatalf("error initializing storage clients: %+v", err) } - mysqlStore, mysqlStoreCh, err := mysqlstore.New(ctx, cfg.SQLStoreConnectionString) + entClient, err := sqlstore.NewEntClient(cfg.SQLStoreBackend, cfg.SQLStoreConnectionString) + if err != nil { + logrus.Fatalf("could not create ent client: %+v", err) + } + + mysqlStore, mysqlStoreCh, err := sqlstore.New(ctx, entClient) if err != nil { logrus.Fatalf("error initializing mysql client: %+v", err) } diff --git a/compose.psql.yml b/compose.psql.yml new file mode 100644 index 00000000..1fa5121e --- /dev/null +++ b/compose.psql.yml @@ -0,0 +1,26 @@ +# Copyright 2023 The Archivista Contributors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +services: + psql: + image: postgres:16 + restart: always + environment: + POSTGRES_USER: testify + POSTGRES_PASSWORD: example + + archivista: + environment: + ARCHIVISTA_SQL_STORE_BACKEND: PSQL + ARCHIVISTA_SQL_STORE_CONNECTION_STRING: postgresql://testify:example@psql?sslmode=disable diff --git a/go.mod b/go.mod index bc53a589..03a354e8 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-multierror v1.1.1 github.com/kelseyhightower/envconfig v1.4.0 + github.com/lib/pq v1.10.7 github.com/minio/minio-go v6.0.14+incompatible github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/go.sum b/go.sum index af4e97d5..d19794c1 100644 --- a/go.sum +++ b/go.sum @@ -116,6 +116,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora/v3 v3.0.0/go.mod h1:vsR12bk5grlLvLXAYrBsb5Oc/N+LxAlxggSjiwMnCUc= github.com/matryer/moq v0.2.7/go.mod h1:kITsx543GOENm48TUAQyJ9+SAvFSr7iGQXPoth/VUBk= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= diff --git a/internal/config/config.go b/internal/config/config.go index 7e0763d9..92d29892 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -32,6 +32,7 @@ type Config struct { SPIFFEAddress string `default:"unix:///tmp/spire-agent/public/api.sock" desc:"SPIFFE server address" split_words:"true"` SPIFFETrustedServerId string `default:"" desc:"Trusted SPIFFE server ID; defaults to any" split_words:"true"` SQLStoreConnectionString string `default:"root:example@tcp(db)/testify" desc:"SQL store connection string" split_words:"true"` + SQLStoreBackend string `default:"MYSQL" desc:"SQL backend to use. Options are MYSQL, PSQL" split_words:"true"` StorageBackend string `default:"" desc:"Backend to use for attestation storage. Options are FILE, BLOB, or empty string for disabled." split_words:"true"` FileServeOn string `default:"" desc:"What address to serve files on. Only valid when using FILE storage backend." split_words:"true"` diff --git a/internal/metadatastorage/sqlstore/client.go b/internal/metadatastorage/sqlstore/client.go new file mode 100644 index 00000000..45a81d91 --- /dev/null +++ b/internal/metadatastorage/sqlstore/client.go @@ -0,0 +1,72 @@ +// Copyright 2023 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlstore + +import ( + "fmt" + "strings" + "time" + + "ariga.io/sqlcomment" + "entgo.io/ent/dialect" + "entgo.io/ent/dialect/sql" + "github.com/go-sql-driver/mysql" + "github.com/testifysec/archivista/ent" + + _ "github.com/lib/pq" +) + +// NewEntClient creates an ent client for use in the sqlmetadata store. +// Valid backends are MYSQL and PSQL. +func NewEntClient(sqlBackend string, connectionString string) (*ent.Client, error) { + var entDialect string + switch strings.ToUpper(sqlBackend) { + case "MYSQL": + dbConfig, err := mysql.ParseDSN(connectionString) + if err != nil { + return nil, fmt.Errorf("could not parse mysql connection string: %w", err) + } + + // this tells the go-sql-driver to parse times from mysql to go's time.Time + // see https://github.com/go-sql-driver/mysql#timetime-support for details + dbConfig.ParseTime = true + entDialect = dialect.MySQL + connectionString = dbConfig.FormatDSN() + case "PSQL": + entDialect = dialect.Postgres + default: + return nil, fmt.Errorf("unknown sql backend: %s", sqlBackend) + } + + drv, err := sql.Open(entDialect, connectionString) + if err != nil { + return nil, fmt.Errorf("could not open sql connection: %w", err) + } + + db := drv.DB() + db.SetMaxIdleConns(10) + db.SetMaxOpenConns(100) + db.SetConnMaxLifetime(3 * time.Minute) + sqlcommentDrv := sqlcomment.NewDriver(drv, + sqlcomment.WithDriverVerTag(), + sqlcomment.WithTags(sqlcomment.Tags{ + sqlcomment.KeyApplication: "archivista", + sqlcomment.KeyFramework: "net/http", + }), + ) + + client := ent.NewClient(ent.Driver(sqlcommentDrv)) + return client, nil +} diff --git a/internal/metadatastorage/mysqlstore/mysql.go b/internal/metadatastorage/sqlstore/store.go similarity index 88% rename from internal/metadatastorage/mysqlstore/mysql.go rename to internal/metadatastorage/sqlstore/store.go index bb52cb8e..3fc0d34c 100644 --- a/internal/metadatastorage/mysqlstore/mysql.go +++ b/internal/metadatastorage/sqlstore/store.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package mysqlstore +package sqlstore import ( "context" @@ -22,10 +22,7 @@ import ( "fmt" "time" - "ariga.io/sqlcomment" - "entgo.io/ent/dialect/sql" "github.com/digitorus/timestamp" - "github.com/go-sql-driver/mysql" "github.com/sirupsen/logrus" "github.com/testifysec/archivista/ent" "github.com/testifysec/archivista/internal/metadatastorage" @@ -47,34 +44,7 @@ type Store struct { client *ent.Client } -func New(ctx context.Context, connectionstring string) (*Store, <-chan error, error) { - dbConfig, err := mysql.ParseDSN(connectionstring) - if err != nil { - return nil, nil, err - } - - dbConfig.ParseTime = true - connectionstring = dbConfig.FormatDSN() - drv, err := sql.Open("mysql", connectionstring) - if err != nil { - return nil, nil, err - } - sqlcommentDrv := sqlcomment.NewDriver(drv, - sqlcomment.WithDriverVerTag(), - sqlcomment.WithTags(sqlcomment.Tags{ - sqlcomment.KeyApplication: "archivista", - sqlcomment.KeyFramework: "net/http", - }), - ) - - // TODO make sure these take affect in sqlcommentDrv - db := drv.DB() - db.SetMaxIdleConns(10) - db.SetMaxOpenConns(100) - db.SetConnMaxLifetime(3 * time.Minute) - - client := ent.NewClient(ent.Driver(sqlcommentDrv)) - +func New(ctx context.Context, client *ent.Client) (*Store, <-chan error, error) { errCh := make(chan error) go func() {