Skip to content

Commit

Permalink
feat: add ability to distribute witness through archivista
Browse files Browse the repository at this point in the history 
Support use case of private archivista deployments where getting witness
may be difficult. Also support use case of folks having custom builds of
witness to distribute.

Signed-off-by: Mikhail Swift <[email protected]>
  • Loading branch information
@mikhailswift
mikhailswift committed Feb 6, 2024
1 parent 7525b6d commit 3e2f929
Show file tree
Hide file tree
Showing 6 changed files with 561 additions and 18 deletions.
20 changes: 19 additions & 1 deletion cmd/archivista/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ import (
"github.com/in-toto/archivista/internal/objectstorage/blobstore"
"github.com/in-toto/archivista/internal/objectstorage/filestore"
"github.com/in-toto/archivista/internal/server"
"github.com/in-toto/archivista/internal/witnessdistro"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/sirupsen/logrus"
)
Expand All @@ -56,6 +57,7 @@ func main() {
defer cancel()

startTime := time.Now()
serverOpts := make([]server.Option, 0)

logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime))
now := time.Now()
Expand All @@ -81,6 +83,7 @@ func main() {
if err != nil {
logrus.Fatalf("error initializing storage clients: %+v", err)
}
serverOpts = append(serverOpts, server.WithObjectStore(fileStore))

entClient, err := sqlstore.NewEntClient(
cfg.SQLStoreBackend,
Expand All @@ -96,6 +99,7 @@ func main() {
if err != nil {
logrus.Fatalf("error initializing mysql client: %+v", err)
}
serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore))

logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients")

Expand All @@ -104,9 +108,23 @@ func main() {
// ********************************************************************************
now = time.Now()

// initialize witness distro store
if cfg.EnableWitnessDistro {
wds, err := witnessdistro.New(witnessdistro.WithConfigFile(cfg.WitnessDistroConfig))
if err != nil {
logrus.Fatalf("could not load witness distro store: %+v", err)
}

serverOpts = append(serverOpts, server.WithWitnessDistroStore(wds))
}

// initialize the server
sqlClient := sqlStore.GetClient()
server := server.New(sqlStore, fileStore, cfg, sqlClient)
serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient))
server, err := server.New(cfg, serverOpts...)
if err != nil {
logrus.Fatalf("could not create archivista server: %+v", err)
}

listenAddress := cfg.ListenOn
listenAddress = strings.ToLower(strings.TrimSpace(listenAddress))
Expand Down
3 changes: 3 additions & 0 deletions internal/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,9 @@ type Config struct {

EnableGraphql bool `default:"TRUE" desc:"*** Enable GraphQL Endpoint" split_words:"true"`
GraphqlWebClientEnable bool `default:"TRUE" desc:"Enable GraphiQL, the GraphQL web client" split_words:"true"`

EnableWitnessDistro bool `default:"FALSE" desc:"*** Enable Witness Distribution Endpoints" split_words:"true"`
WitnessDistroConfig string `default:"/tmp/witness/config.yaml" desc:"Location of the config describing available versions of witness" split_words:"true"`
}

// Process reads config from env
Expand Down
206 changes: 196 additions & 10 deletions internal/server/server.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ import (
"fmt"
"io"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"

"entgo.io/contrib/entgql"
Expand All @@ -33,15 +36,18 @@ import (
_ "github.com/in-toto/archivista/docs"
"github.com/in-toto/archivista/ent"
"github.com/in-toto/archivista/internal/config"
"github.com/in-toto/archivista/internal/witnessdistro"
"github.com/in-toto/archivista/pkg/api"
"github.com/sirupsen/logrus"
httpSwagger "github.com/swaggo/http-swagger/v2"
)

type Server struct {
metadataStore Storer
objectStore StorerGetter
router *mux.Router
metadataStore Storer
objectStore StorerGetter
witnessDistroStore witnessdistro.Store
router *mux.Router
sqlClient *ent.Client
}

type Storer interface {
Expand All @@ -57,16 +63,48 @@ type StorerGetter interface {
Getter
}

func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sqlClient *ent.Client) Server {
type Option func(*Server)

func WithMetadataStore(metadataStore Storer) Option {
return func(s *Server) {
s.metadataStore = metadataStore
}
}

func WithObjectStore(objectStore StorerGetter) Option {
return func(s *Server) {
s.objectStore = objectStore
}
}

func WithEntSqlClient(sqlClient *ent.Client) Option {
return func(s *Server) {
s.sqlClient = sqlClient
}
}

func WithWitnessDistroStore(wds witnessdistro.Store) Option {
return func(s *Server) {
s.witnessDistroStore = wds
}
}

func New(cfg *config.Config, opts ...Option) (Server, error) {
r := mux.NewRouter()
s := &Server{metadataStore, objectStore, nil}
s := Server{
router: r,
}

for _, opt := range opts {
opt(&s)
}

// TODO: remove from future version (v0.6.0) endpoint with version
r.HandleFunc("/download/{gitoid}", s.DownloadHandler)
r.HandleFunc("/upload", s.UploadHandler)
if cfg.EnableGraphql {
r.Handle("/query", s.Query(sqlClient))
r.Handle("/v1/query", s.Query(sqlClient))
r.Handle("/query", s.Query(s.sqlClient))
r.Handle("/v1/query", s.Query(s.sqlClient))
}

r.HandleFunc("/v1/download/{gitoid}", s.DownloadHandler)
Expand All @@ -76,11 +114,16 @@ func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sql
playground.Handler("Archivista", "/v1/query"),
)
}
r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
s.router = r

return *s
if cfg.EnableWitnessDistro {
r.HandleFunc("/v1/witness", s.WitnessAllVersionsHandler)
r.HandleFunc("/v1/witness/", s.WitnessAllVersionsHandler)
r.HandleFunc("/v1/witness/{version}", s.WitnessVersionHandler)
r.HandleFunc("/v1/witness/{version}/{distribution}", s.DownloadWitnessHandler)
}

r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler)
return s, nil
}

// @title Archivista API
Expand Down Expand Up @@ -233,3 +276,146 @@ func (s *Server) Query(sqlclient *ent.Client) *handler.Server {
srv.Use(entgql.Transactioner{TxOpener: sqlclient})
return srv
}

// @Summary Witness List Versions
// @Description retrieves details about all available versions of witness
// @Produce json
// @Success 200 {object} witnessdistro.VersionList
// @Failure 500 {object} string
// @Failure 400 {object} string
// @Router /v1/witness/ [get]
func (s *Server) WitnessAllVersionsHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest)
return
}

allVersions := s.witnessDistroStore.Versions()
allVersionsJson, err := json.Marshal(allVersions)
if err != nil {
http.Error(w, fmt.Errorf("could not marshal witness versions: %w", err).Error(), http.StatusInternalServerError)
return
}

if _, err := io.Copy(w, bytes.NewReader(allVersionsJson)); err != nil {
http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError)
return
}

w.Header().Set("Content-Type", "application/json")
}

// @Summary Witness Version Details
// @Description retrieves details about a specified version of witness
// @Produce json
// @Param version path string true "version of witness"
// @Success 200 {object} witnessdistro.Version
// @Failure 500 {object} string
// @Failure 404 {object} nil
// @Failure 400 {object} string
// @Router /v1/witness/{version} [get]
func (s *Server) WitnessVersionHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest)
return
}

vars := mux.Vars(r)
if vars == nil {
http.Error(w, "version parameter is required", http.StatusBadRequest)
return
}

versionString := vars["version"]
if len(versionString) == 0 {
http.Error(w, "version parameter is required", http.StatusBadRequest)
return
}

version, ok := s.witnessDistroStore.Version(versionString)
if !ok {
http.Error(w, "version not found", http.StatusNotFound)
return
}

versionJson, err := json.Marshal(version)
if err != nil {
http.Error(w, fmt.Errorf("could not marshal witness distros: %w", err).Error(), http.StatusInternalServerError)
return
}

if _, err := io.Copy(w, bytes.NewReader(versionJson)); err != nil {
http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError)
return
}

w.Header().Set("Content-Type", "application/json")
}

// @Summary Download Witness
// @Description downloads a specified distribution of witness
// @Produce octet-stream
// @Param version path string true "version of witness to download"
// @Param distribution path string true "distribution of witness to download"
// @Success 200 {object} octet-stream
// @Failure 500 {object} string
// @Failure 404 {object} nil
// @Failure 400 {object} string
// @Router /v1/witness/{version}/{distribution} [get]
func (s *Server) DownloadWitnessHandler(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest)
return
}

vars := mux.Vars(r)
if vars == nil {
http.Error(w, "version and distribution parameter is required", http.StatusBadRequest)
return
}

versionString := vars["version"]
if len(versionString) == 0 {
http.Error(w, "version parameter is required", http.StatusBadRequest)
return
}

distroString := vars["distribution"]
if len(distroString) == 0 {
http.Error(w, "distribution parameter is required", http.StatusBadRequest)
return
}

distro, ok := s.witnessDistroStore.Distribution(versionString, distroString)
if !ok {
http.Error(w, "distribution of witness not found", http.StatusNotFound)
return
}

file, err := os.Open(distro.FileLocation)
if err != nil {
http.Error(w, "could not read distribution file", http.StatusBadRequest)
return
}

defer func() {
if err := file.Close(); err != nil {
logrus.Errorf(fmt.Sprintf("failed to close witness distribution file %s: %+v", distro.FileLocation, err))
}
}()

fileInfo, err := file.Stat()
if err != nil {
http.Error(w, "could not stat distribution file", http.StatusBadRequest)
return
}

fileName := filepath.Base(distro.FileLocation)
w.Header().Set("Content-Type", "application/octet-stream")
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%v", fileName))
w.Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10))
if _, err := io.Copy(w, file); err != nil {
http.Error(w, fmt.Errorf("could not send witness distribution: %w", err).Error(), http.StatusInternalServerError)
return
}
}
Loading

0 comments on commit 3e2f929

Please sign in to comment.