diff --git a/README.md b/README.md index 012711a7..c934c57b 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ Archivista is configured through environment variables currently. | ARCHIVISTA_SQL_STORE_CONNECTION_STRING | root:example@tcp(db)/testify | SQL store connection string | | ARCHIVISTA_STORAGE_BACKEND | | Backend to use for attestation storage. Options are FILE, BLOB, or empty string for disabled. | | ARCHIVISTA_FILE_SERVE_ON | | What address to serve files on. Only valid when using FILE storage backend. | -| ARCHIVISTA_FILE_DIR | /tmp/archivista/ | Directory to store and serve files. Only valid when using FILE storage backend. | +| ARCHIVISTA_FILE_DIR | /tmp/archivista/ | Directory to store and serve files. Only valid when using FILE storage backend. | | ARCHIVISTA_BLOB_STORE_ENDPOINT | 127.0.0.1:9000 | URL endpoint for blob storage. Only valid when using BLOB storage backend. | | ARCHIVISTA_BLOB_STORE_CREDENTIAL_TYPE | | Blob store credential type. Options are IAM or ACCESS_KEY. | | ARCHIVISTA_BLOB_STORE_ACCESS_KEY_ID | | Blob store access key id. Only valid when using BLOB storage backend. | @@ -89,6 +89,8 @@ Archivista is configured through environment variables currently. | ARCHIVISTA_BLOB_STORE_BUCKET_NAME | | Bucket to use for storage. Only valid when using BLOB storage backend. | | ARCHIVISTA_ENABLE_GRAPHQL | TRUE | Enable GraphQL Endpoint | | ARCHIVISTA_GRAPHQL_WEB_CLIENT_ENABLE | TRUE | Enable GraphiQL, the GraphQL web client | +| ARCHIVISTA_ENABLE_ARTIFACT_STORE | FALSE | Enable Artifact Store Endpoints | +| ARCHIVISTA_ARTIFACT_STORE_CONFIG | /tmp/artifacts/config.yaml | Location of the config describing available artifacts | ## Using Archivista diff --git a/cmd/archivista/main.go b/cmd/archivista/main.go index 7ab1c7a1..0f5c4c82 100644 --- a/cmd/archivista/main.go +++ b/cmd/archivista/main.go @@ -32,6 +32,7 @@ import ( nested "github.com/antonfisher/nested-logrus-formatter" "github.com/gorilla/handlers" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/internal/metadatastorage/sqlstore" "github.com/in-toto/archivista/internal/objectstorage/blobstore" @@ -56,6 +57,7 @@ func main() { defer cancel() startTime := time.Now() + serverOpts := make([]server.Option, 0) logrus.Infof("executing phase 1: get config from environment (time since start: %s)", time.Since(startTime)) now := time.Now() @@ -81,6 +83,7 @@ func main() { if err != nil { logrus.Fatalf("error initializing storage clients: %+v", err) } + serverOpts = append(serverOpts, server.WithObjectStore(fileStore)) entClient, err := sqlstore.NewEntClient( cfg.SQLStoreBackend, @@ -96,6 +99,7 @@ func main() { if err != nil { logrus.Fatalf("error initializing mysql client: %+v", err) } + serverOpts = append(serverOpts, server.WithMetadataStore(sqlStore)) logrus.WithField("duration", time.Since(now)).Infof("completed phase 3: initializing storage clients") @@ -104,9 +108,23 @@ func main() { // ******************************************************************************** now = time.Now() + // initialize the artifact store + if cfg.EnableArtifactStore { + wds, err := artifactstore.New(artifactstore.WithConfigFile(cfg.ArtifactStoreConfig)) + if err != nil { + logrus.Fatalf("could not create the artifact store: %+v", err) + } + + serverOpts = append(serverOpts, server.WithArtifactStore(wds)) + } + // initialize the server sqlClient := sqlStore.GetClient() - server := server.New(sqlStore, fileStore, cfg, sqlClient) + serverOpts = append(serverOpts, server.WithEntSqlClient(sqlClient)) + server, err := server.New(cfg, serverOpts...) + if err != nil { + logrus.Fatalf("could not create archivista server: %+v", err) + } listenAddress := cfg.ListenOn listenAddress = strings.ToLower(strings.TrimSpace(listenAddress)) diff --git a/docs/docs.go b/docs/docs.go index 7dbeabbf..fcafe3cc 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -81,6 +81,194 @@ const docTemplate = `{ } } }, + "/v1/artifacts": { + "get": { + "description": "retrieves details about all available Artifacts", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List all Artifacts", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Artifact" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}": { + "get": { + "description": "retrieves details about all available versions of a specified artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List Artifact Versions", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}/{version}": { + "get": { + "description": "retrieves details about a specified version of an artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "Artifact Version Details", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/artifactstore.Version" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "objecpec" + } + } + } + } + }, + "/v1/download/artifact/{name}/{version}/{distribution}": { + "get": { + "description": "downloads a specified distribution of an artifact", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Artifacts" + ], + "summary": "Download Artifact", + "parameters": [ + { + "type": "string", + "description": "name of artifact", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact to download", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "distribution of artifact to download", + "name": "distribution", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/v1/download/{gitoid}": { "get": { "description": "download an attestation", @@ -178,6 +366,39 @@ const docTemplate = `{ "archivista.Resolver": { "type": "object" }, + "artifactstore.Artifact": { + "type": "object", + "properties": { + "versions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + } + }, + "artifactstore.Distribution": { + "type": "object", + "properties": { + "sha256digest": { + "type": "string" + } + } + }, + "artifactstore.Version": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "distributions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Distribution" + } + } + } + }, "dsse.Envelope": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 323f0c11..09e7a6bc 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -73,6 +73,194 @@ } } }, + "/v1/artifacts": { + "get": { + "description": "retrieves details about all available Artifacts", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List all Artifacts", + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Artifact" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}": { + "get": { + "description": "retrieves details about all available versions of a specified artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "List Artifact Versions", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, + "/v1/artifacts/{name}/{version}": { + "get": { + "description": "retrieves details about a specified version of an artifact", + "produces": [ + "application/json" + ], + "tags": [ + "Artifacts" + ], + "summary": "Artifact Version Details", + "parameters": [ + { + "type": "string", + "description": "artifact name", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact", + "name": "version", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/artifactstore.Version" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "objecpec" + } + } + } + } + }, + "/v1/download/artifact/{name}/{version}/{distribution}": { + "get": { + "description": "downloads a specified distribution of an artifact", + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Artifacts" + ], + "summary": "Download Artifact", + "parameters": [ + { + "type": "string", + "description": "name of artifact", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "version of artifact to download", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "distribution of artifact to download", + "name": "distribution", + "in": "path", + "required": true + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "type": "file" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "type": "string" + } + }, + "404": { + "description": "Not Found" + }, + "500": { + "description": "Internal Server Error", + "schema": { + "type": "string" + } + } + } + } + }, "/v1/download/{gitoid}": { "get": { "description": "download an attestation", @@ -170,6 +358,39 @@ "archivista.Resolver": { "type": "object" }, + "artifactstore.Artifact": { + "type": "object", + "properties": { + "versions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Version" + } + } + } + }, + "artifactstore.Distribution": { + "type": "object", + "properties": { + "sha256digest": { + "type": "string" + } + } + }, + "artifactstore.Version": { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "distributions": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/artifactstore.Distribution" + } + } + } + }, "dsse.Envelope": { "type": "object", "properties": { @@ -249,4 +470,4 @@ ] } } -} +} \ No newline at end of file diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 9a77dcbc..39c4d533 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -6,6 +6,27 @@ definitions: type: object archivista.Resolver: type: object + artifactstore.Artifact: + properties: + versions: + additionalProperties: + $ref: '#/definitions/artifactstore.Version' + type: object + type: object + artifactstore.Distribution: + properties: + sha256digest: + type: string + type: object + artifactstore.Version: + properties: + description: + type: string + distributions: + additionalProperties: + $ref: '#/definitions/artifactstore.Distribution' + type: object + type: object dsse.Envelope: properties: payload: @@ -107,6 +128,92 @@ paths: schema: $ref: '#/definitions/api.StoreResponse' summary: Upload + /v1/artifacts: + get: + description: retrieves details about all available Artifacts + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: + $ref: '#/definitions/artifactstore.Artifact' + type: object + "400": + description: Bad Request + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List all Artifacts + tags: + - Artifacts + /v1/artifacts/{name}: + get: + description: retrieves details about all available versions of a specified artifact + parameters: + - description: artifact name + in: path + name: name + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + additionalProperties: + $ref: '#/definitions/artifactstore.Version' + type: object + "400": + description: Bad Request + schema: + type: string + "500": + description: Internal Server Error + schema: + type: string + summary: List Artifact Versions + tags: + - Artifacts + /v1/artifacts/{name}/{version}: + get: + description: retrieves details about a specified version of an artifact + parameters: + - description: artifact name + in: path + name: name + required: true + type: string + - description: version of artifact + in: path + name: version + required: true + type: string + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/artifactstore.Version' + "400": + description: Bad Request + schema: + type: string + "404": + description: Not Found + "500": + description: Internal Server Error + schema: + type: objecpec + summary: Artifact Version Details + tags: + - Artifacts /v1/download/{gitoid}: get: description: download an attestation @@ -136,6 +243,45 @@ paths: summary: Download tags: - attestation + /v1/download/artifact/{name}/{version}/{distribution}: + get: + description: downloads a specified distribution of an artifact + parameters: + - description: name of artifact + in: path + name: name + required: true + type: string + - description: version of artifact to download + in: path + name: version + required: true + type: string + - description: distribution of artifact to download + in: path + name: distribution + required: true + type: string + produces: + - application/octet-stream + responses: + "200": + description: OK + schema: + type: file + "400": + description: Bad Request + schema: + type: string + "404": + description: Not Found + "500": + description: Internal Server Error + schema: + type: string + summary: Download Artifact + tags: + - Artifacts /v1/query: post: description: GraphQL query diff --git a/internal/artifactstore/artifactstore.go b/internal/artifactstore/artifactstore.go new file mode 100644 index 00000000..ba2d6f0b --- /dev/null +++ b/internal/artifactstore/artifactstore.go @@ -0,0 +1,205 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifactstore + +import ( + "crypto" + "errors" + "fmt" + "os" + "strings" + + "github.com/in-toto/go-witness/cryptoutil" + "gopkg.in/yaml.v3" +) + +// Config represents the available artifacts within the store +type Config struct { + Artifacts map[string]Artifact `json:"artifacts"` +} + +// Artifact represents an artifact and it's available versions in the store +type Artifact struct { + Versions map[string]Version `json:"versions"` +} + +// Version represents a version of an Artifact (ex v0.2.0) with the available Distributions of version +type Version struct { + Distributions map[string]Distribution `json:"distributions"` + Description string `json:"description"` +} + +// Distribution is a specific distribution of a Version of an Artifact(ex linux-amd64) +type Distribution struct { + FileLocation string `json:"-"` + SHA256Digest string `json:"sha256digest"` +} + +// Store is an artifact store served from Archivista +type Store struct { + config Config +} + +type Option func(*Store) error + +// WithConfig creates a Store with the provided config +func WithConfig(config Config) Option { + return func(as *Store) error { + as.config = config + return nil + } +} + +// WithConfigFile creates a Store with a config loaded from a yaml file on disk +func WithConfigFile(configPath string) Option { + return func(as *Store) error { + configBytes, err := os.ReadFile(configPath) + if err != nil { + return err + } + + config := Config{} + if err := yaml.Unmarshal(configBytes, &config); err != nil { + return err + } + + return WithConfig(config)(as) + } +} + +// New creates a new Store with the provided options +func New(opts ...Option) (Store, error) { + as := Store{} + errs := make([]error, 0) + for _, opt := range opts { + if err := opt(&as); err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return Store{}, errors.Join(errs...) + } + + if err := verifyConfig(as); err != nil { + return Store{}, err + } + + return as, nil +} + +// verifyConfig ensures that each file exists on disk and that the sha256sum of the +// files on disk match those of the config +func verifyConfig(as Store) error { + errs := make([]error, 0) + for artifactName, artifact := range as.config.Artifacts { + for versionString, version := range artifact.Versions { + for distroString, distro := range version.Distributions { + if _, err := os.Stat(distro.FileLocation); err != nil { + errs = append(errs, fmt.Errorf("%v version %v-%v does not exist on disk: %w", artifactName, versionString, distroString, err)) + continue + } + + digestSet, err := cryptoutil.CalculateDigestSetFromFile(distro.FileLocation, []crypto.Hash{crypto.SHA256}) + if err != nil { + errs = append(errs, fmt.Errorf("could not calculate sha256 digest for %v version %v-%v: %w", artifactName, versionString, distroString, err)) + } + + sha256Digest := digestSet[cryptoutil.DigestValue{Hash: crypto.SHA256, GitOID: false}] + if !strings.EqualFold(sha256Digest, distro.SHA256Digest) { + errs = append(errs, fmt.Errorf("sha256 digest of %v version %v-%v does not match config: got %v, expected %v", artifactName, versionString, distroString, sha256Digest, distro.SHA256Digest)) + } + } + } + } + + if len(errs) > 0 { + return errors.Join(errs...) + } + + return nil +} + +// Artifacts returns a copy of all the Store's Artifacts +func (as Store) Artifacts() map[string]Artifact { + out := make(map[string]Artifact) + for artifactString, artifact := range as.config.Artifacts { + out[artifactString] = artifact + } + + return out +} + +// Versions returns all of the available Versions for an Artifact. +func (as Store) Versions(artifact string) (map[string]Version, bool) { + out := make(map[string]Version) + a, ok := as.config.Artifacts[artifact] + if !ok { + return out, false + } + + for verString, version := range a.Versions { + out[verString] = version + } + + return out, ok +} + +// Version returns a specific Version for an artifact, if it exists +func (as Store) Version(artifact, version string) (Version, bool) { + a, ok := as.config.Artifacts[artifact] + if !ok { + return Version{}, false + } + + v, ok := a.Versions[version] + return v, ok +} + +// Distributions returns all of the available Distributions for a specified Version of an Artifact +func (as Store) Distributions(artifact, version string) (map[string]Distribution, bool) { + out := make(map[string]Distribution) + a, ok := as.config.Artifacts[artifact] + if !ok { + return out, false + } + + vers, ok := a.Versions[version] + if !ok { + return out, ok + } + + for distroString, distro := range vers.Distributions { + out[distroString] = distro + } + + return out, true +} + +// Distribution returns the entry for a specific distribution for a specific version +func (as Store) Distribution(artifact, version, distribution string) (Distribution, bool) { + a, ok := as.config.Artifacts[artifact] + if !ok { + return Distribution{}, false + } + + vers, ok := a.Versions[version] + if !ok { + return Distribution{}, false + } + + distro, ok := vers.Distributions[distribution] + return distro, ok +} diff --git a/internal/artifactstore/artifactstore_test.go b/internal/artifactstore/artifactstore_test.go new file mode 100644 index 00000000..605ae1e5 --- /dev/null +++ b/internal/artifactstore/artifactstore_test.go @@ -0,0 +1,88 @@ +// Copyright 2024 The Archivista Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package artifactstore + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func createTestConfigFile(t *testing.T, workingDir, version, distroFilePath, distroDigest string) string { + testConfig := `artifacts: + witness: + versions: + ` + version + `: + description: test + distributions: + linux: + filelocation: ` + distroFilePath + ` + sha256digest: ` + distroDigest + testConfigFilePath := filepath.Join(workingDir, "config.yaml") + testConfigFile, err := os.Create(testConfigFilePath) + require.NoError(t, err) + _, err = testConfigFile.WriteString(testConfig) + require.NoError(t, err) + require.NoError(t, testConfigFile.Close()) + return testConfigFilePath +} + +func TestStore(t *testing.T) { + workingDir := t.TempDir() + testDistroFilePath := filepath.Join(workingDir, "test") + testDistroDigest := "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + testVersion := "v0.1.0" + testDistroFile, err := os.Create(testDistroFilePath) + require.NoError(t, err) + _, err = testDistroFile.Write([]byte("test")) + require.NoError(t, err) + require.NoError(t, testDistroFile.Close()) + + t.Run("all good", func(t *testing.T) { + testArtifactName := "witness" + testDistroName := "linux" + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, testDistroFilePath, testDistroDigest) + as, err := New(WithConfigFile(testConfigFilePath)) + require.NoError(t, err) + artifacts := as.Artifacts() + assert.Len(t, artifacts, 1) + versions, ok := as.Versions(testArtifactName) + assert.True(t, ok) + assert.Len(t, versions, 1) + version, ok := as.Version(testArtifactName, testVersion) + assert.True(t, ok) + assert.Len(t, version.Distributions, 1) + testDistro, ok := as.Distribution(testArtifactName, testVersion, testDistroName) + assert.True(t, ok) + assert.Equal(t, testDistro.FileLocation, testDistroFilePath) + assert.Equal(t, testDistro.SHA256Digest, testDistroDigest) + }) + + t.Run("wrong file path", func(t *testing.T) { + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, "garbage", testDistroDigest) + _, err := New(WithConfigFile(testConfigFilePath)) + assert.Error(t, err) + + }) + + t.Run("wrong file digest", func(t *testing.T) { + testConfigFilePath := createTestConfigFile(t, workingDir, testVersion, testDistroFilePath, "garbage") + _, err := New(WithConfigFile(testConfigFilePath)) + assert.Error(t, err) + }) +} diff --git a/internal/config/config.go b/internal/config/config.go index bbdc3ae9..dd708fcb 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -51,6 +51,9 @@ type Config struct { EnableGraphql bool `default:"TRUE" desc:"*** Enable GraphQL Endpoint" split_words:"true"` GraphqlWebClientEnable bool `default:"TRUE" desc:"Enable GraphiQL, the GraphQL web client" split_words:"true"` + + EnableArtifactStore bool `default:"FALSE" desc:"*** Enable Artifact Store Endpoints" split_words:"true"` + ArtifactStoreConfig string `default:"/tmp/artifacts/config.yaml" desc:"Location of the config describing available artifacts" split_words:"true"` } // Process reads config from env diff --git a/internal/server/server.go b/internal/server/server.go index 4cc8347f..165639f5 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -22,6 +22,9 @@ import ( "fmt" "io" "net/http" + "os" + "path/filepath" + "strconv" "strings" "entgo.io/contrib/entgql" @@ -32,6 +35,7 @@ import ( "github.com/in-toto/archivista" _ "github.com/in-toto/archivista/docs" "github.com/in-toto/archivista/ent" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" "github.com/sirupsen/logrus" @@ -41,7 +45,9 @@ import ( type Server struct { metadataStore Storer objectStore StorerGetter + artifactStore artifactstore.Store router *mux.Router + sqlClient *ent.Client } type Storer interface { @@ -57,16 +63,48 @@ type StorerGetter interface { Getter } -func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sqlClient *ent.Client) Server { +type Option func(*Server) + +func WithMetadataStore(metadataStore Storer) Option { + return func(s *Server) { + s.metadataStore = metadataStore + } +} + +func WithObjectStore(objectStore StorerGetter) Option { + return func(s *Server) { + s.objectStore = objectStore + } +} + +func WithEntSqlClient(sqlClient *ent.Client) Option { + return func(s *Server) { + s.sqlClient = sqlClient + } +} + +func WithArtifactStore(wds artifactstore.Store) Option { + return func(s *Server) { + s.artifactStore = wds + } +} + +func New(cfg *config.Config, opts ...Option) (Server, error) { r := mux.NewRouter() - s := &Server{metadataStore, objectStore, nil} + s := Server{ + router: r, + } + + for _, opt := range opts { + opt(&s) + } // TODO: remove from future version (v0.6.0) endpoint with version r.HandleFunc("/download/{gitoid}", s.DownloadHandler) r.HandleFunc("/upload", s.UploadHandler) if cfg.EnableGraphql { - r.Handle("/query", s.Query(sqlClient)) - r.Handle("/v1/query", s.Query(sqlClient)) + r.Handle("/query", s.Query(s.sqlClient)) + r.Handle("/v1/query", s.Query(s.sqlClient)) } r.HandleFunc("/v1/download/{gitoid}", s.DownloadHandler) @@ -76,11 +114,16 @@ func New(metadataStore Storer, objectStore StorerGetter, cfg *config.Config, sql playground.Handler("Archivista", "/v1/query"), ) } - r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) - s.router = r - return *s + if cfg.EnableArtifactStore { + r.HandleFunc("/v1/artifacts", s.AllArtifactsHandler) + r.HandleFunc("/v1/artifacts/{name}", s.ArtifactAllVersionsHandler) + r.HandleFunc("/v1/artifacts/{name}/{version}", s.ArtifactVersionHandler) + r.HandleFunc("/v1/download/artifact/{name}/{version}/{distribution}", s.DownloadArtifactHandler) + } + r.PathPrefix("/swagger/").Handler(httpSwagger.WrapHandler) + return s, nil } // @title Archivista API @@ -233,3 +276,211 @@ func (s *Server) Query(sqlclient *ent.Client) *handler.Server { srv.Use(entgql.Transactioner{TxOpener: sqlclient}) return srv } + +// @Summary List all Artifacts +// @Description retrieves details about all available Artifacts +// @Produce json +// @Success 200 {object} map[string]artifactstore.Artifact +// @Failure 500 {object} string +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts [get] +func (s *Server) AllArtifactsHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + allArtifacts := s.artifactStore.Artifacts() + allArtifactsJson, err := json.Marshal(allArtifacts) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact versions: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(allArtifactsJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + +} + +// @Summary List Artifact Versions +// @Description retrieves details about all available versions of a specified artifact +// @Produce json +// @Param name path string true "artifact name" +// @Success 200 {object} map[string]artifactstore.Version +// @Failure 500 {object} string +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts/{name} [get] +func (s *Server) ArtifactAllVersionsHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + artifactName := vars["name"] + if len(artifactName) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + artifactVersions, ok := s.artifactStore.Versions(artifactName) + if !ok { + http.Error(w, "artifact not found", http.StatusNotFound) + return + } + + artifactVersionsJson, err := json.Marshal(artifactVersions) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact versions: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(artifactVersionsJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") +} + +// @Summary Artifact Version Details +// @Description retrieves details about a specified version of an artifact +// @Produce json +// @Param name path string true "artifact name" +// @Param version path string true "version of artifact" +// @Success 200 {object} artifactstore.Version +// @Failure 500 {objecpec} string +// @Failure 404 {object} nil +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/artifacts/{name}/{version} [get] +func (s *Server) ArtifactVersionHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "name and version parameters are required", http.StatusBadRequest) + return + } + + artifactString := vars["name"] + if len(artifactString) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + versionString := vars["version"] + if len(versionString) == 0 { + http.Error(w, "version parameter is required", http.StatusBadRequest) + return + } + + version, ok := s.artifactStore.Version(artifactString, versionString) + if !ok { + http.Error(w, "version not found", http.StatusNotFound) + return + } + + versionJson, err := json.Marshal(version) + if err != nil { + http.Error(w, fmt.Errorf("could not marshal artifact distros: %w", err).Error(), http.StatusInternalServerError) + return + } + + if _, err := io.Copy(w, bytes.NewReader(versionJson)); err != nil { + http.Error(w, fmt.Errorf("could not send json response: %w", err).Error(), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") +} + +// @Summary Download Artifact +// @Description downloads a specified distribution of an artifact +// @Produce octet-stream +// @Param name path string true "name of artifact" +// @Param version path string true "version of artifact to download" +// @Param distribution path string true "distribution of artifact to download" +// @Success 200 {file} octet-stream +// @Failure 500 {object} string +// @Failure 404 {object} nil +// @Failure 400 {object} string +// @Tags Artifacts +// @Router /v1/download/artifact/{name}/{version}/{distribution} [get] +func (s *Server) DownloadArtifactHandler(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, fmt.Sprintf("%s is an unsupported method", r.Method), http.StatusBadRequest) + return + } + + vars := mux.Vars(r) + if vars == nil { + http.Error(w, "version and distribution parameter is required", http.StatusBadRequest) + return + } + + artifactString := vars["name"] + if len(artifactString) == 0 { + http.Error(w, "name parameter is required", http.StatusBadRequest) + return + } + + versionString := vars["version"] + if len(versionString) == 0 { + http.Error(w, "version parameter is required", http.StatusBadRequest) + return + } + + distroString := vars["distribution"] + if len(distroString) == 0 { + http.Error(w, "distribution parameter is required", http.StatusBadRequest) + return + } + + distro, ok := s.artifactStore.Distribution(artifactString, versionString, distroString) + if !ok { + http.Error(w, "distribution of artifact not found", http.StatusNotFound) + return + } + + file, err := os.Open(distro.FileLocation) + if err != nil { + http.Error(w, "could not read distribution file", http.StatusBadRequest) + return + } + + defer func() { + if err := file.Close(); err != nil { + logrus.Errorf(fmt.Sprintf("failed to close artifact distribution file %s: %+v", distro.FileLocation, err)) + } + }() + + fileInfo, err := file.Stat() + if err != nil { + http.Error(w, "could not stat distribution file", http.StatusBadRequest) + return + } + + fileName := filepath.Base(distro.FileLocation) + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%v", fileName)) + w.Header().Set("Content-Length", strconv.FormatInt(fileInfo.Size(), 10)) + if _, err := io.Copy(w, file); err != nil { + http.Error(w, fmt.Errorf("could not send artifact distribution: %w", err).Error(), http.StatusInternalServerError) + return + } +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go index 92a40856..420455cb 100644 --- a/internal/server/server_test.go +++ b/internal/server/server_test.go @@ -21,10 +21,13 @@ import ( "io" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "github.com/gorilla/mux" + "github.com/in-toto/archivista/internal/artifactstore" "github.com/in-toto/archivista/internal/config" "github.com/in-toto/archivista/pkg/api" "github.com/stretchr/testify/mock" @@ -103,20 +106,26 @@ func (ut *UTServerSuite) SetupTest() { ut.mockedStorer = new(StorerMock) ut.mockedStorerGetter = new(StorerGetterMock) ut.mockedResposeRecorder = new(ResponseRecorderMock) - ut.testServer = Server{ut.mockedStorer, ut.mockedStorerGetter, nil} + ut.testServer = Server{ + metadataStore: ut.mockedStorer, + objectStore: ut.mockedStorerGetter, + artifactStore: ut.testArtifactStore(), + } } func (ut *UTServerSuite) Test_New() { cfg := new(config.Config) cfg.EnableGraphql = true cfg.GraphqlWebClientEnable = true - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -124,6 +133,7 @@ func (ut *UTServerSuite) Test_New() { allPaths = append(allPaths, pathTemplate) return nil }) + if err != nil { ut.FailNow(err.Error()) } @@ -141,13 +151,15 @@ func (ut *UTServerSuite) Test_New_EnableGraphQL_False() { cfg := new(config.Config) cfg.EnableGraphql = false cfg.GraphqlWebClientEnable = true - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -173,13 +185,15 @@ func (ut *UTServerSuite) Test_New_GraphqlWebClientEnable_False() { cfg := new(config.Config) cfg.EnableGraphql = true cfg.GraphqlWebClientEnable = false - ut.testServer = New(ut.mockedStorer, ut.mockedStorerGetter, cfg, nil) + var err error + ut.testServer, err = New(cfg, WithMetadataStore(ut.mockedStorer), WithObjectStore(ut.mockedStorerGetter)) + ut.NoError(err) ut.NotNil(ut.testServer) router := ut.testServer.Router() ut.NotNil(router) allPaths := []string{} - err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + err = router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { pathTemplate, err := route.GetPathTemplate() if err != nil { ut.FailNow(err.Error()) @@ -386,3 +400,93 @@ func (ut *UTServerSuite) Test_DownloadHandler_NotFound() { ut.Equal(http.StatusNotFound, ut.mockedResposeRecorder.Code) ut.Nil(ut.mockedResposeRecorder.Body) } + +func (ut *UTServerSuite) Test_AllArtifactsHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts", nil) + + ut.testServer.AllArtifactsHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "witness") + ut.Contains(w.Body.String(), "v0.1.0") + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactAllVersionsHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness"}) + + ut.testServer.ArtifactAllVersionsHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "v0.1.0") + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactVersionHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness/v0.1.0", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0"}) + + ut.testServer.ArtifactVersionHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "linux-x64") +} + +func (ut *UTServerSuite) Test_ArtifactVersionHandler_NotFound() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/artifacts/witness/v0.3.0", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.3.0"}) + + ut.testServer.ArtifactVersionHandler(w, request) + ut.Equal(http.StatusNotFound, w.Code) + ut.Contains(w.Body.String(), "version not found") +} + +func (ut *UTServerSuite) Test_DownloadArtifactHandler() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/download/artifact/witness/v0.1.0/linux-x64", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0", "distribution": "linux-x64"}) + + ut.testServer.DownloadArtifactHandler(w, request) + ut.Equal(http.StatusOK, w.Code) + ut.Contains(w.Body.String(), "test") +} + +func (ut *UTServerSuite) Test_DownloadArtifactHandler_NotFound() { + w := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/v1/download/artifact/witness/v0.1.0/linux-arm", nil) + request = mux.SetURLVars(request, map[string]string{"name": "witness", "version": "v0.1.0", "distribution": "linux-arm"}) + + ut.testServer.DownloadArtifactHandler(w, request) + ut.Equal(http.StatusNotFound, w.Code) + ut.Contains(w.Body.String(), "distribution of artifact not found") +} + +func (ut *UTServerSuite) testArtifactStore() artifactstore.Store { + testDir := ut.T().TempDir() + testDistroFilePath := filepath.Join(testDir, "witness-v0.1.0-linux-x64") + ut.NoError(os.WriteFile(testDistroFilePath, []byte("test"), 0644)) + + config := artifactstore.Config{ + Artifacts: map[string]artifactstore.Artifact{ + "witness": { + Versions: map[string]artifactstore.Version{ + "v0.1.0": { + Description: "some description", + Distributions: map[string]artifactstore.Distribution{ + "linux-x64": { + SHA256Digest: "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08", + FileLocation: testDistroFilePath, + }, + }, + }, + }, + }, + }, + } + + wds, err := artifactstore.New(artifactstore.WithConfig(config)) + ut.NoError(err) + return wds +}