Add the remaining common packages (#6)

This finishes the migration of common code.

It would possibly be worth teasing appart errhandling so that the
b2cspecific error handling remains in avid.

Similarly, azblob could be re-factored to eliminate the need to move
scannedstatus.

This seem like minor things we can deal with later

AB#8372

Co-authored-by: Robin Bryce <[email protected]>

azblob/credentials.go

@@ -0,0 +1,76 @@
+// Package azblob reads/writes files to Azure
+// blob storage in Chunks.
+package azblob
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"time"
+
+	"github.com/Azure/azure-sdk-for-go/profiles/latest/storage/mgmt/storage"
+	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/go-autorest/autorest/azure/auth"
+
+	"github.com/rkvst/go-rkvstcommon/logger"
+	"github.com/rkvst/go-rkvstcommon/secrets"
+)
+
+const (
+	listKeyExpand  = ""
+	tryTimeoutSecs = 30
+)
+
+// credentials gets credentials from env or file
+func credentials(
+	accountName string,
+	resourceGroup string,
+	subscription string,
+) (*secrets.Secrets, *SharedKeyCredential, error) {
+
+	logger.Sugar.Infof(
+		"Attempt environment auth with accountName/resourceGroup/subscription: %s/%s/%s",
+		accountName, resourceGroup, subscription,
+	)
+
+	if accountName == "" || resourceGroup == "" || subscription == "" {
+		return nil, nil, errors.New("missing authentication variables")
+	}
+
+	authorizer, err := auth.NewAuthorizerFromEnvironment()
+	if err != nil {
+		logger.Sugar.Infof("failed NewAuthorizerFromEnvironment: %v", err)
+		return nil, nil, err
+	}
+	accountClient := storage.NewAccountsClient(subscription)
+	accountClient.Authorizer = authorizer
+
+	// Set up a client context to call Azure with
+	ctx, cancel := context.WithTimeout(context.Background(), tryTimeoutSecs*time.Second)
+
+	// Even though ctx will be expired, it is good practice to call its
+	// cancelation function in any case. Failure to do so may keep the
+	// context and its parent alive longer than necessary.
+	defer cancel()
+
+	blobkeys, err := accountClient.ListKeys(ctx, resourceGroup, accountName, listKeyExpand)
+	if err != nil {
+		logger.Sugar.Infof("failed to list blob keys: %v", err)
+		return nil, nil, err
+	}
+
+	nkeys := len(*blobkeys.Keys)
+
+	if nkeys < 1 {
+		return nil, nil, errors.New("no keys found for storage account")
+	}
+	secret := &secrets.Secrets{
+		Account: accountName,
+		URL:     fmt.Sprintf("https://%s.blob.core.windows.net/", accountName),
+		Key:     *(((*blobkeys.Keys)[0]).Value),
+	}
+	cred, err := azStorageBlob.NewSharedKeyCredential(secret.Account, secret.Key)
+	logger.Sugar.Infof("Credential accountName: %s", cred.AccountName())
+
+	return secret, cred, err
+}

azblob/download.go

@@ -0,0 +1,195 @@
+// Package azblob reads/writes files to Azure
+// blob storage in Chunks.
+package azblob
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"net/textproto"
+	"strconv"
+
+	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+
+	"github.com/rkvst/go-rkvstcommon/logger"
+	"github.com/rkvst/go-rkvstcommon/scannedstatus"
+)
+
+const (
+	// metadata keys
+	ContentKey = "content_type"
+	HashKey    = "hash"
+	MimeKey    = "mime_type"
+	SizeKey    = "size"
+	TimeKey    = "time_accepted"
+)
+
+// getTags gets tags from blob storage
+func (azp *Storer) getTags(
+	ctx context.Context,
+	identity string,
+) (map[string]string, error) {
+
+	var err error
+	logger.Sugar.Debugf("getTags BlockBlob URL %s", identity)
+
+	blobClient, err := azp.containerClient.NewBlobClient(identity)
+	if err != nil {
+		logger.Sugar.Debugf("getTags BlockBlob Client %s error: %v", identity, err)
+		return nil, ErrorFromError(err)
+	}
+	resp, err := blobClient.GetTags(ctx, nil)
+	if err != nil {
+		logger.Sugar.Debugf("getTags BlockBlob URL %s error: %v", identity, err)
+		return nil, ErrorFromError(err)
+	}
+	logger.Sugar.Debugf("getTags BlockBlob tagSet: %v", resp.BlobTagSet)
+	tags := make(map[string]string, len(resp.BlobTagSet))
+	for _, tag := range resp.BlobTagSet {
+		tags[*tag.Key] = *tag.Value
+	}
+	logger.Sugar.Debugf("getTags BlockBlob URL %s tags: %v", identity, tags)
+	return tags, nil
+}
+
+// getMetadata gets metadata from blob storage
+func (azp *Storer) getMetadata(
+	ctx context.Context,
+	identity string,
+) (map[string]string, error) {
+	logger.Sugar.Debugf("getMetadata BlockBlob URL %s", identity)
+
+	blobClient, err := azp.containerClient.NewBlobClient(identity)
+	if err != nil {
+		return nil, ErrorFromError(err)
+	}
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	resp, err := blobClient.GetProperties(ctx, nil)
+	if err != nil {
+		return nil, ErrorFromError(err)
+	}
+	logger.Sugar.Debugf("getMetadata BlockBlob URL %v", resp.Metadata)
+	return resp.Metadata, nil
+}
+
+type ReaderResponse struct {
+	Reader            io.ReadCloser
+	HashValue         string
+	MimeType          string
+	Size              int64
+	Tags              map[string]string
+	TimestampAccepted string
+	ScannedStatus     string
+	ScannedBadReason  string
+	ScannedTimestamp  string
+
+	BlobClient *azStorageBlob.BlobClient
+}
+
+// Reader creates a reader.
+func (azp *Storer) Reader(
+	ctx context.Context,
+	identity string,
+	opts ...Option,
+) (*ReaderResponse, error) {
+
+	var err error
+
+	options := &StorerOptions{}
+	for _, opt := range opts {
+		opt(options)
+	}
+
+	logger.Sugar.Debugf("Reader BlockBlob URL %s", identity)
+
+	resp := &ReaderResponse{}
+	var blobAccessConditions azStorageBlob.BlobAccessConditions
+	if options.leaseID != "" {
+		blobAccessConditions = azStorageBlob.BlobAccessConditions{
+			LeaseAccessConditions: &azStorageBlob.LeaseAccessConditions{
+				LeaseID: &options.leaseID,
+			},
+		}
+	}
+
+	if len(options.tags) > 0 || options.getTags {
+		logger.Sugar.Debugf("Get tags")
+		tags, tagsErr := azp.getTags(
+			ctx,
+			identity,
+		)
+		if tagsErr != nil {
+			logger.Sugar.Infof("cannot get tags: %v", tagsErr)
+			return nil, tagsErr
+		}
+		resp.Tags = tags
+	}
+
+	for k, requiredValue := range options.tags {
+		blobValue, ok := resp.Tags[k]
+		if !ok {
+			logger.Sugar.Infof("tag %s is not specified on blob", k)
+			return nil, NewStatusError(fmt.Sprintf("tag %s is not specified on blob", k), http.StatusForbidden)
+		}
+		if blobValue != requiredValue {
+			logger.Sugar.Infof("blob has different Tag %s than required %s", blobValue, requiredValue)
+			return nil, NewStatusError(fmt.Sprintf("blob has different Tag %s than required %s", blobValue, requiredValue), http.StatusForbidden)
+		}
+	}
+
+	if options.getMetadata == OnlyMetadata || options.getMetadata == BothMetadataAndBlob {
+		metaData, metadataErr := azp.getMetadata(
+			ctx,
+			identity,
+		)
+		if metadataErr != nil {
+			logger.Sugar.Infof("cannot get metadata: %v", metadataErr)
+			return nil, metadataErr
+		}
+		logger.Sugar.Debugf("blob metadata %v", metaData)
+		size, parseErr := strconv.ParseInt(metaData[textproto.CanonicalMIMEHeaderKey(SizeKey)], 10, 64)
+		if parseErr != nil {
+			logger.Sugar.Infof("cannot get size value: %v", parseErr)
+			return nil, parseErr
+		}
+		resp.Size = size
+		resp.HashValue = metaData[textproto.CanonicalMIMEHeaderKey(HashKey)]
+		resp.MimeType = metaData[textproto.CanonicalMIMEHeaderKey(MimeKey)]
+		resp.TimestampAccepted = metaData[textproto.CanonicalMIMEHeaderKey(TimeKey)]
+		resp.ScannedStatus = scannedstatus.FromString(metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Key)]).String()
+		resp.ScannedBadReason = metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.BadReason)]
+		resp.ScannedTimestamp = metaData[textproto.CanonicalMIMEHeaderKey(scannedstatus.Timestamp)]
+	}
+
+	if options.getMetadata == OnlyMetadata {
+		return resp, nil
+	}
+
+	logger.Sugar.Debugf("Creating New io.Reader")
+	resp.BlobClient, err = azp.containerClient.NewBlobClient(identity)
+	if err != nil {
+		return nil, ErrorFromError(err)
+	}
+	countToEnd := int64(azStorageBlob.CountToEnd)
+	get, err := resp.BlobClient.Download(
+		ctx,
+		&azStorageBlob.BlobDownloadOptions{
+			BlobAccessConditions: &blobAccessConditions,
+			Count:                &countToEnd,
+		},
+	)
+	if err != nil && err == io.EOF { // nolint
+		logger.Sugar.Infof("cannot get blob body: %v", err)
+		return nil, ErrorFromError(err)
+	}
+	resp.Reader = get.Body(nil)
+	return resp, nil
+}
+
+func (r *ReaderResponse) DownloadToWriter(w io.Writer) error {
+	defer r.Reader.Close()
+	_, err := io.Copy(w, r.Reader)
+	return err
+}

azblob/error.go

@@ -0,0 +1,61 @@
+// Package azblob reads/writes files to Azure
+// blob storage in Chunks.
+package azblob
+
+import (
+	"errors"
+	"net/http"
+
+	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/rkvst/go-rkvstcommon/logger"
+)
+
+// HTTPError error type with info about http.StatusCode
+type HTTPError interface {
+	Error() string
+	StatusCode() int
+	Unwrap() error
+}
+
+type Error struct {
+	err        error
+	statusCode int
+}
+
+func NewStatusError(text string, statusCode int) *Error {
+	return &Error{
+		err:        errors.New(text),
+		statusCode: statusCode,
+	}
+}
+
+func ErrorFromError(err error) *Error {
+	return &Error{err: err}
+}
+
+func (e *Error) Error() string {
+	return e.err.Error()
+}
+func (e *Error) Unwrap() error {
+	return e.err
+}
+
+// StatusCode returns status code for failing request or 500 if code is not available on the error
+func (e *Error) StatusCode() int {
+
+	var terr *azStorageBlob.StorageError
+	if errors.As(e.err, &terr) {
+		resp := terr.Response()
+		if resp.Body != nil {
+			defer resp.Body.Close()
+		}
+		logger.Sugar.Debugf("Azblob StatusCode %d", resp.StatusCode)
+		return resp.StatusCode
+	}
+	if e.statusCode != 0 {
+		logger.Sugar.Debugf("Return statusCode %d", e.statusCode)
+		return e.statusCode
+	}
+	logger.Sugar.Debugf("Return InternalServerError")
+	return http.StatusInternalServerError
+}

azblob/hashingreader.go

@@ -0,0 +1,41 @@
+package azblob
+
+import (
+	"hash"
+	"io"
+
+	"github.com/rkvst/go-rkvstcommon/logger"
+)
+
+type hashingReader struct {
+	hasher hash.Hash
+	size   int64
+	part   io.Reader
+}
+
+// Implement reader interface and hash and size file while reading so we can
+// retrieve the metadata once the reading is done
+func (up *hashingReader) Read(bytes []byte) (int, error) {
+	length, err := up.part.Read(bytes)
+	if err != nil && err != io.EOF { //nolint https://github.com/golang/go/issues/39155
+		logger.Sugar.Errorf("could not read file: %v", err)
+		return 0, err
+	}
+	if length == 0 {
+		logger.Sugar.Debugf("finished reading %d bytes", up.size)
+		return length, err
+	}
+	logger.Sugar.Debugf("Read %d bytes (%d)", length, up.size)
+	_, herr := up.hasher.Write(bytes[:length])
+	if herr != nil {
+		logger.Sugar.Errorf("failed to hash")
+		return length, herr
+	}
+	up.size += int64(length)
+	if err == io.EOF { //nolint https://github.com/golang/go/issues/39155
+		// we've got all of it
+		logger.Sugar.Debugf("finished reading %d bytes", up.size)
+		return length, err
+	}
+	return length, nil
+}