Dev/robin/8656 list blobs

Taskfile.yml

@@ -26,6 +26,11 @@ includes:
 
 tasks:
 
+  test:preflight:
+    desc: do all necessary cleanup and preparation for a test run
+    cmds:
+      - task: azurite:preflight
+
   qa-basic:
     desc: |
       format the source and run all the quality checks

azblob/delete.go

@@ -0,0 +1,37 @@
+package azblob
+
+import (
+	"context"
+	"errors"
+
+	msazblob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/datatrails/go-datatrails-common/logger"
+)
+
+// Delete the identified blob
+func (azp *Storer) Delete(
+	ctx context.Context,
+	identity string,
+) error {
+	logger.Sugar.Debugf("Delete blob %s", identity)
+
+	blockBlobClient, err := azp.containerClient.NewBlockBlobClient(identity)
+	if err != nil {
+		logger.Sugar.Infof("Cannot get block blob client blob: %v", err)
+		return ErrorFromError(err)
+	}
+
+	_, err = blockBlobClient.Delete(ctx, nil)
+	var terr *msazblob.StorageError
+	if errors.As(err, &terr) {
+		resp := terr.Response()
+		if resp.Body != nil {
+			defer resp.Body.Close()
+		}
+		if resp.StatusCode == 404 {
+			return nil
+		}
+	}
+
+	return err
+}

azblob/download.go

@@ -164,7 +164,10 @@ func (azp *Storer) Reader(
 	normaliseReaderResponseErr(err, resp)
 	if err == nil {
 		// We *always* copy the metadata into the response
-		downloadReaderResponse(get, resp)
+		err = downloadReaderResponse(get, resp)
+		if err != nil {
+			return resp, err
+		}
 
 		// for backwards compat, we only process the metadata on request
 		if options.getMetadata == BothMetadataAndBlob {

azblob/list.go

@@ -47,3 +47,56 @@ func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*azSt
 
 	return filteredBlobs, err
 }
+
+type ListerResponse struct {
+	Marker ListMarker // nil if no more pages
+	Prefix string
+
+	// Standard request status things
+	StatusCode int // For If- header fails, err can be nil and code can be 304
+	Status     string
+
+	Items []*azStorageBlob.BlobItemInternal
+}
+
+func (azp *Storer) List(ctx context.Context, opts ...Option) (*ListerResponse, error) {
+
+	options := &StorerOptions{}
+	for _, opt := range opts {
+		opt(options)
+	}
+	o := azStorageBlob.ContainerListBlobsFlatOptions{
+		Marker: options.listMarker,
+	}
+	if options.listPrefix != "" {
+		o.Prefix = &options.listPrefix
+	}
+	if options.listIncludeTags {
+		o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemTags)
+	}
+	if options.listIncludeMetadata {
+		o.Include = append(o.Include, azStorageBlob.ListBlobsIncludeItemMetadata)
+	}
+
+	// TODO: v1.21 feature which would be great
+	// if options.listDelim != "" {
+	// }
+	r := &ListerResponse{}
+	pager := azp.containerClient.ListBlobsFlat(&o)
+	if !pager.NextPage(ctx) {
+		return r, nil
+	}
+	resp := pager.PageResponse()
+	r.Status = resp.RawResponse.Status
+	r.StatusCode = resp.RawResponse.StatusCode
+
+	if resp.Prefix != nil {
+		r.Prefix = *resp.Prefix
+	}
+
+	// Note: we pass on the azure type otherwise we would be copying for no good
+	// reason. let the caller decided how to deal with that
+	r.Items = resp.Segment.BlobItems
+
+	return r, nil
+}

azblob/options.go

@@ -43,10 +43,50 @@ type StorerOptions struct {
 	etagCondition  ETagCondition // ETagMatch || ETagNoneMatch
 	sinceCondition IfSinceCondition
 	since          *time.Time
+	// Options for List()
+	listPrefix string
+	listDelim  string
+	listMarker ListMarker
+	// extra data model items to include in the respponse
+	listIncludeTags     bool
+	listIncludeMetadata bool
+	// There are more, but these are all we need for now
 }
+type ListMarker *string
 
 type Option func(*StorerOptions)
 
+func WithListPrefix(prefix string) Option {
+	return func(a *StorerOptions) {
+		a.listPrefix = prefix
+	}
+}
+
+// TODO: this is an sdk v1.2.1 feature
+func WithListDelim(delim string) Option {
+	return func(a *StorerOptions) {
+		a.listDelim = delim
+	}
+}
+
+func WithListMarker(marker ListMarker) Option {
+	return func(a *StorerOptions) {
+		a.listMarker = marker
+	}
+}
+
+func WithListTags() Option {
+	return func(a *StorerOptions) {
+		a.listIncludeTags = true
+	}
+}
+
+func WithListMetadata() Option {
+	return func(a *StorerOptions) {
+		a.listIncludeMetadata = true
+	}
+}
+
 func WithModifiedSince(since *time.Time) Option {
 	return func(a *StorerOptions) {
 		if since == nil {

azblob/put.go

@@ -17,10 +17,6 @@ func (azp *Storer) Put(
 	source io.ReadSeekCloser,
 	opts ...Option,
 ) (*WriteResponse, error) {
-	err := azp.checkContainer(ctx)
-	if err != nil {
-		return nil, err
-	}
 	logger.Sugar.Debugf("Create or replace BlockBlob %s", identity)
 
 	options := &StorerOptions{}

azblob/readresponse.go

@@ -16,7 +16,8 @@ type ReaderResponse struct {
 	Reader            io.ReadCloser
 	HashValue         string
 	MimeType          string
-	Size              int64
+	ContentLength     int64
+	Size              int64 // MIME size
 	Tags              map[string]string
 	TimestampAccepted string
 	ScannedStatus     string
@@ -78,16 +79,31 @@ func normaliseReaderResponseErr(err error, rr *ReaderResponse) {
 
 // downloadReaderResponse copies accross the azure sdk response values that are
 // meaningful to our supported api
-func downloadReaderResponse(r azStorageBlob.BlobDownloadResponse, rr *ReaderResponse) {
+func downloadReaderResponse(r azStorageBlob.BlobDownloadResponse, rr *ReaderResponse) error {
 	rr.Status = r.RawResponse.Status
 	rr.StatusCode = r.RawResponse.StatusCode
+
+	rr.LastModified = r.LastModified
+	rr.ETag = r.ETag
+	rr.Metadata = r.Metadata
+
 	value, ok := r.RawResponse.Header[xMsErrorCodeHeader]
 	if ok && len(value) > 0 {
 		rr.XMsErrorCode = value[0]
 	}
-	rr.LastModified = r.LastModified
-	rr.ETag = r.ETag
-	rr.Metadata = r.Metadata
+
+	s, ok := r.RawResponse.Header["Content-Length"]
+	if !ok {
+		return nil
+	}
+	if len(s) == 0 {
+		rr.ContentLength = 0
+		return nil
+	}
+
+	var err error
+	rr.ContentLength, err = strconv.ParseInt(s[0], 10, 64)
+	return err
 }
 
 // readerResponseMetadata processes and conditions values from the metadata we have specific support for.

azblob/writeresponse.go

@@ -70,6 +70,7 @@ func uploadWriteResponse(r azStorageBlob.BlockBlobUploadResponse) *WriteResponse
 	}
 	w.Status = r.RawResponse.Status
 	w.StatusCode = r.RawResponse.StatusCode
+	w.LastModified = r.LastModified
 	value, ok := r.RawResponse.Header[xMsErrorCodeHeader]
 	if ok && len(value) > 0 {
 		w.XMsErrorCode = value[0]

taskfiles/Taskfile_azurite.yml

@@ -17,6 +17,13 @@ vars:
   AZURITE_IMAGE: '{{.AZURITE_IMAGE | default "mcr.microsoft.com/azure-storage/azurite"}}'
 
 tasks:
+  preflight:
+    desc: stops, cleans and re-starts the emulator providing a clean state
+    summary: |
+      stops, cleans and re-starts the emulator providing a clean state
+    cmds:
+      - task: cleanup
+      - task: start
   start:
     desc: start azurite azure local storage emulator in a named docker container
     summary: |