Dev/robin/9402 blob tags to facilitate efficient batch signing (#68)

* Add filter-by-tags support to azblobs AB#9402 * update reader interface * review: improve function doc comment --------- Co-authored-by: Robin Bryce <[email protected]>
datatrails · May 3, 2024 · 298c4f1 · 298c4f1
1 parent 498b5bf
commit 298c4f1
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 21 deletions.
diff --git a/azblob/list.go b/azblob/list.go
@@ -12,41 +12,92 @@ import (
 )
 
 // Count counts the number of blobs filtered by the given tags filter
-func (azp *Storer) Count(ctx context.Context, tagsFilter string) (int64, error) {
+func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) (int64, error) {
 
 	logger.Sugar.Debugf("Count")
 
-	blobs, err := azp.FilteredList(ctx, tagsFilter)
+	r, err := azp.FilteredList(ctx, tagsFilter, opts...)
 	if err != nil {
 		return 0, err
 	}
 
-	return int64(len(blobs)), nil
+	return int64(len(r.Items)), nil
+}
+
+type FilterResponse struct {
+	Marker ListMarker // nil if no more pages
+
+	// Standard request status things
+	StatusCode int // For If- header fails, err can be nil and code can be 304
+	Status     string
+
+	Items []*azStorageBlob.FilterBlobItem
 }
 
 // FilteredList returns a list of blobs filtered on their tag values.
 //
-// tagsFilter example: "dog='germanshepherd' and penguin='emperorpenguin'"
-// Returns all blobs with the specific tag filter
-func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string) ([]*azStorageBlob.FilterBlobItem, error) {
-	logger.Sugar.Debugf("FilteredList")
+// tagsFilter examples:
+//
+//		 All tenants with more than one massif
+//		     "firstindex">'0000000000000000'
+//
+//		 All tenants whose logs have been updated since a particular idtimestamp
+//		     "lastid > '018e84dbbb6513a6'"
+//
+//	 note: in the case where you are making up the id timestamp from a time
+//	 reading, set the least significant 24 bits to zero and use the hex encoding
+//	 of the resulting value
+//
+//		All blobs in a storage account
+//			"cat='tiger' AND penguin='emperorpenguin'"
+//		All blobs in a specific container
+//			"@container='zoo' AND cat='tiger' AND penguin='emperorpenguin'"
+//
+// See also: https://learn.microsoft.com/en-us/rest/api/storageservices/find-blobs-by-tags-container?tabs=microsoft-entra-id
+//
+// Returns all blobs with the specific tag filter.
+func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ...Option) (*FilterResponse, error) {
+	span, ctx := tracing.StartSpanFromContext(ctx, "FilteredList")
+	defer span.Finish()
 
-	var filteredBlobs []*azStorageBlob.FilterBlobItem
 	var err error
 
-	result, err := azp.serviceClient.FindBlobsByTags(
-		ctx,
-		&azStorageBlob.ServiceFilterBlobsOptions{
-			Where: &tagsFilter,
-		},
-	)
+	options := &StorerOptions{}
+	for _, opt := range opts {
+		opt(options)
+	}
+
+	if options.listMarker != nil {
+		span.SetTag("marker", *options.listMarker)
+	}
+	o := &azStorageBlob.ServiceFilterBlobsOptions{
+		Marker: options.listMarker,
+		Where:  &tagsFilter,
+	}
+
+	if options.listMaxResults > 0 {
+		o.MaxResults = &options.listMaxResults
+		span.SetTag("maxResults", options.listMaxResults)
+	}
+
+	resp, err := azp.serviceClient.FindBlobsByTags(ctx, o)
 	if err != nil {
-		return filteredBlobs, err
+		return nil, err
+	}
+
+	r := &FilterResponse{
+		StatusCode: resp.RawResponse.StatusCode,
+		Status:     resp.RawResponse.Status,
+		Marker:     resp.NextMarker,
+		Items:      resp.Blobs,
 	}
 
-	filteredBlobs = result.Blobs
+	r.Marker = resp.NextMarker
+	if r.Marker != nil {
+		span.SetTag("nextmarker", *r.Marker)
+	}
 
-	return filteredBlobs, err
+	return r, err
 }
 
 type ListerResponse struct {

diff --git a/azblob/reader.go b/azblob/reader.go
@@ -20,7 +20,7 @@ type Reader interface {
 		identity string,
 		opts ...Option,
 	) (*ReaderResponse, error)
-
+	FilteredList(ctx context.Context, tagsFilter string, opts ...Option) (*FilterResponse, error)
 	List(ctx context.Context, opts ...Option) (*ListerResponse, error)
 }
 

diff --git a/azblob/storerazurite.go b/azblob/storerazurite.go
@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"strings"
 
 	azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
 	"github.com/datatrails/go-datatrails-common/logger"
@@ -74,6 +75,9 @@ func NewDev(cfg DevConfig, container string) (*Storer, error) {
 		return nil, err
 	}
 
+	// normalise trailing slash
+	cfg.URL = strings.TrimSuffix(cfg.URL, "/") + "/"
+
 	azp := &Storer{
 		AccountName:   cfg.AccountName,
 		ResourceGroup: azuriteResourceGroup, // just for logging
@@ -84,9 +88,7 @@ func NewDev(cfg DevConfig, container string) (*Storer, error) {
 	}
 
 	azp.containerURL = fmt.Sprintf(
-		"%s%s",
-		cfg.URL,
-		container,
+		"%s%s", cfg.URL, container,
 	)
 	azp.serviceClient, err = azStorageBlob.NewServiceClientWithSharedKey(
 		cfg.URL,