From 139dc55317ce7ba5641c1b7c35ca0b5f9fbd8eaf Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Thu, 18 Jul 2024 15:08:29 +0100 Subject: [PATCH 1/7] Flaky blobs caps systemtest AB#5311 --- azblob/list.go | 33 ++++++++++++++++++++++++--------- azblob/logger.go | 11 +++++++++++ azblob/storer.go | 2 ++ 3 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 azblob/logger.go diff --git a/azblob/list.go b/azblob/list.go index 48f845c..178e078 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -13,15 +13,28 @@ import ( // Count counts the number of blobs filtered by the given tags filter func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) (int64, error) { - - logger.Sugar.Debugf("Count") - - r, err := azp.FilteredList(ctx, tagsFilter, opts...) - if err != nil { - return 0, err - } - - return int64(len(r.Items)), nil + log := logger.Sugar.FromContext(ctx) + defer log.close() + + log.Debugf("Count") + + var count int64 + var m string + opts = append(opts, WithListMarker(&m)) + + for { + r, err := azp.FilteredList(ctx, tagsFilter, opts...) + if err != nil { + return 0, err + } + count += int64(len(r.Items)) + log.Debugf("Count %d (%v)", count, r.Marker) + if r.Marker == nil || *r.Marker == "" { + break + } + } + log.Debugf("Count %d", count) + return count, nil } type FilterResponse struct { @@ -57,6 +70,8 @@ type FilterResponse struct { // // Returns all blobs with the specific tag filter. func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ...Option) (*FilterResponse, error) { + log := logger.Sugar.FromContext(ctx) + defer log.close() span, ctx := tracing.StartSpanFromContext(ctx, "FilteredList") defer span.Finish() diff --git a/azblob/logger.go b/azblob/logger.go new file mode 100644 index 0000000..15db966 --- /dev/null +++ b/azblob/logger.go @@ -0,0 +1,11 @@ +package azbus + +import ( + "github.com/datatrails/go-datatrails-common/logger" +) + +const ( + DebugLevel = logger.DebugLevel +) + +type Logger = logger.Logger diff --git a/azblob/storer.go b/azblob/storer.go index 51822fd..8aa582a 100644 --- a/azblob/storer.go +++ b/azblob/storer.go @@ -27,6 +27,8 @@ type Storer struct { containerURL string containerClient *ContainerClient serviceClient *ServiceClient + + log Logger } // New returns new az blob read/write object From 328c6924a0ac670f74dccd08837a3b029f330b3e Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 12:00:01 +0100 Subject: [PATCH 2/7] fixup! Flaky blobs caps systemtest --- azblob/list.go | 16 ++++++++-------- azblob/logger.go | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/azblob/list.go b/azblob/list.go index 178e078..a226894 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -14,24 +14,25 @@ import ( // Count counts the number of blobs filtered by the given tags filter func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) (int64, error) { log := logger.Sugar.FromContext(ctx) - defer log.close() + defer log.Close() log.Debugf("Count") var count int64 - var m string - opts = append(opts, WithListMarker(&m)) + var m ListMarker for { + opts = append(opts, WithListMarker(m)) r, err := azp.FilteredList(ctx, tagsFilter, opts...) if err != nil { return 0, err } count += int64(len(r.Items)) log.Debugf("Count %d (%v)", count, r.Marker) - if r.Marker == nil || *r.Marker == "" { + if r.Marker == nil || *r.Marker == "" { break } + m = r.Marker } log.Debugf("Count %d", count) return count, nil @@ -71,7 +72,7 @@ type FilterResponse struct { // Returns all blobs with the specific tag filter. func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ...Option) (*FilterResponse, error) { log := logger.Sugar.FromContext(ctx) - defer log.close() + defer log.Close() span, ctx := tracing.StartSpanFromContext(ctx, "FilteredList") defer span.Finish() @@ -83,7 +84,7 @@ func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ... } if options.listMarker != nil { - span.SetTag("marker", *options.listMarker) + span.SetTag("marker", options.listMarker) } o := &azStorageBlob.ServiceFilterBlobsOptions{ Marker: options.listMarker, @@ -107,12 +108,11 @@ func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ... Items: resp.Blobs, } - r.Marker = resp.NextMarker if r.Marker != nil { span.SetTag("nextmarker", *r.Marker) } - return r, err + return r, nil } type ListerResponse struct { diff --git a/azblob/logger.go b/azblob/logger.go index 15db966..22c99b7 100644 --- a/azblob/logger.go +++ b/azblob/logger.go @@ -1,4 +1,4 @@ -package azbus +package azblob import ( "github.com/datatrails/go-datatrails-common/logger" From 6a93dcbcf3ef7571e389e1e8e1f1be27cbdc3540 Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 14:08:16 +0100 Subject: [PATCH 3/7] fixup! fixup! Flaky blobs caps systemtest --- azblob/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azblob/list.go b/azblob/list.go index a226894..6c9cb40 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -84,7 +84,7 @@ func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ... } if options.listMarker != nil { - span.SetTag("marker", options.listMarker) + span.SetTag("marker", *options.listMarker) } o := &azStorageBlob.ServiceFilterBlobsOptions{ Marker: options.listMarker, From 16ecc403ee192e464c8867e68022f1dea405a2b6 Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 14:17:47 +0100 Subject: [PATCH 4/7] fixup! fixup! fixup! Flaky blobs caps systemtest --- azblob/list.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/azblob/list.go b/azblob/list.go index 6c9cb40..e7e0603 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -16,8 +16,6 @@ func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) log := logger.Sugar.FromContext(ctx) defer log.Close() - log.Debugf("Count") - var count int64 var m ListMarker @@ -28,13 +26,11 @@ func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) return 0, err } count += int64(len(r.Items)) - log.Debugf("Count %d (%v)", count, r.Marker) if r.Marker == nil || *r.Marker == "" { break } m = r.Marker } - log.Debugf("Count %d", count) return count, nil } From 026f492f4508fedc60c94c78f2d97f9a4777304c Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 14:46:06 +0100 Subject: [PATCH 5/7] fixup! fixup! fixup! fixup! Flaky blobs caps systemtest --- azblob/list.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/azblob/list.go b/azblob/list.go index e7e0603..655aaae 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -13,15 +13,12 @@ import ( // Count counts the number of blobs filtered by the given tags filter func (azp *Storer) Count(ctx context.Context, tagsFilter string, opts ...Option) (int64, error) { - log := logger.Sugar.FromContext(ctx) - defer log.Close() var count int64 var m ListMarker for { - opts = append(opts, WithListMarker(m)) - r, err := azp.FilteredList(ctx, tagsFilter, opts...) + r, err := azp.FilteredList(ctx, tagsFilter, append(opts, WithListMarker(m))...) if err != nil { return 0, err } From 0f3bf5852de7353d22f26b4b5c5b6bebc555b905 Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 15:03:50 +0100 Subject: [PATCH 6/7] fixup! fixup! fixup! fixup! fixup! Flaky blobs caps systemtest --- azblob/list.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/azblob/list.go b/azblob/list.go index 655aaae..ec84d9c 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -64,8 +64,6 @@ type FilterResponse struct { // // Returns all blobs with the specific tag filter. func (azp *Storer) FilteredList(ctx context.Context, tagsFilter string, opts ...Option) (*FilterResponse, error) { - log := logger.Sugar.FromContext(ctx) - defer log.Close() span, ctx := tracing.StartSpanFromContext(ctx, "FilteredList") defer span.Finish() From 17930893fcaacb8205a6dbccc2fdbf89534172aa Mon Sep 17 00:00:00 2001 From: Paul Hewlett Date: Wed, 14 Aug 2024 15:17:32 +0100 Subject: [PATCH 7/7] fixup! fixup! fixup! fixup! fixup! fixup! Flaky blobs caps systemtest --- azblob/list.go | 1 - 1 file changed, 1 deletion(-) diff --git a/azblob/list.go b/azblob/list.go index ec84d9c..d954b77 100644 --- a/azblob/list.go +++ b/azblob/list.go @@ -7,7 +7,6 @@ import ( azStorageBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/datatrails/go-datatrails-common/logger" "github.com/datatrails/go-datatrails-common/tracing" )