From 3fb36d9ae22c3c450e454074b673ea26ce09a48f Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Tue, 21 May 2024 18:24:31 +0200 Subject: [PATCH] Allow gofail trigger to fail as long as the member stops running This is required for compaction based failpoint, to allow the traffic send compaction request causing etcd to crash before failpoint executes the trigger. Signed-off-by: Marek Siarkowicz --- tests/robustness/failpoint/gofail.go | 32 +++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go index 2e85798bacc..b6218edb9af 100644 --- a/tests/robustness/failpoint/gofail.go +++ b/tests/robustness/failpoint/gofail.go @@ -90,30 +90,38 @@ func (f goPanicFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logg lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) continue } - if !member.IsRunning() { - // TODO: Check member logs that etcd not running is caused panic caused by proper gofailpoint. - break - } - if f.trigger != nil { + break + } + + if f.trigger != nil { + for member.IsRunning() { + select { + case <-ctx.Done(): + return reports, ctx.Err() + default: + } var r []report.ClientReport lg.Info("Triggering gofailpoint", zap.String("failpoint", f.Name())) r, err = f.trigger.Trigger(ctx, t, member, clus, baseTime, ids) if err != nil { lg.Info("gofailpoint trigger failed", zap.String("failpoint", f.Name()), zap.Error(err)) + continue } if r != nil { reports = append(reports, r...) } + break } - lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name)) - err = member.Wait(ctx) - if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { - lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err)) - return reports, fmt.Errorf("member didn't exit as expected: %v", err) - } - lg.Info("Member exited as expected", zap.String("member", member.Config().Name)) } + lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name)) + err = member.Wait(ctx) + if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { + lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err)) + return reports, fmt.Errorf("member didn't exit as expected: %v", err) + } + lg.Info("Member exited as expected", zap.String("member", member.Config().Name)) + if lazyfs := member.LazyFS(); lazyfs != nil { lg.Info("Removing data that was not fsynced") err := lazyfs.ClearCache(ctx)