From 4533c7f3e8305427e46d2490000a594d251d67ae Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Tue, 21 May 2024 18:24:31 +0200 Subject: [PATCH] Allow gofail trigger to fail as long as the member stops running This is required for compaction based failpoint, to allow the traffic send compaction request causing etcd to crash before failpoint executes the trigger. Signed-off-by: Marek Siarkowicz --- tests/robustness/failpoint/gofail.go | 35 +++++++++++++++++----------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/tests/robustness/failpoint/gofail.go b/tests/robustness/failpoint/gofail.go index 2e85798bacc1..3c1320da5a41 100644 --- a/tests/robustness/failpoint/gofail.go +++ b/tests/robustness/failpoint/gofail.go @@ -90,30 +90,37 @@ func (f goPanicFailpoint) Inject(ctx context.Context, t *testing.T, lg *zap.Logg lg.Info("goFailpoint setup failed", zap.String("failpoint", f.Name()), zap.Error(err)) continue } - if !member.IsRunning() { - // TODO: Check member logs that etcd not running is caused panic caused by proper gofailpoint. - break - } - if f.trigger != nil { - var r []report.ClientReport + break + } + + if f.trigger != nil { + for member.IsRunning() { + select { + case <-ctx.Done(): + return reports, ctx.Err() + default: + } lg.Info("Triggering gofailpoint", zap.String("failpoint", f.Name())) - r, err = f.trigger.Trigger(ctx, t, member, clus, baseTime, ids) + r, err := f.trigger.Trigger(ctx, t, member, clus, baseTime, ids) if err != nil { lg.Info("gofailpoint trigger failed", zap.String("failpoint", f.Name()), zap.Error(err)) + continue } if r != nil { reports = append(reports, r...) } + break } - lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name)) - err = member.Wait(ctx) - if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { - lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err)) - return reports, fmt.Errorf("member didn't exit as expected: %v", err) - } - lg.Info("Member exited as expected", zap.String("member", member.Config().Name)) } + lg.Info("Waiting for member to exit", zap.String("member", member.Config().Name)) + err = member.Wait(ctx) + if err != nil && !strings.Contains(err.Error(), "unexpected exit code") { + lg.Info("Member didn't exit as expected", zap.String("member", member.Config().Name), zap.Error(err)) + return reports, fmt.Errorf("member didn't exit as expected: %v", err) + } + lg.Info("Member exited as expected", zap.String("member", member.Config().Name)) + if lazyfs := member.LazyFS(); lazyfs != nil { lg.Info("Removing data that was not fsynced") err := lazyfs.ClearCache(ctx)