Add a simple test ensuring membership is properly bootstrapped

[serathius]

This simple test shows that approach presented in #16084 to just switch validation to just use v3 store is incorrect

If run on top of the PR it results in panic

 panic: failed to delete member from store [recovered]
   panic: execute job failed
 
 goroutine 209 [running]:
 go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x2?, 0x4?, {0x0?, 0x0?, 0xc00051c9e0?})
   go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
 go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0006300d0, {0xc00064c400, 0x2, 0x2})
   go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
 go.uber.org/zap.(*Logger).Panic(0x1056348?, {0x1064328?, 0xe82200?}, {0xc00064c400, 0x2, 0x2})
   go.uber.org/[email protected]/logger.go:284 +0x51
 go.etcd.io/etcd/pkg/v3/schedule.(*fifo).executeJob.func1()
   go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:202 +0x2c5
 panic({0xe82200?, 0xc000642090?})
   runtime/panic.go:914 +0x21f
 go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x2?, 0x198f848?, {0x0?, 0x0?, 0xc0001ec000?})
   go.uber.org/[email protected]/zapcore/entry.go:196 +0x54
 go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000630000, {0xc00064c300, 0x2, 0x2})
   go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
 go.uber.org/zap.(*Logger).Panic(0xc0004ec460?, {0x107b7e6?, 0xc?}, {0xc00064c300, 0x2, 0x2})
   go.uber.org/[email protected]/logger.go:284 +0x51
 go.etcd.io/etcd/server/v3/etcdserver/api/membership.mustDeleteMemberFromStore(0xc1503f80a2d9d159?, {0x1205f88, 0xc0004ec460}, 0xc3f540?)
   go.etcd.io/etcd/server/v3/etcdserver/api/membership/storev2.go:104 +0x250
 go.etcd.io/etcd/server/v3/etcdserver/api/membership.(*RaftCluster).RemoveMember(0xc0004ec070, 0xc000136000?, 0x0)
   go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go:436 +0xc9
 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyConfChange(0xc000156a00, {0x1, 0x1, {0x0, 0x0, 0x0}, 0xd7588c010d69b305}, 0xc000136000, 0x0)
   go.etcd.io/etcd/server/v3/etcdserver/server.go:2039 +0x25a
 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).apply(0xc000156a00, {0xc0001da5d0, 0x5, 0xc000326160?}, 0x2?, 0xc000210120)
   go.etcd.io/etcd/server/v3/etcdserver/server.go:1844 +0x52a
 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyEntries(0xc000156a00, 0xc000136000, 0xc0002be780?)
   go.etcd.io/etcd/server/v3/etcdserver/server.go:1131 +0x265
 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyAll(0xc000156a00, 0xc000136000, 0xc0002be780)
   go.etcd.io/etcd/server/v3/etcdserver/server.go:919 +0x57
 go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).run.func8({0x0, 0x0})
   go.etcd.io/etcd/server/v3/etcdserver/server.go:842 +0x1f
 go.etcd.io/etcd/pkg/v3/schedule.job.Do(...)
   go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:41
 go.etcd.io/etcd/pkg/v3/schedule.(*fifo).executeJob(0x0?, {0x11f36b0?, 0xc000120030?}, 0x0?)
   go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:206 +0x7d
 go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc0005221c0)
   go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:187 +0x109
 created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler in goroutine 191
   go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:101 +0x16f

Issue is cause by skipping validation of entries from last snapshot.

[serathius]

/retest

[serathius]

cc @ahrtr @fuweid @chaochn47 @siyuanfoundation

[fuweid]

LGTM

ctl_v3_member_test.go:310:
                Error Trace:    /home/fuwei/go/src/go.etcd.io/etcd/tests/e2e/ctl_v3_member_test.go:310
                Error:          Received unexpected error:
                                match not found.  Set EXPECT_DEBUG for more info Errs: [unexpected exit code [2] after running [/home/fuwei/go/src/go.etcd.io/etcd/bin/etcd --name TestRemoveNonExistingMember-test-0 --listen-client-urls http://localhost:20000 --advertise-client-urls http://localhost:20000 --listen-peer-urls http://localhost:20001 --initial-advertise-peer-urls http://localhost:20001 --initial-cluster-token new --data-dir /tmp/TestRemoveNonExistingMember3670107214/002 --snapshot-count 10000 --initial-cluster TestRemoveNonExistingMember-test-0=http://localhost:20001 --initial-cluster-state new]], last lines:

                                goroutine 50 [running]:
                                go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x2?, 0x3?, {0x0?, 0x0?, 0xc0007400a0?})
                                        go.uber.org/[email protected]/zapcore/entry.go:196 +0x65
                                go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc0007440d0, {0xc000742100, 0x2, 0x2})
                                        go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
                                go.uber.org/zap.(*Logger).Panic(0x10da0a2?, {0x10e7885?, 0xf19800?}, {0xc000742100, 0x2, 0x2})
                                        go.uber.org/[email protected]/logger.go:284 +0x59
                                go.etcd.io/etcd/pkg/v3/schedule.(*fifo).executeJob.func1()
                                        go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:202 +0x2cb
                                panic({0xf19800, 0xc00072c0b0})
                                        runtime/panic.go:884 +0x213
                                go.uber.org/zap/zapcore.CheckWriteAction.OnWrite(0x2?, 0x7fbfdbc7f0e8?, {0x0?, 0x0?, 0xc00008c1c0?})
                                        go.uber.org/[email protected]/zapcore/entry.go:196 +0x65
                                go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000744000, {0xc000742000, 0x2, 0x2})
                                        go.uber.org/[email protected]/zapcore/entry.go:262 +0x3ec
                                go.uber.org/zap.(*Logger).Panic(0xc000324150?, {0x10ff498?, 0xc?}, {0xc000742000, 0x2, 0x2})
                                        go.uber.org/[email protected]/logger.go:284 +0x59
                                go.etcd.io/etcd/server/v3/etcdserver/api/membership.mustDeleteMemberFromStore(0xc0003241f8?, {0x128b4f8, 0xc000324150}, 0xc0006f1120?)
                                        go.etcd.io/etcd/server/v3/etcdserver/api/membership/storev2.go:104 +0x25c
                                go.etcd.io/etcd/server/v3/etcdserver/api/membership.(*RaftCluster).RemoveMember(0xc0003241c0, 0x1279b60?, 0x0)
                                        go.etcd.io/etcd/server/v3/etcdserver/api/membership/cluster.go:436 +0xd8
                                go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyConfChange(0xc0007ad500, {0x1, 0x1, {0x0, 0x0, 0x0}, 0xd7588c01e226ed05}, 0xc000488d80, 0x0)
                                        go.etcd.io/etcd/server/v3/etcdserver/server.go:2039 +0x27a
                                go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).apply(0xc0007ad500, {0xc0005c2c90, 0x5, 0x0?}, 0x0?, 0xc000596420)
                                        go.etcd.io/etcd/server/v3/etcdserver/server.go:1844 +0x536
                                go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyEntries(0xc0007ad500, 0xc000488d80, 0xc000000000?)
                                        go.etcd.io/etcd/server/v3/etcdserver/server.go:1131 +0x26b
                                go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).applyAll(0xc0007ad500, 0xc000488d80, 0xc000000000)
                                        go.etcd.io/etcd/server/v3/etcdserver/server.go:919 +0x65
                                go.etcd.io/etcd/server/v3/etcdserver.(*EtcdServer).run.func8({0xc0000980b8, 0xc000596240})
                                        go.etcd.io/etcd/server/v3/etcdserver/server.go:842 +0x25
                                go.etcd.io/etcd/pkg/v3/schedule.job.Do(...)
                                        go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:41
                                go.etcd.io/etcd/pkg/v3/schedule.(*fifo).executeJob(0xc00056b790?, {0x1279ae8?, 0xc0000101c8?}, 0x0?)
                                        go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:206 +0x88
                                go.etcd.io/etcd/pkg/v3/schedule.(*fifo).run(0xc0007b81c0)
                                        go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:187 +0x11a
                                created by go.etcd.io/etcd/pkg/v3/schedule.NewFIFOScheduler
                                        go.etcd.io/etcd/pkg/[email protected]/schedule/schedule.go:101 +0x18b
                Test:           TestRemoveNonExistingMember
    logger.go:130: 2023-11-24T23:10:47.683+0800 INFO    closing test cluster...
    logger.go:130: 2023-11-24T23:10:47.683+0800 INFO    closing server...       {"name": "TestRemoveNonExistingMember-test-0"}
    logger.go:130: 2023-11-24T23:10:47.683+0800 INFO    removing directory      {"data-dir": "/tmp/TestRemoveNonExistingMember3670107214/002"}
    logger.go:130: 2023-11-24T23:10:47.683+0800 INFO    closed test cluster.
--- FAIL: TestRemoveNonExistingMember (0.17s)

[ahrtr]

This PR is correct by itself. But it doesn't prove that the approach in #16084 is incorrect. Of course, it's still open to discussion if you have concern.

#16084 isn't complete, please see #17019