From 7bd431b94aa9f201c8a52c99fc7a0dcd8f18c6eb Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 13 Jan 2025 14:02:27 +0100 Subject: [PATCH] Runn Issue17529 100 times --- tests/robustness/scenarios/scenarios.go | 328 ++++++++++++------------ 1 file changed, 161 insertions(+), 167 deletions(-) diff --git a/tests/robustness/scenarios/scenarios.go b/tests/robustness/scenarios/scenarios.go index c3c6dd23d05..202f9e2c0e8 100644 --- a/tests/robustness/scenarios/scenarios.go +++ b/tests/robustness/scenarios/scenarios.go @@ -15,20 +15,12 @@ package scenarios import ( - "path/filepath" "testing" - "time" - "github.com/stretchr/testify/require" - - "go.etcd.io/etcd/api/v3/version" - "go.etcd.io/etcd/client/pkg/v3/fileutil" - "go.etcd.io/etcd/server/v3/etcdserver" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/client" "go.etcd.io/etcd/tests/v3/robustness/failpoint" "go.etcd.io/etcd/tests/v3/robustness/options" - "go.etcd.io/etcd/tests/v3/robustness/random" "go.etcd.io/etcd/tests/v3/robustness/traffic" ) @@ -71,175 +63,177 @@ type TestScenario struct { } func Exploratory(_ *testing.T) []TestScenario { - randomizableOptions := []e2e.EPClusterOption{ - options.WithClusterOptionGroups( - options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)}, - options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)}, - options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), - } - - mixedVersionOptionChoices := []random.ChoiceWeight[options.ClusterOptions]{ - // 60% with all members of current version - {Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60}, - // 10% with 2 members of current version, 1 member last version, leader is current version - {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, - // 10% with 2 members of current version, 1 member last version, leader is last version - {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, - // 10% with 2 members of last version, 1 member current version, leader is last version - {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, - // 10% with 2 members of last version, 1 member current version, leader is current version - {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, - } - mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices)) - - baseOptions := []e2e.EPClusterOption{ - options.WithSnapshotCount(50, 100, 1000), - options.WithSubsetOptions(randomizableOptions...), - e2e.WithGoFailEnabled(true), - // Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints. - options.WithCompactionBatchLimit(10, 100, 1000), - e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), - } - - if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { - baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries)) - } + //randomizableOptions := []e2e.EPClusterOption{ + // options.WithClusterOptionGroups( + // options.ClusterOptions{options.WithTickMs(29), options.WithElectionMs(271)}, + // options.ClusterOptions{options.WithTickMs(101), options.WithElectionMs(521)}, + // options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), + //} + // + //mixedVersionOptionChoices := []random.ChoiceWeight[options.ClusterOptions]{ + // // 60% with all members of current version + // {Choice: options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, Weight: 60}, + // // 10% with 2 members of current version, 1 member last version, leader is current version + // {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, + // // 10% with 2 members of current version, 1 member last version, leader is last version + // {Choice: options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, + // // 10% with 2 members of last version, 1 member current version, leader is last version + // {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, Weight: 10}, + // // 10% with 2 members of last version, 1 member current version, leader is current version + // {Choice: options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, Weight: 10}, + //} + //mixedVersionOption := options.WithClusterOptionGroups(random.PickRandom[options.ClusterOptions](mixedVersionOptionChoices)) + // + //baseOptions := []e2e.EPClusterOption{ + // options.WithSnapshotCount(50, 100, 1000), + // options.WithSubsetOptions(randomizableOptions...), + // e2e.WithGoFailEnabled(true), + // // Set low minimal compaction batch limit to allow for triggering multi batch compaction failpoints. + // options.WithCompactionBatchLimit(10, 100, 1000), + // e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), + //} + // + //if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { + // baseOptions = append(baseOptions, options.WithSnapshotCatchUpEntries(100, etcdserver.DefaultSnapshotCatchUpEntries)) + //} scenarios := []TestScenario{} - for _, tp := range trafficProfiles { - name := filepath.Join(tp.Name, "ClusterOfSize1") - clusterOfSize1Options := baseOptions - clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1)) - scenarios = append(scenarios, TestScenario{ - Name: name, - Traffic: tp.Traffic, - Profile: tp.Profile, - Cluster: *e2e.NewConfig(clusterOfSize1Options...), - }) - } - - for _, tp := range trafficProfiles { - name := filepath.Join(tp.Name, "ClusterOfSize3") - clusterOfSize3Options := baseOptions - clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true)) - clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true)) - if fileutil.Exist(e2e.BinPath.EtcdLastRelease) { - clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption) - } - scenarios = append(scenarios, TestScenario{ - Name: name, - Traffic: tp.Traffic, - Profile: tp.Profile, - Cluster: *e2e.NewConfig(clusterOfSize3Options...), - }) - } - if e2e.BinPath.LazyFSAvailable() { - newScenarios := scenarios - for _, s := range scenarios { - // LazyFS increases the load on CPU, so we run it with more lightweight case. - if s.Profile.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 { - lazyfsCluster := s.Cluster - lazyfsCluster.LazyFSEnabled = true - newScenarios = append(newScenarios, TestScenario{ - Name: filepath.Join(s.Name, "LazyFS"), - Failpoint: s.Failpoint, - Cluster: lazyfsCluster, - Traffic: s.Traffic, - Profile: s.Profile.WithoutCompaction(), - Watch: s.Watch, - }) - } - } - scenarios = newScenarios - } + //for _, tp := range trafficProfiles { + // name := filepath.Join(tp.Name, "ClusterOfSize1") + // clusterOfSize1Options := baseOptions + // clusterOfSize1Options = append(clusterOfSize1Options, e2e.WithClusterSize(1)) + // scenarios = append(scenarios, TestScenario{ + // Name: name, + // Traffic: tp.Traffic, + // Profile: tp.Profile, + // Cluster: *e2e.NewConfig(clusterOfSize1Options...), + // }) + //} + // + //for _, tp := range trafficProfiles { + // name := filepath.Join(tp.Name, "ClusterOfSize3") + // clusterOfSize3Options := baseOptions + // clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true)) + // clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true)) + // if fileutil.Exist(e2e.BinPath.EtcdLastRelease) { + // clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption) + // } + // scenarios = append(scenarios, TestScenario{ + // Name: name, + // Traffic: tp.Traffic, + // Profile: tp.Profile, + // Cluster: *e2e.NewConfig(clusterOfSize3Options...), + // }) + //} + //if e2e.BinPath.LazyFSAvailable() { + // newScenarios := scenarios + // for _, s := range scenarios { + // // LazyFS increases the load on CPU, so we run it with more lightweight case. + // if s.Profile.MinimalQPS <= 100 && s.Cluster.ClusterSize == 1 { + // lazyfsCluster := s.Cluster + // lazyfsCluster.LazyFSEnabled = true + // newScenarios = append(newScenarios, TestScenario{ + // Name: filepath.Join(s.Name, "LazyFS"), + // Failpoint: s.Failpoint, + // Cluster: lazyfsCluster, + // Traffic: s.Traffic, + // Profile: s.Profile.WithoutCompaction(), + // Watch: s.Watch, + // }) + // } + // } + // scenarios = newScenarios + //} return scenarios } func Regression(t *testing.T) []TestScenario { - v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) - require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd) + //v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) + //require.NoErrorf(t, err, "Failed checking etcd version binary, binary: %q", e2e.BinPath.Etcd) scenarios := []TestScenario{} - scenarios = append(scenarios, TestScenario{ - Name: "Issue14370", - Failpoint: failpoint.RaftBeforeSavePanic, - Profile: traffic.LowTraffic, - Traffic: traffic.EtcdPutDeleteLease, - Cluster: *e2e.NewConfig( - e2e.WithClusterSize(1), - e2e.WithGoFailEnabled(true), - ), - }) - scenarios = append(scenarios, TestScenario{ - Name: "Issue14685", - Failpoint: failpoint.DefragBeforeCopyPanic, - Profile: traffic.LowTraffic, - Traffic: traffic.EtcdPutDeleteLease, - Cluster: *e2e.NewConfig( - e2e.WithClusterSize(1), - e2e.WithGoFailEnabled(true), - ), - }) - scenarios = append(scenarios, TestScenario{ - Name: "Issue13766", - Failpoint: failpoint.KillFailpoint, - Profile: traffic.HighTrafficProfile, - Traffic: traffic.EtcdPut, - Cluster: *e2e.NewConfig( - e2e.WithSnapshotCount(100), - ), - }) - scenarios = append(scenarios, TestScenario{ - Name: "Issue15220", - Watch: client.WatchConfig{ - RequestProgress: true, - }, - Profile: traffic.LowTraffic, - Traffic: traffic.EtcdPutDeleteLease, - Failpoint: failpoint.KillFailpoint, - Cluster: *e2e.NewConfig( - e2e.WithClusterSize(1), - ), - }) - scenarios = append(scenarios, TestScenario{ - Name: "Issue17529", - Profile: traffic.HighTrafficProfile, - Traffic: traffic.Kubernetes, - Failpoint: failpoint.SleepBeforeSendWatchResponse, - Cluster: *e2e.NewConfig( - e2e.WithClusterSize(1), - e2e.WithGoFailEnabled(true), - options.WithSnapshotCount(100), - ), - }) - - scenarios = append(scenarios, TestScenario{ - Name: "Issue17780", - Profile: traffic.LowTraffic.WithoutCompaction(), - Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, - Traffic: traffic.Kubernetes, - Cluster: *e2e.NewConfig( - e2e.WithClusterSize(1), - e2e.WithCompactionBatchLimit(300), - e2e.WithSnapshotCount(1000), - e2e.WithGoFailEnabled(true), - ), - }) - if v.Compare(version.V3_5) >= 0 { - opts := []e2e.EPClusterOption{ - e2e.WithSnapshotCount(100), - e2e.WithPeerProxy(true), - e2e.WithIsPeerTLS(true), - } - if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { - opts = append(opts, e2e.WithSnapshotCatchUpEntries(100)) - } + //scenarios = append(scenarios, TestScenario{ + // Name: "Issue14370", + // Failpoint: failpoint.RaftBeforeSavePanic, + // Profile: traffic.LowTraffic, + // Traffic: traffic.EtcdPutDeleteLease, + // Cluster: *e2e.NewConfig( + // e2e.WithClusterSize(1), + // e2e.WithGoFailEnabled(true), + // ), + //}) + //scenarios = append(scenarios, TestScenario{ + // Name: "Issue14685", + // Failpoint: failpoint.DefragBeforeCopyPanic, + // Profile: traffic.LowTraffic, + // Traffic: traffic.EtcdPutDeleteLease, + // Cluster: *e2e.NewConfig( + // e2e.WithClusterSize(1), + // e2e.WithGoFailEnabled(true), + // ), + //}) + //scenarios = append(scenarios, TestScenario{ + // Name: "Issue13766", + // Failpoint: failpoint.KillFailpoint, + // Profile: traffic.HighTrafficProfile, + // Traffic: traffic.EtcdPut, + // Cluster: *e2e.NewConfig( + // e2e.WithSnapshotCount(100), + // ), + //}) + //scenarios = append(scenarios, TestScenario{ + // Name: "Issue15220", + // Watch: client.WatchConfig{ + // RequestProgress: true, + // }, + // Profile: traffic.LowTraffic, + // Traffic: traffic.EtcdPutDeleteLease, + // Failpoint: failpoint.KillFailpoint, + // Cluster: *e2e.NewConfig( + // e2e.WithClusterSize(1), + // ), + //}) + for i := 0; i < 100; i++ { scenarios = append(scenarios, TestScenario{ - Name: "Issue15271", - Failpoint: failpoint.BlackholeUntilSnapshot, + Name: "Issue17529", Profile: traffic.HighTrafficProfile, - Traffic: traffic.EtcdPut, - Cluster: *e2e.NewConfig(opts...), + Traffic: traffic.Kubernetes, + Failpoint: failpoint.SleepBeforeSendWatchResponse, + Cluster: *e2e.NewConfig( + e2e.WithClusterSize(1), + e2e.WithGoFailEnabled(true), + options.WithSnapshotCount(100), + ), }) } + + //scenarios = append(scenarios, TestScenario{ + // Name: "Issue17780", + // Profile: traffic.LowTraffic.WithoutCompaction(), + // Failpoint: failpoint.BatchCompactBeforeSetFinishedCompactPanic, + // Traffic: traffic.Kubernetes, + // Cluster: *e2e.NewConfig( + // e2e.WithClusterSize(1), + // e2e.WithCompactionBatchLimit(300), + // e2e.WithSnapshotCount(1000), + // e2e.WithGoFailEnabled(true), + // ), + //}) + //if v.Compare(version.V3_5) >= 0 { + // opts := []e2e.EPClusterOption{ + // e2e.WithSnapshotCount(100), + // e2e.WithPeerProxy(true), + // e2e.WithIsPeerTLS(true), + // } + // if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { + // opts = append(opts, e2e.WithSnapshotCatchUpEntries(100)) + // } + // scenarios = append(scenarios, TestScenario{ + // Name: "Issue15271", + // Failpoint: failpoint.BlackholeUntilSnapshot, + // Profile: traffic.HighTrafficProfile, + // Traffic: traffic.EtcdPut, + // Cluster: *e2e.NewConfig(opts...), + // }) + //} return scenarios }