From 0507c53db39304ccb47df0b1e925823a89333447 Mon Sep 17 00:00:00 2001 From: Lan Liang Date: Mon, 19 Aug 2024 05:42:25 +0000 Subject: [PATCH] migrate experimental-enable-lease-checkpoint and experimental-enable-lease-checkpoint-persist flag to feature gate. Signed-off-by: Lan Liang --- server/embed/config.go | 4 +-- server/embed/config_test.go | 53 ++++++++++++++++++++++++++------ server/embed/etcd.go | 1 - server/etcdmain/help.go | 4 +-- server/etcdserver/server.go | 5 +-- server/features/etcd_features.go | 23 ++++++++++++-- 6 files changed, 70 insertions(+), 20 deletions(-) diff --git a/server/embed/config.go b/server/embed/config.go index b10c5dc52c6..b5eaddb2127 100644 --- a/server/embed/config.go +++ b/server/embed/config.go @@ -1070,11 +1070,11 @@ func (cfg *Config) Validate() error { } } - if !cfg.ExperimentalEnableLeaseCheckpointPersist && cfg.ExperimentalEnableLeaseCheckpoint { + if !cfg.ServerFeatureGate.Enabled(features.EnableLeaseCheckpointPersist) && cfg.ServerFeatureGate.Enabled(features.EnableLeaseCheckpoint) { cfg.logger.Warn("Detected that checkpointing is enabled without persistence. Consider enabling experimental-enable-lease-checkpoint-persist") } - if cfg.ExperimentalEnableLeaseCheckpointPersist && !cfg.ExperimentalEnableLeaseCheckpoint { + if cfg.ServerFeatureGate.Enabled(features.EnableLeaseCheckpointPersist) && !cfg.ServerFeatureGate.Enabled(features.EnableLeaseCheckpoint) { return fmt.Errorf("setting experimental-enable-lease-checkpoint-persist requires experimental-enable-lease-checkpoint") } diff --git a/server/embed/config_test.go b/server/embed/config_test.go index cf18a5ec55c..98f9ffb2891 100644 --- a/server/embed/config_test.go +++ b/server/embed/config_test.go @@ -103,8 +103,10 @@ func TestConfigFileFeatureGates(t *testing.T) { { name: "default", expectedFeatures: map[featuregate.Feature]bool{ - features.DistributedTracing: false, - features.StopGRPCServiceOnDefrag: false, + features.DistributedTracing: false, + features.StopGRPCServiceOnDefrag: false, + features.EnableLeaseCheckpoint: false, + features.EnableLeaseCheckpointPersist: false, }, }, { @@ -126,32 +128,40 @@ func TestConfigFileFeatureGates(t *testing.T) { name: "can set feature gate to true from experimental flag", experimentalStopGRPCServiceOnDefrag: "true", expectedFeatures: map[featuregate.Feature]bool{ - features.StopGRPCServiceOnDefrag: true, - features.DistributedTracing: false, + features.StopGRPCServiceOnDefrag: true, + features.DistributedTracing: false, + features.EnableLeaseCheckpoint: false, + features.EnableLeaseCheckpointPersist: false, }, }, { name: "can set feature gate to false from experimental flag", experimentalStopGRPCServiceOnDefrag: "false", expectedFeatures: map[featuregate.Feature]bool{ - features.StopGRPCServiceOnDefrag: false, - features.DistributedTracing: false, + features.StopGRPCServiceOnDefrag: false, + features.DistributedTracing: false, + features.EnableLeaseCheckpoint: false, + features.EnableLeaseCheckpointPersist: false, }, }, { name: "can set feature gate to true from feature gate flag", serverFeatureGatesJSON: "StopGRPCServiceOnDefrag=true", expectedFeatures: map[featuregate.Feature]bool{ - features.StopGRPCServiceOnDefrag: true, - features.DistributedTracing: false, + features.StopGRPCServiceOnDefrag: true, + features.DistributedTracing: false, + features.EnableLeaseCheckpoint: false, + features.EnableLeaseCheckpointPersist: false, }, }, { name: "can set feature gate to false from feature gate flag", serverFeatureGatesJSON: "StopGRPCServiceOnDefrag=false", expectedFeatures: map[featuregate.Feature]bool{ - features.StopGRPCServiceOnDefrag: false, - features.DistributedTracing: false, + features.StopGRPCServiceOnDefrag: false, + features.DistributedTracing: false, + features.EnableLeaseCheckpoint: false, + features.EnableLeaseCheckpointPersist: false, }, }, } @@ -563,6 +573,12 @@ func TestPeerURLsMapAndTokenFromSRV(t *testing.T) { } } +func SetFeatureGateDuringTest(cfg Config, fgname string, value bool) { + if err := cfg.ServerFeatureGate.(featuregate.MutableFeatureGate).Set(fmt.Sprintf("%s=%v", fgname, true)); err != nil { + panic(err) + } +} + func TestLeaseCheckpointValidate(t *testing.T) { tcs := []struct { name string @@ -575,6 +591,14 @@ func TestLeaseCheckpointValidate(t *testing.T) { return *NewConfig() }, }, + { + name: "Enabling checkpoint leases with feature gate should pass", + configFunc: func() Config { + cfg := *NewConfig() + SetFeatureGateDuringTest(cfg, "EnableLeaseCheckpoint", true) + return cfg + }, + }, { name: "Enabling checkpoint leases should pass", configFunc: func() Config { @@ -592,6 +616,15 @@ func TestLeaseCheckpointValidate(t *testing.T) { return cfg }, }, + { + name: "Enabling checkpoint leases and persist with feature gate should pass", + configFunc: func() Config { + cfg := *NewConfig() + SetFeatureGateDuringTest(cfg, "EnableLeaseCheckpoint", true) + SetFeatureGateDuringTest(cfg, "EnableLeaseCheckpointPersist", true) + return cfg + }, + }, { name: "Enabling checkpoint leases persist without checkpointing itself should fail", configFunc: func() Config { diff --git a/server/embed/etcd.go b/server/embed/etcd.go index 2970a804d11..838da244b84 100644 --- a/server/embed/etcd.go +++ b/server/embed/etcd.go @@ -213,7 +213,6 @@ func StartEtcd(inCfg *Config) (e *Etcd, err error) { EnableGRPCGateway: cfg.EnableGRPCGateway, ExperimentalEnableDistributedTracing: cfg.ExperimentalEnableDistributedTracing, UnsafeNoFsync: cfg.UnsafeNoFsync, - EnableLeaseCheckpoint: cfg.ExperimentalEnableLeaseCheckpoint, LeaseCheckpointPersist: cfg.ExperimentalEnableLeaseCheckpointPersist, CompactionBatchLimit: cfg.ExperimentalCompactionBatchLimit, CompactionSleepInterval: cfg.ExperimentalCompactionSleepInterval, diff --git a/server/etcdmain/help.go b/server/etcdmain/help.go index 23c531b720c..f2faa71b16f 100644 --- a/server/etcdmain/help.go +++ b/server/etcdmain/help.go @@ -281,7 +281,7 @@ Experimental feature: Enable leader to periodically check followers compaction hashes. --experimental-compact-hash-check-time '1m' Duration of time between leader checks followers compaction hashes. - --experimental-enable-lease-checkpoint 'false' + --experimental-enable-lease-checkpoint 'false'.It's deprecated, and will be decommissioned in v3.7. Use '--feature-gates=EnableLeaseCheckpoint=true' instead. ExperimentalEnableLeaseCheckpoint enables primary lessor to persist lease remainingTTL to prevent indefinite auto-renewal of long lived leases. --experimental-compaction-batch-limit 1000 ExperimentalCompactionBatchLimit sets the maximum revisions deleted in each compaction batch. @@ -305,7 +305,7 @@ Experimental feature: Sets the sleep interval between each compaction batch. --experimental-downgrade-check-time Duration of time between two downgrade status checks. - --experimental-enable-lease-checkpoint-persist 'false' + --experimental-enable-lease-checkpoint-persist 'false'.It's deprecated, and will be decommissioned in v3.7. Use '--feature-gates=EnableLeaseCheckpointPersist=true' instead. Enable persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. Requires experimental-enable-lease-checkpoint to be enabled. --experimental-memory-mlock Enable to enforce etcd pages (in particular bbolt) to stay in RAM. diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index 0600a31b896..27e7b9d0a4c 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -63,6 +63,7 @@ import ( "go.etcd.io/etcd/server/v3/etcdserver/errors" "go.etcd.io/etcd/server/v3/etcdserver/txn" serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" + "go.etcd.io/etcd/server/v3/features" "go.etcd.io/etcd/server/v3/lease" "go.etcd.io/etcd/server/v3/lease/leasehttp" serverstorage "go.etcd.io/etcd/server/v3/storage" @@ -350,7 +351,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { srv.lessor = lease.NewLessor(srv.Logger(), srv.be, srv.cluster, lease.LessorConfig{ MinLeaseTTL: int64(math.Ceil(minTTL.Seconds())), CheckpointInterval: cfg.LeaseCheckpointInterval, - CheckpointPersist: cfg.LeaseCheckpointPersist, + CheckpointPersist: cfg.ServerFeatureGate.Enabled(features.EnableLeaseCheckpointPersist), ExpiredLeasesRetryInterval: srv.Cfg.ReqTimeout(), }) @@ -395,7 +396,7 @@ func NewServer(cfg config.ServerConfig) (srv *EtcdServer, err error) { } srv.uberApply = srv.NewUberApplier() - if srv.Cfg.EnableLeaseCheckpoint { + if srv.FeatureEnabled(features.EnableLeaseCheckpoint) { // setting checkpointer enables lease checkpoint feature. srv.lessor.SetCheckpointer(func(ctx context.Context, cp *pb.LeaseCheckpointRequest) error { if !srv.ensureLeadership() { diff --git a/server/features/etcd_features.go b/server/features/etcd_features.go index d6804f2d048..8d869fc2b48 100644 --- a/server/features/etcd_features.go +++ b/server/features/etcd_features.go @@ -45,18 +45,35 @@ const ( // alpha: v3.6 // main PR: https://github.com/etcd-io/etcd/pull/18279 StopGRPCServiceOnDefrag featuregate.Feature = "StopGRPCServiceOnDefrag" + + // EnableLeaseCheckpoint enables leader to send regular checkpoints to other members to prevent reset of remaining TTL on leader change. + // owner: @ + // alpha: v3.6 + EnableLeaseCheckpoint featuregate.Feature = "EnableLeaseCheckpoint" + // EnableLeaseCheckpointPersist enables persisting remainingTTL to prevent indefinite auto-renewal of long lived leases. Always enabled in v3.6. Should be used to ensure smooth upgrade from v3.5 clusters with this feature enabled. + // Requires EnableLeaseCheckpoint featuragate to be enabled. + // Deprecated in v3.6. + // TODO: Delete in v3.7 + // owner: @serathius + // alpha: v3.6 + // main PR: https://github.com/etcd-io/etcd/pull/13508 + EnableLeaseCheckpointPersist featuregate.Feature = "EnableLeaseCheckpointPersist" ) var ( DefaultEtcdServerFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{ - DistributedTracing: {Default: false, PreRelease: featuregate.Alpha}, - StopGRPCServiceOnDefrag: {Default: false, PreRelease: featuregate.Alpha}, + DistributedTracing: {Default: false, PreRelease: featuregate.Alpha}, + StopGRPCServiceOnDefrag: {Default: false, PreRelease: featuregate.Alpha}, + EnableLeaseCheckpoint: {Default: false, PreRelease: featuregate.Alpha}, + EnableLeaseCheckpointPersist: {Default: false, PreRelease: featuregate.Alpha}, } // ExperimentalFlagToFeatureMap is the map from the cmd line flags of experimental features // to their corresponding feature gates. // Deprecated: only add existing experimental features here. DO NOT use for new features. ExperimentalFlagToFeatureMap = map[string]featuregate.Feature{ - "experimental-stop-grpc-service-on-defrag": StopGRPCServiceOnDefrag, + "experimental-stop-grpc-service-on-defrag": StopGRPCServiceOnDefrag, + "experimental-enable-lease-checkpoint": EnableLeaseCheckpoint, + "experimental-enable-lease-checkpoint-persist": EnableLeaseCheckpointPersist, } )