From bab04c0e2ca0d6901fc41eddc967cfc36b75fa9c Mon Sep 17 00:00:00 2001
From: Joshua Zhang <joshuazh@microsoft.com>
Date: Wed, 8 Jan 2025 15:07:31 +0800
Subject: [PATCH] Avoid deadlock in etcd.Close when stopping during
 bootstrapping

Signed-off-by: Joshua Zhang <joshuazh@microsoft.com>
---
 server/embed/serve.go                 |  5 ++--
 server/etcdserver/server.go           |  1 +
 tests/integration/embed/embed_test.go | 40 +++++++++++++++++++++++++++
 3 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/server/embed/serve.go b/server/embed/serve.go
index a73ba58b1b5..daa65937523 100644
--- a/server/embed/serve.go
+++ b/server/embed/serve.go
@@ -102,6 +102,9 @@ func (sctx *serveCtx) serve(
 ) (err error) {
 	logger := defaultLog.New(io.Discard, "etcdhttp", 0)
 
+	// Make sure serversC is closed even if we prematurely exit the function.
+	defer close(sctx.serversC)
+
 	select {
 	case <-s.StoppingNotify():
 		return errors.New("server is stopping")
@@ -121,8 +124,6 @@ func (sctx *serveCtx) serve(
 	servElection := v3election.NewElectionServer(v3c)
 	servLock := v3lock.NewLockServer(v3c)
 
-	// Make sure serversC is closed even if we prematurely exit the function.
-	defer close(sctx.serversC)
 	var gwmux *gw.ServeMux
 	if s.Cfg.EnableGRPCGateway {
 		// GRPC gateway connects to grpc server via connection provided by grpc dial.
diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go
index 778e786c3af..e6cee5b2650 100644
--- a/server/etcdserver/server.go
+++ b/server/etcdserver/server.go
@@ -1776,6 +1776,7 @@ func (s *EtcdServer) publishV3(timeout time.Duration) {
 			ClientUrls: s.attributes.ClientURLs,
 		},
 	}
+	// gofail: var beforePublishing struct{}
 	lg := s.Logger()
 	for {
 		select {
diff --git a/tests/integration/embed/embed_test.go b/tests/integration/embed/embed_test.go
index 7a3d11f1cd4..22b82b10eee 100644
--- a/tests/integration/embed/embed_test.go
+++ b/tests/integration/embed/embed_test.go
@@ -219,3 +219,43 @@ func TestEmbedEtcdAutoCompactionRetentionRetained(t *testing.T) {
 	assert.Equal(t, durationToCompare, autoCompactionRetention)
 	e.Close()
 }
+
+func TestEmbedEtcdStopDuringBootstrapping(t *testing.T) {
+	integration2.BeforeTest(t, integration2.WithFailpoint("beforePublishing", `sleep("2s")`))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+
+		cfg := embed.NewConfig()
+		urls := newEmbedURLs(false, 2)
+		setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]})
+		cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd")
+
+		e, err := embed.StartEtcd(cfg)
+		require.NoError(t, err)
+		defer e.Close()
+
+		go func() {
+			time.Sleep(time.Second)
+			e.Server.Stop()
+			t.Log("Stopped server during bootstrapping")
+		}()
+
+		select {
+		case <-e.Server.ReadyNotify():
+			t.Log("Server is ready!")
+		case <-e.Server.StopNotify():
+			t.Log("Server is stopped")
+		case <-time.After(20 * time.Second):
+			e.Server.Stop() // trigger a shutdown
+			t.Error("Server took too long to start!")
+		}
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Error("timeout in bootstrapping etcd")
+	}
+}