Avoid deadlock in etcd.Close when stopping during bootstrapping

Signed-off-by: Joshua Zhang <[email protected]>

server/embed/serve.go

@@ -102,6 +102,9 @@ func (sctx *serveCtx) serve(
 ) (err error) {
 	logger := defaultLog.New(io.Discard, "etcdhttp", 0)
 
+	// Make sure serversC is closed even if we prematurely exit the function.
+	defer close(sctx.serversC)
+
 	select {
 	case <-s.StoppingNotify():
 		return errors.New("server is stopping")
@@ -121,8 +124,6 @@ func (sctx *serveCtx) serve(
 	servElection := v3election.NewElectionServer(v3c)
 	servLock := v3lock.NewLockServer(v3c)
 
-	// Make sure serversC is closed even if we prematurely exit the function.
-	defer close(sctx.serversC)
 	var gwmux *gw.ServeMux
 	if s.Cfg.EnableGRPCGateway {
 		// GRPC gateway connects to grpc server via connection provided by grpc dial.

server/etcdserver/server.go

@@ -1776,6 +1776,7 @@ func (s *EtcdServer) publishV3(timeout time.Duration) {
 			ClientUrls: s.attributes.ClientURLs,
 		},
 	}
+	// gofail: var beforePublishing struct{}
 	lg := s.Logger()
 	for {
 		select {

tests/integration/embed/embed_test.go

@@ -219,3 +219,43 @@ func TestEmbedEtcdAutoCompactionRetentionRetained(t *testing.T) {
 	assert.Equal(t, durationToCompare, autoCompactionRetention)
 	e.Close()
 }
+
+func TestEmbedEtcdStopDuringBootstrapping(t *testing.T) {
+	integration2.BeforeTest(t, integration2.WithFailpoint("beforePublishing", `sleep("2s")`))
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+
+		cfg := embed.NewConfig()
+		urls := newEmbedURLs(false, 2)
+		setupEmbedCfg(cfg, []url.URL{urls[0]}, []url.URL{urls[1]})
+		cfg.Dir = filepath.Join(t.TempDir(), "embed-etcd")
+
+		e, err := embed.StartEtcd(cfg)
+		require.NoError(t, err)
+		defer e.Close()
+
+		go func() {
+			time.Sleep(time.Second)
+			e.Server.Stop()
+			t.Log("Stopped server during bootstrapping")
+		}()
+
+		select {
+		case <-e.Server.ReadyNotify():
+			t.Log("Server is ready!")
+		case <-e.Server.StopNotify():
+			t.Log("Server is stopped")
+		case <-time.After(20 * time.Second):
+			e.Server.Stop() // trigger a shutdown
+			t.Error("Server took too long to start!")
+		}
+	}()
+
+	select {
+	case <-done:
+	case <-time.After(10 * time.Second):
+		t.Error("timeout in bootstrapping etcd")
+	}
+}