diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go
index 65bb495edf8..7a1dcc4df30 100644
--- a/cmd/pd-server/main.go
+++ b/cmd/pd-server/main.go
@@ -93,7 +93,10 @@ func main() {
 
 	// Creates server.
 	ctx, cancel := context.WithCancel(context.Background())
-	serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, swaggerserver.NewHandler, autoscaling.NewHandler}
+	serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, autoscaling.NewHandler}
+	if swaggerserver.Enabled() {
+		serviceBuilders = append(serviceBuilders, swaggerserver.NewHandler)
+	}
 	serviceBuilders = append(serviceBuilders, dashboard.GetServiceBuilders()...)
 	svr, err := server.CreateServer(ctx, cfg, serviceBuilders...)
 	if err != nil {
diff --git a/pkg/swaggerserver/swagger_handler.go b/pkg/swaggerserver/swagger_handler.go
deleted file mode 100644
index 69cff3d2751..00000000000
--- a/pkg/swaggerserver/swagger_handler.go
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2020 TiKV Project Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-//go:build swagger_server
-// +build swagger_server
-
-package swaggerserver
-
-import (
-	"net/http"
-
-	httpSwagger "github.com/swaggo/http-swagger"
-	_ "github.com/tikv/pd/docs/swagger"
-)
-
-func handler() http.Handler {
-	return httpSwagger.Handler()
-}
diff --git a/pkg/swaggerserver/swaggerserver.go b/pkg/swaggerserver/swaggerserver.go
index e2bac01bf0f..d95659ff570 100644
--- a/pkg/swaggerserver/swaggerserver.go
+++ b/pkg/swaggerserver/swaggerserver.go
@@ -12,12 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+//go:build swagger_server
+// +build swagger_server
+
 package swaggerserver
 
 import (
 	"context"
 	"net/http"
 
+	httpSwagger "github.com/swaggo/http-swagger"
+	_ "github.com/tikv/pd/docs/swagger"
 	"github.com/tikv/pd/server"
 )
 
@@ -32,9 +37,14 @@ var (
 	}
 )
 
+// Enabled return true if swagger server is disabled.
+func Enabled() bool {
+	return true
+}
+
 // NewHandler creates a HTTP handler for Swagger.
 func NewHandler(context.Context, *server.Server) (http.Handler, server.ServiceGroup, error) {
 	swaggerHandler := http.NewServeMux()
-	swaggerHandler.Handle(swaggerPrefix, handler())
+	swaggerHandler.Handle(swaggerPrefix, httpSwagger.Handler())
 	return swaggerHandler, swaggerServiceGroup, nil
 }
diff --git a/pkg/swaggerserver/empty_handler.go b/pkg/swaggerserver/swaggerserver_disable.go
similarity index 64%
rename from pkg/swaggerserver/empty_handler.go
rename to pkg/swaggerserver/swaggerserver_disable.go
index 888593b215e..c0e390bfdfc 100644
--- a/pkg/swaggerserver/empty_handler.go
+++ b/pkg/swaggerserver/swaggerserver_disable.go
@@ -1,4 +1,4 @@
-// Copyright 2020 TiKV Project Authors.
+// Copyright 2023 TiKV Project Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,12 +18,18 @@
 package swaggerserver
 
 import (
-	"io"
+	"context"
 	"net/http"
+
+	"github.com/tikv/pd/server"
 )
 
-func handler() http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		_, _ = io.WriteString(w, "Swagger UI is not built. Try `make` without `SWAGGER=1`.\n")
-	})
+// Enabled return false if swagger server is disabled.
+func Enabled() bool {
+	return false
+}
+
+// NewHandler creates a HTTP handler for Swagger.
+func NewHandler(context.Context, *server.Server) (http.Handler, server.ServiceGroup, error) {
+	return nil, server.ServiceGroup{}, nil
 }
diff --git a/tests/cluster.go b/tests/cluster.go
index 2bafe3e08f4..d0478a333e9 100644
--- a/tests/cluster.go
+++ b/tests/cluster.go
@@ -82,7 +82,10 @@ func NewTestServer(ctx context.Context, cfg *config.Config) (*TestServer, error)
 	if err != nil {
 		return nil, err
 	}
-	serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, swaggerserver.NewHandler, autoscaling.NewHandler}
+	serviceBuilders := []server.HandlerBuilder{api.NewHandler, apiv2.NewV2Handler, autoscaling.NewHandler}
+	if swaggerserver.Enabled() {
+		serviceBuilders = append(serviceBuilders, swaggerserver.NewHandler)
+	}
 	serviceBuilders = append(serviceBuilders, dashboard.GetServiceBuilders()...)
 	svr, err := server.CreateServer(ctx, cfg, serviceBuilders...)
 	if err != nil {
diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go
index 7b36618e62c..7df52240298 100644
--- a/tests/server/api/api_test.go
+++ b/tests/server/api/api_test.go
@@ -377,6 +377,15 @@ func (suite *middlewareTestSuite) TestRateLimitMiddleware() {
 	}
 }
 
+func (suite *middlewareTestSuite) TestSwaggerUrl() {
+	leader := suite.cluster.GetServer(suite.cluster.GetLeader())
+	req, _ := http.NewRequest(http.MethodGet, leader.GetAddr()+"/swagger/ui/index", nil)
+	resp, err := dialClient.Do(req)
+	suite.NoError(err)
+	suite.True(resp.StatusCode == http.StatusNotFound)
+	resp.Body.Close()
+}
+
 func (suite *middlewareTestSuite) TestAuditPrometheusBackend() {
 	leader := suite.cluster.GetServer(suite.cluster.GetLeader())
 	input := map[string]interface{}{