diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go
index 2fc58dd9b1c..71fc267ab01 100644
--- a/cmd/antrea-agent/agent.go
+++ b/cmd/antrea-agent/agent.go
@@ -231,6 +231,7 @@ func run(o *Options) error {
 		k8sClient,
 		crdClient,
 		ovsBridgeClient,
+		ovsctl.NewClient(o.config.OVSBridge),
 		ofClient,
 		routeClient,
 		ifaceStore,
diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go
index a8d993a68de..8a0afc77132 100644
--- a/pkg/agent/agent.go
+++ b/pkg/agent/agent.go
@@ -95,6 +95,7 @@ type Initializer struct {
 	client                clientset.Interface
 	crdClient             versioned.Interface
 	ovsBridgeClient       ovsconfig.OVSBridgeClient
+	ovsCtlClient          ovsctl.OVSCtlClient
 	ofClient              openflow.Client
 	routeClient           route.Interface
 	wireGuardClient       wireguard.Interface
@@ -122,6 +123,7 @@ func NewInitializer(
 	k8sClient clientset.Interface,
 	crdClient versioned.Interface,
 	ovsBridgeClient ovsconfig.OVSBridgeClient,
+	ovsCtlClient ovsctl.OVSCtlClient,
 	ofClient openflow.Client,
 	routeClient route.Interface,
 	ifaceStore interfacestore.InterfaceStore,
@@ -142,6 +144,7 @@ func NewInitializer(
 ) *Initializer {
 	return &Initializer{
 		ovsBridgeClient:       ovsBridgeClient,
+		ovsCtlClient:          ovsCtlClient,
 		client:                k8sClient,
 		crdClient:             crdClient,
 		ifaceStore:            ifaceStore,
@@ -371,17 +374,20 @@ func (i *Initializer) initInterfaceStore() error {
 }
 
 func (i *Initializer) restorePortConfigs() error {
-	ovsCtlClient := ovsctl.NewClient(i.ovsBridge)
 	interfaces := i.ifaceStore.ListInterfaces()
 	for _, intf := range interfaces {
 		switch intf.Type {
 		case interfacestore.IPSecTunnelInterface:
 			fallthrough
 		case interfacestore.TrafficControlInterface:
-			if err := ovsCtlClient.SetPortNoFlood(int(intf.OFPort)); err != nil {
-				return fmt.Errorf("failed to set port %s with no-flood: %w", intf.InterfaceName, err)
+			if intf.OFPort < 0 {
+				klog.InfoS("Skipped setting no-flood for port due to invalid ofPort", "port", intf.InterfaceName, "ofport", intf.OFPort)
+				continue
+			}
+			if err := i.ovsCtlClient.SetPortNoFlood(int(intf.OFPort)); err != nil {
+				return fmt.Errorf("failed to set no-flood for port %s: %w", intf.InterfaceName, err)
 			}
-			klog.InfoS("Set port no-flood successfully", "PortName", intf.InterfaceName)
+			klog.InfoS("Set no-flood for port", "port", intf.InterfaceName)
 		}
 	}
 	return nil
diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go
index 266b80ff86a..b1a2411877f 100644
--- a/pkg/agent/agent_test.go
+++ b/pkg/agent/agent_test.go
@@ -37,6 +37,7 @@ import (
 	"antrea.io/antrea/pkg/agent/types"
 	"antrea.io/antrea/pkg/ovs/ovsconfig"
 	ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing"
+	ovsctltest "antrea.io/antrea/pkg/ovs/ovsctl/testing"
 	"antrea.io/antrea/pkg/util/env"
 	"antrea.io/antrea/pkg/util/ip"
 )
@@ -518,3 +519,77 @@ func mockConfigureLinkAddress(returnedErr error) func() {
 		configureLinkAddresses = originalConfigureLinkAddresses
 	}
 }
+
+func TestRestorePortConfigs(t *testing.T) {
+	ipsecTunnelInterface := interfacestore.NewIPSecTunnelInterface("antrea-ipsec1",
+		ovsconfig.GeneveTunnel,
+		"node1",
+		net.ParseIP("1.1.1.1"),
+		"abcdefg",
+		"node1")
+	ipsecTunnelInterface.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 11, PortUUID: "uuid1"}
+	tunnelInterface := interfacestore.NewTunnelInterface(defaultTunInterfaceName,
+		ovsconfig.GeneveTunnel,
+		0,
+		net.ParseIP("1.1.1.10"),
+		true)
+	tunnelInterface.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 12}
+	trafficControlInterface1 := interfacestore.NewTrafficControlInterface("antrea-tap1")
+	trafficControlInterface1.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 13, PortUUID: "uuid3"}
+	trafficControlInterface2 := interfacestore.NewTrafficControlInterface("antrea-tap2")
+	trafficControlInterface2.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: -1, PortUUID: "uuid3"}
+
+	tests := []struct {
+		name                string
+		existingInterfaces  []*interfacestore.InterfaceConfig
+		expectedOVSCtlCalls func(client *ovsctltest.MockOVSCtlClientMockRecorder)
+		expectedErr         string
+	}{
+		{
+			name: "success",
+			existingInterfaces: []*interfacestore.InterfaceConfig{
+				ipsecTunnelInterface,
+				tunnelInterface,
+				trafficControlInterface1,
+				trafficControlInterface2,
+			},
+			expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) {
+				client.SetPortNoFlood(11).Return(nil)
+				client.SetPortNoFlood(13).Return(nil)
+			},
+		},
+		{
+			name: "fail",
+			existingInterfaces: []*interfacestore.InterfaceConfig{
+				&interfacestore.InterfaceConfig{InterfaceName: "antrea-tap1",
+					Type:          interfacestore.TrafficControlInterface,
+					OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 10, PortUUID: "uuid3"},
+				},
+			},
+			expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) {
+				client.SetPortNoFlood(10).Return(fmt.Errorf("server unavailable"))
+			},
+			expectedErr: "failed to set no-flood for port antrea-tap1: server unavailable",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			controller := mock.NewController(t)
+			defer controller.Finish()
+			mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(controller)
+			ifaceStore := interfacestore.NewInterfaceStore()
+			initializer := &Initializer{
+				ifaceStore:   ifaceStore,
+				ovsCtlClient: mockOVSCtlClient,
+			}
+			ifaceStore.Initialize(tt.existingInterfaces)
+			tt.expectedOVSCtlCalls(mockOVSCtlClient.EXPECT())
+			err := initializer.restorePortConfigs()
+			if tt.expectedErr == "" {
+				assert.NoError(t, err)
+			} else {
+				assert.ErrorContains(t, err, tt.expectedErr)
+			}
+		})
+	}
+}
diff --git a/pkg/agent/interfacestore/interface_cache_test.go b/pkg/agent/interfacestore/interface_cache_test.go
index 84c61c2bc18..ba7b4ea770a 100644
--- a/pkg/agent/interfacestore/interface_cache_test.go
+++ b/pkg/agent/interfacestore/interface_cache_test.go
@@ -40,7 +40,6 @@ func TestNewInterfaceStore(t *testing.T) {
 	t.Run("testGatewayInterface", testGatewayInterface)
 	t.Run("testTunnelInterface", testTunnelInterface)
 	t.Run("testUplinkInterface", testUplinkInterface)
-	t.Run("testTrafficControlInterface", testTrafficControlInterface)
 	t.Run("testExternalEntityInterface", testEntityInterface)
 }
 
@@ -166,16 +165,6 @@ func testUplinkInterface(t *testing.T) {
 	testGeneralInterface(t, uplinkInterface, UplinkInterface)
 }
 
-func testTrafficControlInterface(t *testing.T) {
-	tcInterface := NewTrafficControlInterface("tc0")
-	tcInterface.IPs = []net.IP{hostIP}
-	tcInterface.OVSPortConfig = &OVSPortConfig{
-		OFPort:   17,
-		PortUUID: "1234567890",
-	}
-	testGeneralInterface(t, tcInterface, TrafficControlInterface)
-}
-
 func testEntityInterface(t *testing.T) {
 	store := NewInterfaceStore()
 	portConfig := &OVSPortConfig{OFPort: 18, PortUUID: "123456789"}