Skip to content

Commit

Permalink
Provide generic API for OVS register match/set value
Browse files Browse the repository at this point in the history
1. Use RegMark/RegField in OpenFlow Match or Load actions instead of passing the register ID and range.
2. The usage for OVS registers(including xxreg) should be predefined in regmarks.go.
3. Use reg0[0..3] to indicate the source of packet, and release reg0[4..15] for other usages.

Signed-off-by: wenyingd <wenyingd@vmware.com>
  • Loading branch information
wenyingd committed Aug 12, 2021
1 parent 40685f9 commit 6bc4a10
Show file tree
Hide file tree
Showing 22 changed files with 846 additions and 403 deletions.
12 changes: 6 additions & 6 deletions docs/design/ovs-pipeline.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ that Node and `<BRIDGE_NAME>` is the name of the bridge created by Antrea
We use 2 32-bit OVS registers to carry information throughout the pipeline:

* reg0 (NXM_NX_REG0):
- bits [0..15] are used to store the traffic source (from tunnel: 0, from
- bits [0..3] are used to store the traffic source (from tunnel: 0, from
local gateway: 1, from local Pod: 2). It is set in the [ClassifierTable].
- bit 16 is used to indicate whether the destination MAC address of a packet
is "known", i.e. corresponds to an entry in [L2ForwardingCalcTable], which
Expand Down Expand Up @@ -621,8 +621,8 @@ be SNAT'd with a SNAT IP configured on the Node. In the latter case, the flow
also rewrites the destination MAC to the local gateway interface MAC.

```text
table=70, priority=190,ip,reg0=0x2/0xffff actions=goto_table:71
table=70, priority=190,ip,reg0=0/0xffff actions=mod_dl_dst:e2:e5:a4:9b:1c:b1,goto_table:71
table=70, priority=190,ip,reg0=0x2/0xf actions=goto_table:71
table=70, priority=190,ip,reg0=0/0xf actions=mod_dl_dst:e2:e5:a4:9b:1c:b1,goto_table:71
```

### SNATTable (71)
Expand All @@ -639,7 +639,7 @@ Egress applied, to [L2ForwardingCalcTable]. Such traffic will be SNAT'd with
the default SNAT IP (by an iptables masquerade rule).

```text
table=71, priority=190,ct_state=+new+trk,ip,reg0=0/0xffff actions=drop
table=71, priority=190,ct_state=+new+trk,ip,reg0=0/0xf actions=drop
table=71, priority=0 actions=goto_table:80
```

Expand Down Expand Up @@ -685,7 +685,7 @@ IP stack should have already decremented TTL if that is needed.
If you dump the flows for this table, you should see flows like the following:

```text
1. table=72, priority=210,ip,reg0=0x1/0xffff, actions=goto_table:80
1. table=72, priority=210,ip,reg0=0x1/0xf, actions=goto_table:80
2. table=72, priority=200,ip, actions=dec_ttl,goto_table:80
3. table=72, priority=0, actions=goto_table:80
```
Expand Down Expand Up @@ -884,7 +884,7 @@ which are not dropped because of Network Policies. If you dump the flows for thi
table, you should see something like this:

```text
1. table=105, priority=200,ct_state=+new+trk,ip,reg0=0x1/0xffff actions=ct(commit,table=110,zone=65520,exec(load:0x20->NXM_NX_CT_MARK[]))
1. table=105, priority=200,ct_state=+new+trk,ip,reg0=0x1/0xf actions=ct(commit,table=110,zone=65520,exec(load:0x20->NXM_NX_CT_MARK[]))
2. table=105, priority=190,ct_state=+new+trk,ip actions=ct(commit,table=110,zone=65520)
3. table=105, priority=0 actions=goto_table:110
```
Expand Down
31 changes: 17 additions & 14 deletions pkg/agent/controller/networkpolicy/packetin.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,24 +59,27 @@ func (c *Controller) HandlePacketIn(pktIn *ofctrl.PacketIn) error {

matches := pktIn.GetMatches()
// Get custom reasons in this packet-in.
match := getMatchRegField(matches, uint32(openflow.CustomReasonMarkReg))
customReasons, err := getInfoInReg(match, openflow.CustomReasonMarkRange.ToNXRange())
match := getMatchRegField(matches, openflow.CustomReasonField)
customReasons, err := getInfoInReg(match, openflow.CustomReasonField.GetRange().ToNXRange())
if err != nil {
return fmt.Errorf("received error while unloading customReason from reg: %v", err)
}

// Use reasons to choose operations.
if customReasons&openflow.CustomReasonLogging == openflow.CustomReasonLogging {
var checkCustomReason = func(customReasonMark *binding.RegMark) bool {
return customReasons&customReasonMark.GetValue() == customReasonMark.GetValue()
}
if checkCustomReason(openflow.CustomReasonLoggingRegMark) {
if err := c.logPacket(pktIn); err != nil {
return err
}
}
if customReasons&openflow.CustomReasonReject == openflow.CustomReasonReject {
if checkCustomReason(openflow.CustomReasonRejectRegMark) {
if err := c.rejectRequest(pktIn); err != nil {
return err
}
}
if customReasons&openflow.CustomReasonDeny == openflow.CustomReasonDeny {
if checkCustomReason(openflow.CustomReasonDenyRegMark) {
if err := c.storeDenyConnection(pktIn); err != nil {
return err
}
Expand All @@ -85,26 +88,26 @@ func (c *Controller) HandlePacketIn(pktIn *ofctrl.PacketIn) error {
}

// getMatchRegField returns match to the regNum register.
func getMatchRegField(matchers *ofctrl.Matchers, regNum uint32) *ofctrl.MatchField {
return matchers.GetMatchByName(fmt.Sprintf("NXM_NX_REG%d", regNum))
func getMatchRegField(matchers *ofctrl.Matchers, field *binding.RegField) *ofctrl.MatchField {
return matchers.GetMatchByName(field.GetNXFieldName())
}

// getMatch receives ofctrl matchers and table id, match field.
// Modifies match field to Ingress/Egress register based on tableID.
func getMatch(matchers *ofctrl.Matchers, tableID binding.TableIDType, disposition uint32) *ofctrl.MatchField {
// Get match from CNPDenyConjIDReg if disposition is not allow.
if disposition != openflow.DispositionAllow {
return getMatchRegField(matchers, uint32(openflow.CNPDenyConjIDReg))
return getMatchRegField(matchers, openflow.CNPDenyConjIDField)
}
// Get match from ingress/egress reg if disposition is allow
for _, table := range append(openflow.GetAntreaPolicyEgressTables(), openflow.EgressRuleTable) {
if tableID == table {
return getMatchRegField(matchers, uint32(openflow.EgressReg))
return getMatchRegField(matchers, openflow.TFEgressConjIDField)
}
}
for _, table := range append(openflow.GetAntreaPolicyIngressTables(), openflow.IngressRuleTable) {
if tableID == table {
return getMatchRegField(matchers, uint32(openflow.IngressReg))
return getMatchRegField(matchers, openflow.TFIngressConjIDField)
}
}
return nil
Expand All @@ -131,8 +134,8 @@ func getNetworkPolicyInfo(pktIn *ofctrl.PacketIn, c *Controller, ob *logInfo) er
ob.tableName = openflow.GetFlowTableName(tableID)

// Get disposition Allow or Drop
match = getMatchRegField(matchers, uint32(openflow.DispositionMarkReg))
info, err := getInfoInReg(match, openflow.APDispositionMarkRange.ToNXRange())
match = getMatchRegField(matchers, openflow.APDispositionField)
info, err := getInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange())
if err != nil {
return fmt.Errorf("received error while unloading disposition from reg: %v", err)
}
Expand Down Expand Up @@ -325,8 +328,8 @@ func (c *Controller) storeDenyConnection(pktIn *ofctrl.PacketIn) error {
// Get table ID
tableID := binding.TableIDType(pktIn.TableId)
// Get disposition Allow, Drop or Reject
match = getMatchRegField(matchers, uint32(openflow.DispositionMarkReg))
id, err := getInfoInReg(match, openflow.APDispositionMarkRange.ToNXRange())
match = getMatchRegField(matchers, openflow.APDispositionField)
id, err := getInfoInReg(match, openflow.APDispositionField.GetRange().ToNXRange())
if err != nil {
return fmt.Errorf("error when getting disposition from reg: %v", err)
}
Expand Down
12 changes: 6 additions & 6 deletions pkg/agent/controller/traceflow/packetin.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
obs = append(obs, *ob)
}
// Collect egress conjunctionID and get NetworkPolicy from cache.
if match := getMatchRegField(matchers, uint32(openflow.EgressReg)); match != nil {
if match := getMatchRegField(matchers, openflow.TFEgressConjIDField); match != nil {
egressInfo, err := getRegValue(match, nil)
if err != nil {
return nil, nil, nil, err
Expand All @@ -191,7 +191,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
}

// Collect ingress conjunctionID and get NetworkPolicy from cache.
if match := getMatchRegField(matchers, uint32(openflow.IngressReg)); match != nil {
if match := getMatchRegField(matchers, openflow.TFIngressConjIDField); match != nil {
ingressInfo, err := getRegValue(match, nil)
if err != nil {
return nil, nil, nil, err
Expand All @@ -207,7 +207,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
// Get drop table.
if tableID == uint8(openflow.EgressMetricTable) || tableID == uint8(openflow.IngressMetricTable) {
ob := getNetworkPolicyObservation(tableID, tableID == uint8(openflow.IngressMetricTable))
if match := getMatchRegField(matchers, uint32(openflow.CNPDenyConjIDReg)); match != nil {
if match := getMatchRegField(matchers, openflow.CNPDenyConjIDField); match != nil {
notAllowConjInfo, err := getRegValue(match, nil)
if err != nil {
return nil, nil, nil, err
Expand Down Expand Up @@ -239,7 +239,7 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
}
}
var outputPort uint32
if match := getMatchRegField(matchers, uint32(openflow.PortCacheReg)); match != nil {
if match := getMatchRegField(matchers, openflow.TargetOFPortField); match != nil {
outputPort, err = getRegValue(match, nil)
if err != nil {
return nil, nil, nil, err
Expand Down Expand Up @@ -271,8 +271,8 @@ func (c *Controller) parsePacketIn(pktIn *ofctrl.PacketIn) (*crdv1alpha1.Tracefl
return tf, &nodeResult, capturedPacket, nil
}

func getMatchRegField(matchers *ofctrl.Matchers, regNum uint32) *ofctrl.MatchField {
return matchers.GetMatchByName(fmt.Sprintf("NXM_NX_REG%d", regNum))
func getMatchRegField(matchers *ofctrl.Matchers, field *binding.RegField) *ofctrl.MatchField {
return matchers.GetMatchByName(field.GetNXFieldName())
}

func getMatchTunnelDstField(matchers *ofctrl.Matchers, isIPv6 bool) *ofctrl.MatchField {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ func (cs *ConntrackConnectionStore) AddOrUpdateConn(conn *flowexporter.Connectio
klog.V(4).Infof("Antrea flow updated: %v", existingConn)
} else {
cs.fillPodInfo(conn)
if conn.Mark == openflow.ServiceCTMark {
if conn.Mark == openflow.ServiceCTMark.GetValue() {
clusterIP := conn.DestinationServiceAddress.String()
svcPort := conn.DestinationServicePort
protocol, err := lookupServiceProtocol(conn.FlowKey.Protocol)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func TestConntrackConnectionStore_AddOrUpdateConn(t *testing.T) {
StartTime: refTime.Add(-(time.Second * 50)),
StopTime: refTime,
FlowKey: tuple4,
Mark: openflow.ServiceCTMark,
Mark: openflow.ServiceCTMark.GetValue(),
IsPresent: true,
}
// Create copy of old conntrack flow for testing purposes.
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/flowexporter/exporter/exporter.go
Original file line number Diff line number Diff line change
Expand Up @@ -705,7 +705,7 @@ func (exp *flowExporter) findFlowType(conn flowexporter.Connection) uint8 {
return 0
}
if exp.nodeRouteController.IPInPodSubnets(conn.FlowKey.SourceAddress) {
if conn.Mark == openflow.ServiceCTMark || exp.nodeRouteController.IPInPodSubnets(conn.FlowKey.DestinationAddress) {
if conn.Mark == openflow.ServiceCTMark.GetValue() || exp.nodeRouteController.IPInPodSubnets(conn.FlowKey.DestinationAddress) {
if conn.SourcePodName == "" || conn.DestinationPodName == "" {
return ipfixregistry.FlowTypeInterNode
}
Expand Down
6 changes: 2 additions & 4 deletions pkg/agent/openflow/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -1040,8 +1040,7 @@ func (c *client) SendTCPPacketOut(

// Reject response packet should bypass ConnTrack
if isReject {
name := fmt.Sprintf("%s%d", binding.NxmFieldReg, marksReg)
packetOutBuilder = packetOutBuilder.AddLoadAction(name, uint64(CustomReasonReject), CustomReasonMarkRange)
packetOutBuilder = packetOutBuilder.AddLoadRegMark(CustomReasonRejectRegMark)
}

packetOutObj := packetOutBuilder.Done()
Expand Down Expand Up @@ -1079,8 +1078,7 @@ func (c *client) SendICMPPacketOut(

// Reject response packet should bypass ConnTrack
if isReject {
name := fmt.Sprintf("%s%d", binding.NxmFieldReg, marksReg)
packetOutBuilder = packetOutBuilder.AddLoadAction(name, uint64(CustomReasonReject), CustomReasonMarkRange)
packetOutBuilder = packetOutBuilder.AddLoadRegMark(CustomReasonRejectRegMark)
}

packetOutObj := packetOutBuilder.Done()
Expand Down
128 changes: 128 additions & 0 deletions pkg/agent/openflow/fields.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright 2019 Antrea Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package openflow

import binding "antrea.io/antrea/pkg/ovs/openflow"

// Fields using reg.
var (
// reg0 (NXM_NX_REG0)
// reg0[0..3]: Field to mark the packet source. Marks in this field include,
// - 0: from the tunnel port
// - 1: from antrea-gw0
// - 2: from the local Pods
// - 4: from the Bridge interface
// - 5: from the uplink interface
PktSourceField = binding.NewRegField(0, 0, 3, "PacketSource")
FromTunnelRegMark = binding.NewRegMark(PktSourceField, 0)
FromGatewayRegMark = binding.NewRegMark(PktSourceField, 1)
FromLocalRegMark = binding.NewRegMark(PktSourceField, 2)
FromUplinkRegMark = binding.NewRegMark(PktSourceField, 4)
FromBridgeRegMark = binding.NewRegMark(PktSourceField, 5)
// reg0[16]: Mark to indicate the ofPort number of an interface is found.
OFPortFoundRegMark = binding.NewOneBitRegMark(0, 16, "OFPortFound")
// reg0[17]: Mark to indicate the packet needs to be SNATed with Node's IP.
SNATNodeIPRegMark = binding.NewOneBitRegMark(0, 17, "SNATWithNodeIP")
// reg0[18]: Mark to indicate the packet needs DNAT to virtual IP.
// If a packet uses HairpinRegMark, it will be output to the port where it enters OVS pipeline in L2ForwardingOutTable.
HairpinRegMark = binding.NewOneBitRegMark(0, 18, "Hairpin")
// reg0[19]: Mark to indicate the packet's MAC address needs to be rewritten.
RewriteMACRegMark = binding.NewOneBitRegMark(0, 19, "RewriteMAC")
// reg0[20]: Mark to indicate the packet is denied(Drop/Reject).
CnpDenyRegMark = binding.NewOneBitRegMark(0, 20, "CNPDeny")
// reg0[21..22]: Field to indicate disposition of Antrea Policy. It could have more bits to support more disposition
// that Antrea policy support in the future.
// Marks in this field include,
// - 0b00: allow
// - 0b01: drop
// - 0b10: reject
APDispositionField = binding.NewRegField(0, 21, 22, "APDisposition")
DispositionAllowRegMark = binding.NewRegMark(APDispositionField, DispositionAllow)
DispositionDropRegMark = binding.NewRegMark(APDispositionField, DispositionDrop)
DispositionRejRegMark = binding.NewRegMark(APDispositionField, DispositionRej)
// reg0[24..26]: Field to indicate the reasons of sending packet to the controller.
// Marks in this field include,
// - 0b001: logging
// - 0b010: reject
// - 0b100: deny (used by Flow Exporter)
CustomReasonField = binding.NewRegField(0, 24, 26, "PacketInReason")
CustomReasonLoggingRegMark = binding.NewRegMark(CustomReasonField, CustomReasonLogging)
CustomReasonRejectRegMark = binding.NewRegMark(CustomReasonField, CustomReasonReject)
CustomReasonDenyRegMark = binding.NewRegMark(CustomReasonField, CustomReasonDeny)

// reg1(NXM_NX_REG1)
// Field to cache the ofPort of the OVS interface where to output packet.
TargetOFPortField = binding.NewRegField(1, 0, 31, "TargetOFPort")

// reg2(NXM_NX_REG2)
// Field to help swap values in two different flow fields in the OpenFlow actions. This field is only used in func
// `arpResponderStaticFlow`.
SwapField = binding.NewRegField(2, 0, 31, "SwapValue")

// reg3(NXM_NX_REG3)
// Field to store the selected Service Endpoint IP
EndpointIPField = binding.NewRegField(3, 0, 31, "EndpointIP")
// Field to store the conjunction ID which is for "deny" rule in CNP. It shares the same register with EndpointIPField,
// since the service selection will finish when a packet hitting NetworkPolicy related rules.
CNPDenyConjIDField = binding.NewRegField(3, 0, 31, "CNPDenyConjunctionID")

// reg4(NXM_NX_REG4)
// reg4[0..15]: Field to store the selected Service Endpoint port.
EndpointPortField = binding.NewRegField(4, 0, 15, "EndpointPort")
// reg4[16..18]: Field to store the state of a packet accessing a Service. Marks in this field include,
// - 0b001: packet need to do service selection.
// - 0b010: packet has done service selection.
// - 0b011: packet has done service selection and the selection result needs to be cached.
ServiceEPStateField = binding.NewRegField(4, 16, 18, "EndpointState")
EpToSelectRegMark = binding.NewRegMark(ServiceEPStateField, 0b001)
EpSelectedRegMark = binding.NewRegMark(ServiceEPStateField, 0b010)
EpToLearnRegMark = binding.NewRegMark(ServiceEPStateField, 0b011)
// reg4[0..18]: Field to store the union value of Endpoint port and Endpoint status. It is used as a single match
// when needed.
EpUnionField = binding.NewRegField(4, 0, 18, "EndpointUnion")

// reg5(NXM_NX_REG5)
// Field to cache the Egress conjunction ID hit by TraceFlow packet.
TFEgressConjIDField = binding.NewRegField(5, 0, 31, "TFEgressConjunctionID")

// reg(N6XM_NX_REG6)
// Field to store the Ingress conjunction ID hit by TraceFlow packet.
TFIngressConjIDField = binding.NewRegField(6, 0, 31, "TFIngressConjunctionID")
)

// Fields using xxreg.
var (
// xxreg3(NXM_NX_XXREG3)
// xxreg3: Field to cache Endpoint IPv6 address. It occupies reg12-reg15 in the meanwhile.
EndpointIP6Field = binding.NewXXRegField(3, 0, 127)
)

// Marks using CT.
var (
// Mark to indicate the connection is initiated through the host gateway interface
// (i.e. for which the first packet of the connection was received through the gateway).
FromGatewayCTMark = binding.NewCtMark(0x20, 0, 31)
// Mark to indicate DNAT is performed on the connection for Service.
ServiceCTMark = binding.NewCtMark(0x21, 0, 31)
)

// Fields using CT label.
var (
// Field to store the ingress rule ID.
IngressRuleCTLabelField = binding.NewCtLabelField(0, 31, "ingressRuleCtLabel")

// Field to store the egress rule ID.
EgressRuleCTLabelField = binding.NewCtLabelField(32, 63, "egressRuleCtLabel")
)
Loading

0 comments on commit 6bc4a10

Please sign in to comment.