AL2 5.15 Update EFA driver to 2.13.0

Signed-off-by: Michael Margolin <mrgolin@amazon.com>
Signed-off-by: Mahmoud Adam <mngyadam@amazon.com>

Author: mrgolin

drivers/amazon/net/efa/Makefile

@@ -7,6 +7,6 @@
 obj-$(CONFIG_AMAZON_EFA_INFINIBAND) += efa.o
 
 efa-y := efa_com.o efa_com_cmd.o efa_gdr.o efa_main.o efa_neuron.o efa_p2p.o
-efa-y += efa_verbs.o
+efa-y += efa_verbs.o efa_data_verbs.o
 
 efa-$(CONFIG_SYSFS) += efa_sysfs.o

drivers/amazon/net/efa/efa.h

@@ -44,6 +44,8 @@ struct efa_stats {
 	atomic64_t create_ah_err;
 	atomic64_t mmap_err;
 	atomic64_t keep_alive_rcvd;
+	atomic64_t alloc_mr_err;
+	atomic64_t get_dma_mr_err;
 };
 
 struct efa_dev {
@@ -58,7 +60,10 @@ struct efa_dev {
 	u64 mem_bar_len;
 	u64 db_bar_addr;
 	u64 db_bar_len;
+	u8 __iomem *mem_bar;
+	u8 __iomem *db_bar;
 
+	unsigned int num_irq_vectors;
 	int admin_msix_vector_idx;
 	struct efa_irq admin_irq;
 
@@ -70,6 +75,11 @@ struct efa_dev {
 
 	/* Only stores CQs with interrupts enabled */
 	struct xarray cqs_xa;
+	u16 uarn;
+	struct efa_qp **qp_table;
+	/* Protects against simultaneous QPs insertion or removal */
+	spinlock_t qp_table_lock;
+	u32 qp_table_mask;
 };
 
 struct efa_ucontext {
@@ -93,10 +103,32 @@ struct efa_mr_interconnect_info {
 
 struct efa_mr {
 	struct ib_mr ibmr;
-	struct ib_umem *umem;
-	struct efa_mr_interconnect_info ic_info;
-	struct efa_p2pmem *p2pmem;
-	u64 p2p_ticket;
+	union {
+		/* Used only by kernel MRs */
+		struct {
+			u64 *pages_list;
+			u32 pages_list_length;
+			dma_addr_t pages_list_dma_addr;
+			u32 num_pages;
+		};
+
+		/* Used only by User MRs */
+		struct {
+			struct ib_umem *umem;
+			struct efa_mr_interconnect_info ic_info;
+			struct efa_p2pmem *p2pmem;
+			u64 p2p_ticket;
+		};
+	};
+};
+
+struct efa_sub_cq {
+	u8 *buf;
+	u32 cqe_size;
+	u32 queue_mask;
+	u32 ref_cnt;
+	u32 consumed_cnt;
+	int phase;
 };
 
 struct efa_cq {
@@ -110,6 +142,57 @@ struct efa_cq {
 	u16 cq_idx;
 	/* NULL when no interrupts requested */
 	struct efa_eq *eq;
+	u8 *buf;
+	size_t buf_size;
+	struct efa_sub_cq *sub_cq_arr;
+	u16 num_sub_cqs;
+	u32 *db;
+	/* Index of next sub cq idx to poll. This is used to guarantee fairness for sub cqs */
+	u16 next_poll_idx;
+	/* Protects the access to the CQ and CQ to QP association */
+	u16 cc; /* Consumer Counter */
+	u8 cmd_sn;
+	spinlock_t lock;
+};
+
+struct efa_wq {
+	u64 *wrid;
+	/* wrid_idx_pool: Pool of free indexes in the wrid array, used to select the
+	 * wrid entry to be used to hold the next tx packet's context.
+	 * At init time, entry N will hold value N, as OOO tx-completions arrive,
+	 * the value stored in a given entry might not equal the entry's index.
+	 */
+	u32 *wrid_idx_pool;
+	/* wrid_idx_pool_next: Index of the next entry to use in wrid_idx_pool. */
+	u32 wrid_idx_pool_next;
+	u32 max_sge;
+	u32 max_wqes;
+	u32 queue_mask;
+	u32 *db;
+	u32 wqes_posted;
+	u32 wqes_completed;
+	/* Producer counter */
+	u32 pc;
+	int phase;
+	u16 sub_cq_idx;
+	/* Synchronizes access to the WQ on datapath */
+	spinlock_t lock;
+};
+
+struct efa_rq {
+	struct efa_wq wq;
+	u8 *buf;
+	size_t buf_size;
+};
+
+struct efa_sq {
+	struct efa_wq wq;
+	u8 *desc;
+	u32 desc_offset;
+	u32 max_inline_data;
+	u32 max_rdma_sges;
+	u32 max_batch_wr;
+	enum ib_sig_type sig_type;
 };
 
 struct efa_qp {
@@ -131,6 +214,8 @@ struct efa_qp {
 	u32 max_send_sge;
 	u32 max_recv_sge;
 	u32 max_inline_data;
+	struct efa_sq sq;
+	struct efa_rq rq;
 };
 
 struct efa_ah {
@@ -172,6 +257,9 @@ struct ib_mr *efa_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 start,
 				     u64 length, u64 virt_addr,
 				     int fd, int access_flags,
 				     struct ib_udata *udata);
+struct ib_mr *efa_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+struct ib_mr *efa_alloc_fast_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
+				u32 max_num_sg);
 int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
 int efa_get_port_immutable(struct ib_device *ibdev, port_t port_num,
 			   struct ib_port_immutable *immutable);
@@ -184,6 +272,16 @@ int efa_create_ah(struct ib_ah *ibah,
 		  struct rdma_ah_init_attr *init_attr,
 		  struct ib_udata *udata);
 int efa_destroy_ah(struct ib_ah *ibah, u32 flags);
+int efa_post_send(struct ib_qp *ibqp,
+		  const struct ib_send_wr *wr,
+		  const struct ib_send_wr **bad_wr);
+int efa_post_recv(struct ib_qp *ibqp,
+		  const struct ib_recv_wr *wr,
+		  const struct ib_recv_wr **bad_wr);
+int efa_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc);
+int efa_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);
+int efa_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
+		  unsigned int *sg_offset);
 int efa_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 		  int qp_attr_mask, struct ib_udata *udata);
 enum rdma_link_layer efa_port_link_layer(struct ib_device *ibdev,

drivers/amazon/net/efa/efa_admin_cmds_defs.h

@@ -30,7 +30,8 @@ enum efa_admin_aq_opcode {
 	EFA_ADMIN_DEALLOC_UAR                       = 17,
 	EFA_ADMIN_CREATE_EQ                         = 18,
 	EFA_ADMIN_DESTROY_EQ                        = 19,
-	EFA_ADMIN_MAX_OPCODE                        = 19,
+	EFA_ADMIN_ALLOC_MR                          = 20,
+	EFA_ADMIN_MAX_OPCODE                        = 20,
 };
 
 enum efa_admin_aq_feature_id {
@@ -110,7 +111,10 @@ struct efa_admin_create_qp_cmd {
 	 *    virtual (IOVA returned by MR registration)
 	 * 1 : rq_virt - If set, RQ ring base address is
 	 *    virtual (IOVA returned by MR registration)
-	 * 7:2 : reserved - MBZ
+	 * 2 : unsolicited_write_recv - If set, work requests
+	 *    will not be consumed for incoming RDMA write with
+	 *    immediate
+	 * 7:3 : reserved - MBZ
 	 */
 	u8 flags;
 
@@ -147,8 +151,11 @@ struct efa_admin_create_qp_cmd {
 	/* UAR number */
 	u16 uar;
 
+	/* Requested service level for the QP, 0 is the default SL */
+	u8 sl;
+
 	/* MBZ */
-	u16 reserved;
+	u8 reserved;
 
 	/* MBZ */
 	u32 reserved2;
@@ -456,6 +463,41 @@ struct efa_admin_dereg_mr_resp {
 	struct efa_admin_acq_common_desc acq_common_desc;
 };
 
+/*
+ * Allocation of MemoryRegion, required for QP working with Virtual
+ * Addresses in kernel verbs semantics, ready for fast registration use.
+ */
+struct efa_admin_alloc_mr_cmd {
+	/* Common Admin Queue descriptor */
+	struct efa_admin_aq_common_desc aq_common_desc;
+
+	/* Protection Domain */
+	u16 pd;
+
+	/* MBZ */
+	u16 reserved1;
+
+	/* Maximum number of pages this MR supports. */
+	u32 max_pages;
+};
+
+struct efa_admin_alloc_mr_resp {
+	/* Common Admin Queue completion descriptor */
+	struct efa_admin_acq_common_desc acq_common_desc;
+
+	/*
+	 * L_Key, to be used in conjunction with local buffer references in
+	 * SQ and RQ WQE, or with virtual RQ/CQ rings
+	 */
+	u32 l_key;
+
+	/*
+	 * R_Key, to be used in RDMA messages to refer to remotely accessed
+	 * memory region
+	 */
+	u32 r_key;
+};
+
 struct efa_admin_create_cq_cmd {
 	struct efa_admin_aq_common_desc aq_common_desc;
 
@@ -480,8 +522,8 @@ struct efa_admin_create_cq_cmd {
 	 */
 	u8 cq_caps_2;
 
-	/* completion queue depth in # of entries. must be power of 2 */
-	u16 cq_depth;
+	/* Sub completion queue depth in # of entries. must be power of 2 */
+	u16 sub_cq_depth;
 
 	/* EQ number assigned to this cq */
 	u16 eqn;
@@ -516,8 +558,8 @@ struct efa_admin_create_cq_resp {
 
 	u16 cq_idx;
 
-	/* actual cq depth in number of entries */
-	u16 cq_actual_depth;
+	/* actual sub cq depth in number of entries */
+	u16 sub_cq_actual_depth;
 
 	/* CQ doorbell address, as offset to PCIe DB BAR */
 	u32 db_offset;
@@ -575,6 +617,8 @@ struct efa_admin_basic_stats {
 	u64 rx_pkts;
 
 	u64 rx_drops;
+
+	u64 qkey_viol;
 };
 
 struct efa_admin_messages_stats {
@@ -663,12 +707,26 @@ struct efa_admin_feature_device_attr_desc {
 	 *    polling is supported
 	 * 3 : rdma_write - If set, RDMA Write is supported
 	 *    on TX queues
-	 * 31:4 : reserved - MBZ
+	 * 4 : unsolicited_write_recv - If set, unsolicited
+	 *    write with imm. receive is supported
+	 * 31:5 : reserved - MBZ
 	 */
 	u32 device_caps;
 
 	/* Max RDMA transfer size in bytes */
 	u32 max_rdma_size;
+
+	/* Unique global ID for an EFA device */
+	u64 guid;
+
+	/* The device maximum link speed in Gbit/sec */
+	u16 max_link_speed_gbps;
+
+	/* MBZ */
+	u16 reserved0;
+
+	/* MBZ */
+	u32 reserved1;
 };
 
 struct efa_admin_feature_queue_attr_desc {
@@ -1009,6 +1067,7 @@ struct efa_admin_host_info {
 /* create_qp_cmd */
 #define EFA_ADMIN_CREATE_QP_CMD_SQ_VIRT_MASK                BIT(0)
 #define EFA_ADMIN_CREATE_QP_CMD_RQ_VIRT_MASK                BIT(1)
+#define EFA_ADMIN_CREATE_QP_CMD_UNSOLICITED_WRITE_RECV_MASK BIT(2)
 
 /* modify_qp_cmd */
 #define EFA_ADMIN_MODIFY_QP_CMD_QP_STATE_MASK               BIT(0)
@@ -1044,10 +1103,10 @@ struct efa_admin_host_info {
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RNR_RETRY_MASK   BIT(1)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_DATA_POLLING_128_MASK BIT(2)
 #define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_RDMA_WRITE_MASK  BIT(3)
+#define EFA_ADMIN_FEATURE_DEVICE_ATTR_DESC_UNSOLICITED_WRITE_RECV_MASK BIT(4)
 
 /* create_eq_cmd */
 #define EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS_MASK       GENMASK(4, 0)
-#define EFA_ADMIN_CREATE_EQ_CMD_VIRT_MASK                   BIT(6)
 #define EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS_MASK      BIT(0)
 
 /* host_info */

drivers/amazon/net/efa/efa_admin_defs.h

@@ -1,6 +1,6 @@
 /* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */
 /*
- * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
+ * Copyright 2018-2024 Amazon.com, Inc. or its affiliates. All rights reserved.
  */
 
 #ifndef _EFA_ADMIN_H_
@@ -96,7 +96,7 @@ struct efa_admin_acq_entry {
 struct efa_admin_aenq_common_desc {
 	u16 group;
 
-	u16 syndrom;
+	u16 syndrome;
 
 	/*
 	 * 0 : phase