diff --git a/kernel-headers/rdma/mana-abi.h b/kernel-headers/rdma/mana-abi.h index 32cbbfc80..c9a241766 100644 --- a/kernel-headers/rdma/mana-abi.h +++ b/kernel-headers/rdma/mana-abi.h @@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp { __u32 reserved; }; +enum mana_ib_create_rc_qp_flags { + MANA_IB_RC_QP_FIXED_WQE = 1 << 0, + MANA_IB_RC_MMQ_CREATE = 1 << 1, +}; + struct mana_ib_create_rc_qp { __aligned_u64 queue_buf[4]; __u32 queue_size[4]; + __aligned_u64 mmq_buf; + __u32 mmq_size; + __u32 comp_mask; }; struct mana_ib_create_rc_qp_resp { __u32 queue_id[4]; + __u32 mmq_id; + __u32 reserved; }; struct mana_ib_create_uc_qp { @@ -100,6 +110,7 @@ struct mana_ib_create_qp_rss_resp { enum mana_ib_ucontext_support { MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0, + MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1, }; struct mana_ib_alloc_ucontext_resp { diff --git a/providers/mana/gdma.h b/providers/mana/gdma.h index 1f797c7d3..0c1dd8320 100644 --- a/providers/mana/gdma.h +++ b/providers/mana/gdma.h @@ -68,7 +68,7 @@ struct gdma_sge { struct rdma_recv_oob { uint32_t psn_start:24; uint32_t reserved1:8; - uint32_t psn_range:24; + uint32_t msn:24; uint32_t reserved2:8; }; /* HW DATA */ diff --git a/providers/mana/mana.h b/providers/mana/mana.h index 98318d1c0..4934d6970 100644 --- a/providers/mana/mana.h +++ b/providers/mana/mana.h @@ -18,6 +18,7 @@ #define INLINE_OOB_SMALL_SIZE 8 #define INLINE_OOB_LARGE_SIZE 24 +#define INLINE_OOB_EXTRA_LARGE_SIZE 32 #define GDMA_WQE_ALIGNMENT_UNIT_SIZE 32 @@ -73,9 +74,21 @@ static inline uint32_t get_wqe_size(uint32_t sge) return align(wqe_size, GDMA_WQE_ALIGNMENT_UNIT_SIZE); } -static inline uint32_t get_large_wqe_size(uint32_t sge) +static inline uint32_t get_large_fixed_wqe_size(uint32_t sge) { - uint32_t wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_LARGE_SIZE; + uint32_t wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_EXTRA_LARGE_SIZE; + + return roundup_pow_of_two(wqe_size); +} + +static inline uint32_t get_large_wqe_size(uint32_t sge, uint32_t wqe_size_in_bu) +{ + uint32_t wqe_size; + + if (wqe_size_in_bu) + return wqe_size_in_bu * GDMA_WQE_ALIGNMENT_UNIT_SIZE; + + wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_LARGE_SIZE; return align(wqe_size, GDMA_WQE_ALIGNMENT_UNIT_SIZE); } @@ -108,6 +121,7 @@ struct mana_gdma_queue { uint32_t size; uint32_t prod_idx; uint32_t cons_idx; + uint8_t wqe_size_in_bu; void *db_page; void *buffer; diff --git a/providers/mana/qp.c b/providers/mana/qp.c index 752959bbd..f94a4468a 100644 --- a/providers/mana/qp.c +++ b/providers/mana/qp.c @@ -222,17 +222,17 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty switch (type) { case USER_RNIC_SEND_QUEUE_REQUESTER: /* WQE must have at least one SGE */ - /* For write with imm we need one extra SGE */ - sges = max(1U, attr->cap.max_send_sge) + 1; - size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges)); + sges = max(1U, attr->cap.max_send_sge); + size = align_hw_size(attr->cap.max_send_wr * get_large_fixed_wqe_size(sges)); break; case USER_RNIC_SEND_QUEUE_RESPONDER: if (attr->qp_type == IBV_QPT_RC) size = align_hw_size(MANA_PAGE_SIZE); break; case USER_RNIC_RECV_QUEUE_REQUESTER: + sges = max(1U, attr->cap.max_send_sge); if (attr->qp_type == IBV_QPT_RC) - size = align_hw_size(MANA_PAGE_SIZE); + size = align_hw_size(attr->cap.max_send_wr * get_wqe_size(sges)); break; case USER_RNIC_RECV_QUEUE_RESPONDER: /* WQE must have at least one SGE */ @@ -240,8 +240,8 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges)); break; case USER_RNIC_SEND_QUEUE_MM: - sges = 2; - size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges)); + sges = 1; + size = align_hw_size(attr->cap.max_send_wr * get_large_fixed_wqe_size(sges)); break; default: return 0; @@ -256,6 +256,7 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd, struct ibv_qp_init_attr *attr) { + struct mana_context *ctx = to_mctx(ibpd->context); struct mana_ib_create_rc_qp_resp *qp_resp_drv; struct mana_create_rc_qp_resp qp_resp = {}; struct mana_ib_create_rc_qp *qp_cmd_drv; @@ -272,6 +273,13 @@ static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd, qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size; } + if (ctx->comp_mask & MANA_IB_UCNTX_RC_EXT_SUPPORT) { + qp_cmd_drv->comp_mask |= MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE; + qp_cmd_drv->mmq_buf = + (uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer; + qp_cmd_drv->mmq_size = qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size; + } + ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd, sizeof(qp_cmd), &qp_resp.ibv_resp, sizeof(qp_resp)); @@ -286,6 +294,8 @@ static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd, qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i]; } + qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id = qp_resp_drv->mmq_id; + return 0; } @@ -355,6 +365,7 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd, struct ibv_qp_init_attr *attr) { struct mana_context *ctx = to_mctx(ibpd->context); + uint8_t wqe_size_in_bu; struct mana_qp *qp; int ret, i; @@ -393,6 +404,10 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd, } } + wqe_size_in_bu = get_large_fixed_wqe_size(max(1U, attr->cap.max_send_sge)) + / GDMA_WQE_ALIGNMENT_UNIT_SIZE; + qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].wqe_size_in_bu = wqe_size_in_bu; + ret = mana_create_cmd_qp(qp, ibpd, attr); if (ret) { errno = ret; diff --git a/providers/mana/wr.c b/providers/mana/wr.c index e4d7c38ae..3ea394151 100644 --- a/providers/mana/wr.c +++ b/providers/mana/wr.c @@ -249,7 +249,7 @@ gdma_post_sq_wqe(struct mana_gdma_queue *wq, struct ibv_sge *sgl, struct rdma_se } total_sge = num_sge + (oob_sge ? 1 : 0); - wqe_size = get_large_wqe_size(total_sge); + wqe_size = get_large_wqe_size(total_sge, wq->wqe_size_in_bu); ret = gdma_get_current_wqe(wq, INLINE_OOB_LARGE_SIZE, wqe_size, wqe); if (ret) @@ -299,6 +299,7 @@ mana_ib_post_send_request(struct mana_qp *qp, struct ibv_send_wr *wr, struct rdma_recv_oob recv_oob = {0}; recv_oob.psn_start = qp->sq_psn; + recv_oob.msn = qp->sq_ssn; ret = gdma_post_rq_wqe(mana_ib_get_rreq(qp), wr->sg_list, &recv_oob, num_sge, GDMA_WORK_REQ_CHECK_SN, &gdma_wqe); if (ret) {