Skip to content

providers/mana: Extend MANA RC QP support for UAPI #1795

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
TaranovK wants to merge 2 commits into
base: master
Choose a base branch
from
+51 −10
Open

providers/mana: Extend MANA RC QP support for UAPI #1795

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c30a641
Update kernel headers
Sep 3, 2026
d82b030
providers/mana: Fixed Wqe and MMQ for rc qp
Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions kernel-headers/rdma/mana-abi.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,23 @@ struct mana_ib_create_qp_resp {
__u32 reserved;
};

enum mana_ib_create_rc_qp_flags {
MANA_IB_RC_QP_FIXED_WQE = 1 << 0,
MANA_IB_RC_MMQ_CREATE = 1 << 1,
};

struct mana_ib_create_rc_qp {
__aligned_u64 queue_buf[4];
__u32 queue_size[4];
__aligned_u64 mmq_buf;
__u32 mmq_size;
__u32 comp_mask;
};

struct mana_ib_create_rc_qp_resp {
__u32 queue_id[4];
__u32 mmq_id;
__u32 reserved;
};

struct mana_ib_create_uc_qp {
Expand Down Expand Up @@ -100,6 +110,7 @@ struct mana_ib_create_qp_rss_resp {

enum mana_ib_ucontext_support {
MANA_IB_UCNTX_ALLOC_PDN_SUPPORT = 1 << 0,
MANA_IB_UCNTX_RC_EXT_SUPPORT = 1 << 1,
};

struct mana_ib_alloc_ucontext_resp {
Expand Down
2 changes: 1 addition & 1 deletion providers/mana/gdma.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ struct gdma_sge {
struct rdma_recv_oob {
uint32_t psn_start:24;
uint32_t reserved1:8;
uint32_t psn_range:24;
uint32_t msn:24;
uint32_t reserved2:8;
}; /* HW DATA */

Expand Down
18 changes: 16 additions & 2 deletions providers/mana/mana.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

#define INLINE_OOB_SMALL_SIZE 8
#define INLINE_OOB_LARGE_SIZE 24
#define INLINE_OOB_EXTRA_LARGE_SIZE 32

#define GDMA_WQE_ALIGNMENT_UNIT_SIZE 32

Expand Down Expand Up @@ -73,9 +74,21 @@ static inline uint32_t get_wqe_size(uint32_t sge)
return align(wqe_size, GDMA_WQE_ALIGNMENT_UNIT_SIZE);
}

static inline uint32_t get_large_wqe_size(uint32_t sge)
static inline uint32_t get_large_fixed_wqe_size(uint32_t sge)
{
uint32_t wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_LARGE_SIZE;
uint32_t wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_EXTRA_LARGE_SIZE;

return roundup_pow_of_two(wqe_size);
}

static inline uint32_t get_large_wqe_size(uint32_t sge, uint32_t wqe_size_in_bu)
{
uint32_t wqe_size;

if (wqe_size_in_bu)
return wqe_size_in_bu * GDMA_WQE_ALIGNMENT_UNIT_SIZE;

wqe_size = sge * SGE_SIZE + DMA_OOB_SIZE + INLINE_OOB_LARGE_SIZE;

return align(wqe_size, GDMA_WQE_ALIGNMENT_UNIT_SIZE);
}
Expand Down Expand Up @@ -108,6 +121,7 @@ struct mana_gdma_queue {
uint32_t size;
uint32_t prod_idx;
uint32_t cons_idx;
uint8_t wqe_size_in_bu;

void *db_page;
void *buffer;
Expand Down
27 changes: 21 additions & 6 deletions providers/mana/qp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,26 +222,26 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty
switch (type) {
case USER_RNIC_SEND_QUEUE_REQUESTER:
/* WQE must have at least one SGE */
/* For write with imm we need one extra SGE */
sges = max(1U, attr->cap.max_send_sge) + 1;
size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
sges = max(1U, attr->cap.max_send_sge);
size = align_hw_size(attr->cap.max_send_wr * get_large_fixed_wqe_size(sges));
break;
case USER_RNIC_SEND_QUEUE_RESPONDER:
if (attr->qp_type == IBV_QPT_RC)
size = align_hw_size(MANA_PAGE_SIZE);
break;
case USER_RNIC_RECV_QUEUE_REQUESTER:
sges = max(1U, attr->cap.max_send_sge);
if (attr->qp_type == IBV_QPT_RC)
size = align_hw_size(MANA_PAGE_SIZE);
size = align_hw_size(attr->cap.max_send_wr * get_wqe_size(sges));
break;
case USER_RNIC_RECV_QUEUE_RESPONDER:
/* WQE must have at least one SGE */
sges = max(1U, attr->cap.max_recv_sge);
size = align_hw_size(attr->cap.max_recv_wr * get_wqe_size(sges));
break;
case USER_RNIC_SEND_QUEUE_MM:
sges = 2;
size = align_hw_size(attr->cap.max_send_wr * get_large_wqe_size(sges));
sges = 1;
size = align_hw_size(attr->cap.max_send_wr * get_large_fixed_wqe_size(sges));
break;
default:
return 0;
Expand All @@ -256,6 +256,7 @@ static uint32_t get_queue_size(struct ibv_qp_init_attr *attr, enum user_queue_ty
static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
struct ibv_qp_init_attr *attr)
{
struct mana_context *ctx = to_mctx(ibpd->context);
struct mana_ib_create_rc_qp_resp *qp_resp_drv;
struct mana_create_rc_qp_resp qp_resp = {};
struct mana_ib_create_rc_qp *qp_cmd_drv;
Expand All @@ -272,6 +273,13 @@ static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
qp_cmd_drv->queue_size[i] = qp->rnic_qp.queues[i].size;
}

if (ctx->comp_mask & MANA_IB_UCNTX_RC_EXT_SUPPORT) {
qp_cmd_drv->comp_mask |= MANA_IB_RC_QP_FIXED_WQE | MANA_IB_RC_MMQ_CREATE;
qp_cmd_drv->mmq_buf =
(uintptr_t)qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].buffer;
qp_cmd_drv->mmq_size = qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].size;
}

ret = ibv_cmd_create_qp(ibpd, &qp->ibqp.qp, attr, &qp_cmd.ibv_cmd,
sizeof(qp_cmd), &qp_resp.ibv_resp,
sizeof(qp_resp));
Expand All @@ -286,6 +294,8 @@ static int mana_create_cmd_qp_rc(struct mana_qp *qp, struct ibv_pd *ibpd,
qp->rnic_qp.queues[i].id = qp_resp_drv->queue_id[i];
}

qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_MM].id = qp_resp_drv->mmq_id;

return 0;
}

Expand Down Expand Up @@ -355,6 +365,7 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
struct ibv_qp_init_attr *attr)
{
struct mana_context *ctx = to_mctx(ibpd->context);
uint8_t wqe_size_in_bu;
struct mana_qp *qp;
int ret, i;

Expand Down Expand Up @@ -393,6 +404,10 @@ static struct ibv_qp *mana_create_qp_rnic(struct ibv_pd *ibpd,
}
}

wqe_size_in_bu = get_large_fixed_wqe_size(max(1U, attr->cap.max_send_sge))
/ GDMA_WQE_ALIGNMENT_UNIT_SIZE;
qp->rnic_qp.queues[USER_RNIC_SEND_QUEUE_REQUESTER].wqe_size_in_bu = wqe_size_in_bu;

ret = mana_create_cmd_qp(qp, ibpd, attr);
if (ret) {
errno = ret;
Expand Down
3 changes: 2 additions & 1 deletion providers/mana/wr.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ gdma_post_sq_wqe(struct mana_gdma_queue *wq, struct ibv_sge *sgl, struct rdma_se
}

total_sge = num_sge + (oob_sge ? 1 : 0);
wqe_size = get_large_wqe_size(total_sge);
wqe_size = get_large_wqe_size(total_sge, wq->wqe_size_in_bu);

ret = gdma_get_current_wqe(wq, INLINE_OOB_LARGE_SIZE, wqe_size, wqe);
if (ret)
Expand Down Expand Up @@ -299,6 +299,7 @@ mana_ib_post_send_request(struct mana_qp *qp, struct ibv_send_wr *wr,
struct rdma_recv_oob recv_oob = {0};

recv_oob.psn_start = qp->sq_psn;
recv_oob.msn = qp->sq_ssn;
ret = gdma_post_rq_wqe(mana_ib_get_rreq(qp), wr->sg_list,
&recv_oob, num_sge, GDMA_WORK_REQ_CHECK_SN, &gdma_wqe);
if (ret) {
Expand Down