Skip to content

Implement dedicated MSIx for each vPort #1

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: develop
Choose a base branch
from
Open

Implement dedicated MSIx for each vPort #1

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
9ff9927
net:mana: Create seperate EQs for each vport
longlimsft Jun 19, 2025
837f860
net:mana: Query device and decides msi sharing for EQs
longlimsft Jun 19, 2025
09fcf8e
net:mana: Introduce new gic context and refcount for interrupt context
longlimsft Jun 19, 2025
45e5260
net:mana: Use gic functions to allocate glocal EQs
longlimsft Jun 19, 2025
ca930b2
net:mana: Create or get interrupt for each EQ when creating vPort
longlimsft Jun 19, 2025
4846d29
RDMA/mana_ib: Create interrupts on EQs
longlimsft Jun 19, 2025
ca870d9
net:mana: Indicate the driver now supports dedicated EQs/MSIs for eac…
longlimsft Sep 5, 2025
5acd774
staged
longlimsft Sep 9, 2025
b520bd0
staged
longlimsft Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 23 additions & 5 deletions drivers/infiniband/hw/mana/main.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,10 @@ void mana_ib_uncfg_vport(struct mana_ib_dev *dev, struct mana_ib_pd *pd,
pd->vport_use_count--;
WARN_ON(pd->vport_use_count < 0);

if (!pd->vport_use_count)
if (!pd->vport_use_count) {
mana_destroy_eq(mpc);
mana_uncfg_vport(mpc);
}

mutex_unlock(&pd->vport_mutex);
}
Expand Down Expand Up @@ -54,15 +56,21 @@ int mana_ib_cfg_vport(struct mana_ib_dev *dev, u32 port, struct mana_ib_pd *pd,
return err;
}

mutex_unlock(&pd->vport_mutex);

pd->tx_shortform_allowed = mpc->tx_shortform_allowed;
pd->tx_vp_offset = mpc->tx_vp_offset;
err = mana_create_eq(mpc);
if (err) {
mana_uncfg_vport(mpc);
pd->vport_use_count--;
}

mutex_unlock(&pd->vport_mutex);

ibdev_dbg(&dev->ib_dev, "vport handle %llx pdid %x doorbell_id %x\n",
mpc->port_handle, pd->pdn, doorbell_id);

return 0;
return err;
}

int mana_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
Expand Down Expand Up @@ -730,6 +738,7 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev)
{
struct gdma_context *gc = mdev_to_gc(mdev);
struct gdma_queue_spec spec = {};
struct gdma_irq_context *gic;
int err, i;

spec.type = GDMA_EQ;
Expand All @@ -740,6 +749,8 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev)
spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
spec.eq.msix_index = 0;

gic = gdma_get_gic(gc, false, 0, 0, &spec.eq.msix_index);

err = mana_gd_create_mana_eq(&gc->mana_ib, &spec, &mdev->fatal_err_eq);
if (err)
return err;
Expand All @@ -753,6 +764,9 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev)
spec.eq.callback = NULL;
for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) {
spec.eq.msix_index = (i + 1) % gc->num_msix_usable;

gic = gdma_get_gic(gc, false, 0, 0, &spec.eq.msix_index);

err = mana_gd_create_mana_eq(mdev->gdma_dev, &spec, &mdev->eqs[i]);
if (err)
goto destroy_eqs;
Expand All @@ -772,12 +786,16 @@ int mana_ib_create_eqs(struct mana_ib_dev *mdev)
void mana_ib_destroy_eqs(struct mana_ib_dev *mdev)
{
struct gdma_context *gc = mdev_to_gc(mdev);
int i;
int i, msi;

mana_gd_destroy_queue(gc, mdev->fatal_err_eq);
gdma_put_gic(gc, false, 0);

for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++)
for (i = 0; i < mdev->ib_dev.num_comp_vectors; i++) {
mana_gd_destroy_queue(gc, mdev->eqs[i]);
msi = (i + 1) % gc->num_msix_usable;
gdma_put_gic(gc, false, msi);
}

kfree(mdev->eqs);
}
Expand Down
4 changes: 2 additions & 2 deletions drivers/infiniband/hw/mana/qp.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ static int mana_ib_create_qp_rss(struct ib_qp *ibqp, struct ib_pd *pd,
cq_spec.gdma_region = cq->queue.gdma_region;
cq_spec.queue_size = cq->cqe * COMP_ENTRY_SIZE;
cq_spec.modr_ctx_id = 0;
eq = &mpc->ac->eqs[cq->comp_vector];
eq = &mpc->eqs[cq->comp_vector % mpc->num_queues];
cq_spec.attached_eq = eq->eq->id;

ret = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_RQ,
Expand Down Expand Up @@ -341,7 +341,7 @@ static int mana_ib_create_qp_raw(struct ib_qp *ibqp, struct ib_pd *ibpd,
cq_spec.queue_size = send_cq->cqe * COMP_ENTRY_SIZE;
cq_spec.modr_ctx_id = 0;
eq_vec = send_cq->comp_vector;
eq = &mpc->ac->eqs[eq_vec];
eq = &mpc->eqs[eq_vec % mpc->num_queues];
cq_spec.attached_eq = eq->eq->id;

err = mana_create_wq_obj(mpc, mpc->port_handle, GDMA_SQ, &wq_spec,
Expand Down
Loading