Skip to content

Commit

Permalink
RDMA: Support more than 255 rdma ports
Browse files Browse the repository at this point in the history 
Current code uses many different types when dealing with a port of a RDMA
device: u8, unsigned int and u32. Switch to u32 to clean up the logic.

This allows us to make (at least) the core view consistent and use the
same type. Unfortunately not all places can be converted. Many uverbs
functions expect port to be u8 so keep those places in order not to break
UAPIs.  HW/Spec defined values must also not be changed.

With the switch to u32 we now can support devices with more than 255
ports. U32_MAX is reserved to make control logic a bit easier to deal
with. As a device with U32_MAX ports probably isn't going to happen any
time soon this seems like a non issue.

When a device with more than 255 ports is created uverbs will report the
RDMA device as having 255 ports as this is the max currently supported.

The verbs interface is not changed yet because the IBTA spec limits the
port size in too many places to be u8 and all applications that relies in
verbs won't be able to cope with this change. At this stage, we are
extending the interfaces that are using vendor channel solely

Once the limitation is lifted mlx5 in switchdev mode will be able to have
thousands of SFs created by the device. As the only instance of an RDMA
device that reports more than 255 ports will be a representor device and
it exposes itself as a RAW Ethernet only device CM/MAD/IPoIB and other
ULPs aren't effected by this change and their sysfs/interfaces that are
exposes to userspace can remain unchanged.

While here cleanup some alignment issues and remove unneeded sanity
checks (mainly in rdmavt),

Link: https://lore.kernel.org/r/20210301070420.439400-1-leon@kernel.org
Signed-off-by: Mark Bloch <mbloch@nvidia.com>
Signed-off-by: Leon Romanovsky <leonro@nvidia.com>
Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
  • Loading branch information
@mark-bloch @jgunthorpe
mark-bloch authored and jgunthorpe committed Mar 26, 2021
1 parent 847d19a commit 1fb7f89
Show file tree
Hide file tree
Showing 107 changed files with 776 additions and 776 deletions.
84 changes: 38 additions & 46 deletions drivers/infiniband/core/cache.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ struct ib_gid_table {
u32 default_gid_indices;
};

static void dispatch_gid_change_event(struct ib_device *ib_dev, u8 port)
static void dispatch_gid_change_event(struct ib_device *ib_dev, u32 port)
{
struct ib_event event;

Expand Down Expand Up @@ -197,7 +197,7 @@ int ib_cache_gid_parse_type_str(const char *buf)
}
EXPORT_SYMBOL(ib_cache_gid_parse_type_str);

static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u8 port)
static struct ib_gid_table *rdma_gid_table(struct ib_device *device, u32 port)
{
return device->port_data[port].cache.gid;
}
Expand Down Expand Up @@ -237,10 +237,10 @@ static void put_gid_ndev(struct rcu_head *head)
static void free_gid_entry_locked(struct ib_gid_table_entry *entry)
{
struct ib_device *device = entry->attr.device;
u8 port_num = entry->attr.port_num;
u32 port_num = entry->attr.port_num;
struct ib_gid_table *table = rdma_gid_table(device, port_num);

dev_dbg(&device->dev, "%s port=%d index=%d gid %pI6\n", __func__,
dev_dbg(&device->dev, "%s port=%u index=%d gid %pI6\n", __func__,
port_num, entry->attr.index, entry->attr.gid.raw);

write_lock_irq(&table->rwlock);
Expand Down Expand Up @@ -282,7 +282,7 @@ static void free_gid_work(struct work_struct *work)
struct ib_gid_table_entry *entry =
container_of(work, struct ib_gid_table_entry, del_work);
struct ib_device *device = entry->attr.device;
u8 port_num = entry->attr.port_num;
u32 port_num = entry->attr.port_num;
struct ib_gid_table *table = rdma_gid_table(device, port_num);

mutex_lock(&table->lock);
Expand Down Expand Up @@ -379,15 +379,15 @@ static int add_roce_gid(struct ib_gid_table_entry *entry)
* @ix: GID entry index to delete
*
*/
static void del_gid(struct ib_device *ib_dev, u8 port,
static void del_gid(struct ib_device *ib_dev, u32 port,
struct ib_gid_table *table, int ix)
{
struct roce_gid_ndev_storage *ndev_storage;
struct ib_gid_table_entry *entry;

lockdep_assert_held(&table->lock);

dev_dbg(&ib_dev->dev, "%s port=%d index=%d gid %pI6\n", __func__, port,
dev_dbg(&ib_dev->dev, "%s port=%u index=%d gid %pI6\n", __func__, port,
ix, table->data_vec[ix]->attr.gid.raw);

write_lock_irq(&table->rwlock);
Expand Down Expand Up @@ -543,7 +543,7 @@ static void make_default_gid(struct net_device *dev, union ib_gid *gid)
addrconf_ifid_eui48(&gid->raw[8], dev);
}

static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
static int __ib_cache_gid_add(struct ib_device *ib_dev, u32 port,
union ib_gid *gid, struct ib_gid_attr *attr,
unsigned long mask, bool default_gid)
{
Expand Down Expand Up @@ -587,7 +587,7 @@ static int __ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
return ret;
}

int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
int ib_cache_gid_add(struct ib_device *ib_dev, u32 port,
union ib_gid *gid, struct ib_gid_attr *attr)
{
unsigned long mask = GID_ATTR_FIND_MASK_GID |
Expand All @@ -598,7 +598,7 @@ int ib_cache_gid_add(struct ib_device *ib_dev, u8 port,
}

static int
_ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
_ib_cache_gid_del(struct ib_device *ib_dev, u32 port,
union ib_gid *gid, struct ib_gid_attr *attr,
unsigned long mask, bool default_gid)
{
Expand Down Expand Up @@ -627,7 +627,7 @@ _ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
return ret;
}

int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
int ib_cache_gid_del(struct ib_device *ib_dev, u32 port,
union ib_gid *gid, struct ib_gid_attr *attr)
{
unsigned long mask = GID_ATTR_FIND_MASK_GID |
Expand All @@ -638,7 +638,7 @@ int ib_cache_gid_del(struct ib_device *ib_dev, u8 port,
return _ib_cache_gid_del(ib_dev, port, gid, attr, mask, false);
}

int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u8 port,
int ib_cache_gid_del_all_netdev_gids(struct ib_device *ib_dev, u32 port,
struct net_device *ndev)
{
struct ib_gid_table *table;
Expand Down Expand Up @@ -683,7 +683,7 @@ const struct ib_gid_attr *
rdma_find_gid_by_port(struct ib_device *ib_dev,
const union ib_gid *gid,
enum ib_gid_type gid_type,
u8 port, struct net_device *ndev)
u32 port, struct net_device *ndev)
{
int local_index;
struct ib_gid_table *table;
Expand Down Expand Up @@ -734,7 +734,7 @@ EXPORT_SYMBOL(rdma_find_gid_by_port);
*
*/
const struct ib_gid_attr *rdma_find_gid_by_filter(
struct ib_device *ib_dev, const union ib_gid *gid, u8 port,
struct ib_device *ib_dev, const union ib_gid *gid, u32 port,
bool (*filter)(const union ib_gid *gid, const struct ib_gid_attr *,
void *),
void *context)
Expand Down Expand Up @@ -818,7 +818,7 @@ static void release_gid_table(struct ib_device *device,
kfree(table);
}

static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
static void cleanup_gid_table_port(struct ib_device *ib_dev, u32 port,
struct ib_gid_table *table)
{
int i;
Expand All @@ -834,7 +834,7 @@ static void cleanup_gid_table_port(struct ib_device *ib_dev, u8 port,
mutex_unlock(&table->lock);
}

void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u32 port,
struct net_device *ndev,
unsigned long gid_type_mask,
enum ib_cache_gid_default_mode mode)
Expand Down Expand Up @@ -867,7 +867,7 @@ void ib_cache_gid_set_default_gid(struct ib_device *ib_dev, u8 port,
}
}

static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,
static void gid_table_reserve_default(struct ib_device *ib_dev, u32 port,
struct ib_gid_table *table)
{
unsigned int i;
Expand All @@ -884,7 +884,7 @@ static void gid_table_reserve_default(struct ib_device *ib_dev, u8 port,

static void gid_table_release_one(struct ib_device *ib_dev)
{
unsigned int p;
u32 p;

rdma_for_each_port (ib_dev, p) {
release_gid_table(ib_dev, ib_dev->port_data[p].cache.gid);
Expand All @@ -895,7 +895,7 @@ static void gid_table_release_one(struct ib_device *ib_dev)
static int _gid_table_setup_one(struct ib_device *ib_dev)
{
struct ib_gid_table *table;
unsigned int rdma_port;
u32 rdma_port;

rdma_for_each_port (ib_dev, rdma_port) {
table = alloc_gid_table(
Expand All @@ -915,7 +915,7 @@ static int _gid_table_setup_one(struct ib_device *ib_dev)

static void gid_table_cleanup_one(struct ib_device *ib_dev)
{
unsigned int p;
u32 p;

rdma_for_each_port (ib_dev, p)
cleanup_gid_table_port(ib_dev, p,
Expand Down Expand Up @@ -950,7 +950,7 @@ static int gid_table_setup_one(struct ib_device *ib_dev)
* Returns 0 on success or appropriate error code.
*
*/
int rdma_query_gid(struct ib_device *device, u8 port_num,
int rdma_query_gid(struct ib_device *device, u32 port_num,
int index, union ib_gid *gid)
{
struct ib_gid_table *table;
Expand Down Expand Up @@ -1014,7 +1014,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
unsigned long mask = GID_ATTR_FIND_MASK_GID |
GID_ATTR_FIND_MASK_GID_TYPE;
struct ib_gid_attr gid_attr_val = {.ndev = ndev, .gid_type = gid_type};
unsigned int p;
u32 p;

if (ndev)
mask |= GID_ATTR_FIND_MASK_NETDEV;
Expand Down Expand Up @@ -1043,7 +1043,7 @@ const struct ib_gid_attr *rdma_find_gid(struct ib_device *device,
EXPORT_SYMBOL(rdma_find_gid);

int ib_get_cached_pkey(struct ib_device *device,
u8 port_num,
u32 port_num,
int index,
u16 *pkey)
{
Expand All @@ -1069,9 +1069,8 @@ int ib_get_cached_pkey(struct ib_device *device,
}
EXPORT_SYMBOL(ib_get_cached_pkey);

int ib_get_cached_subnet_prefix(struct ib_device *device,
u8 port_num,
u64 *sn_pfx)
int ib_get_cached_subnet_prefix(struct ib_device *device, u32 port_num,
u64 *sn_pfx)
{
unsigned long flags;

Expand All @@ -1086,10 +1085,8 @@ int ib_get_cached_subnet_prefix(struct ib_device *device,
}
EXPORT_SYMBOL(ib_get_cached_subnet_prefix);

int ib_find_cached_pkey(struct ib_device *device,
u8 port_num,
u16 pkey,
u16 *index)
int ib_find_cached_pkey(struct ib_device *device, u32 port_num,
u16 pkey, u16 *index)
{
struct ib_pkey_cache *cache;
unsigned long flags;
Expand Down Expand Up @@ -1132,10 +1129,8 @@ int ib_find_cached_pkey(struct ib_device *device,
}
EXPORT_SYMBOL(ib_find_cached_pkey);

int ib_find_exact_cached_pkey(struct ib_device *device,
u8 port_num,
u16 pkey,
u16 *index)
int ib_find_exact_cached_pkey(struct ib_device *device, u32 port_num,
u16 pkey, u16 *index)
{
struct ib_pkey_cache *cache;
unsigned long flags;
Expand Down Expand Up @@ -1169,9 +1164,7 @@ int ib_find_exact_cached_pkey(struct ib_device *device,
}
EXPORT_SYMBOL(ib_find_exact_cached_pkey);

int ib_get_cached_lmc(struct ib_device *device,
u8 port_num,
u8 *lmc)
int ib_get_cached_lmc(struct ib_device *device, u32 port_num, u8 *lmc)
{
unsigned long flags;
int ret = 0;
Expand All @@ -1187,8 +1180,7 @@ int ib_get_cached_lmc(struct ib_device *device,
}
EXPORT_SYMBOL(ib_get_cached_lmc);

int ib_get_cached_port_state(struct ib_device *device,
u8 port_num,
int ib_get_cached_port_state(struct ib_device *device, u32 port_num,
enum ib_port_state *port_state)
{
unsigned long flags;
Expand Down Expand Up @@ -1222,7 +1214,7 @@ EXPORT_SYMBOL(ib_get_cached_port_state);
* code.
*/
const struct ib_gid_attr *
rdma_get_gid_attr(struct ib_device *device, u8 port_num, int index)
rdma_get_gid_attr(struct ib_device *device, u32 port_num, int index)
{
const struct ib_gid_attr *attr = ERR_PTR(-ENODATA);
struct ib_gid_table *table;
Expand Down Expand Up @@ -1263,7 +1255,7 @@ ssize_t rdma_query_gid_table(struct ib_device *device,
const struct ib_gid_attr *gid_attr;
ssize_t num_entries = 0, ret;
struct ib_gid_table *table;
unsigned int port_num, i;
u32 port_num, i;
struct net_device *ndev;
unsigned long flags;

Expand Down Expand Up @@ -1361,7 +1353,7 @@ struct net_device *rdma_read_gid_attr_ndev_rcu(const struct ib_gid_attr *attr)
container_of(attr, struct ib_gid_table_entry, attr);
struct ib_device *device = entry->attr.device;
struct net_device *ndev = ERR_PTR(-EINVAL);
u8 port_num = entry->attr.port_num;
u32 port_num = entry->attr.port_num;
struct ib_gid_table *table;
unsigned long flags;
bool valid;
Expand Down Expand Up @@ -1441,7 +1433,7 @@ int rdma_read_gid_l2_fields(const struct ib_gid_attr *attr,
EXPORT_SYMBOL(rdma_read_gid_l2_fields);

static int config_non_roce_gid_cache(struct ib_device *device,
u8 port, int gid_tbl_len)
u32 port, int gid_tbl_len)
{
struct ib_gid_attr gid_attr = {};
struct ib_gid_table *table;
Expand Down Expand Up @@ -1472,7 +1464,7 @@ static int config_non_roce_gid_cache(struct ib_device *device,
}

static int
ib_cache_update(struct ib_device *device, u8 port, bool enforce_security)
ib_cache_update(struct ib_device *device, u32 port, bool enforce_security)
{
struct ib_port_attr *tprops = NULL;
struct ib_pkey_cache *pkey_cache = NULL, *old_pkey_cache;
Expand Down Expand Up @@ -1621,7 +1613,7 @@ EXPORT_SYMBOL(ib_dispatch_event);

int ib_cache_setup_one(struct ib_device *device)
{
unsigned int p;
u32 p;
int err;

rwlock_init(&device->cache_lock);
Expand All @@ -1641,7 +1633,7 @@ int ib_cache_setup_one(struct ib_device *device)

void ib_cache_release_one(struct ib_device *device)
{
unsigned int p;
u32 p;

/*
* The release function frees all the cache elements.
Expand Down
12 changes: 6 additions & 6 deletions drivers/infiniband/core/cm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ static struct attribute *cm_counter_default_attrs[] = {
struct cm_port {
struct cm_device *cm_dev;
struct ib_mad_agent *mad_agent;
u8 port_num;
u32 port_num;
struct list_head cm_priv_prim_list;
struct list_head cm_priv_altr_list;
struct cm_counter_group counter_group[CM_COUNTER_GROUPS];
Expand Down Expand Up @@ -1631,7 +1631,7 @@ static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
req_msg))));
}

static void cm_path_set_rec_type(struct ib_device *ib_device, u8 port_num,
static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
struct sa_path_rec *path, union ib_gid *gid)
{
if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
Expand Down Expand Up @@ -1750,7 +1750,7 @@ static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
static u16 cm_get_bth_pkey(struct cm_work *work)
{
struct ib_device *ib_dev = work->port->cm_dev->ib_device;
u8 port_num = work->port->port_num;
u32 port_num = work->port->port_num;
u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
u16 pkey;
int ret;
Expand Down Expand Up @@ -1778,7 +1778,7 @@ static void cm_opa_to_ib_sgid(struct cm_work *work,
struct sa_path_rec *path)
{
struct ib_device *dev = work->port->cm_dev->ib_device;
u8 port_num = work->port->port_num;
u32 port_num = work->port->port_num;

if (rdma_cap_opa_ah(dev, port_num) &&
(ib_is_opa_gid(&path->sgid))) {
Expand Down Expand Up @@ -4334,7 +4334,7 @@ static int cm_add_one(struct ib_device *ib_device)
unsigned long flags;
int ret;
int count = 0;
unsigned int i;
u32 i;

cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
GFP_KERNEL);
Expand Down Expand Up @@ -4432,7 +4432,7 @@ static void cm_remove_one(struct ib_device *ib_device, void *client_data)
.clr_port_cap_mask = IB_PORT_CM_SUP
};
unsigned long flags;
unsigned int i;
u32 i;

write_lock_irqsave(&cm.device_lock, flags);
list_del(&cm_dev->list);
Expand Down
Loading

0 comments on commit 1fb7f89

Please sign in to comment.