diff --git a/src/uct/ib/base/ib_device.h b/src/uct/ib/base/ib_device.h index dd14b2723bd..962a71cf393 100644 --- a/src/uct/ib/base/ib_device.h +++ b/src/uct/ib/base/ib_device.h @@ -91,17 +91,19 @@ typedef enum uct_ib_roce_version { enum { - UCT_IB_DEVICE_FLAG_MLX4_PRM = UCS_BIT(1), /* Device supports mlx4 PRM */ - UCT_IB_DEVICE_FLAG_MLX5_PRM = UCS_BIT(2), /* Device supports mlx5 PRM */ - UCT_IB_DEVICE_FLAG_MELLANOX = UCS_BIT(3), /* Mellanox device */ - UCT_IB_DEVICE_FLAG_SRQ = UCS_BIT(4), /* Supports SRQ */ - UCT_IB_DEVICE_FLAG_LINK_IB = UCS_BIT(5), /* Require only IB */ - UCT_IB_DEVICE_FLAG_DC_V1 = UCS_BIT(6), /* Device supports DC ver 1 */ - UCT_IB_DEVICE_FLAG_DC_V2 = UCS_BIT(7), /* Device supports DC ver 2 */ - UCT_IB_DEVICE_FLAG_AV = UCS_BIT(8), /* Device supports compact AV */ - UCT_IB_DEVICE_FLAG_DC = UCT_IB_DEVICE_FLAG_DC_V1 | - UCT_IB_DEVICE_FLAG_DC_V2, /* Device supports DC */ - UCT_IB_DEVICE_FAILED = UCS_BIT(9) /* Got fatal error */ + UCT_IB_DEVICE_FLAG_MLX4_PRM = UCS_BIT(1), /* Device supports mlx4 PRM */ + UCT_IB_DEVICE_FLAG_MLX5_PRM = UCS_BIT(2), /* Device supports mlx5 PRM */ + UCT_IB_DEVICE_FLAG_MELLANOX = UCS_BIT(3), /* Mellanox device */ + UCT_IB_DEVICE_FLAG_SRQ = UCS_BIT(4), /* Supports SRQ */ + UCT_IB_DEVICE_FLAG_LINK_IB = UCS_BIT(5), /* Require only IB */ + UCT_IB_DEVICE_FLAG_DC_V1 = UCS_BIT(6), /* Device supports DC ver 1 */ + UCT_IB_DEVICE_FLAG_DC_V2 = UCS_BIT(7), /* Device supports DC ver 2 */ + UCT_IB_DEVICE_FLAG_AV = UCS_BIT(8), /* Device supports compact AV */ + /* Device supports DC */ + UCT_IB_DEVICE_FLAG_DC = UCT_IB_DEVICE_FLAG_DC_V1 | + UCT_IB_DEVICE_FLAG_DC_V2, + UCT_IB_DEVICE_FAILED = UCS_BIT(9), /* Got fatal error */ + UCT_IB_DEVICE_FLAG_MULTIPLANE = UCS_BIT(10) /* Supports multiplane */ }; diff --git a/src/uct/ib/base/ib_iface.c b/src/uct/ib/base/ib_iface.c index eadb79103b0..fcbc852eef6 100644 --- a/src/uct/ib/base/ib_iface.c +++ b/src/uct/ib/base/ib_iface.c @@ -36,8 +36,11 @@ /** * Maximum bandwidth of NDR single path with PCIe Gen5 and RDMA_READ operation. */ -#define UCT_IB_NDR_READ_PATH_BANDWIDTH 38e9 -#define UCT_IB_XDR_READ_PATH_BANDWIDTH 35e9 +#define UCT_IB_NDR_READ_PATH_BANDWIDTH 38e9 +#define UCT_IB_XDR_READ_PATH_BANDWIDTH 35e9 +#define UCT_IB_HIGH_SPEED_NUM_PATHS 2 +#define UCT_IB_PORT_SPEED_UNIT_GBPS 0.1 +#define UCT_IB_MULTIPLANE_XDR_BANDWIDTH_GBPS 800.0 /** * Minimal NDR single path ratio. @@ -1419,6 +1422,42 @@ static unsigned uct_ib_iface_roce_lag_level(uct_ib_iface_t *iface) iface->gid_info.gid_index); } +static double +uct_ib_iface_query_port_speed_gbps(uct_ib_iface_t *iface) +{ +#if HAVE_DECL_IBV_QUERY_PORT_SPEED + uct_ib_device_t *dev = uct_ib_iface_device(iface); + ucs_log_level_t log_level; + uint64_t port_speed; + int ret; + + ret = ibv_query_port_speed(dev->ibv_context, iface->config.port_num, + &port_speed); + if (ret != 0) { + log_level = ((errno == EOPNOTSUPP) || + (errno == EPROTONOSUPPORT) || + (errno == ENOSYS)) ? UCS_LOG_LEVEL_DEBUG : + UCS_LOG_LEVEL_DIAG; + ucs_log(log_level, "ibv_query_port_speed(%s:%d) failed: %m", + uct_ib_device_name(dev), iface->config.port_num); + return 0.0; + } + + return port_speed * UCT_IB_PORT_SPEED_UNIT_GBPS; +#else + return 0.0; +#endif +} + +int uct_ib_iface_is_multiplane_xdr_bw(uct_ib_iface_t *iface) +{ + uct_ib_device_t *dev = uct_ib_iface_device(iface); + + return (dev->flags & UCT_IB_DEVICE_FLAG_MULTIPLANE) && + (ucs_fp_compare(uct_ib_iface_query_port_speed_gbps(iface), + UCT_IB_MULTIPLANE_XDR_BANDWIDTH_GBPS) == 0); +} + static void uct_ib_iface_set_num_paths(uct_ib_iface_t *iface, const uct_ib_iface_config_t *config) { @@ -1426,6 +1465,9 @@ static void uct_ib_iface_set_num_paths(uct_ib_iface_t *iface, if (uct_ib_iface_is_roce(iface)) { /* RoCE - number of paths is RoCE LAG level */ iface->num_paths = uct_ib_iface_roce_lag_level(iface); + if (uct_ib_iface_is_multiplane_xdr_bw(iface)) { + iface->num_paths = UCT_IB_HIGH_SPEED_NUM_PATHS; + } } else { /* IB - number of paths is LMC level */ ucs_assert(iface->path_bits_count > 0); @@ -1434,7 +1476,7 @@ static void uct_ib_iface_set_num_paths(uct_ib_iface_t *iface, if ((iface->num_paths == 1) && (uct_ib_iface_port_active_speed(iface) >= UCT_IB_SPEED_NDR)) { - iface->num_paths = 2; + iface->num_paths = UCT_IB_HIGH_SPEED_NUM_PATHS; } } else { iface->num_paths = config->num_paths; @@ -2091,15 +2133,22 @@ uct_ib_iface_estimate_path_bw(uct_ib_iface_t *iface, double path_ratio = 1.0; uct_ep_operation_t op = UCT_ATTR_VALUE(PERF, perf_attr, operation, OPERATION, UCT_EP_OP_LAST); + const int roce_lag = uct_ib_iface_is_roce(iface) && + (uct_ib_iface_roce_lag_level(iface) > 1); - if (uct_ib_iface_is_roce(iface) && - (uct_ib_iface_roce_lag_level(iface) > 1)) { + if (roce_lag) { path_ratio = 1.0 / iface_attr->dev_num_paths; - } else if (uct_ep_op_is_get(op)) { - if (uct_ib_iface_port_is_ndr(iface)) { + } + + if (uct_ep_op_is_get(op)) { + if (roce_lag && uct_ib_iface_is_multiplane_xdr_bw(iface)) { + max_path_bandwidth = UCT_IB_XDR_READ_PATH_BANDWIDTH; + path_ratio = ucs_min(path_ratio, + UCT_IB_XDR_READ_PATH_RATIO); + } else if (!roce_lag && uct_ib_iface_port_is_ndr(iface)) { max_path_bandwidth = UCT_IB_NDR_READ_PATH_BANDWIDTH; path_ratio = UCT_IB_NDR_READ_PATH_RATIO; - } else if (uct_ib_iface_port_is_xdr(iface)) { + } else if (!roce_lag && uct_ib_iface_port_is_xdr(iface)) { max_path_bandwidth = UCT_IB_XDR_READ_PATH_BANDWIDTH; path_ratio = UCT_IB_XDR_READ_PATH_RATIO; } @@ -2112,32 +2161,15 @@ static uct_ppn_bandwidth_t uct_ib_iface_estimate_bandwidth(uct_ib_iface_t *iface, const uct_iface_attr_t *iface_attr) { -#if HAVE_DECL_IBV_QUERY_PORT_SPEED - uct_ib_device_t *dev = uct_ib_iface_device(iface); - ucs_log_level_t log_level; - uint64_t port_speed; - double wire_speed; - int ret; + const double port_speed_gbps = + uct_ib_iface_query_port_speed_gbps(iface); - ret = ibv_query_port_speed(dev->ibv_context, iface->config.port_num, - &port_speed); - if (ret != 0) { - log_level = ((errno == EOPNOTSUPP) || - (errno == EPROTONOSUPPORT) || - (errno == ENOSYS)) ? UCS_LOG_LEVEL_DEBUG : - UCS_LOG_LEVEL_DIAG; - ucs_log(log_level, - "ibv_query_port_speed("UCT_IB_IFACE_FMT", port_num=%d) failed:" - " %m", UCT_IB_IFACE_ARG(iface), iface->config.port_num); + if (port_speed_gbps == 0.0) { return iface_attr->bandwidth; } - /* Convert port speed (in 100 Mb/s granularity) to bandwidth in bytes/s. */ - wire_speed = (double)port_speed * 1e8 / 8.0; - return uct_ib_iface_get_bandwidth(iface, wire_speed); -#else - return iface_attr->bandwidth; -#endif + return uct_ib_iface_get_bandwidth(iface, + port_speed_gbps * 1e9 / 8.0); } ucs_status_t @@ -2178,7 +2210,9 @@ uct_ib_iface_estimate_perf(uct_iface_h iface, uct_perf_attr_t *perf_attr) if (perf_attr->field_mask & UCT_PERF_ATTR_FIELD_BANDWIDTH) { perf_attr->bandwidth = uct_ib_iface_estimate_bandwidth(ib_iface, &iface_attr); - if (uct_ep_op_is_get(op) && uct_ib_iface_port_is_xdr(ib_iface)) { + if (uct_ep_op_is_get(op) && + (uct_ib_iface_port_is_xdr(ib_iface) || + uct_ib_iface_is_multiplane_xdr_bw(ib_iface))) { max_bandwidth = perf_attr->bandwidth.shared * iface_attr.dev_num_paths * UCT_IB_XDR_READ_PATH_RATIO; perf_attr->bandwidth.shared = ucs_min(perf_attr->bandwidth.shared, diff --git a/src/uct/ib/base/ib_iface.h b/src/uct/ib/base/ib_iface.h index 05f4616d998..7d1b2ed0d80 100644 --- a/src/uct/ib/base/ib_iface.h +++ b/src/uct/ib/base/ib_iface.h @@ -571,6 +571,12 @@ ucs_status_t uct_ib_iface_query(uct_ib_iface_t *iface, uct_iface_attr_t *iface_attr); +/** + * @return Nonzero if a multiplane interface reports XDR-equivalent bandwidth. + */ +int uct_ib_iface_is_multiplane_xdr_bw(uct_ib_iface_t *iface); + + ucs_status_t uct_ib_iface_estimate_perf(uct_iface_h tl_iface, uct_perf_attr_t *perf_attr); diff --git a/src/uct/ib/mlx5/dv/ib_mlx5_ifc.h b/src/uct/ib/mlx5/dv/ib_mlx5_ifc.h index c7b32cedac2..3707ee18541 100644 --- a/src/uct/ib/mlx5/dv/ib_mlx5_ifc.h +++ b/src/uct/ib/mlx5/dv/ib_mlx5_ifc.h @@ -533,7 +533,8 @@ struct uct_ib_mlx5_cmd_hca_cap_2_bits { uint8_t log_conn_track_granularity[0x5]; uint8_t reserved_at_d0[0x3]; uint8_t log_conn_track_max_alloc[0x5]; - uint8_t reserved_at_d8[0x3]; + uint8_t multiplane[0x1]; + uint8_t reserved_at_d9[0x2]; uint8_t log_max_conn_track_offload[0x5]; uint8_t cross_vhca_object_to_object_supported[0x20]; diff --git a/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c b/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c index 9feac14e3dc..1e19d377566 100644 --- a/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c +++ b/src/uct/ib/mlx5/dv/ib_mlx5dv_md.c @@ -2516,6 +2516,9 @@ ucs_status_t uct_ib_mlx5_devx_md_open_common(const char *name, size_t size, if (status == UCS_OK) { cap_2 = UCT_IB_MLX5DV_ADDR_OF(query_hca_cap_out, cap_2_out, capability); uct_ib_mlx5_devx_check_mkey_by_name(md, cap_2, dev); + if (UCT_IB_MLX5DV_GET(cmd_hca_cap_2, cap_2, multiplane)) { + dev->flags |= UCT_IB_DEVICE_FLAG_MULTIPLANE; + } } else { cap_2 = NULL; } diff --git a/test/gtest/uct/ib/test_ib.cc b/test/gtest/uct/ib/test_ib.cc index 73f9aefee6a..67511ea3afa 100644 --- a/test/gtest/uct/ib/test_ib.cc +++ b/test/gtest/uct/ib/test_ib.cc @@ -5,6 +5,7 @@ */ #include +#include #ifdef HAVE_MLX5_DV extern "C" { #include @@ -80,6 +81,81 @@ void test_uct_ib::send_recv_short() { size_t test_uct_ib::m_ib_am_handler_counter = 0; +class test_uct_ib_perf : public test_uct_ib { +protected: + void create_connected_entities() override + { + m_e1 = uct_test::create_entity(0); + m_entities.push_back(m_e1); + } + + static uct_perf_attr_t init_perf_attr(uct_ep_operation_t op) + { + uct_perf_attr_t perf_attr = {}; + + perf_attr.field_mask = UCT_PERF_ATTR_FIELD_OPERATION | + UCT_PERF_ATTR_FIELD_LOCAL_MEMORY_TYPE | + UCT_PERF_ATTR_FIELD_REMOTE_MEMORY_TYPE | + UCT_PERF_ATTR_FIELD_LOCAL_SYS_DEVICE | + UCT_PERF_ATTR_FIELD_REMOTE_SYS_DEVICE | + UCT_PERF_ATTR_FIELD_BANDWIDTH | + UCT_PERF_ATTR_FIELD_PATH_BANDWIDTH; + perf_attr.operation = op; + perf_attr.local_memory_type = UCS_MEMORY_TYPE_HOST; + perf_attr.remote_memory_type = UCS_MEMORY_TYPE_HOST; + perf_attr.local_sys_device = UCS_SYS_DEVICE_ID_UNKNOWN; + perf_attr.remote_sys_device = UCS_SYS_DEVICE_ID_UNKNOWN; + + return perf_attr; + } + + void check_get_put_bandwidth(bool expect_equal_bandwidth) + { + uct_ib_iface_t *iface = ucs_derived_of(m_e1->iface(), + uct_ib_iface_t); + uct_perf_attr_t get_perf = init_perf_attr(UCT_EP_OP_GET_ZCOPY); + uct_perf_attr_t put_perf = init_perf_attr(UCT_EP_OP_PUT_ZCOPY); + + if (!(m_e1->iface_attr().cap.flags & UCT_IFACE_FLAG_GET_ZCOPY) || + !(m_e1->iface_attr().cap.flags & UCT_IFACE_FLAG_PUT_ZCOPY)) { + UCS_TEST_SKIP_R("requires PUT and GET zcopy"); + } + + ASSERT_UCS_OK(uct_iface_estimate_perf(m_e1->iface(), &get_perf)); + ASSERT_UCS_OK(uct_iface_estimate_perf(m_e1->iface(), &put_perf)); + + if (uct_ib_iface_is_multiplane_xdr_bw(iface)) { + if (expect_equal_bandwidth) { + EXPECT_DOUBLE_EQ(put_perf.bandwidth.shared, + get_perf.bandwidth.shared); + EXPECT_DOUBLE_EQ(put_perf.path_bandwidth.shared, + get_perf.path_bandwidth.shared); + } else { + EXPECT_GT(put_perf.bandwidth.shared, + get_perf.bandwidth.shared); + EXPECT_GT(put_perf.path_bandwidth.shared, + get_perf.path_bandwidth.shared); + } + } else { + EXPECT_DOUBLE_EQ(put_perf.bandwidth.shared, + get_perf.bandwidth.shared); + } + } +}; + +UCS_TEST_P(test_uct_ib_perf, get_path_bandwidth, "IB_NUM_PATHS?=auto") +{ + check_get_put_bandwidth(false); +} + +UCS_TEST_P(test_uct_ib_perf, get_path_bandwidth_explicit_num_paths, + "IB_NUM_PATHS=4") +{ + check_get_put_bandwidth(true); +} + +UCT_INSTANTIATE_IB_TEST_CASE(test_uct_ib_perf); + class test_uct_ib_addr : public test_uct_ib { public: uct_ib_iface_config_t *ib_config() {