derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
450 lines
21 KiB
JSON
450 lines
21 KiB
JSON
{
|
|
"system_configurations": {
|
|
"name": "RCCL-AINIC",
|
|
"description": "AINIC test plan. Group A: NetIbMPI + GIN component/unit tests (2 nodes x 1 GPU = 2 ranks). Group B: NCCL_NET plugin x knobs sweep on rccl-tests perf (2 nodes x 8 GPU = 16 ranks), including out-of-the-box autodetect. Group C: IONIC hardware counter validation. Run on the AINIC cluster with --system ainic."
|
|
},
|
|
"paths": {
|
|
"workdir": "${WORKDIR:-$PWD}",
|
|
"rocm_path": "${ROCM_PATH:-/it-share/rccl-ci/rocm_devel}",
|
|
"mpi_path": "${MPI_PATH:-/it-share/ompi-5.0.8}",
|
|
"test_binary_dir": "${RCCL_TESTS_BIN:-/it-share/rccl-ci/rccl-tests-2.30.4/bin}"
|
|
},
|
|
"auto_detect_hosts": true,
|
|
"mpi_args": {
|
|
"ainic": "--bind-to numa --mca pml ob1 --mca osc ^ucx --mca btl tcp,self --mca btl_tcp_if_include eno0"
|
|
},
|
|
"env_variables": {
|
|
"NCCL_DEBUG": "INFO",
|
|
"HSA_NO_SCRATCH_RECLAIM": "1"
|
|
},
|
|
"system_env_variables": {
|
|
"ainic": {
|
|
"NCCL_SOCKET_IFNAME": "eno0",
|
|
"RCCL_AINIC_ROCE": "1",
|
|
"IONIC_LOCKFREE": "all",
|
|
"NCCL_IB_HCA": "rdma,roce"
|
|
}
|
|
},
|
|
"build_configuration": {
|
|
"install_flags": ["--debug", "-t", "--amdgpu_targets", "gfx950", "--log-trace", "--enable-mpi-tests", "--no_clean"],
|
|
"cmake_options": "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON"
|
|
},
|
|
|
|
"test_configurations": {
|
|
|
|
"a_base": {
|
|
"is_gtest": true,
|
|
"binary": "rccl-UnitTestsMPI",
|
|
"num_ranks": 2,
|
|
"num_nodes": 2,
|
|
"num_gpus": 1,
|
|
"timeout": 600,
|
|
"env_variables": {
|
|
"NCCL_IB_MERGE_NICS": "1",
|
|
"NCCL_IB_MERGE_VFS": "1",
|
|
"NCCL_DMABUF_ENABLE": "1"
|
|
}
|
|
},
|
|
|
|
"a_netib_component": {
|
|
"extends": "a_base",
|
|
"description": "A1-A3: NetIb plugin init, properties, data path.",
|
|
"tests": [
|
|
{
|
|
"name": "A1_NetIB_InitializePlugin",
|
|
"description": "Verifies IB-CAST plugin init and libionic.so load on AINIC.",
|
|
"test_filter": "NetIbMPITest.InitializePlugin"
|
|
},
|
|
{
|
|
"name": "A2_NetIB_GetDeviceProperties",
|
|
"description": "Verifies IONIC devices (rdma0+) are visible and properties valid.",
|
|
"test_filter": "NetIbMPITest.GetDeviceProperties"
|
|
},
|
|
{
|
|
"name": "A3_NetIB_SimpleSendRecv",
|
|
"description": "Verifies basic send/recv over AINIC data path (no collectives).",
|
|
"test_filter": "NetIbMPITest.SimpleSendRecv"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_cts_workaround": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "A4: CTS-off P2P stress (16x64=1024 outstanding). Do NOT bump CTS_QP_DEPTH to 256 or it will time out.",
|
|
"env_variables": {
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1",
|
|
"CTS_NUM_CONNS": "16",
|
|
"CTS_QP_DEPTH": "64",
|
|
"CTS_SNAP_EVERY": "4"
|
|
},
|
|
"tests": [
|
|
{
|
|
"name": "A4_NetIB_CtsDepthStress_CtsOff",
|
|
"description": "CTS-off P2P stress path passes cleanly (1024 outstanding recvs drain fully).",
|
|
"test_filter": "NetIbMPITest.CtsDepthStress"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_cts_depth32": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "A5: CTS-on at depth 32 (expected PASS). With CTS ON, CTS_QP_DEPTH=32 is well UNDER the AINIC CTS match-table overflow boundary, so this must PASS cleanly.",
|
|
"env_variables": {
|
|
"RCCL_IB_P2P_DISABLE_CTS": "0",
|
|
"CTS_NUM_CONNS": "16",
|
|
"CTS_QP_DEPTH": "32",
|
|
"CTS_SNAP_EVERY": "4"
|
|
},
|
|
"tests": [
|
|
{
|
|
"name": "A5_NetIB_CtsDepthStress_CtsOn_Depth32",
|
|
"description": "CTS ON at depth 32: under the CTS match-table overflow boundary, so this must PASS.",
|
|
"test_filter": "NetIbMPITest.CtsDepthStress"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_cast_base": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "CAST base: full WRR env required or tests GTEST_SKIP. No tests here; children extend.",
|
|
"env_variables": {
|
|
"RCCL_IB_QP_SCHED_ENABLE": "1",
|
|
"RCCL_IB_QP_SCHED_WRR_ENABLE": "1",
|
|
"RCCL_IB_QP_SCHED_WEIGHT": "0",
|
|
"RCCL_IB_QP_SCHED_UPDATE_INTERVAL": "50",
|
|
"RCCL_IB_QP_SCHED_RESET_INTERVAL": "60000",
|
|
"RCCL_IB_QP_SCHED_SPLIT_DATA_MIN": "65536",
|
|
"NCCL_IB_QPS_PER_CONNECTION": "2",
|
|
"NCCL_IB_SPLIT_DATA_ON_QPS": "1"
|
|
}
|
|
},
|
|
|
|
"a_cast_wrr": {
|
|
"extends": "a_cast_base",
|
|
"description": "A6/A7: CAST multi-QP WRR scheduler.",
|
|
"tests": [
|
|
{
|
|
"name": "A6_Cast_EqualWeightsTwoQPsTokenCounts",
|
|
"description": "WRR scheduler balances 2 QPs without corruption (token table sane).",
|
|
"test_filter": "NetIbMPITest.CastEqualWeightsTwoQPsTokenCounts"
|
|
},
|
|
{
|
|
"name": "A7_Cast_SingleQPBypassesWrr",
|
|
"description": "Single-QP connection bypasses WRR scheduler initialisation (keep sched enabled; just drop to 1 QP).",
|
|
"test_filter": "NetIbMPITest.CastSingleQPBypassesWrr",
|
|
"env_variables": {
|
|
"NCCL_IB_QPS_PER_CONNECTION": "1",
|
|
"NCCL_IB_SPLIT_DATA_ON_QPS": "0"
|
|
}
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_fault_inject": {
|
|
"extends": "a_cast_base",
|
|
"timeout": 600,
|
|
"description": "A8: per-QP fault injection on CAST path.",
|
|
"tests": [
|
|
{
|
|
"name": "A8_Cast_FaultInjQpErrorIsFatal",
|
|
"description": "QP fault is detected (fatal) and does not corrupt other QPs on CAST path.",
|
|
"test_filter": "NetIbMPITest.FaultInjCastQpErrorIsFatal"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_gin": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "A9-A13: GIN one-sided proxy tests. Require NCCL_NET=ib-cast.",
|
|
"env_variables": {
|
|
"NCCL_NET": "ib-cast"
|
|
},
|
|
"tests": [
|
|
{
|
|
"name": "A9_Gin_IPutBasic",
|
|
"description": "GIN one-sided iput RDMA write over AINIC.",
|
|
"test_filter": "*GinMPITest.IPutBasic*"
|
|
},
|
|
{
|
|
"name": "A10_Gin_IGetBasic",
|
|
"description": "GIN one-sided iget RDMA read over AINIC.",
|
|
"test_filter": "*GinMPITest.IGetBasic*"
|
|
},
|
|
{
|
|
"name": "A11_Gin_IPutSignal",
|
|
"description": "GIN put-with-signal (atomic inc/add) completion semantics.",
|
|
"test_filter": "*GinMPITest.IPutSignalInc*:*GinMPITest.IPutSignalAdd*"
|
|
},
|
|
{
|
|
"name": "A12_Gin_MultipleInflight",
|
|
"description": "Multiple outstanding GIN iput/iget complete without corruption.",
|
|
"test_filter": "*GinMPITest.MultipleInflightIPuts*:*GinMPITest.MultipleInflightIGets*"
|
|
},
|
|
{
|
|
"name": "A13_Gin_IPutSignalStress10k",
|
|
"description": "Stress: 10k put-with-signal ops, signal counter must equal 10000 (no drops).",
|
|
"test_filter": "*GinMPIStressTest.IPutSignalStress10k*"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_gin_enable_gate": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "A14: ROCM-27070 regression. On AINIC autodetect, NCCL_GIN_ENABLE=0 must gate GIN init early (comm init succeeds, no GIN assigned, no enumeration segfault).",
|
|
"env_variables": {
|
|
"NCCL_GIN_ENABLE": "0"
|
|
},
|
|
"tests": [
|
|
{
|
|
"name": "A14_Gin_EnableZeroSkipsInit",
|
|
"description": "NCCL_GIN_ENABLE=0 skips GIN plugin init; comm init succeeds with no GIN backend assigned.",
|
|
"test_filter": "GinInitMPITest.GinEnableZeroSkipsInit"
|
|
}
|
|
]
|
|
},
|
|
|
|
"a_gin_ainic_backend": {
|
|
"extends": "a_base",
|
|
"timeout": 600,
|
|
"description": "A15: ROCM-27070 regression. On AINIC autodetect (NCCL_NET unset), GIN must select the ib-cast backend, never the generic IB proxy that triggers a second device enumeration.",
|
|
"tests": [
|
|
{
|
|
"name": "A15_Gin_AinicSelectsCastBackend",
|
|
"description": "GIN on AINIC uses IbCastGinIb, not the generic ncclGinIbProxy (no dual enumeration).",
|
|
"test_filter": "GinInitMPITest.AinicSelectsCastBackend"
|
|
}
|
|
]
|
|
},
|
|
|
|
"b_base": {
|
|
"description": "Sweep base: derived suites run the full 1024-16G range for both allreduce and alltoall. Sets NO shared env_variables so each sweep runs against stock defaults plus its own per-suite knobs, and NO group-level command_args by design - every test sets its own explicit, complete command_args so nothing is silently concatenated/duplicated.",
|
|
"is_gtest": false,
|
|
"num_ranks": 16,
|
|
"num_nodes": 2,
|
|
"num_gpus": 8,
|
|
"timeout": 600,
|
|
"env_variables": {}
|
|
},
|
|
|
|
"b1_ib": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B1: baseline plain RoCE, no AINIC/ionic/CTS.",
|
|
"env_variables": { "NCCL_NET": "IB" },
|
|
"tests": [
|
|
{ "name": "B1_alltoall_IB", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "Baseline plain RoCE alltoall over the full 1024-16G range - confirms HW + MPI healthy, independent of the AINIC/CAST path." },
|
|
{ "name": "B1_allreduce_IB", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "Baseline plain RoCE allreduce." }
|
|
]
|
|
},
|
|
|
|
"b_alltoall_cast_full": {
|
|
"is_gtest": false,
|
|
"num_ranks": 16,
|
|
"num_nodes": 2,
|
|
"num_gpus": 8,
|
|
"timeout": 600,
|
|
"command_args": "-b 1024 -e 16G -f 2 -g 1 -n 20 -c 1 -w 2",
|
|
"description": "Full-range CAST alltoall (1K..16G): single-QP, WRR off, CTS offload off, GDR flush off.",
|
|
"env_variables": {
|
|
"NCCL_NET": "IB-CAST",
|
|
"NCCL_IB_QPS_PER_CONNECTION": "1",
|
|
"RCCL_IB_QPS_PER_P2P": "1",
|
|
"RCCL_IB_QP_SCHED_ENABLE": "0",
|
|
"NCCL_IB_PCI_RELAXED_ORDERING": "1",
|
|
"RCCL_CTS_OFFLOAD_ENABLED": "0",
|
|
"NCCL_NET_OPTIONAL_RECV_COMPLETION": "1",
|
|
"NCCL_GDR_FLUSH_DISABLE": "1",
|
|
"NCCL_DEBUG": "WARN"
|
|
},
|
|
"tests": [
|
|
{ "name": "B2_alltoall_CAST_1K_16G", "binary": "alltoall_perf", "description": "CAST alltoall full range 1K..16G (single-QP, CTS offload off)." }
|
|
]
|
|
},
|
|
|
|
"b2_cast_wrr_ctsoff": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B3: explicit CAST, WRR ON, CTS OFF.",
|
|
"env_variables": {
|
|
"NCCL_NET": "IB-CAST",
|
|
"RCCL_IB_QP_SCHED_ENABLE": "1",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1"
|
|
},
|
|
"tests": [
|
|
{ "name": "B3_alltoall_CAST_WRR_CTSoff", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CAST WRR ON / CTS OFF - validates the multi-QP CAST path on AINIC over the full 1024-16G range." },
|
|
{ "name": "B3_allreduce_CAST_WRR_CTSoff", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CAST WRR ON / CTS OFF allreduce." }
|
|
]
|
|
},
|
|
|
|
"b3_cast_nowrr_ctsoff": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B4: CAST, WRR OFF, CTS OFF (isolate scheduler).",
|
|
"env_variables": {
|
|
"NCCL_NET": "IB-CAST",
|
|
"RCCL_IB_QP_SCHED_ENABLE": "0",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1"
|
|
},
|
|
"tests": [
|
|
{ "name": "B4_alltoall_CAST_noWRR_CTSoff", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CAST without WRR - isolates the scheduler, ionic QP path only, over the full 1024-16G range." },
|
|
{ "name": "B4_allreduce_CAST_noWRR_CTSoff", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CAST without WRR allreduce." }
|
|
]
|
|
},
|
|
|
|
"b4_rocmib_ctsoff": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B5: ROCM-IB alias, CTS OFF (stable AINIC RoCEv2 path).",
|
|
"env_variables": {
|
|
"NCCL_NET": "ROCM-IB",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1"
|
|
},
|
|
"tests": [
|
|
{ "name": "B5_alltoall_ROCMIB_CTSoff", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "AINIC alias path (sched OFF, CTS OFF) over the full 1024-16G range." },
|
|
{ "name": "B5_allreduce_ROCMIB_CTSoff", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "AINIC alias allreduce." }
|
|
]
|
|
},
|
|
|
|
"b5_rocmib_ctson": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B6: ROCM-IB alias, CTS ON on P2P.",
|
|
"env_variables": {
|
|
"NCCL_NET": "ROCM-IB",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "0"
|
|
},
|
|
"tests": [
|
|
{ "name": "B6_alltoall_ROCMIB_CTSon", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "ROCM-IB alltoall with CTS ON on P2P connections." },
|
|
{ "name": "B6_allreduce_ROCMIB_CTSon", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "ROCM-IB allreduce with CTS ON." }
|
|
]
|
|
},
|
|
|
|
"b6_cast_wrr_ctson": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B7: CAST, WRR ON + CTS requested ON (CTS should be forced off when sched active).",
|
|
"env_variables": {
|
|
"NCCL_NET": "IB-CAST",
|
|
"RCCL_IB_QP_SCHED_ENABLE": "1",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "0"
|
|
},
|
|
"tests": [
|
|
{ "name": "B7_alltoall_CAST_WRR_CTSon", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "WRR ON + CTS requested - verifies CTS is forced OFF under an active scheduler, over the full 1024-16G range." },
|
|
{ "name": "B7_allreduce_CAST_WRR_CTSon", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "WRR ON + CTS requested allreduce." }
|
|
]
|
|
},
|
|
|
|
"b7_autodetect": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B8: out-of-the-box autodetect. NCCL_NET intentionally omitted (do NOT set to '').",
|
|
"tests": [
|
|
{ "name": "B8_alltoall_autodetect", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "Out-of-the-box: auto-select netIbCast on AINIC HW with stock defaults, over the full 1024-16G range (autodetect resolves to CAST)." },
|
|
{ "name": "B8_allreduce_autodetect", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "Out-of-the-box autodetect allreduce." }
|
|
]
|
|
},
|
|
|
|
"b7b_autodetect_off": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B9: AINIC forced OFF (RCCL_AINIC_ROCE=0); expect fallback to plain IB/RoCE.",
|
|
"env_variables": {
|
|
"RCCL_AINIC_ROCE": "0"
|
|
},
|
|
"tests": [
|
|
{ "name": "B9_alltoall_autodetect_off", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "AINIC forced OFF - verifies fallback to plain IB/RoCE on the same HW, over the full 1024-16G range for parity with the rest of the sweep." },
|
|
{ "name": "B9_allreduce_autodetect_off", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "AINIC forced OFF allreduce." }
|
|
]
|
|
},
|
|
|
|
"b8_no_cts_offload": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B10: ROCM-IB with CTS offload fully disabled.",
|
|
"env_variables": {
|
|
"NCCL_NET": "ROCM-IB",
|
|
"RCCL_CTS_OFFLOAD_ENABLED": "0"
|
|
},
|
|
"tests": [
|
|
{ "name": "B10_alltoall_ROCMIB_noOffload", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CTS offload disabled - isolates the ionic path without the CTS handshake, over the full 1024-16G range." },
|
|
{ "name": "B10_allreduce_ROCMIB_noOffload", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "CTS offload disabled allreduce." }
|
|
]
|
|
},
|
|
|
|
"b9_cts_force_override": {
|
|
"extends": "b_base",
|
|
"timeout": 600,
|
|
"description": "B11: PR #4657 force-override precedence. RCCL_CTS_OFFLOAD_ENABLED=1 is the strongest override and forces CTS offload on ALL connections including P2P, overriding RCCL_IB_P2P_DISABLE_CTS=1. Exercises the `enabled && (userForced || !(isP2p && IbP2pDisableCts))` userForced branch that no other B suite covers.",
|
|
"env_variables": {
|
|
"NCCL_NET": "ROCM-IB",
|
|
"RCCL_CTS_OFFLOAD_ENABLED": "1",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1"
|
|
},
|
|
"tests": [
|
|
{ "name": "B11_alltoall_ROCMIB_CTSforceOverride", "binary": "alltoall_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "PR #4657: RCCL_CTS_OFFLOAD_ENABLED=1 overrides IB_P2P_DISABLE_CTS=1, forcing CTS on P2P - validates the force-override precedence." },
|
|
{ "name": "B11_allreduce_ROCMIB_CTSforceOverride", "binary": "all_reduce_perf", "command_args": "-b 1024 -e 16G -f 2 -g 1", "description": "PR #4657 force-override on collective transport: allreduce expected PASS (CTS is supported on collective connections)." }
|
|
]
|
|
},
|
|
|
|
"b_large_alltoall_repro": {
|
|
"extends": "b_base",
|
|
"command_args": "-b 1K -e 16G -f 2 -g 1",
|
|
"description": "Dedicated large-size alltoall guard for the AINIC data path: CAST and ROCM-IB swept over the full 1K-16G range with a plain-IB control. Test names use the 1K_16G suffix to match the actual -b 1K -e 16G range.",
|
|
"tests": [
|
|
{ "name": "B12_alltoall_IB_1K_16G_control", "binary": "alltoall_perf", "env_variables": { "NCCL_NET": "IB" }, "description": "CONTROL: plain RoCE alltoall over 1K-16G (HW health check)." },
|
|
{ "name": "B12_alltoall_CAST_1K_16G", "binary": "alltoall_perf", "env_variables": { "NCCL_NET": "IB-CAST", "RCCL_IB_P2P_DISABLE_CTS": "1" }, "description": "CAST alltoall over 1K-16G." },
|
|
{ "name": "B12_alltoall_ROCMIB_1K_16G", "binary": "alltoall_perf", "env_variables": { "NCCL_NET": "ROCM-IB", "RCCL_IB_P2P_DISABLE_CTS": "1" }, "description": "ROCM-IB alltoall over 1K-16G (CAST path via alias)." }
|
|
]
|
|
},
|
|
|
|
"c_counters": {
|
|
"is_gtest": false,
|
|
"num_ranks": 16,
|
|
"num_nodes": 2,
|
|
"num_gpus": 8,
|
|
"timeout": 600,
|
|
"env_variables": {
|
|
"RCCL_TESTS_NET_COUNTER_ENABLE": "1",
|
|
"RCCL_IB_P2P_DISABLE_CTS": "1"
|
|
},
|
|
"tests": [
|
|
{ "name": "C1_alltoall_counters_CAST", "binary": "alltoall_perf", "command_args": "-b 1M -e 128M -f 2 -g 1", "env_variables": { "NCCL_NET": "IB-CAST" }, "description": "IONIC counters (tx_rdma_ucast_bytes) non-zero and BW > 10 GB/s on the IB-CAST backend. Capped at 128M to keep the counter run fast; large-size coverage lives in b_large_alltoall_repro." },
|
|
{ "name": "C1_allreduce_counters_CAST", "binary": "all_reduce_perf", "command_args": "-b 1G -e 8G -f 2 -g 1", "env_variables": { "NCCL_NET": "IB-CAST" }, "description": "IONIC counter validation on allreduce over the IB-CAST backend." },
|
|
{ "name": "C1_alltoall_counters_ROCMIB", "binary": "alltoall_perf", "command_args": "-b 1M -e 128M -f 2 -g 1", "env_variables": { "NCCL_NET": "ROCM-IB" }, "description": "IONIC counters (tx_rdma_ucast_bytes) non-zero and BW > 10 GB/s on the ROCM-IB backend. Capped at 128M to keep the counter run fast; large-size coverage lives in b_large_alltoall_repro." },
|
|
{ "name": "C1_allreduce_counters_ROCMIB", "binary": "all_reduce_perf", "command_args": "-b 1G -e 8G -f 2 -g 1", "env_variables": { "NCCL_NET": "ROCM-IB" }, "description": "IONIC counter validation on allreduce over the ROCM-IB backend." }
|
|
]
|
|
}
|
|
},
|
|
|
|
"test_suites": [
|
|
{ "name": "A1-A3. Component - NetIb core", "config": "a_netib_component", "enabled": true },
|
|
{ "name": "A4. Component - CTS workaround (off)", "config": "a_cts_workaround", "enabled": true },
|
|
{ "name": "A5. Component - CTS depth 32 (on, passes)", "config": "a_cts_depth32", "enabled": true },
|
|
{ "name": "A6-A7. Component - CAST WRR", "config": "a_cast_wrr", "enabled": true },
|
|
{ "name": "A8. Component - Fault injection", "config": "a_fault_inject", "enabled": true },
|
|
{ "name": "A9-A13. Component - GIN proxy", "config": "a_gin", "enabled": true },
|
|
{ "name": "A14. GIN enable gate (ROCM-27070)", "config": "a_gin_enable_gate", "enabled": true },
|
|
{ "name": "A15. GIN AINIC backend (ROCM-27070)", "config": "a_gin_ainic_backend", "enabled": true },
|
|
|
|
{ "name": "B1. IB baseline", "config": "b1_ib", "enabled": true },
|
|
{ "name": "B2. CAST alltoall full (1K-16G)", "config": "b_alltoall_cast_full", "enabled": true },
|
|
{ "name": "B3. CAST WRR / CTS off", "config": "b2_cast_wrr_ctsoff","enabled": true },
|
|
{ "name": "B4. CAST no-WRR / CTS off", "config": "b3_cast_nowrr_ctsoff","enabled": true },
|
|
{ "name": "B5. ROCM-IB / CTS off", "config": "b4_rocmib_ctsoff", "enabled": true },
|
|
{ "name": "B6. ROCM-IB / CTS on", "config": "b5_rocmib_ctson", "enabled": true },
|
|
{ "name": "B7. CAST WRR / CTS on", "config": "b6_cast_wrr_ctson", "enabled": true },
|
|
{ "name": "B8. Autodetect (out-of-box)", "config": "b7_autodetect", "enabled": true },
|
|
{ "name": "B9. Autodetect off (fallback)", "config": "b7b_autodetect_off","enabled": true },
|
|
{ "name": "B10. ROCM-IB / no CTS offload", "config": "b8_no_cts_offload", "enabled": true },
|
|
{ "name": "B11. CTS force-override (PR#4657)", "config": "b9_cts_force_override", "enabled": true },
|
|
{ "name": "B12. alltoall full-range guard", "config": "b_large_alltoall_repro", "enabled": true },
|
|
|
|
{ "name": "C1. IONIC counter validation", "config": "c_counters", "enabled": true }
|
|
]
|
|
}
|