derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
3252 lines
133 KiB
C++
3252 lines
133 KiB
C++
/*************************************************************************
|
|
* Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved.
|
|
*
|
|
* See LICENSE.txt for license information
|
|
************************************************************************/
|
|
|
|
// MPI tests for the device-side GIN proxy backend. Each test launches a real
|
|
// gin.{put|putValue|waitSignal|...} kernel against a real proxy thread + IB,
|
|
// and validates the wire-level result on the receiving rank.
|
|
|
|
#include "MPITestBase.hpp"
|
|
#include "ResourceGuards.hpp"
|
|
#include "TestChecks.hpp"
|
|
|
|
#include "nccl_device.h"
|
|
|
|
#include <algorithm>
|
|
#include <cstdint>
|
|
#include <cstdlib>
|
|
#include <cstring>
|
|
#include <hip/hip_runtime.h>
|
|
#include <ios>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#ifdef MPI_TESTS_ENABLED
|
|
|
|
using namespace RCCLTestGuards;
|
|
|
|
namespace {
|
|
|
|
std::string ginEnvDisabledReason() {
|
|
if (const char* e = std::getenv("NCCL_GIN_ENABLE"); e && std::strcmp(e, "0") == 0)
|
|
return "GIN explicitly disabled by environment (NCCL_GIN_ENABLE=0)";
|
|
return "";
|
|
}
|
|
|
|
// GIN type requested for this run (2=proxy, 4=rocshmem-gda); 0 if unset.
|
|
int requestedGinType() {
|
|
const char* t = std::getenv("NCCL_GIN_TYPE");
|
|
return t ? std::atoi(t) : 0;
|
|
}
|
|
|
|
std::string ginTypeReason() {
|
|
const char* ginType = std::getenv("NCCL_GIN_TYPE");
|
|
if (!ginType)
|
|
return "GIN type not set (required NCCL_GIN_TYPE=2 [proxy] or 4 [rocshmem-gda])";
|
|
int t = requestedGinType();
|
|
if (t != 2 && t != 4)
|
|
return std::string("Invalid GIN type: ") + ginType + " (required NCCL_GIN_TYPE=2 [proxy] or 4 [rocshmem-gda])";
|
|
return "";
|
|
}
|
|
|
|
std::string cuMemReason() {
|
|
const char* cumem = std::getenv("NCCL_CUMEM_ENABLE");
|
|
if (!cumem || std::strcmp(cumem, "1") != 0)
|
|
return "Symmetric memory required (NCCL_CUMEM_ENABLE=1)";
|
|
return "";
|
|
}
|
|
|
|
// Number of MPI ranks co-located on this rank's node.
|
|
int nodeLocalRanks() {
|
|
MPI_Comm nodeComm;
|
|
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &nodeComm);
|
|
int n = 0;
|
|
MPI_Comm_size(nodeComm, &n);
|
|
MPI_Comm_free(&nodeComm);
|
|
return n;
|
|
}
|
|
|
|
// Single-node runs need intranet mode -- otherwise the topology pruner
|
|
// removes the NET node and GIN has no path to bind.
|
|
std::string intranetReason() {
|
|
int worldSize = 0;
|
|
MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
|
|
if (nodeLocalRanks() != worldSize) return "";
|
|
const char* intra = std::getenv("RCCL_ENABLE_INTRANET");
|
|
if (!intra || std::strcmp(intra, "1") != 0)
|
|
return "Intranet mode required for single-node run (RCCL_ENABLE_INTRANET=1)";
|
|
return "";
|
|
}
|
|
|
|
// Skip if all ranks share a node -- IB would silently loopback.
|
|
std::string crossNodeReason() {
|
|
int worldSize = 0;
|
|
MPI_Comm_size(MPI_COMM_WORLD, &worldSize);
|
|
if (nodeLocalRanks() == worldSize)
|
|
return "Cross-node test requires ranks on >=2 physical nodes";
|
|
return "";
|
|
}
|
|
|
|
// First failing prerequisite, or "" if all met.
|
|
std::string ginProxyTestSkipReason() {
|
|
for (auto check : {ginEnvDisabledReason, ginTypeReason, cuMemReason, intranetReason}) {
|
|
if (auto reason = check(); !reason.empty()) return reason;
|
|
}
|
|
return "";
|
|
}
|
|
|
|
// Initialized reqs with ginConnectionType=FULL; default for all tests here.
|
|
ncclDevCommRequirements defaultGinReqs() {
|
|
ncclDevCommRequirements r = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER;
|
|
r.ginConnectionType = NCCL_GIN_CONNECTION_FULL;
|
|
return r;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
// Producer: thread 0 of block 0 issues one put with a SignalInc; CTA flushes.
|
|
__global__ void putBasicProducerKernel(
|
|
ncclWindow_t srcWin, size_t srcOff,
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
size_t bytes, ncclGinSignal_t sigIdx, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff,
|
|
srcWin, srcOff,
|
|
bytes,
|
|
ncclGin_SignalInc{sigIdx});
|
|
}
|
|
// Drain the posted GFD before the kernel exits.
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Consumer: whole CTA cooperatively waits for the signal to reach the target.
|
|
__global__ void putBasicConsumerKernel(
|
|
ncclGinSignal_t sigIdx, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
// Combined producer + consumer for alltoall: thread 0 puts to every non-self
|
|
// peer (one slot each), the CTA flushes, then waits for the same number of
|
|
// signal increments to arrive from peers.
|
|
__global__ void alltoallKernel(
|
|
ncclWindow_t sendWin,
|
|
ncclWindow_t recvWin,
|
|
size_t bytesPerSlot,
|
|
int nRanks, int myRank,
|
|
size_t slotStrideBytes,
|
|
ncclGinSignal_t sigIdx,
|
|
uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
auto team = ncclTeamWorld(devComm);
|
|
// Send slot p of our send buffer into peer p's recv slot for our rank.
|
|
for (int p = 0; p < nRanks; ++p) {
|
|
if (p == myRank) continue;
|
|
gin.put(team, p,
|
|
recvWin, /*dstOff=*/(size_t)myRank * slotStrideBytes,
|
|
sendWin, /*srcOff=*/(size_t)p * slotStrideBytes,
|
|
bytesPerSlot,
|
|
ncclGin_SignalInc{sigIdx});
|
|
}
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
class GinMPIDeviceTests : public MPITestBase {
|
|
protected:
|
|
// Minimal 64-byte put + waitSignal round-trip from rank 0 to rank 1.
|
|
// Used by the Invalid_*Pool tests to confirm comm bring-up + the GIN
|
|
// data path still work after the runtime clamps an oversized pool.
|
|
void runBasicPutSelfCheck() {
|
|
// Bring up the comm + stream from the fixture.
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Tiny geometry: full-buffer transfer, signal 0, peer is rank 1.
|
|
constexpr size_t kBufBytes = 64;
|
|
constexpr size_t kTransferBytes = 64;
|
|
constexpr ncclGinSignal_t kSigIdx = 0;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst on every rank; freed on scope exit.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective symmetric windows so the device side can address
|
|
// peer memory through srcWin/dstWin.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up the GIN device comm (1 barrier slot, 1 signal cell).
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Stage a deterministic byte pattern in the source buffer; dst stays zero.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
hostSrc[i] = static_cast<uint8_t>(0xA0 + (i & 0x3F));
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Sync so neither rank launches before the other has finished setup.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 0 puts the payload + bumps the signal; rank 1 waits on it.
|
|
if (rank == 0) {
|
|
putBasicProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, /*srcOff=*/0,
|
|
dstWin, /*dstOff=*/0,
|
|
kTransferBytes, kSigIdx, kPeer,
|
|
devComm);
|
|
} else {
|
|
putBasicConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Sync so rank 1's verify isn't racing rank 0's kernel completion.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 1 reads dst back and checks every byte landed.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
ASSERT_EQ(hostSrc[i], hostResult[i]) << "byte " << i << " mismatched";
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
// Smallest end-to-end exercise of the device put -> proxy -> IB -> peer
|
|
// signal chain, with non-zero src/dst/signal offsets so address-arithmetic
|
|
// regressions surface as a verification mismatch.
|
|
TEST_F(GinMPIDeviceTests, Put_BasicAndOffsets) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// 8 KiB symmetric buffers; 4 KiB transfer at non-zero src/dst offsets;
|
|
// signal at non-zero index. Forces every offset to be exercised.
|
|
constexpr size_t kBufBytes = 8 * 1024;
|
|
constexpr size_t kTransferBytes = 4 * 1024;
|
|
constexpr size_t kSrcOff = 4 * 1024;
|
|
constexpr size_t kDstOff = 2 * 1024;
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst on every rank.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows over the symmetric buffers.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN with 2 signals so kSigIdx=1 is a valid in-pool index.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Stage source pattern in [kSrcOff, kSrcOff+kTransferBytes); rest is zero.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
hostSrc[kSrcOff + i] = static_cast<uint8_t>(0x40 + (i & 0xFF));
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Sync so neither rank launches its kernel before setup is done globally.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 0 puts payload + bumps signal; rank 1 waits on the same signal.
|
|
if (rank == 0) {
|
|
putBasicProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff,
|
|
dstWin, kDstOff,
|
|
kTransferBytes, kSigIdx, kPeer,
|
|
devComm);
|
|
} else {
|
|
putBasicConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Sync before verify; both ranks have finished their kernel.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 1 verifies: payload landed exactly in [kDstOff, +kTransferBytes),
|
|
// and bytes before/after that range are still zero.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
const uint8_t expected = static_cast<uint8_t>(0x40 + (i & 0xFF));
|
|
ASSERT_EQ(expected, hostResult[kDstOff + i])
|
|
<< "byte " << i << " in [dstOff, dstOff+xfer) differs";
|
|
}
|
|
for (size_t i = 0; i < kDstOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " before dstOff was unexpectedly written";
|
|
}
|
|
for (size_t i = kDstOff + kTransferBytes; i < kBufBytes; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " after dstOff+xfer was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
|
|
// Same wire-level put as Put_BasicAndOffsets, but requires the two ranks to
|
|
// live on different physical nodes so IB actually traverses the fabric
|
|
// instead of falling back to a single-node loopback path.
|
|
TEST_F(GinMPIDeviceTests, Put_CrossNode) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// Skip on single-node runs; otherwise we'd just be retesting loopback.
|
|
if (auto reason = crossNodeReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Same geometry as Put_BasicAndOffsets so the comparison is apples-to-apples.
|
|
constexpr size_t kBufBytes = 8 * 1024;
|
|
constexpr size_t kTransferBytes = 4 * 1024;
|
|
constexpr size_t kSrcOff = 4 * 1024;
|
|
constexpr size_t kDstOff = 2 * 1024;
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst on every rank.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows over the symmetric buffers.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN with 2 signals so kSigIdx=1 is valid.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Stage source pattern in [kSrcOff, kSrcOff+kTransferBytes); rest zero.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
hostSrc[kSrcOff + i] = static_cast<uint8_t>(0x40 + (i & 0xFF));
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 0 puts + signals; rank 1 waits.
|
|
if (rank == 0) {
|
|
putBasicProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff,
|
|
dstWin, kDstOff,
|
|
kTransferBytes, kSigIdx, kPeer,
|
|
devComm);
|
|
} else {
|
|
putBasicConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Verify payload + zero-tails on rank 1.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
const uint8_t expected = static_cast<uint8_t>(0x40 + (i & 0xFF));
|
|
ASSERT_EQ(expected, hostResult[kDstOff + i])
|
|
<< "byte " << i << " in [dstOff, dstOff+xfer) differs";
|
|
}
|
|
for (size_t i = 0; i < kDstOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " before dstOff was unexpectedly written";
|
|
}
|
|
for (size_t i = kDstOff + kTransferBytes; i < kBufBytes; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " after dstOff+xfer was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
|
|
// Producer: putValue carries the 8-byte payload inside the GFD itself
|
|
// (no source MR is dereferenced); also bumps a signal so the receiver
|
|
// can wait synchronously.
|
|
__global__ void putValueInlineProducerKernel(
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
uint64_t value, ncclGinSignal_t sigIdx, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.putValue<uint64_t>(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff, value,
|
|
ncclGin_SignalInc{sigIdx});
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Consumer: just waits for the inline-value signal.
|
|
__global__ void putValueInlineConsumerKernel(
|
|
ncclGinSignal_t sigIdx, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
// Sends a single 8-byte uint64_t inline (no source buffer / MR involved)
|
|
// and verifies it lands intact at the requested offset on the peer.
|
|
TEST_F(GinMPIDeviceTests, PutValue_Inline) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// kValue exercises all three pieces of the inline 4+2+2 byte split,
|
|
// so any field-reconstruction regression in the host proxy surfaces.
|
|
constexpr size_t kBufBytes = 4 * 1024;
|
|
constexpr size_t kDstOff = 1 * 1024;
|
|
constexpr uint64_t kValue = 0x123456789ABCDEF0ULL;
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric dst on every rank (no src needed: value is inline).
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register dst window collectively.
|
|
ncclWindow_t dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN with 2 signals so kSigIdx=1 is valid.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Zero dst so any spurious write outside the 8-byte landing surfaces.
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 0 sends the inline value + signal; rank 1 waits.
|
|
if (rank == 0) {
|
|
putValueInlineProducerKernel<<<1, 32, 0, stream>>>(
|
|
dstWin, kDstOff, kValue, kSigIdx, kPeer, devComm);
|
|
} else {
|
|
putValueInlineConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 1 verifies the 8 bytes at dstOff match kValue and nothing else moved.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
uint64_t got = 0;
|
|
std::memcpy(&got, hostResult.data() + kDstOff, sizeof(got));
|
|
ASSERT_EQ(kValue, got)
|
|
<< "inline value mismatch (4+2+2 split likely corrupted)";
|
|
|
|
for (size_t i = 0; i < kDstOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " before dstOff was unexpectedly written";
|
|
}
|
|
for (size_t i = kDstOff + sizeof(uint64_t); i < kBufBytes; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " after dstOff+8 was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
|
|
// Producer: thread 0 issues a zero-byte gin.signal (no src/dst windows).
|
|
// The GFD becomes a non-inline put with size=0, hasInline=false; the proxy
|
|
// dispatches it through the signal-only iputSignal path
|
|
// (gin_host_proxy.cc:271-273).
|
|
__global__ void signalNoPayloadProducerKernel(
|
|
ncclGinSignal_t sigIdx, int peer, struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.signal(ncclTeamWorld(devComm), peer, ncclGin_SignalInc{sigIdx});
|
|
}
|
|
// Drain the posted GFD before the kernel exits.
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Consumer: whole CTA cooperatively waits for the signal to reach the target.
|
|
__global__ void signalNoPayloadConsumerKernel(
|
|
ncclGinSignal_t sigIdx, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
// Bare-minimum signal RTT: rank 0 issues a zero-byte gin.signal that only
|
|
// bumps rank 1's signal cell -- no data, no src/dst windows. It's the
|
|
// latency-floor primitive the barrier is built on and exercises the proxy's
|
|
// signal-only path. Signal index is non-zero so a regression that drops the
|
|
// index multiplier would land at cell 0 and the consumer's waitSignal(1)
|
|
// would never observe the bump.
|
|
TEST_F(GinMPIDeviceTests, Signal_NoPayload) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr int kPeer = 1; // rank 0 -> rank 1
|
|
|
|
// No buffers, no windows: gin.signal is a zero-byte put. The runtime's
|
|
// own signal pool is the only memory touched by the GFD; it's allocated
|
|
// and registered by ncclDevCommCreate based on ginSignalCount.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Sync so neither rank launches its kernel before setup is done globally.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Producer (rank 0) emits a single signal; consumer (rank 1) waits for
|
|
// signal cell kSigIdx to reach 1. The successful return of waitSignal on
|
|
// the consumer IS the assertion -- there is no payload to verify.
|
|
if (rank == 0) {
|
|
signalNoPayloadProducerKernel<<<1, 32, 0, stream>>>(kSigIdx, kPeer, devComm);
|
|
} else {
|
|
signalNoPayloadConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Both kernels are done; rank 1's signal cell is settled.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Validates the NCCL 2.29.7 defect (AICOMRCCL-1115): "Fix a 16-bit overflow of
|
|
// signal and counter ids with GIN proxy." Pre-fix, the proxy GPU-flush
|
|
// descriptor packed signalId/counterId into uint16_t fields
|
|
// (gin_proxy_device_host_common.h), so any id >= 65536 truncated to (id &
|
|
// 0xFFFF). The fix widens those fields to uint32_t bitfields (signalId : 24,
|
|
// counterId : 23) and grows the pools/validation to match. Here rank 0 bumps a
|
|
// signal whose index is exactly one past the old 16-bit range; rank 1 waits on
|
|
// that same index. With the fix the bump lands at cell 65536 and waitSignal
|
|
// returns; pre-fix the id truncates to cell 0, the waiter on 65536 never
|
|
// observes it, and the consumer hangs (surfaced as a timeout) -- exactly the
|
|
// discrimination we want. This reuses the Signal_NoPayload kernels unchanged;
|
|
// only the index and pool size differ from the in-range control above.
|
|
TEST_F(GinMPIDeviceTests, Signal_HighIdNoOverflow) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Index one past the old uint16_t range. ginSignalCount must exceed it so the
|
|
// cell is a valid in-pool index (post-fix default pool is 512Ki).
|
|
constexpr ncclGinSignal_t kSigIdx = 65536; // > 0xFFFF
|
|
constexpr uint32_t kSignalPoolCount = 65537;
|
|
constexpr int kPeer = 1; // rank 0 -> rank 1
|
|
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = kSignalPoolCount;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Producer (rank 0) bumps signal cell kSigIdx; consumer (rank 1) waits for it
|
|
// to reach 1. The consumer's successful return IS the assertion -- pre-fix it
|
|
// never returns because the bump truncates to cell 0.
|
|
if (rank == 0) {
|
|
signalNoPayloadProducerKernel<<<1, 32, 0, stream>>>(kSigIdx, kPeer, devComm);
|
|
} else {
|
|
signalNoPayloadConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Producer-only kernel: rank 0 issues a put with no remote action and a
|
|
// CounterInc local action, then waits on its OWN counter. The counter is
|
|
// bumped by the local proxy thread when the IB CQE for the put lands
|
|
// (gin_host_proxy.cc:147-152, inside proxyGinPollCompletions). That's a
|
|
// distinct path from the remote-signal write dispatched via iputSignal
|
|
// (gin_host_proxy.cc:271-273) which the prior put/signal tests exercise.
|
|
__global__ void waitCounterLocalProducerKernel(
|
|
ncclWindow_t srcWin, size_t srcOff,
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
size_t bytes, ncclGinCounter_t cntIdx, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff, srcWin, srcOff, bytes,
|
|
ncclGin_None{}, // no remote action (no signal)
|
|
ncclGin_CounterInc{cntIdx}); // local: bump cntIdx on IB CQE
|
|
}
|
|
// CTA-collective wait; only thread 0 spins on the cell. Once it returns,
|
|
// the proxy thread has observed the IB CQE for the put and bumped the
|
|
// counter.
|
|
gin.waitCounter(ncclCoopCta(), cntIdx, /*least=*/1);
|
|
}
|
|
|
|
// Exercises the local IB-completion -> counter-bump path. Rank 1 is a
|
|
// passive RDMA target (no kernel) - the payload lands in its dst window
|
|
// via the IB plugin. The successful return of waitCounter(1) on rank 0
|
|
// (gated by hipStreamSynchronize) IS the assertion.
|
|
TEST_F(GinMPIDeviceTests, WaitCounter_Local) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// 8 KiB symmetric buffers; 4 KiB payload at non-zero src/dst offsets so
|
|
// the put is a realistic data move. The counter (not the payload) is
|
|
// what we verify -- this test is about the LOCAL completion path.
|
|
constexpr size_t kBufBytes = 8 * 1024;
|
|
constexpr size_t kTransferBytes = 4 * 1024;
|
|
constexpr size_t kSrcOff = 4 * 1024;
|
|
constexpr size_t kDstOff = 2 * 1024;
|
|
constexpr ncclGinCounter_t kCntIdx = 1; // non-zero counter index
|
|
constexpr int kPeer = 1;
|
|
|
|
// Symmetric src + dst (every rank allocates because window registration
|
|
// is collective for SYMMETRIC-mode windows).
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// ginCounterCount=2 so kCntIdx=1 is in range; no ginSignalCount needed
|
|
// (the put has no remote-signal action). railGinBarrierCount=1 because
|
|
// the runtime allocates per-CTA barrier state for any 1-CTA launch.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginCounterCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Known fill on src so the put has real bytes to move; both ranks zero
|
|
// their dst as a baseline. We don't verify the destination -- this test
|
|
// is about the local counter, not data delivery.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0xCC);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Sync so neither rank launches its kernel before setup is done globally.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Only rank 0 launches a kernel; the local counter lives on rank 0. Rank
|
|
// 1 is purely the RDMA target -- the IB plugin lands the payload in its
|
|
// dst window passively.
|
|
if (rank == 0) {
|
|
waitCounterLocalProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff, dstWin, kDstOff,
|
|
kTransferBytes, kCntIdx, kPeer, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Sync so ScopeGuards (window/comm teardown is collective) see both
|
|
// ranks past the kernel phase.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Producer kernel: rank 0 issues a single put carrying BOTH a remote
|
|
// SignalInc action and a local CounterInc action, then waits on its OWN
|
|
// counter. The remote SignalInc bumps the peer's signal cell when the IB
|
|
// write lands at the target (gin_host_proxy.cc:271-273); the local
|
|
// CounterInc bumps rank 0's counter when the IB CQE for the put is
|
|
// observed by the local proxy (gin_host_proxy.cc:147-152). One put,
|
|
// two completion sites.
|
|
__global__ void waitCounterAndSignalProducerKernel(
|
|
ncclWindow_t srcWin, size_t srcOff,
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
size_t bytes,
|
|
ncclGinSignal_t sigIdx, ncclGinCounter_t cntIdx, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff, srcWin, srcOff, bytes,
|
|
ncclGin_SignalInc{sigIdx}, // remote: bump peer's signal cell
|
|
ncclGin_CounterInc{cntIdx}); // local: bump cntIdx on IB CQE
|
|
}
|
|
// CTA-collective wait gating kernel exit on local CQ completion. On the
|
|
// proxy backend, the local CQE strictly follows the remote SignalInc
|
|
// landing, so once this returns the consumer's waitSignal is unblocked
|
|
// (or already past).
|
|
gin.waitCounter(ncclCoopCta(), cntIdx, /*least=*/1);
|
|
}
|
|
|
|
// Consumer kernel: rank 1 waits on its signal cell to reach 1. No put is
|
|
// issued from this side -- the IB plugin lands rank 0's payload + signal
|
|
// passively.
|
|
__global__ void waitCounterAndSignalConsumerKernel(
|
|
ncclGinSignal_t sigIdx, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
// Exercises both completion sites of a single gin.put with combined
|
|
// (SignalInc, CounterInc) actions:
|
|
// - Remote signal notification: rank 1's waitSignal returns when the
|
|
// IB write of the SignalInc lands on its signal cell.
|
|
// - Local IB-CQE -> counter bump: rank 0's waitCounter returns when the
|
|
// local proxy thread has observed the CQE for the put.
|
|
// Together with the post-sync host-side payload check on rank 1, this
|
|
// proves the same put delivered (a) bytes, (b) remote signal, and
|
|
// (c) local counter -- all driven by one gin.put.
|
|
TEST_F(GinMPIDeviceTests, WaitCounterAndSignal) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// 8 KiB symmetric buffers; 4 KiB payload at non-zero src/dst offsets so
|
|
// the put is a realistic data move. Both signal and counter indices are
|
|
// non-zero so a regression that drops the index multiplier would land
|
|
// at cell 0 and the waits would never observe the bump.
|
|
constexpr size_t kBufBytes = 8 * 1024;
|
|
constexpr size_t kTransferBytes = 4 * 1024;
|
|
constexpr size_t kSrcOff = 4 * 1024;
|
|
constexpr size_t kDstOff = 2 * 1024;
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr ncclGinCounter_t kCntIdx = 1;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Symmetric src + dst on every rank (window registration is collective
|
|
// for SYMMETRIC-mode windows).
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// ginSignalCount=2 and ginCounterCount=2 so kSigIdx=1 and kCntIdx=1 are
|
|
// both in range. railGinBarrierCount=1 covers the 1-CTA launch.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
reqs.ginCounterCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Deterministic byte pattern in src so the post-sync data check on
|
|
// rank 1 catches wrong-bytes / wrong-offset regressions. Both ranks
|
|
// zero their dst as a baseline.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
hostSrc[kSrcOff + i] = static_cast<uint8_t>(0x40 + (i & 0xFF));
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Sync so neither rank launches before setup is done globally.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 0 puts + bumps remote signal + bumps local counter; rank 1 waits
|
|
// for the signal. Each side's gin wait IS its assertion (counter on
|
|
// rank 0, signal on rank 1); the host-side memcmp on rank 1 below
|
|
// verifies the payload.
|
|
if (rank == 0) {
|
|
waitCounterAndSignalProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff, dstWin, kDstOff,
|
|
kTransferBytes, kSigIdx, kCntIdx, kPeer, devComm);
|
|
} else {
|
|
waitCounterAndSignalConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Sync so rank 1's verify isn't racing rank 0's kernel completion.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 1 verifies the payload landed at [kDstOff, kDstOff+kTransferBytes)
|
|
// and nothing outside that range was touched.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
for (size_t i = 0; i < kDstOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " before dstOff was unexpectedly written";
|
|
}
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
ASSERT_EQ(hostSrc[kSrcOff + i], hostResult[kDstOff + i])
|
|
<< "payload byte " << i << " mismatched";
|
|
}
|
|
for (size_t i = kDstOff + kTransferBytes; i < kBufBytes; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " after dstOff+kTransferBytes was unexpectedly written";
|
|
}
|
|
}
|
|
|
|
// Final sync so collective teardown (ScopeGuards) see both ranks past
|
|
// the verification phase.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Collective kernel: every rank runs the same code. The barrier composes
|
|
// signal + waitSignal + an epoch counter (gin_barrier__funcs.h:42-60) -- each
|
|
// sync sends SignalInc to every other rank and waits for the local signal
|
|
// cell to reach a higher epoch, so consecutive syncs don't collide on the
|
|
// same expected value. The ncclTeamTagRail{} overload routes to
|
|
// comm.railGinBarrier and the rail team automatically.
|
|
__global__ void barrier2RanksKernel(int iters, struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
// barrierIndex must be < railGinBarrierCount (we set =1, so 0).
|
|
ncclGinBarrierSession<ncclCoopCta> bar{
|
|
ncclCoopCta(), gin, ncclTeamTagRail{}, /*barrierIndex=*/0};
|
|
for (int i = 0; i < iters; i++) {
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed);
|
|
}
|
|
}
|
|
|
|
// First test that actually USES the railGinBarrier resource (prior tests
|
|
// only provisioned it). Kernel completion after kIters rounds proves both
|
|
// ranks moved through the barrier in lockstep -- if the epoch math broke,
|
|
// iter 1 would deadlock waiting for an epoch the peer never reaches.
|
|
TEST_F(GinMPIDeviceTests, Barrier_TwoRanks) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Large enough that a stuck-at-zero epoch would deadlock at iter 1
|
|
// (not silently pass on iter 0); small enough to stay fast. Each round
|
|
// sends 1 SignalInc per peer (1 in the 2-rank case) and waits for the
|
|
// corresponding signal cell to reach the new epoch.
|
|
constexpr int kIters = 16;
|
|
|
|
// railGinBarrierCount=1 covers our 1-CTA launch (barrierIndex=0). The
|
|
// barrier session uses the rail-team GIN barrier pool that the runtime
|
|
// allocates via ncclGinBarrierCreateRequirement; its signal cells live
|
|
// at (handle.signal0 + barrierIndex), separate from the user-facing
|
|
// ginSignalCount pool.
|
|
//
|
|
// ginForceEnable is REQUIRED. ncclDevCommCreateInternal only activates
|
|
// GIN (populating ginHandles[]/ginSignalBase) when nNodes>1 OR
|
|
// ginForceEnable OR ginSignalCount!=0 OR ginCounterCount!=0 -- the gate
|
|
// intentionally ignores railGinBarrierCount. Without this flag a
|
|
// single-node test that asks only for railGinBarrierCount gets NULL
|
|
// ginHandles[0], and the barrier's internal waitSignal -> getSignalPtr
|
|
// dispatch dereferences a NULL ncclGinProxyGpuCtx and faults at (nil)
|
|
// on the GPU. Other tests dodge the gate accidentally by setting
|
|
// ginSignalCount/ginCounterCount non-zero; this one has neither so we
|
|
// ask for GIN explicitly.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginForceEnable = true;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// All ranks finished setup before any kernel launches.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Both ranks launch the same kernel -- barrier is collective, no
|
|
// producer/consumer split. Each rank's iter i sends SignalInc to the
|
|
// other rank and waits for its own signal cell to reach epoch+1; if the
|
|
// barrier raced past the peer (epoch broken), iter i+1 would either spin
|
|
// forever or read a stale epoch. Kernel completion means all kIters
|
|
// rounds stayed in lockstep.
|
|
barrier2RanksKernel<<<1, 32, 0, stream>>>(kIters, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
// Successful return of the kernel IS the assertion: signal + waitSignal
|
|
// + epoch all worked together for kIters rounds.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Direct tests for ncclBarrierSession (barrier.h), separate from the alltoall
|
|
// kernels that use it incidentally: BarrierSession_LsaOnly (LSA-only subset,
|
|
// ncclTeamTagLsa, no GIN) and BarrierSession_Hybrid (world-team: inner LSA +
|
|
// outer rail-GIN).
|
|
|
|
// LSA-team collective for the LSA-only subset. Each round every rank stamps the
|
|
// iteration into its slot of each peer's window, barriers, then checks all
|
|
// peers' stamps are visible. acq_rel publishes our writes and acquires theirs;
|
|
// a broken barrier leaves a stale slot in dErr (or deadlocks on a broken epoch).
|
|
__global__ void barrierSessionLsaKernel(
|
|
ncclWindow_t win, size_t off, int iters, int* dErr,
|
|
struct ncclDevComm devComm) {
|
|
// Dedicated LSA barrier session reads comm.lsaBarrier (sized by reqs.lsaBarrierCount, no GIN); the composite
|
|
// ncclBarrierSession(ncclTeamTagLsa) reads the 0-sized comm.hybridLsaBarrier and overruns for LSA teams >2 ranks.
|
|
ncclLsaBarrierSession<ncclCoopCta> bar{
|
|
ncclCoopCta(), devComm, ncclTeamTagLsa(), /*index=*/blockIdx.x};
|
|
ncclTeam lsa = ncclTeamLsa(devComm);
|
|
int* myBuf = static_cast<int*>(ncclGetLocalPointer(win, off));
|
|
for (int it = 1; it <= iters; ++it) {
|
|
if (threadIdx.x == 0) {
|
|
for (int p = 0; p < lsa.nRanks; ++p) {
|
|
int* peer = static_cast<int*>(ncclGetLsaPointer(win, off, p));
|
|
peer[lsa.rank] = it;
|
|
}
|
|
}
|
|
// Publish our writes to LSA peers and acquire theirs (acq_rel so the loads
|
|
// below are ordered after every peer's store, not relaxed).
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_acq_rel);
|
|
if (threadIdx.x == 0) {
|
|
for (int p = 0; p < lsa.nRanks; ++p) {
|
|
if (myBuf[p] != it) atomicExch(dErr, it);
|
|
}
|
|
}
|
|
// Gate the next round's overwrites on all peers having read this round.
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_acquire);
|
|
}
|
|
}
|
|
|
|
// Exercises the LSA-only subset in isolation. Needs symmetric memory and >=2
|
|
// ranks co-located on a node; it does NOT require the GIN proxy, so it gates on
|
|
// cuMem + a node-local size check rather than the full GIN prerequisites.
|
|
TEST_F(GinMPIDeviceTests, BarrierSession_LsaOnly) {
|
|
if (auto reason = cuMemReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
// Needs >=2 co-located ranks; otherwise the LSA team is size 1 (vacuous).
|
|
if (nodeLocalRanks() < 2)
|
|
GTEST_SKIP() << "Requires >=2 ranks co-located on a node for a non-trivial LSA team";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int nRanks = -1;
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
// LSA-only path: start from the plain initializer (GIN disabled) and ask only
|
|
// for the inner LSA barrier pool, so ncclDevCommCreate does not activate GIN.
|
|
ncclDevCommRequirements reqs = NCCL_DEV_COMM_REQUIREMENTS_INITIALIZER;
|
|
reqs.lsaBarrierCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// One int slot per possible LSA peer (<= nRanks). Symmetric window so peers
|
|
// can store into our buffer via ncclGetLsaPointer.
|
|
constexpr int kIters = 16;
|
|
const size_t bytes = static_cast<size_t>(nRanks) * sizeof(int);
|
|
void* dBuf = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dBuf, bytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dBuf) (void)ncclMemFree(dBuf);
|
|
});
|
|
|
|
ncclWindow_t win = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dBuf, bytes, &win, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (win) (void)ncclCommWindowDeregister(comm, win);
|
|
});
|
|
|
|
// Device-side error flag: set to the failing iteration if a peer's write
|
|
// was not visible after the barrier.
|
|
int* dErr = nullptr;
|
|
ASSERT_MPI_EQ(hipSuccess, hipMalloc(&dErr, sizeof(int)));
|
|
auto errCleanup = makeScopeGuard([&]() {
|
|
if (dErr) (void)hipFree(dErr);
|
|
});
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemset(dErr, 0, sizeof(int)));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemset(dBuf, 0, bytes));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Collective: every rank runs the same kernel on its node's LSA team.
|
|
barrierSessionLsaKernel<<<1, 64, 0, stream>>>(win, /*off=*/0, kIters, dErr, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
int hErr = 0;
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(&hErr, dErr, sizeof(int), hipMemcpyDeviceToHost));
|
|
ASSERT_EQ(0, hErr) << "LSA barrier failed to make a peer's write visible at iteration " << hErr;
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// Collective over the world team: composes the inner LSA barrier with the
|
|
// outer rail-GIN barrier (ncclTeamTagWorld ctor). Mirrors Barrier_TwoRanks'
|
|
// "completion == success" philosophy -- a broken inner or outer epoch
|
|
// deadlocks at iter 1 -- but routes through both substrates. On single node
|
|
// the outer rail arm degenerates; multi-node it crosses rails.
|
|
__global__ void barrierSessionHybridKernel(int iters, struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
ncclBarrierSession<ncclCoopCta> bar{
|
|
ncclCoopCta(), ncclTeamTagWorld(), gin, /*index=*/blockIdx.x};
|
|
for (int i = 0; i < iters; i++) {
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed);
|
|
}
|
|
}
|
|
|
|
TEST_F(GinMPIDeviceTests, BarrierSession_Hybrid) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int nRanks = -1;
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
constexpr int kIters = 16;
|
|
|
|
// The world-team ncclBarrierSession sizes its hybrid LSA+rail-GIN barriers from
|
|
// reqs.barrierCount (not lsa/railGinBarrierCount); 0 hangs the rail arm. ginForceEnable
|
|
// activates GIN so the outer barrier's waitSignal has a valid ctx (see Barrier_TwoRanks).
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.barrierCount = 1;
|
|
reqs.ginForceEnable = true;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// All ranks run the same collective barrier loop; completion means every
|
|
// round's inner LSA + outer rail-GIN epochs stayed in lockstep.
|
|
barrierSessionHybridKernel<<<1, 32, 0, stream>>>(kIters, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// SignalAdd_AndShadow
|
|
// Walks the signal-shadow API family end-to-end. SignalAdd (variable add)
|
|
// takes a different proxy host path than SignalInc (+1):
|
|
// gin_proxy.h -> ncclGinProxyOpWithSignalAdd. The shadow-side calls
|
|
// (readSignal, increaseSignalShadow, waitSignalMeetShadow,
|
|
// waitSignalFollowShadow) are not exercised by any other test. The shadow
|
|
// lives in GPU memory at comm.ginSignalShadows + ... so it persists across
|
|
// kernel launches on the same rank; only increaseSignalShadow and
|
|
// resetSignal mutate it.
|
|
//
|
|
// Phases (rank 0 = producer, rank 1 = consumer):
|
|
// P1: producer 1x put(SignalAdd+5, CounterInc); waitCounter(1).
|
|
// consumer waitSignal(5); readSignal -> must be 5.
|
|
// P2: producer 3x put(SignalAdd+7, CounterInc); waitCounter(4).
|
|
// consumer waitSignalFollowShadow(leastDelta=1, &b, &d) -> b=0,
|
|
// d=26 (signal moved 0->26 since the last shadow snapshot). Then
|
|
// increaseSignalShadow(+100): shadow := 126.
|
|
// P3: producer 1x put(SignalAdd+100); no counter; drained via flush.
|
|
// consumer waitSignalMeetShadow returns when signal >= 126.
|
|
// P4: producer readCounter -> must be 4 (4 counter-bearing puts; P3
|
|
// carried no counter).
|
|
//
|
|
// Why waitCounter at the end of P1/P2 producer kernels: local CQ
|
|
// completion (counter bump) implies the remote SignalAdd write for that
|
|
// put has landed on the consumer's signal cell. Without it, the
|
|
// inter-phase MPI_Barrier would not guarantee in-flight signal writes
|
|
// have reached the peer, and the consumer could observe an intermediate
|
|
// signal value (e.g. 5 instead of 26 in P2).
|
|
//
|
|
// P2 ordering quirk: waitSignalFollowShadow(leastDelta=1) returns as soon
|
|
// as signal > shadow. If P2's puts and the consumer's wait launched
|
|
// concurrently, the consumer could observe signal=5 (still just P1) and
|
|
// return with delta=5 -- the d==26 assertion would fail spuriously. So
|
|
// P2 producer runs to completion first, then an MPI_Barrier, then the P2
|
|
// consumer launches. P1 and P3 don't have this issue and run concurrently.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
// Producer with counter. Used by P1 (nPuts=1, signalDelta=5) and P2
|
|
// (nPuts=3, signalDelta=7). waitCounter at the end gates exit on local CQ
|
|
// completion for all `nPuts` puts, which on the proxy backend implies the
|
|
// remote SignalAdd writes have landed at the peer.
|
|
__global__ void signalAddProducerWithCounterKernel(
|
|
ncclWindow_t srcWin, size_t srcOff,
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
size_t bytes,
|
|
ncclGinSignal_t sigIdx, uint64_t signalDelta,
|
|
ncclGinCounter_t cntIdx, uint64_t counterExpected,
|
|
int nPuts, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
for (int i = 0; i < nPuts; i++) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff, srcWin, srcOff, bytes,
|
|
ncclGin_SignalAdd{sigIdx, signalDelta}, // remote: bump peer's signal by delta
|
|
ncclGin_CounterInc{cntIdx}); // local: bump cntIdx on IB CQE
|
|
}
|
|
}
|
|
gin.waitCounter(ncclCoopCta(), cntIdx, counterExpected);
|
|
}
|
|
|
|
// Producer without counter. Used by P3: a single put carrying only
|
|
// ncclGin_SignalAdd{sig, 100}. flush() drains posted GFDs before exit so
|
|
// the kernel doesn't return while the SignalAdd is still in flight on the
|
|
// proxy.
|
|
__global__ void signalAddProducerNoCounterKernel(
|
|
ncclWindow_t srcWin, size_t srcOff,
|
|
ncclWindow_t dstWin, size_t dstOff,
|
|
size_t bytes,
|
|
ncclGinSignal_t sigIdx, uint64_t signalDelta,
|
|
int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, dstOff, srcWin, srcOff, bytes,
|
|
ncclGin_SignalAdd{sigIdx, signalDelta});
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Producer P4: read the counter back. Must be 4 = 1 (P1) + 3 (P2); P3 had
|
|
// no counter so it cannot have bumped it.
|
|
__global__ void signalAddProducerReadCounterKernel(
|
|
ncclGinCounter_t cntIdx, uint64_t* outCounter,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
*outCounter = gin.readCounter(cntIdx);
|
|
}
|
|
}
|
|
|
|
// Consumer P1: waitSignal(>=5) then readSignal. After P1's single
|
|
// SignalAdd+5 the signal cell is exactly 5, so readSignal must return 5.
|
|
__global__ void signalAddConsumerPhase1Kernel(
|
|
ncclGinSignal_t sigIdx, uint64_t least,
|
|
uint64_t* outReadSignal,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, least);
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
*outReadSignal = gin.readSignal(sigIdx);
|
|
}
|
|
}
|
|
|
|
// Consumer P2: waitSignalFollowShadow(leastDelta=1) returns once
|
|
// signal > shadow by >= 1; P2 producer's waitCounter(4) guarantees signal
|
|
// has reached 5+3*7=26 here, so before==0 (initial shadow) and delta==26.
|
|
// Then increaseSignalShadow(+100) raises shadow from 26 to 126 for P3.
|
|
__global__ void signalAddConsumerPhase2Kernel(
|
|
ncclGinSignal_t sigIdx,
|
|
uint64_t leastDelta,
|
|
uint64_t shadowBump,
|
|
uint64_t* outBefore, uint64_t* outDelta,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
uint64_t before = 0, delta = 0;
|
|
gin.waitSignalFollowShadow(ncclCoopCta(), sigIdx, leastDelta, &before, &delta);
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
*outBefore = before;
|
|
*outDelta = delta;
|
|
gin.increaseSignalShadow(sigIdx, shadowBump);
|
|
}
|
|
}
|
|
|
|
// Consumer P3: shadow was raised to 126 in P2; this waits for the
|
|
// producer's SignalAdd+100 to bring signal from 26 -> 126.
|
|
__global__ void signalAddConsumerPhase3Kernel(
|
|
ncclGinSignal_t sigIdx, struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignalMeetShadow(ncclCoopCta(), sigIdx);
|
|
}
|
|
|
|
TEST_F(GinMPIDeviceTests, SignalAdd_AndShadow) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// The put is just a vehicle for the SignalAdd + CounterInc actions; 8
|
|
// bytes is the smallest reasonable payload. Bytes correctness is covered
|
|
// by Put_BasicAndOffsets and is not asserted here.
|
|
constexpr size_t kBufBytes = 64;
|
|
constexpr size_t kTransferBytes = 8;
|
|
constexpr size_t kSrcOff = 0;
|
|
constexpr size_t kDstOff = 0;
|
|
constexpr ncclGinSignal_t kSigIdx = 1; // non-zero so signal-pool indexing is exercised
|
|
constexpr ncclGinCounter_t kCntIdx = 1;
|
|
constexpr int kPeer = 1; // rank 0 -> rank 1
|
|
|
|
// Symmetric src + dst (window registration is collective for
|
|
// SYMMETRIC-mode windows, so every rank allocates).
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// sigIdx=1 and cntIdx=1 -> ginSignalCount/ginCounterCount must be >= 2.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
reqs.ginCounterCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Producer needs a known fill (bytes aren't asserted but the put must
|
|
// have something to move). Consumer's dst gets zeroed.
|
|
if (rank == 0) {
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0xA5);
|
|
ASSERT_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
}
|
|
std::vector<uint8_t> hostZero(kBufBytes, 0);
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostZero.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Per-rank result buffer. Layout (uint64_t slots):
|
|
// producer (rank 0): [0] readCounter (P4)
|
|
// consumer (rank 1): [0] readSignal (P1), [1] before (P2), [2] delta (P2)
|
|
// Initialized to a sentinel so an unwritten slot is obvious in failure logs.
|
|
constexpr size_t kResultSlots = 4;
|
|
constexpr uint64_t kSentinel = 0xFFFFFFFFFFFFFFFFull;
|
|
uint64_t* dResults = nullptr;
|
|
ASSERT_EQ(hipSuccess, hipMalloc(&dResults, kResultSlots * sizeof(uint64_t)));
|
|
auto resultsCleanup = makeScopeGuard([&]() { if (dResults) (void)hipFree(dResults); });
|
|
std::vector<uint64_t> hostInit(kResultSlots, kSentinel);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(dResults, hostInit.data(), kResultSlots * sizeof(uint64_t),
|
|
hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// --- P1: producer 1x SignalAdd+5/CounterInc, consumer waitSignal(5) +
|
|
// readSignal. Concurrent: the consumer's waitSignal blocks until
|
|
// the put lands, and no later producer activity is in flight that
|
|
// could push signal>5.
|
|
if (rank == 0) {
|
|
signalAddProducerWithCounterKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff, dstWin, kDstOff, kTransferBytes,
|
|
kSigIdx, /*signalDelta=*/5,
|
|
kCntIdx, /*counterExpected=*/1,
|
|
/*nPuts=*/1, kPeer, devComm);
|
|
} else {
|
|
signalAddConsumerPhase1Kernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*least=*/5, &dResults[0], devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// --- P2: producer 3x SignalAdd+7/CounterInc, ending with waitCounter(4).
|
|
// MUST complete BEFORE the consumer's waitSignalFollowShadow runs
|
|
// (see banner: leastDelta=1 could otherwise observe signal=5 from
|
|
// P1 and return with delta=5 instead of 26).
|
|
if (rank == 0) {
|
|
signalAddProducerWithCounterKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff, dstWin, kDstOff, kTransferBytes,
|
|
kSigIdx, /*signalDelta=*/7,
|
|
kCntIdx, /*counterExpected=*/4,
|
|
/*nPuts=*/3, kPeer, devComm);
|
|
ASSERT_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
}
|
|
MPI_Barrier(MPI_COMM_WORLD); // gate consumer P2 on producer P2 done
|
|
|
|
if (rank == 1) {
|
|
signalAddConsumerPhase2Kernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*leastDelta=*/1, /*shadowBump=*/100,
|
|
&dResults[1], &dResults[2], devComm);
|
|
ASSERT_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
}
|
|
MPI_Barrier(MPI_COMM_WORLD); // shadow on rank 1 is now 26+100=126
|
|
|
|
// --- P3: producer 1x SignalAdd+100 (no counter, drained via flush);
|
|
// consumer waitSignalMeetShadow returns when signal >= 126.
|
|
// Concurrent: consumer is gated on the SignalAdd landing.
|
|
if (rank == 0) {
|
|
signalAddProducerNoCounterKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, kSrcOff, dstWin, kDstOff, kTransferBytes,
|
|
kSigIdx, /*signalDelta=*/100,
|
|
kPeer, devComm);
|
|
} else {
|
|
signalAddConsumerPhase3Kernel<<<1, 32, 0, stream>>>(kSigIdx, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// --- P4: producer reads counter back. The 4 counter-bearing puts (P1: 1,
|
|
// P2: 3) were all waited for via waitCounter in their kernels;
|
|
// P3 had no counter. Counter must be exactly 4.
|
|
if (rank == 0) {
|
|
signalAddProducerReadCounterKernel<<<1, 1, 0, stream>>>(
|
|
kCntIdx, &dResults[0], devComm);
|
|
ASSERT_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
}
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Pull results back per-rank and assert. These branches are
|
|
// rank-divergent, so use plain ASSERT_EQ (ASSERT_MPI_EQ would deadlock on
|
|
// its internal MPI_Allreduce).
|
|
std::vector<uint64_t> hostResults(kResultSlots, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResults.data(), dResults, kResultSlots * sizeof(uint64_t),
|
|
hipMemcpyDeviceToHost));
|
|
if (rank == 0) {
|
|
ASSERT_EQ(4ull, hostResults[0]) << "P4 readCounter must equal 4 (1 from P1 + 3 from P2)";
|
|
} else {
|
|
ASSERT_EQ(5ull, hostResults[0]) << "P1 readSignal must equal 5";
|
|
ASSERT_EQ(0ull, hostResults[1]) << "P2 waitSignalFollowShadow before must be 0";
|
|
ASSERT_EQ(26ull, hostResults[2]) << "P2 waitSignalFollowShadow delta must be 26";
|
|
}
|
|
|
|
// ScopeGuards run in reverse order on return.
|
|
}
|
|
|
|
// Producer: drives BOTH symPtr-form shims in a single kernel:
|
|
// 1) gin.put<float>(team, peer, dstSym, srcSym, nElts, SignalInc) -- the
|
|
// typed-pointer put shim (gin__funcs.h:168) unpacks
|
|
// ncclSymPtr<T>::{window,offset} and dispatches to the window-form put
|
|
// with byte-size = nElts*sizeof(T).
|
|
// 2) gin.putValue<uint32_t>(team, peer, valDstSym, value, SignalInc) --
|
|
// the typed-pointer putValue shim (gin__funcs.h:231) unpacks valDstSym
|
|
// and dispatches to the window-form putValue with the value carried
|
|
// inline in the GFD itself. This drives the 4-byte inline branch of
|
|
// buildGfd (only inlineLow.inlineValLow set, inlineValLow2/inlineValHigh
|
|
// stay zero because sizeof(T) is not > 4 and not > 6 -- gin_proxy.h:84-95),
|
|
// distinct from the uint64_t 4+2+2 split exercised by PutValue_Inline.
|
|
// Both posts bump kSigIdx by 1 via SignalInc; the consumer waits on 2.
|
|
__global__ void symPtrProducerKernel(
|
|
ncclSymPtr<float> dstSym, ncclSymPtr<float> srcSym, size_t nElts,
|
|
ncclSymPtr<uint32_t> valDstSym, uint32_t inlineValue,
|
|
ncclGinSignal_t sigIdx, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstSym, srcSym, nElts,
|
|
ncclGin_SignalInc{sigIdx});
|
|
gin.putValue<uint32_t>(ncclTeamWorld(devComm), peer,
|
|
valDstSym, inlineValue,
|
|
ncclGin_SignalInc{sigIdx});
|
|
}
|
|
// Drain both posted GFDs before exit.
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Consumer: whole CTA waits for both signal increments from the two posts.
|
|
__global__ void symPtrConsumerKernel(
|
|
ncclGinSignal_t sigIdx, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigIdx, expectedSignalValue);
|
|
}
|
|
|
|
// Exercises the typed-pointer shims (ncclSymPtr<T>) for both gin.put and
|
|
// gin.putValue. Rank 0 sends a 512-float array via symPtr put and a 4-byte
|
|
// uint32 sentinel via symPtr putValue<uint32_t>; rank 1 verifies both
|
|
// land at their respective offsets and nothing else moved.
|
|
TEST_F(GinMPIDeviceTests, SymPtr_PutAndPutValue) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Layout: src window holds a 512-float array starting at srcOff. dst
|
|
// window receives the float array at dstOff (deliberately != srcOff to
|
|
// exercise non-zero offsets), and the 4-byte uint32 inline value lands
|
|
// at valOff (well past the float region so a misrouted put surfaces as
|
|
// a verification mismatch). No src memory is needed for the inline
|
|
// putValue -- the 4 bytes ride inside the GFD itself.
|
|
constexpr size_t kBufBytes = 8 * 1024;
|
|
constexpr size_t kNElts = 512;
|
|
constexpr size_t kSrcOff = 1 * 1024; // bytes
|
|
constexpr size_t kDstOff = 2 * 1024; // bytes
|
|
constexpr size_t kValOff = 6 * 1024; // bytes (non-overlapping)
|
|
constexpr uint32_t kInlineValue = 0xCAFEBABEu;
|
|
constexpr ncclGinSignal_t kSigIdx = 1;
|
|
constexpr int kPeer = 1;
|
|
|
|
static_assert(kSrcOff + kNElts * sizeof(float) <= kBufBytes, "src overflow");
|
|
static_assert(kDstOff + kNElts * sizeof(float) <= kBufBytes, "dst overflow");
|
|
static_assert(kValOff + sizeof(uint32_t) <= kBufBytes, "val overflow");
|
|
static_assert(kValOff >= kDstOff + kNElts * sizeof(float),
|
|
"val region must not overlap dst float region");
|
|
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// ginSignalCount=2 so kSigIdx=1 is in range.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 2;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Rank 0 fills its src float array with a deterministic pattern. Both
|
|
// ranks zero their dst so any spurious write outside [dstOff, dstOff+xfer)
|
|
// or near kValOff surfaces as a verification mismatch.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
if (rank == 0) {
|
|
float* srcFloats = reinterpret_cast<float*>(hostSrc.data() + kSrcOff);
|
|
for (size_t i = 0; i < kNElts; i++) {
|
|
srcFloats[i] = static_cast<float>(i) + 0.5f;
|
|
}
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Sync so neither rank launches its kernel before setup is done globally.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Producer issues both the float-array put and the inline uint32 putValue
|
|
// from a single kernel; consumer waitSignal(2) drains both. Two posts
|
|
// each bump kSigIdx by 1 via SignalInc, so the final value is 2.
|
|
if (rank == 0) {
|
|
ncclSymPtr<float> srcSym(srcWin, kSrcOff);
|
|
ncclSymPtr<float> dstSym(dstWin, kDstOff);
|
|
ncclSymPtr<uint32_t> valDstSym(dstWin, kValOff);
|
|
symPtrProducerKernel<<<1, 32, 0, stream>>>(
|
|
dstSym, srcSym, kNElts,
|
|
valDstSym, kInlineValue,
|
|
kSigIdx, kPeer, devComm);
|
|
} else {
|
|
symPtrConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/2, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Receiver-side verification.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
// 1) The float array landed at dstOff and matches rank 0's source.
|
|
const float* gotFloats = reinterpret_cast<const float*>(hostResult.data() + kDstOff);
|
|
for (size_t i = 0; i < kNElts; i++) {
|
|
const float expected = static_cast<float>(i) + 0.5f;
|
|
ASSERT_EQ(expected, gotFloats[i])
|
|
<< "float[" << i << "] mismatch at dstOff via symPtr put";
|
|
}
|
|
// 2) The 4-byte uint32 inline value landed at valOff. These bytes came
|
|
// from the inline GFD slot (putValue<uint32_t>), not from any source
|
|
// MR -- the symPtr shim only carried the destination offset.
|
|
uint32_t gotInline = 0;
|
|
std::memcpy(&gotInline, hostResult.data() + kValOff, sizeof(gotInline));
|
|
ASSERT_EQ(kInlineValue, gotInline)
|
|
<< "inline uint32 mismatch at valOff (symPtr putValue path)";
|
|
// 3) Nothing was written outside the two destination regions.
|
|
for (size_t i = 0; i < kDstOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " before float-dst region was unexpectedly written";
|
|
}
|
|
for (size_t i = kDstOff + kNElts * sizeof(float); i < kValOff; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " between float-dst and val-dst was unexpectedly written";
|
|
}
|
|
for (size_t i = kValOff + sizeof(uint32_t); i < kBufBytes; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "byte " << i << " after val-dst region was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
|
|
// Reference single-node alltoall: every rank puts its slice for every peer
|
|
// into that peer's recvbuf via gin.put + SignalInc, then waits on its own
|
|
// signal cell for nRanks increments. Ported one-for-one from
|
|
// examples/06_device_api/02_gin_alltoall_pure/main.cu so a regression in
|
|
// put + signal + barrier + flush composed under realistic load surfaces here.
|
|
__global__ void alltoallPureKernel(
|
|
ncclWindow_t sendwin, size_t sendoffset,
|
|
ncclWindow_t recvwin, size_t recvoffset,
|
|
size_t count, struct ncclDevComm devComm) {
|
|
constexpr int ginContext = 0;
|
|
unsigned int signalIndex = blockIdx.x;
|
|
ncclGin gin{devComm, ginContext};
|
|
// Capture current signal cell so a count-sweep that reuses the same
|
|
// kernel/devComm doesn't conflate increments from past iterations.
|
|
uint64_t signalValue = gin.readSignal(signalIndex);
|
|
|
|
// Cross-rank sync ensures every rank's sendbuf is registered before any
|
|
// peer reads from it via gin.put below.
|
|
ncclGinBarrierSession<ncclCoopCta> bar{
|
|
ncclCoopCta(), gin, ncclTeamWorld(devComm),
|
|
devComm.railGinBarrier, blockIdx.x};
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_acquire, ncclGinFenceLevel::Relaxed);
|
|
|
|
int tid = threadIdx.x + blockIdx.x * blockDim.x;
|
|
int nthreads = blockDim.x * gridDim.x;
|
|
|
|
// Each rank writes its per-peer slice into the destination's recvbuf
|
|
// slot (offset = my rank * size). All puts bump signalIndex by 1 on the
|
|
// destination, so the destination's signal cell receives nRanks
|
|
// increments (one per source rank including self).
|
|
const size_t size = count * sizeof(float);
|
|
for (int r = tid; r < devComm.nRanks; r += nthreads) {
|
|
gin.put(ncclTeamWorld(devComm), r,
|
|
recvwin, recvoffset + devComm.rank * size,
|
|
sendwin, sendoffset + r * size,
|
|
size, ncclGin_SignalInc{signalIndex});
|
|
}
|
|
|
|
int receivingCta = (devComm.rank % nthreads) / blockDim.x;
|
|
if (blockIdx.x == receivingCta)
|
|
gin.waitSignal(ncclCoopCta(), signalIndex, signalValue + devComm.nRanks);
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Single-node 2-8 ranks; count sweep {1, 1024, 1<<16}. Bit-for-bit verifies
|
|
// the deterministic sendbuf[i] = rank*1000 + dst*100 + i pattern landed on
|
|
// the right peer slot. The 1<<16 case (256 KiB / direction / peer) saturates
|
|
// ring credit to exercise the postGfd credit-wait path.
|
|
TEST_F(GinMPIDeviceTests, Alltoall_PureReference) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
// ginSignalCount=1 covers signalIndex=0; railGinBarrierCount=1 matches
|
|
// our single-CTA launch. Both non-zero so GIN activates without
|
|
// ginForceEnable.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// 1 (alignment/tail edges), 1024 (medium), 65536 (saturating).
|
|
const std::vector<size_t> counts = {1, 1024, size_t{1} << 16};
|
|
constexpr int kCTAs = 1; // == railGinBarrierCount
|
|
constexpr int kThreadsPerCTA = 512; // matches the production example launch
|
|
|
|
for (size_t count : counts) {
|
|
SCOPED_TRACE(::testing::Message() << "count=" << count);
|
|
|
|
const size_t totalElements = count * static_cast<size_t>(nRanks);
|
|
const size_t sizeBytes = totalElements * sizeof(float);
|
|
|
|
void* dSend = nullptr;
|
|
void* dRecv = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSend, sizeBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dRecv, sizeBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSend) (void)ncclMemFree(dSend);
|
|
if (dRecv) (void)ncclMemFree(dRecv);
|
|
});
|
|
|
|
ncclWindow_t sendWin = nullptr, recvWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSend, sizeBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dRecv, sizeBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (sendWin) (void)ncclCommWindowDeregister(comm, sendWin);
|
|
if (recvWin) (void)ncclCommWindowDeregister(comm, recvWin);
|
|
});
|
|
|
|
// Per-source/per-dest pattern: every byte of every slice has a unique
|
|
// expected value, so a misrouted slice surfaces as a mismatch.
|
|
std::vector<float> hostSend(totalElements, 0.0f);
|
|
std::vector<float> hostRecv(totalElements, 0.0f);
|
|
for (int dst = 0; dst < nRanks; dst++) {
|
|
for (size_t i = 0; i < count; i++) {
|
|
hostSend[static_cast<size_t>(dst) * count + i] =
|
|
static_cast<float>(rank * 1000 + dst * 100 + static_cast<int>(i));
|
|
}
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dSend, hostSend.data(), sizeBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dRecv, hostRecv.data(), sizeBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
alltoallPureKernel<<<kCTAs, kThreadsPerCTA, 0, stream>>>(
|
|
sendWin, /*sendoffset=*/0,
|
|
recvWin, /*recvoffset=*/0,
|
|
count, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// recvbuf[src*count + i] on rank R should equal src*1000 + R*100 + i.
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostRecv.data(), dRecv, sizeBytes, hipMemcpyDeviceToHost));
|
|
for (int src = 0; src < nRanks; src++) {
|
|
for (size_t i = 0; i < count; i++) {
|
|
const float expected =
|
|
static_cast<float>(src * 1000 + rank * 100 + static_cast<int>(i));
|
|
const float got = hostRecv[static_cast<size_t>(src) * count + i];
|
|
ASSERT_EQ(expected, got)
|
|
<< "rank=" << rank << " src=" << src << " i=" << i;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Hybrid alltoall: LSA stores for intra-node peers + gin.put for cross-node
|
|
// peers, bracketed by entry/exit barriers. Ported from
|
|
// examples/06_device_api/03_gin_alltoall_hybrid/main.cu (HybridAlltoAllKernel).
|
|
// On single-node nNodes==1 so world.nRanks==lsa.nRanks; numRemotePeers==0
|
|
// and the gin.put loops are no-ops -- this run primarily exercises the
|
|
// LSA-store path + entry/exit barrier composition. The cross-node arm is
|
|
// covered by Alltoall_CrossNode per the plan.
|
|
__global__ void alltoallHybridKernel(
|
|
ncclWindow_t sendwin, size_t sendoffset,
|
|
ncclWindow_t recvwin, size_t recvoffset,
|
|
size_t count, struct ncclDevComm devComm) {
|
|
constexpr int ginContext = 0;
|
|
constexpr unsigned int signalIndex = 0;
|
|
ncclGin gin{devComm, ginContext};
|
|
uint64_t signalValue = gin.readSignal(signalIndex);
|
|
|
|
// ncclBarrierSession (not the gin-only variant) routes through both LSA
|
|
// and rail-team GIN under ncclTeamTagWorld(). Single-node degenerates to
|
|
// an LSA-team barrier; multi-node crosses rails.
|
|
ncclBarrierSession<ncclCoopCta> bar{
|
|
ncclCoopCta(), ncclTeamTagWorld(), gin, blockIdx.x};
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_relaxed, ncclGinFenceLevel::Relaxed);
|
|
|
|
int tid = threadIdx.x + blockIdx.x * blockDim.x;
|
|
int nthreads = blockDim.x * gridDim.x;
|
|
|
|
ncclTeam world = ncclTeamWorld(devComm);
|
|
ncclTeam lsa = ncclTeamLsa(devComm);
|
|
const int startLsa = world.rank - lsa.rank;
|
|
const int lsaSize = lsa.nRanks;
|
|
|
|
// Cross-node peers: covers everything outside this node's LSA team
|
|
// (no-op on single-node).
|
|
const size_t size = count * sizeof(float);
|
|
for (int r = tid; r < startLsa; r += nthreads) {
|
|
gin.put(world, r,
|
|
recvwin, recvoffset + world.rank * size,
|
|
sendwin, sendoffset + r * size,
|
|
size, ncclGin_SignalInc{signalIndex});
|
|
}
|
|
for (int r = startLsa + lsaSize + tid; r < world.nRanks; r += nthreads) {
|
|
gin.put(world, r,
|
|
recvwin, recvoffset + world.rank * size,
|
|
sendwin, sendoffset + r * size,
|
|
size, ncclGin_SignalInc{signalIndex});
|
|
}
|
|
|
|
// Intra-node peers: write each LSA peer's slot in their recv buffer
|
|
// directly. ncclGetLocalPointer resolves our own sendbuf;
|
|
// ncclGetLsaPointer resolves an LSA peer's recvbuf into our address space.
|
|
float* sendLocal = (float*)ncclGetLocalPointer(sendwin, sendoffset);
|
|
for (size_t offset = tid; offset < count; offset += nthreads) {
|
|
for (int lp = 0; lp < lsa.nRanks; lp++) {
|
|
int wr = startLsa + lp;
|
|
float* recvPtr = (float*)ncclGetLsaPointer(recvwin, recvoffset, lp);
|
|
recvPtr[world.rank * count + offset] = sendLocal[wr * count + offset];
|
|
}
|
|
}
|
|
|
|
// Only wait for remote peers' increments; LSA peers don't bump the
|
|
// signal cell. On single-node this returns immediately (delta=0).
|
|
int numRemotePeers = world.nRanks - lsa.nRanks;
|
|
gin.waitSignal(ncclCoopCta(), signalIndex, signalValue + numRemotePeers);
|
|
gin.flush(ncclCoopCta());
|
|
|
|
// Final barrier: every rank's LSA stores into peer recvbufs are issued
|
|
// and the local writes from peers are visible before any rank consumes
|
|
// its recvbuf.
|
|
bar.sync(ncclCoopCta(), cuda::memory_order_release, ncclGinFenceLevel::Relaxed);
|
|
}
|
|
|
|
// Same shape and verification as Alltoall_PureReference but exercises the
|
|
// LSA-store path and the dual-pool barrier. On single-node the gin.put arm
|
|
// is a no-op; the test still validates LSA + rail-team-GIN barrier
|
|
// composition + readSignal/waitSignal plumbing with delta=0.
|
|
TEST_F(GinMPIDeviceTests, AlltoallHybrid_Reference) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
// The world-team ncclBarrierSession sizes its hybrid LSA+rail-GIN barriers from
|
|
// reqs.barrierCount (not lsa/railGinBarrierCount); 0 hangs the rail arm.
|
|
// ginSignalCount=1 covers the cross-node signal cell and triggers GIN activation.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.barrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
const std::vector<size_t> counts = {1, 1024, size_t{1} << 16};
|
|
constexpr int kCTAs = 1;
|
|
constexpr int kThreadsPerCTA = 512;
|
|
|
|
for (size_t count : counts) {
|
|
SCOPED_TRACE(::testing::Message() << "count=" << count);
|
|
|
|
const size_t totalElements = count * static_cast<size_t>(nRanks);
|
|
const size_t sizeBytes = totalElements * sizeof(float);
|
|
|
|
void* dSend = nullptr;
|
|
void* dRecv = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSend, sizeBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dRecv, sizeBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSend) (void)ncclMemFree(dSend);
|
|
if (dRecv) (void)ncclMemFree(dRecv);
|
|
});
|
|
|
|
ncclWindow_t sendWin = nullptr, recvWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSend, sizeBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dRecv, sizeBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (sendWin) (void)ncclCommWindowDeregister(comm, sendWin);
|
|
if (recvWin) (void)ncclCommWindowDeregister(comm, recvWin);
|
|
});
|
|
|
|
// Same deterministic pattern as Alltoall_PureReference.
|
|
std::vector<float> hostSend(totalElements, 0.0f);
|
|
std::vector<float> hostRecv(totalElements, 0.0f);
|
|
for (int dst = 0; dst < nRanks; dst++) {
|
|
for (size_t i = 0; i < count; i++) {
|
|
hostSend[static_cast<size_t>(dst) * count + i] =
|
|
static_cast<float>(rank * 1000 + dst * 100 + static_cast<int>(i));
|
|
}
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dSend, hostSend.data(), sizeBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dRecv, hostRecv.data(), sizeBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
alltoallHybridKernel<<<kCTAs, kThreadsPerCTA, 0, stream>>>(
|
|
sendWin, /*sendoffset=*/0,
|
|
recvWin, /*recvoffset=*/0,
|
|
count, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostRecv.data(), dRecv, sizeBytes, hipMemcpyDeviceToHost));
|
|
for (int src = 0; src < nRanks; src++) {
|
|
for (size_t i = 0; i < count; i++) {
|
|
const float expected =
|
|
static_cast<float>(src * 1000 + rank * 100 + static_cast<int>(i));
|
|
const float got = hostRecv[static_cast<size_t>(src) * count + i];
|
|
ASSERT_EQ(expected, got)
|
|
<< "rank=" << rank << " src=" << src << " i=" << i;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Producer: one block per GIN context. Block b uses ginContext=b, puts into
|
|
// its own slot, and signals signal[b] in that context.
|
|
__global__ void multiContextProducerKernel(
|
|
ncclWindow_t srcWin, ncclWindow_t dstWin,
|
|
size_t slotStride, size_t bytes, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, (int)blockIdx.x};
|
|
const size_t off = (size_t)blockIdx.x * slotStride;
|
|
if (threadIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, off,
|
|
srcWin, off,
|
|
bytes,
|
|
ncclGin_SignalInc{(ncclGinSignal_t)blockIdx.x});
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
// Consumer: block b waits on signal[b] in its own context, mirroring the
|
|
// producer-side mapping.
|
|
__global__ void multiContextConsumerKernel(
|
|
uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, (int)blockIdx.x};
|
|
gin.waitSignal(ncclCoopCta(), (ncclGinSignal_t)blockIdx.x, expectedSignalValue);
|
|
}
|
|
|
|
// Drives all NCCL_GIN_MAX_CONTEXTS contexts in parallel, each with its own
|
|
// slot + per-context signal. Confirms every contextId has a working
|
|
// proxy ring + IB QP and that there's no cross-context contamination.
|
|
TEST_F(GinMPIDeviceTests, MultiContext_AllFourRoute) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// 4 per-context slots; 1 KiB payload into each 4 KiB slot. The 3 KiB
|
|
// tail per slot is asserted zero to catch cross-context contamination.
|
|
constexpr int kNumContexts = NCCL_GIN_MAX_CONTEXTS; // 4
|
|
constexpr size_t kSlotStride = 4 * 1024;
|
|
constexpr size_t kTransferBytes = 1 * 1024;
|
|
constexpr size_t kBufBytes = kNumContexts * kSlotStride;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst large enough for all contexts' slots.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows over the symmetric buffers.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN. ginContextCount on reqs is a hint; the authoritative
|
|
// value is env-driven and read back from devComm. Each block uses a
|
|
// signal id == blockIdx.x, so we need kNumContexts signal cells per ctx.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginContextCount = kNumContexts;
|
|
reqs.ginSignalCount = kNumContexts;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Skip if the runtime didn't actually give us the requested number of ctxs.
|
|
if ((int)devComm.ginContextCount != kNumContexts) {
|
|
GTEST_SKIP() << "Test requires " << kNumContexts << " GIN contexts, got "
|
|
<< (int)devComm.ginContextCount
|
|
<< " (set NCCL_GIN_NCONTEXTS=" << kNumContexts << ")";
|
|
}
|
|
|
|
// Stage a distinct pattern (0x10 + b) per context slot so cross-context
|
|
// landings show up as value mismatches rather than missing writes.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (int b = 0; b < kNumContexts; b++) {
|
|
const uint8_t pattern = static_cast<uint8_t>(0x10 + b);
|
|
std::fill_n(hostSrc.begin() + b * kSlotStride, kTransferBytes, pattern);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Launch kNumContexts blocks on each side; one block per context.
|
|
if (rank == 0) {
|
|
multiContextProducerKernel<<<kNumContexts, 32, 0, stream>>>(
|
|
srcWin, dstWin, kSlotStride, kTransferBytes, kPeer, devComm);
|
|
} else {
|
|
multiContextConsumerKernel<<<kNumContexts, 32, 0, stream>>>(
|
|
/*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Verify every context's slot independently: payload range matches
|
|
// pattern, slot tail is still zero.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
for (int b = 0; b < kNumContexts; b++) {
|
|
const uint8_t pattern = static_cast<uint8_t>(0x10 + b);
|
|
const size_t base = (size_t)b * kSlotStride;
|
|
for (size_t i = 0; i < kTransferBytes; i++) {
|
|
ASSERT_EQ(pattern, hostResult[base + i])
|
|
<< "ctx " << b << ": byte " << i
|
|
<< " in transferred range mismatched (expected 0x" << std::hex
|
|
<< (int)pattern << ")";
|
|
}
|
|
for (size_t i = kTransferBytes; i < kSlotStride; i++) {
|
|
ASSERT_EQ(0u, hostResult[base + i])
|
|
<< "ctx " << b << ": byte " << i
|
|
<< " in slot tail was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Exercises the non-power-of-2 context count (3), which forces the modulo
|
|
// arm of the ncclGin ctor. Producer launches 6 blocks; pairs (0,3), (1,4),
|
|
// (2,5) collide on contextId 0/1/2 and each pair writes a disjoint sub-slot.
|
|
// Each producer signals signal 0 of its ctx; consumer waits for value 2.
|
|
// Requires NCCL_GIN_NCONTEXTS=3 (skipped otherwise).
|
|
__global__ void multiContextNpo2ProducerKernel(
|
|
ncclWindow_t srcWin, ncclWindow_t dstWin,
|
|
int numContexts, size_t slotStride, size_t subSlotBytes, int peer,
|
|
struct ncclDevComm devComm) {
|
|
const int ctx = (int)blockIdx.x % numContexts;
|
|
const int subSlotIx = (int)blockIdx.x / numContexts;
|
|
const size_t off = (size_t)ctx * slotStride + (size_t)subSlotIx * subSlotBytes;
|
|
ncclGin gin{devComm, ctx};
|
|
if (threadIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, off,
|
|
srcWin, off,
|
|
subSlotBytes,
|
|
ncclGin_SignalInc{(ncclGinSignal_t)0});
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
__global__ void multiContextNpo2ConsumerKernel(
|
|
uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, (int)blockIdx.x};
|
|
gin.waitSignal(ncclCoopCta(), (ncclGinSignal_t)0, expectedSignalValue);
|
|
}
|
|
|
|
TEST_F(GinMPIDeviceTests, MultiContext_NonPowerOf2) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// 3 ctx slots * 4 KiB stride; 2 sub-slots of 1 KiB per slot (one per
|
|
// producer block hitting that ctx); 2 KiB tail per slot asserted zero.
|
|
constexpr int kNumContexts = 3;
|
|
constexpr int kBlocksPerCtx = 2;
|
|
constexpr int kProducerBlocks = kNumContexts * kBlocksPerCtx;
|
|
constexpr size_t kSubSlotBytes = 1 * 1024;
|
|
constexpr size_t kSlotStride = 4 * 1024;
|
|
constexpr size_t kBufBytes = kNumContexts * kSlotStride;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows over the symmetric buffers.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN with 3 contexts, 1 signal each (signal pools are per-ctx,
|
|
// so signal 0 is independent across the three contexts).
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginContextCount = kNumContexts;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Skip unless the runtime gave us exactly 3 contexts (this test is the
|
|
// only thing exercising the modulo arm of the ctor).
|
|
if ((int)devComm.ginContextCount != kNumContexts) {
|
|
GTEST_SKIP() << "Test requires " << kNumContexts << " GIN contexts, got "
|
|
<< (int)devComm.ginContextCount
|
|
<< " (set NCCL_GIN_NCONTEXTS=" << kNumContexts << ")";
|
|
}
|
|
|
|
// Stage a distinct pattern per producer block (0x10 + b) so a stray
|
|
// landing surfaces as a value mismatch.
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0);
|
|
std::vector<uint8_t> hostDst(kBufBytes, 0);
|
|
for (int b = 0; b < kProducerBlocks; b++) {
|
|
const int ctx = b % kNumContexts;
|
|
const int subSlotIx = b / kNumContexts;
|
|
const size_t off = (size_t)ctx * kSlotStride + (size_t)subSlotIx * kSubSlotBytes;
|
|
const uint8_t pattern = static_cast<uint8_t>(0x10 + b);
|
|
std::fill_n(hostSrc.begin() + off, kSubSlotBytes, pattern);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Producer launches 6 blocks (2 per ctx); consumer launches one block
|
|
// per ctx and waits for value 2 on signal 0 of that ctx.
|
|
if (rank == 0) {
|
|
multiContextNpo2ProducerKernel<<<kProducerBlocks, 32, 0, stream>>>(
|
|
srcWin, dstWin, kNumContexts, kSlotStride, kSubSlotBytes, kPeer, devComm);
|
|
} else {
|
|
multiContextNpo2ConsumerKernel<<<kNumContexts, 32, 0, stream>>>(
|
|
/*expectedSignalValue=*/static_cast<uint64_t>(kBlocksPerCtx), devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Verify both sub-slots of every ctx hold the right pattern, and the
|
|
// tail past the two sub-slots is still zero.
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> hostResult(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
|
|
for (int ctx = 0; ctx < kNumContexts; ctx++) {
|
|
const size_t base = (size_t)ctx * kSlotStride;
|
|
for (int subSlotIx = 0; subSlotIx < kBlocksPerCtx; subSlotIx++) {
|
|
const int b = subSlotIx * kNumContexts + ctx;
|
|
const uint8_t pattern = static_cast<uint8_t>(0x10 + b);
|
|
const size_t off = base + (size_t)subSlotIx * kSubSlotBytes;
|
|
for (size_t i = 0; i < kSubSlotBytes; i++) {
|
|
ASSERT_EQ(pattern, hostResult[off + i])
|
|
<< "ctx " << ctx << " sub-slot " << subSlotIx
|
|
<< " (block " << b << "): byte " << i
|
|
<< " mismatched (expected 0x" << std::hex << (int)pattern << ")";
|
|
}
|
|
}
|
|
const size_t tailStart = base + (size_t)kBlocksPerCtx * kSubSlotBytes;
|
|
for (size_t i = tailStart; i < base + kSlotStride; i++) {
|
|
ASSERT_EQ(0u, hostResult[i])
|
|
<< "ctx " << ctx << ": byte " << (i - base)
|
|
<< " in slot tail was unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Sweep a matrix of aligned + unaligned transfer sizes through the same
|
|
// put kernel; reuses one comm / window pair / signal cell across iters
|
|
// (signal monotonically bumped, consumer waits for iter+1).
|
|
TEST_F(GinMPIDeviceTests, LargeBuffer_Sweep) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Aligned + unaligned matrix; the +1 variants force a trailing fragment
|
|
// so RDMA tail-byte handling regressions surface.
|
|
const std::vector<size_t> kSizes = {
|
|
1,
|
|
64,
|
|
4 * 1024,
|
|
4 * 1024 + 1,
|
|
1 * 1024 * 1024,
|
|
4 * 1024 * 1024,
|
|
16 * 1024 * 1024,
|
|
16 * 1024 * 1024 + 1,
|
|
};
|
|
const size_t kMaxBytes = kSizes.back();
|
|
constexpr ncclGinSignal_t kSigIdx = 0;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Allocate symmetric src/dst sized for the largest sweep entry.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kMaxBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kMaxBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows; reused across all iterations.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kMaxBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kMaxBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN with one signal cell; we'll bump it once per iter.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Stage src once: src[i] = i & 0xFF. Distinct bytes mod 256 catch
|
|
// byte-shift / off-by-one regressions in the RDMA path.
|
|
std::vector<uint8_t> hostSrc(kMaxBytes, 0);
|
|
for (size_t i = 0; i < kMaxBytes; i++) hostSrc[i] = static_cast<uint8_t>(i & 0xFF);
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kMaxBytes, hipMemcpyHostToDevice));
|
|
|
|
// Reusable buffers: zero pattern for clearing dst, scratch for verify.
|
|
const std::vector<uint8_t> hostZero(kMaxBytes, 0);
|
|
std::vector<uint8_t> hostResult(kMaxBytes, 0);
|
|
|
|
uint64_t signalExpected = 0;
|
|
for (size_t iter = 0; iter < kSizes.size(); iter++) {
|
|
const size_t sz = kSizes[iter];
|
|
|
|
// Clear dDst so the previous iter's larger payload doesn't leak into
|
|
// this iter's tail-byte assertions.
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostZero.data(), kMaxBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Each iter bumps signal[0] by 1; consumer waits for the cumulative count.
|
|
signalExpected++;
|
|
if (rank == 0) {
|
|
putBasicProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, /*srcOff=*/0,
|
|
dstWin, /*dstOff=*/0,
|
|
sz, kSigIdx, kPeer, devComm);
|
|
} else {
|
|
putBasicConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, signalExpected, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Rank 1 verifies: payload [0,sz) matches src; tail [sz,kMaxBytes) zero.
|
|
if (rank == 1) {
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostResult.data(), dDst, kMaxBytes, hipMemcpyDeviceToHost));
|
|
|
|
const int payloadCmp = std::memcmp(hostResult.data(), hostSrc.data(), sz);
|
|
ASSERT_EQ(0, payloadCmp)
|
|
<< "size=" << sz << ": payload range [0," << sz
|
|
<< ") differs from source";
|
|
|
|
if (sz < kMaxBytes) {
|
|
const int tailCmp =
|
|
std::memcmp(hostResult.data() + sz, hostZero.data(), kMaxBytes - sz);
|
|
ASSERT_EQ(0, tailCmp)
|
|
<< "size=" << sz << ": bytes in [" << sz << "," << kMaxBytes
|
|
<< ") were unexpectedly written";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Negative test: with NCCL_GIN_ENABLE=0, ncclDevCommCreate must fail.
|
|
TEST_F(GinMPIDeviceTests, Disable_Error) {
|
|
const char* e = std::getenv("NCCL_GIN_ENABLE");
|
|
if (!e || std::strcmp(e, "0") != 0)
|
|
GTEST_SKIP() << "Negative-path test; opt in by setting NCCL_GIN_ENABLE=0";
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// Skip ginProxyTestSkipReason: bare comm bring-up does not call into GIN,
|
|
// so the data-path gates don't apply here.
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
|
|
// ginSignalCount > 0 + ginConnectionType=FULL forces ncclDevCommCreate to
|
|
// attempt GIN bring-up.
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ncclResult_t r = ncclDevCommCreate(comm, &reqs, &devComm);
|
|
|
|
ASSERT_TRUE(r == ncclInvalidUsage || r == ncclInvalidArgument)
|
|
<< "ncclDevCommCreate must fail with a request-rejected error when "
|
|
"NCCL_GIN_ENABLE=0; got result " << (int)r;
|
|
}
|
|
|
|
// With an oversized signal pool the runtime should silently clamp to its
|
|
// internal max; confirm the data path still works after the clamp.
|
|
TEST_F(GinMPIDeviceTests, Invalid_SignalPool) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// Opt in with an oversized signal pool (>= 1 GiB) to exercise the clamp.
|
|
const char* env = std::getenv("NCCL_GIN_SIGNAL_POOL_SIZE");
|
|
if (!env || std::strtoull(env, nullptr, 0) < (1ULL << 30))
|
|
GTEST_SKIP() << "Set NCCL_GIN_SIGNAL_POOL_SIZE>=0x40000000 to opt in";
|
|
|
|
// If clamp + data path are healthy, the basic put round-trip succeeds.
|
|
runBasicPutSelfCheck();
|
|
}
|
|
|
|
// Counterpart of Invalid_SignalPool for the counter pool.
|
|
TEST_F(GinMPIDeviceTests, Invalid_CounterPool) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// Opt in with an oversized counter pool (>= 1 GiB).
|
|
const char* env = std::getenv("NCCL_GIN_COUNTER_POOL_SIZE");
|
|
if (!env || std::strtoull(env, nullptr, 0) < (1ULL << 30))
|
|
GTEST_SKIP() << "Set NCCL_GIN_COUNTER_POOL_SIZE>=0x40000000 to opt in";
|
|
|
|
runBasicPutSelfCheck();
|
|
}
|
|
|
|
// Run a put round-trip, then explicitly tear the comm down and check
|
|
// cleanup returns success. Failure here indicates a leak / unjoined proxy
|
|
// thread / undeleased MR or QP somewhere in ncclGinFinalize.
|
|
TEST_F(GinMPIDeviceTests, Teardown_NoLeaks) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// Inner scope so window/devComm/mem guards fire before we call
|
|
// cleanupTestCommunicator() below (they hold refs to the comm).
|
|
{
|
|
// Setup: comm, stream, geometry.
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
constexpr size_t kBufBytes = 64;
|
|
constexpr ncclGinSignal_t kSigIdx = 0;
|
|
constexpr int kPeer = 1;
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
// Allocate symmetric src/dst.
|
|
void* dSrc = nullptr;
|
|
void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
// Register collective windows.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
// Bring up GIN (1 signal cell suffices for a single round-trip).
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Run a single basic put + waitSignal round-trip.
|
|
if (rank == 0) {
|
|
putBasicProducerKernel<<<1, 32, 0, stream>>>(
|
|
srcWin, /*srcOff=*/0,
|
|
dstWin, /*dstOff=*/0,
|
|
kBufBytes, kSigIdx, kPeer,
|
|
devComm);
|
|
} else {
|
|
putBasicConsumerKernel<<<1, 32, 0, stream>>>(
|
|
kSigIdx, /*expectedSignalValue=*/1, devComm);
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
} // mem/window/devComm guards fire here while comm is still live.
|
|
|
|
// Explicit destroy: success implies ncclGinFinalize ran to completion
|
|
// (proxy thread joined, MR/QP released).
|
|
ASSERT_EQ(ncclSuccess, cleanupTestCommunicator());
|
|
ASSERT_EQ(nullptr, getActiveCommunicator());
|
|
ASSERT_EQ(nullptr, getActiveStream());
|
|
}
|
|
|
|
// Hammer create/destroy in a loop. Catches refcount, mutex, and
|
|
// finalize-order regressions that only surface across multiple lifecycles.
|
|
TEST_F(GinMPIDeviceTests, Init_Destroy_Stress) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
constexpr int kIterations = 10;
|
|
for (int i = 0; i < kIterations; ++i) {
|
|
// Fresh comm each iter.
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator()) << "iter " << i;
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
|
|
// Allocate + register a tiny window to actually trigger the GIN
|
|
// connect machinery (not just bare comm bring-up).
|
|
void* d = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&d, 64));
|
|
|
|
ncclWindow_t win = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, d, 64, &win, NCCL_WIN_COLL_SYMMETRIC));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Tear everything down in reverse order before the next iter.
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclCommWindowDeregister(comm, win));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemFree(d));
|
|
|
|
ASSERT_EQ(ncclSuccess, cleanupTestCommunicator()) << "iter " << i;
|
|
}
|
|
}
|
|
|
|
// Cross-node alltoall using device-side put. Each rank sends one slot to
|
|
// every other rank and waits for nRanks-1 signal increments per iter.
|
|
// Sweeps tiny/medium/saturating sizes through the same comm and buffers.
|
|
TEST_F(GinMPIDeviceTests, Alltoall_CrossNode) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
// Single-node would loopback IB; require real cross-node ranks.
|
|
if (auto reason = crossNodeReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
if (!validateTestPrerequisites(/*min_processes=*/2))
|
|
GTEST_SKIP() << "Requires >=2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
|
|
// Geometry: per-peer slot is sized for the largest sweep entry; smaller
|
|
// counts just use the head of each slot.
|
|
using T = uint32_t;
|
|
static constexpr size_t kCounts[] = {1, 1u << 10, 1u << 20};
|
|
static constexpr size_t kMaxCount = 1u << 20;
|
|
const size_t slotStrideBytes = kMaxCount * sizeof(T);
|
|
const size_t bufBytes = (size_t)nRanks * slotStrideBytes;
|
|
constexpr ncclGinSignal_t kSigIdx = 0;
|
|
|
|
// Allocate symmetric send/recv buffers (one slot per peer).
|
|
void* dSend = nullptr;
|
|
void* dRecv = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSend, bufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dRecv, bufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSend) (void)ncclMemFree(dSend);
|
|
if (dRecv) (void)ncclMemFree(dRecv);
|
|
});
|
|
|
|
// Register collective windows over send/recv.
|
|
ncclWindow_t sendWin = nullptr, recvWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSend, bufBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dRecv, bufBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (sendWin) (void)ncclCommWindowDeregister(comm, sendWin);
|
|
if (recvWin) (void)ncclCommWindowDeregister(comm, recvWin);
|
|
});
|
|
|
|
// Bring up GIN with one signal cell (cumulative across iters).
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Encoding: byte[3] = sender rank, byte[2] = dest rank, byte[1:0] =
|
|
// element index. Lets the receiver decode both source and dest from a slot.
|
|
auto pack = [](int sender, int dest, size_t i) -> T {
|
|
return (T)((((uint32_t)sender & 0xFFu) << 24) |
|
|
(((uint32_t)dest & 0xFFu) << 16) |
|
|
((uint32_t)i & 0xFFFFu));
|
|
};
|
|
|
|
// Stage send buffer: slot p contains values addressed to peer p.
|
|
std::vector<T> hostSend((size_t)nRanks * kMaxCount);
|
|
for (int p = 0; p < nRanks; ++p) {
|
|
for (size_t i = 0; i < kMaxCount; ++i) {
|
|
hostSend[(size_t)p * kMaxCount + i] = pack(rank, p, i);
|
|
}
|
|
}
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dSend, hostSend.data(), bufBytes, hipMemcpyHostToDevice));
|
|
|
|
// Cumulative across iters; the signal cell is never reset between iters.
|
|
std::vector<T> hostRecv((size_t)nRanks * kMaxCount);
|
|
uint64_t expectedSignal = 0;
|
|
|
|
for (size_t count : kCounts) {
|
|
// Kernel skips p == myRank; fill our self slot locally so verify
|
|
// covers all nRanks rows uniformly.
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpyAsync((uint8_t*)dRecv + (size_t)rank * slotStrideBytes,
|
|
(uint8_t*)dSend + (size_t)rank * slotStrideBytes,
|
|
count * sizeof(T),
|
|
hipMemcpyDeviceToDevice,
|
|
stream));
|
|
|
|
// We expect to receive nRanks-1 signal increments per iter.
|
|
expectedSignal += (uint64_t)(nRanks - 1);
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Single combined kernel: put to every peer + flush + waitSignal.
|
|
alltoallKernel<<<1, 32, 0, stream>>>(
|
|
sendWin, recvWin,
|
|
count * sizeof(T),
|
|
nRanks, rank,
|
|
slotStrideBytes,
|
|
kSigIdx, expectedSignal,
|
|
devComm);
|
|
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Verify each row of the recv buffer holds the bytes packed by sender r.
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostRecv.data(), dRecv, bufBytes, hipMemcpyDeviceToHost));
|
|
for (int r = 0; r < nRanks; ++r) {
|
|
for (size_t i = 0; i < count; ++i) {
|
|
T expected = pack(r, rank, i);
|
|
T actual = hostRecv[(size_t)r * kMaxCount + i];
|
|
ASSERT_EQ(expected, actual)
|
|
<< "count=" << count << " sender=" << r << " i=" << i;
|
|
}
|
|
}
|
|
|
|
// Hold all ranks here so a fast peer can't start the next iter and
|
|
// overwrite a slow rank's recvbuf before it has verified.
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
}
|
|
}
|
|
|
|
// =====================================================================
|
|
// Coverage for the new NCCL 2.29.7 GIN device-API features. All run on the
|
|
// device-API NCCL_GIN_TYPE=2 path with cross-node placement (2 ranks on 2
|
|
// nodes). Tests for features that depend on topology/library support that
|
|
// may be unavailable skip gracefully rather than fail.
|
|
// =====================================================================
|
|
|
|
// nLsaTeams / ginType / railedGinType via ncclCommQueryProperties (2.29.7).
|
|
TEST_F(GinMPIDeviceTests, Properties_NLsaTeams) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
|
|
int nRanks = -1;
|
|
ncclCommCount(comm, &nRanks);
|
|
|
|
ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER;
|
|
ASSERT_EQ(ncclSuccess, ncclCommQueryProperties(comm, &props));
|
|
|
|
// Backend type must match the requested NCCL_GIN_TYPE (2=proxy, 4=rocshmem-gda).
|
|
EXPECT_EQ(requestedGinType(), (int)props.ginType);
|
|
|
|
// nLsaTeams = nRanks / lsaSize: >= 1 and must evenly divide nRanks.
|
|
// Skip when the runtime reports nLsaTeams==0 on this configuration.
|
|
if (props.nLsaTeams == 0)
|
|
GTEST_SKIP() << "nLsaTeams reported 0 on this configuration";
|
|
EXPECT_GE(props.nLsaTeams, 1);
|
|
EXPECT_EQ(0, nRanks % props.nLsaTeams)
|
|
<< "nRanks=" << nRanks << " not divisible by nLsaTeams=" << props.nLsaTeams;
|
|
}
|
|
|
|
// Exclusive GIN contexts -- reqs.ginExclusiveContexts (2.29.7).
|
|
TEST_F(GinMPIDeviceTests, MultiContext_Exclusive) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
constexpr int kNumContexts = 2;
|
|
constexpr size_t kSlotStride = 4 * 1024;
|
|
constexpr size_t kTransferBytes = 1 * 1024;
|
|
constexpr size_t kBufBytes = kNumContexts * kSlotStride;
|
|
constexpr int kPeer = 1;
|
|
|
|
void* dSrc = nullptr; void* dDst = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBufBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBufBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
});
|
|
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBufBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBufBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
});
|
|
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.ginExclusiveContexts = true; // <-- feature under test
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginContextCount = kNumContexts;
|
|
reqs.ginSignalCount = kNumContexts;
|
|
ncclDevComm devComm{};
|
|
// Exclusive contexts are carved from the same pool as shared contexts: shared
|
|
// allocations grow from index 0 upward while exclusive ones are reserved from
|
|
// the top down. The pool size is NCCL_GIN_NCONTEXTS (rounded up to the GIN
|
|
// connection count). On a small topology (few connections) the default pool
|
|
// can be fully consumed by shared allocations, leaving none to reserve
|
|
// exclusively. Run with NCCL_GIN_NCONTEXTS large enough to leave headroom
|
|
// (e.g. >= shared usage + ginContextCount; 8 is sufficient on a 2-rank/1-NIC
|
|
// setup). Skip (rather than fail) when the pool has no room on this config.
|
|
ncclResult_t cr = ncclDevCommCreate(comm, &reqs, &devComm);
|
|
if (cr != ncclSuccess)
|
|
GTEST_SKIP() << "Exclusive GIN contexts unavailable on this configuration "
|
|
"(increase NCCL_GIN_NCONTEXTS for more pool headroom); rc=" << (int)cr;
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// The runtime rounds the requested context count up to a multiple of the GIN
|
|
// connection count, so expect ROUNDUP(kNumContexts, ginConnectionCount) rather
|
|
// than an exact match. Kernels below only drive [0, kNumContexts), so extras are harmless.
|
|
const int connCount = (int)devComm.ginConnectionCount;
|
|
const int expectedCtxs = connCount > 0
|
|
? ((kNumContexts + connCount - 1) / connCount) * connCount
|
|
: kNumContexts;
|
|
ASSERT_EQ(expectedCtxs, (int)devComm.ginContextCount)
|
|
<< "exclusive allocation returned " << (int)devComm.ginContextCount
|
|
<< " for " << connCount << " connection(s) (requested " << kNumContexts << ")";
|
|
ASSERT_GE((int)devComm.ginContextCount, kNumContexts);
|
|
|
|
std::vector<uint8_t> hostSrc(kBufBytes, 0), hostDst(kBufBytes, 0);
|
|
for (int b = 0; b < kNumContexts; b++)
|
|
std::fill_n(hostSrc.begin() + b * kSlotStride, kTransferBytes,
|
|
static_cast<uint8_t>(0x20 + b));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hostSrc.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hostDst.data(), kBufBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
if (rank == 0)
|
|
multiContextProducerKernel<<<kNumContexts, 32, 0, stream>>>(
|
|
srcWin, dstWin, kSlotStride, kTransferBytes, kPeer, devComm);
|
|
else
|
|
multiContextConsumerKernel<<<kNumContexts, 32, 0, stream>>>(
|
|
/*expectedSignalValue=*/1, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> res(kBufBytes, 0);
|
|
ASSERT_EQ(hipSuccess, hipMemcpy(res.data(), dDst, kBufBytes, hipMemcpyDeviceToHost));
|
|
for (int b = 0; b < kNumContexts; b++) {
|
|
const uint8_t pat = static_cast<uint8_t>(0x20 + b);
|
|
const size_t base = (size_t)b * kSlotStride;
|
|
for (size_t i = 0; i < kTransferBytes; i++)
|
|
ASSERT_EQ(pat, res[base + i]) << "ctx " << b << " byte " << i;
|
|
}
|
|
}
|
|
}
|
|
|
|
// VA-based GIN signals -- ncclGin_VASignalInc + waitSignal(window, offset) (2.29.7).
|
|
// The other tests use the INDEXED signal API (ncclGin_SignalInc + a pool slot
|
|
// index). This one addresses the signal by its virtual address -- a (window,
|
|
// offset) pair backed by a user-registered symmetric window -- which drives the
|
|
// NCCL_GIN_SIGNAL_TYPE_VA descriptor path instead of NCCL_GIN_SIGNAL_TYPE_INDEXED.
|
|
// On the proxy backend the VA signal is emitted as a separate GFD after the
|
|
// data put (see gin/proxy/gin_proxy.h), so a successful payload+signal here
|
|
// proves the VA signal landed.
|
|
__global__ void putVASignalProducerKernel(
|
|
ncclWindow_t srcWin, ncclWindow_t dstWin, size_t bytes,
|
|
ncclWindow_t sigWin, size_t sigOff, int peer,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
if (threadIdx.x == 0 && blockIdx.x == 0) {
|
|
gin.put(ncclTeamWorld(devComm), peer,
|
|
dstWin, /*dstOff=*/0,
|
|
srcWin, /*srcOff=*/0,
|
|
bytes,
|
|
ncclGin_VASignalInc{sigWin, sigOff}); // <-- VA signal (window,offset)
|
|
}
|
|
gin.flush(ncclCoopCta());
|
|
}
|
|
|
|
__global__ void putVASignalConsumerKernel(
|
|
ncclWindow_t sigWin, size_t sigOff, uint64_t expectedSignalValue,
|
|
struct ncclDevComm devComm) {
|
|
ncclGin gin{devComm, /*ginContext=*/0};
|
|
gin.waitSignal(ncclCoopCta(), sigWin, sigOff, expectedSignalValue); // <-- VA wait
|
|
}
|
|
|
|
TEST_F(GinMPIDeviceTests, VASignal_Put) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
// rocSHMEM-GDA (type 4) implements only INDEXED signals; VA signals are
|
|
// unsupported (Put/PutValue address the signal via indexedSignal.signalId).
|
|
if (requestedGinType() == 4)
|
|
GTEST_SKIP() << "VA signals not supported by rocSHMEM-GDA (NCCL_GIN_TYPE=4)";
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_EQ(2, nRanks);
|
|
|
|
constexpr size_t kBytes = 4096;
|
|
constexpr size_t kSigBytes = 64; // one 8B signal cell lives at offset 0
|
|
constexpr size_t kSigOff = 0;
|
|
constexpr int kPeer = 1;
|
|
|
|
// Separate buffers for payload (src/dst) and the VA signal cell.
|
|
void* dSrc = nullptr; void* dDst = nullptr; void* dSig = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSrc, kBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dDst, kBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSig, kSigBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSrc) (void)ncclMemFree(dSrc);
|
|
if (dDst) (void)ncclMemFree(dDst);
|
|
if (dSig) (void)ncclMemFree(dSig);
|
|
});
|
|
|
|
// The VA signal is addressed through its own symmetric window; the payload
|
|
// uses ordinary symmetric windows.
|
|
ncclWindow_t srcWin = nullptr, dstWin = nullptr, sigWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSrc, kBytes, &srcWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dDst, kBytes, &dstWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSig, kSigBytes, &sigWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (srcWin) (void)ncclCommWindowDeregister(comm, srcWin);
|
|
if (dstWin) (void)ncclCommWindowDeregister(comm, dstWin);
|
|
if (sigWin) (void)ncclCommWindowDeregister(comm, sigWin);
|
|
});
|
|
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
auto devCommCleanup = makeScopeGuard([&]() {
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
});
|
|
|
|
// Stage payload; zero the destination and the VA signal cell on every rank
|
|
// (the signal is incremented from 0 -> 1 by the producer's put).
|
|
std::vector<uint8_t> hs(kBytes, 0), hd(kBytes, 0), hsig(kSigBytes, 0);
|
|
for (size_t i = 0; i < kBytes; i++) hs[i] = static_cast<uint8_t>(0x37 + (i & 0x1F));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSrc, hs.data(), kBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dDst, hd.data(), kBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess, hipMemcpy(dSig, hsig.data(), kSigBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
if (rank == 0)
|
|
putVASignalProducerKernel<<<1, 32, 0, stream>>>(srcWin, dstWin, kBytes, sigWin, kSigOff, kPeer, devComm);
|
|
else
|
|
putVASignalConsumerKernel<<<1, 32, 0, stream>>>(sigWin, kSigOff, /*expectedSignalValue=*/1, devComm);
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
if (rank == 1) {
|
|
std::vector<uint8_t> r(kBytes, 0);
|
|
ASSERT_EQ(hipSuccess, hipMemcpy(r.data(), dDst, kBytes, hipMemcpyDeviceToHost));
|
|
for (size_t i = 0; i < kBytes; i++) ASSERT_EQ(hs[i], r[i]) << "byte " << i;
|
|
}
|
|
}
|
|
|
|
// RAIL / no-cross-rail connection type -- ginConnectionType=RAIL (2.29.7).
|
|
TEST_F(GinMPIDeviceTests, RailConnection_Create) {
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/2))
|
|
GTEST_SKIP() << "Requires exactly 2 ranks";
|
|
|
|
// ginIsRailed reflects the communicator's connection mode (RAIL only when the
|
|
// system cannot cross NICs). Opt in by disabling cross-NIC: NCCL_CROSS_NIC=0.
|
|
const char* crossNic = std::getenv("NCCL_CROSS_NIC");
|
|
if (!crossNic || std::strcmp(crossNic, "0") != 0)
|
|
GTEST_SKIP() << "RAIL connection requires NCCL_CROSS_NIC=0 (rail-only mode)";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
|
|
// RAIL is only valid when the communicator advertises a railed GIN type.
|
|
ncclCommProperties_t props = NCCL_COMM_PROPERTIES_INITIALIZER;
|
|
ASSERT_EQ(ncclSuccess, ncclCommQueryProperties(comm, &props));
|
|
if (props.railedGinType == NCCL_GIN_TYPE_NONE)
|
|
GTEST_SKIP() << "Communicator does not advertise railed GIN (railedGinType=NONE)";
|
|
|
|
ncclDevCommRequirements reqs = defaultGinReqs();
|
|
reqs.ginConnectionType = NCCL_GIN_CONNECTION_RAIL; // <-- feature under test
|
|
reqs.railGinBarrierCount = 1;
|
|
reqs.ginSignalCount = 1;
|
|
ncclDevComm devComm{};
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclDevCommCreate(comm, &reqs, &devComm));
|
|
EXPECT_TRUE(devComm.ginIsRailed) << "devComm should report railed GIN";
|
|
(void)ncclDevCommDestroy(comm, &devComm);
|
|
}
|
|
|
|
|
|
std::string intraNodeSymReason() {
|
|
MPI_Comm nodeComm;
|
|
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &nodeComm);
|
|
int nodeSize = 0;
|
|
MPI_Comm_size(nodeComm, &nodeSize);
|
|
MPI_Comm_free(&nodeComm);
|
|
if (nodeSize < 2)
|
|
return "Symmetric ReduceScatter requires >=2 ranks per node";
|
|
return "";
|
|
}
|
|
|
|
// End-to-end ReduceScatter through the public collective API on symmetric
|
|
// (NCCL_WIN_COLL_SYMMETRIC) buffers. Registering the buffers makes the
|
|
// symmetric-memory RS kernel (RailA2A_LsaLD) eligible, so this drives the
|
|
// production kernel path -- not a hand-written reference kernel.
|
|
TEST_F(GinMPIDeviceTests, ReduceScatter_Symmetric) {
|
|
if (requestedGinType() == 4)
|
|
GTEST_SKIP() << "Skipping symmetric ReduceScatter (RailA2A_LsaLD) for rocSHMEM-GDA";
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (auto reason = crossNodeReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (auto reason = intraNodeSymReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
// 1 (alignment/tail edges), 1024 (medium), 65536 (saturating).
|
|
const std::vector<size_t> counts = {1, 1024, size_t{1} << 16};
|
|
|
|
for (size_t count : counts) {
|
|
SCOPED_TRACE(::testing::Message() << "count=" << count);
|
|
|
|
// Send buffer holds one block per rank; recv holds this rank's block.
|
|
const size_t sendElems = count * static_cast<size_t>(nRanks);
|
|
const size_t sendBytes = sendElems * sizeof(float);
|
|
const size_t recvBytes = count * sizeof(float);
|
|
|
|
void* dSend = nullptr;
|
|
void* dRecv = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSend, sendBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dRecv, recvBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSend) (void)ncclMemFree(dSend);
|
|
if (dRecv) (void)ncclMemFree(dRecv);
|
|
});
|
|
|
|
ncclWindow_t sendWin = nullptr, recvWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSend, sendBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dRecv, recvBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (sendWin) (void)ncclCommWindowDeregister(comm, sendWin);
|
|
if (recvWin) (void)ncclCommWindowDeregister(comm, recvWin);
|
|
});
|
|
|
|
// sendbuf[j] = j + rank. After RS-sum, rank r's output element i is
|
|
// sum over all ranks s of ((r*count + i) + s) -- a closed form we check
|
|
// exactly (values stay < 2^24, so float arithmetic is exact here).
|
|
std::vector<float> hostSend(sendElems);
|
|
for (size_t j = 0; j < sendElems; j++)
|
|
hostSend[j] = static_cast<float>(j + static_cast<size_t>(rank));
|
|
std::vector<float> hostRecv(count, -1.0f);
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dSend, hostSend.data(), sendBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dRecv, hostRecv.data(), recvBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclReduceScatter(dSend, dRecv, count, ncclFloat32, ncclSum, comm, stream));
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Expected[i] on rank r = nRanks*(r*count + i) + nRanks*(nRanks-1)/2.
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostRecv.data(), dRecv, recvBytes, hipMemcpyDeviceToHost));
|
|
const double sumOfRanks = static_cast<double>(nRanks) * (nRanks - 1) / 2.0;
|
|
for (size_t i = 0; i < count; i++) {
|
|
const double base = static_cast<double>(static_cast<size_t>(rank) * count + i);
|
|
const float expected = static_cast<float>(nRanks * base + sumOfRanks);
|
|
ASSERT_EQ(expected, hostRecv[i]) << "rank=" << rank << " i=" << i;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Same as ReduceScatter_Symmetric but exercises ncclAvg, which drives the
|
|
// FuncSumPostDiv post-divide path on the symmetric RS kernel (RailA2A_LsaLD).
|
|
TEST_F(GinMPIDeviceTests, ReduceScatter_Symmetric_Avg) {
|
|
if (requestedGinType() == 4)
|
|
GTEST_SKIP() << "Skipping symmetric ReduceScatter (RailA2A_LsaLD) for rocSHMEM-GDA";
|
|
if (auto reason = ginProxyTestSkipReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (auto reason = crossNodeReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (auto reason = intraNodeSymReason(); !reason.empty())
|
|
GTEST_SKIP() << reason;
|
|
|
|
if (!validateTestPrerequisites(/*min_processes=*/2, /*max_processes=*/8))
|
|
GTEST_SKIP() << "Requires 2-8 ranks";
|
|
|
|
ASSERT_EQ(ncclSuccess, createTestCommunicator());
|
|
ncclComm_t comm = getActiveCommunicator();
|
|
hipStream_t stream = getActiveStream();
|
|
|
|
int rank = -1, nRanks = -1;
|
|
ncclCommUserRank(comm, &rank);
|
|
ncclCommCount(comm, &nRanks);
|
|
ASSERT_GE(nRanks, 2);
|
|
ASSERT_LE(nRanks, 8);
|
|
|
|
const std::vector<size_t> counts = {1, 1024, size_t{1} << 16};
|
|
|
|
for (size_t count : counts) {
|
|
SCOPED_TRACE(::testing::Message() << "count=" << count);
|
|
|
|
const size_t sendElems = count * static_cast<size_t>(nRanks);
|
|
const size_t sendBytes = sendElems * sizeof(float);
|
|
const size_t recvBytes = count * sizeof(float);
|
|
|
|
void* dSend = nullptr;
|
|
void* dRecv = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dSend, sendBytes));
|
|
ASSERT_MPI_EQ(ncclSuccess, ncclMemAlloc(&dRecv, recvBytes));
|
|
auto memCleanup = makeScopeGuard([&]() {
|
|
if (dSend) (void)ncclMemFree(dSend);
|
|
if (dRecv) (void)ncclMemFree(dRecv);
|
|
});
|
|
|
|
ncclWindow_t sendWin = nullptr, recvWin = nullptr;
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dSend, sendBytes, &sendWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclCommWindowRegister(comm, dRecv, recvBytes, &recvWin, NCCL_WIN_COLL_SYMMETRIC));
|
|
auto winCleanup = makeScopeGuard([&]() {
|
|
if (sendWin) (void)ncclCommWindowDeregister(comm, sendWin);
|
|
if (recvWin) (void)ncclCommWindowDeregister(comm, recvWin);
|
|
});
|
|
|
|
std::vector<float> hostSend(sendElems);
|
|
for (size_t j = 0; j < sendElems; j++)
|
|
hostSend[j] = static_cast<float>(j + static_cast<size_t>(rank));
|
|
std::vector<float> hostRecv(count, -1.0f);
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dSend, hostSend.data(), sendBytes, hipMemcpyHostToDevice));
|
|
ASSERT_MPI_EQ(hipSuccess,
|
|
hipMemcpy(dRecv, hostRecv.data(), recvBytes, hipMemcpyHostToDevice));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
ASSERT_MPI_EQ(ncclSuccess,
|
|
ncclReduceScatter(dSend, dRecv, count, ncclFloat32, ncclAvg, comm, stream));
|
|
ASSERT_MPI_EQ(hipSuccess, hipStreamSynchronize(stream));
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
// Avg = sum/nRanks; expected[i] on rank r = (r*count + i) + (nRanks-1)/2.
|
|
// The kernel multiplies by a float 1/nRanks, so allow a small tolerance.
|
|
ASSERT_EQ(hipSuccess,
|
|
hipMemcpy(hostRecv.data(), dRecv, recvBytes, hipMemcpyDeviceToHost));
|
|
const double sumOfRanks = static_cast<double>(nRanks) * (nRanks - 1) / 2.0;
|
|
for (size_t i = 0; i < count; i++) {
|
|
const double base = static_cast<double>(static_cast<size_t>(rank) * count + i);
|
|
const double expected = (nRanks * base + sumOfRanks) / nRanks;
|
|
const double tol = (expected < 0 ? -expected : expected) * 1e-5 + 1e-3;
|
|
ASSERT_NEAR(expected, static_cast<double>(hostRecv[i]), tol)
|
|
<< "rank=" << rank << " i=" << i;
|
|
}
|
|
}
|
|
}
|
|
|
|
#endif // MPI_TESTS_ENABLED
|