derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
555 lines
25 KiB
Bash
Executable File
555 lines
25 KiB
Bash
Executable File
#!/bin/bash
|
|
# Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved.
|
|
|
|
# #################################################
|
|
# global variables
|
|
# #################################################
|
|
ROCM_PATH=${ROCM_PATH:="/opt/rocm"}
|
|
|
|
# Default values
|
|
build_address_sanitizer=false
|
|
build_bfd=false
|
|
build_freorg_bkwdcomp=false
|
|
build_local_gpu_only=false
|
|
build_amdgpu_targets=""
|
|
build_package=false
|
|
build_release=true
|
|
debug_fast=false
|
|
build_static=false
|
|
build_tests=false
|
|
build_verbose=false
|
|
clean_build=true
|
|
dump_asm=false
|
|
enable_code_coverage=false
|
|
enable_ninja=""
|
|
install_dependencies=false
|
|
install_library=false
|
|
install_prefix="${ROCM_PATH}"
|
|
log_trace=false
|
|
num_parallel_jobs=$(nproc)
|
|
openmp_test_enabled=false
|
|
enable_mpi_tests=false
|
|
kernel_resource_use=false
|
|
roctx_enabled=true
|
|
run_tests=false
|
|
run_tests_all=false
|
|
time_trace=false
|
|
use_ninja=false
|
|
force_reduce_pipeline=false
|
|
generate_sym_kernels=true
|
|
device_linker=true
|
|
warp_speed_enabled=true # note that this flag will be overridden to false for non MI350/MI300 platforms
|
|
quiet_warnings=false
|
|
build_rocshmem_support=false
|
|
rocshmem_mono_hash="0e2998b11f99e8302c72f1ac2ce9f2b8c1816587"
|
|
custom_cmake_options=""
|
|
|
|
# #################################################
|
|
# helper functions
|
|
# #################################################
|
|
function display_help()
|
|
{
|
|
echo "RCCL build & installation helper script"
|
|
echo " Options:"
|
|
echo " --address-sanitizer Build with address sanitizer enabled"
|
|
echo " --amdgpu_targets Only compile for specified GPU architecture(s). For multiple targets, separate by ';' (builds for all supported GPU architectures by default)"
|
|
echo " --cmake-options Pass additional CMake options (e.g. --cmake-options \"-DFOO=BAR -DBAZ=ON\")"
|
|
echo " --debug Build debug library"
|
|
echo " --debug-fast Build debug library with lto optimization disabled (fast build times)"
|
|
echo " -d|--dependencies Install RCCL dependencies"
|
|
echo " --device-linker Build with assembly-extract device linker (default)"
|
|
echo " --disable-roctx Build without ROCTX logging"
|
|
echo " --disable-sym-kernels Disable symmetric memory kernels"
|
|
echo " --disable-warp-speed Disable WARP_SPEED kernel optimizations"
|
|
echo " --dump-asm Disassemble code and dump assembly with inline code"
|
|
echo " -c|--enable-code-coverage Enable code coverage"
|
|
echo " --enable_backtrace Build with custom backtrace support"
|
|
echo " --enable-mpi-tests Enable MPI-based tests (requires --debug and MPI installation; set MPI_PATH if not in /opt/ompi)"
|
|
echo " -f|--fast Quick-build RCCL (local gpu arch only, no backtrace)"
|
|
echo " --force-reduce-pipeline Force reduce_copy sw pipeline to be used for every reduce-based collectives and datatypes"
|
|
echo " -h|--help Prints this help message"
|
|
echo " -i|--install Install RCCL library (see --prefix argument below)"
|
|
echo " -j|--jobs Specify how many parallel compilation jobs to run ($num_parallel_jobs by default)"
|
|
echo " --kernel-resource-use Dump GPU kernel resource usage (e.g., VGPRs, scratch, spill) at link stage"
|
|
echo " -l|--local_gpu_only Only compile for local GPU architecture"
|
|
echo " --log-trace Build with log trace enabled (i.e. NCCL_DEBUG=TRACE)"
|
|
echo " --no_clean Don't delete files if they already exist"
|
|
echo " --no-device-linker Disable device linker, use standard -fgpu-rdc"
|
|
echo " --openmp-test-enable Enable OpenMP in rccl unit tests"
|
|
echo " -p|--package_build Build RCCL package"
|
|
echo " --prefix Specify custom directory to install RCCL to (default: \`/opt/rocm\`)"
|
|
echo " -q|--quiet-warnings Suppress majority of compiler warnings (not recommended)"
|
|
echo " --rocshmem Build with rocSHMEM support"
|
|
echo " --run_tests_all Run all rccl unit tests (must be built already)"
|
|
echo " -r|--run_tests_quick Run small subset of rccl unit tests (must be built already)"
|
|
echo " --static Build RCCL as a static library instead of shared library"
|
|
echo " -t|--tests_build Build rccl unit tests, but do not run"
|
|
echo " --time-trace Plot the build time of RCCL (requires \`ninja-build\` package installed on the system)"
|
|
echo " --ninja Use the Ninja generator instead of Make (requires \`ninja-build\`; recommended for multi-arch builds)"
|
|
echo " --verbose Show compile commands"
|
|
echo ""
|
|
echo " Available RCCL-specific CMake options for --cmake-options:"
|
|
echo " -DBUILD_PLUGIN_EXAMPLES=ON Build plugin example libraries: net, tuner, profiler, env (default: OFF)"
|
|
echo " -DDWORDX4_INTRINSICS=OFF Disable dwordx4 intrinsics (default: ON)"
|
|
echo " -DENABLE_COMPRESS=OFF Disable GPU code compression (default: ON)"
|
|
echo " -DENABLE_IFC=ON Enable indirect function call (default: OFF)"
|
|
echo " -DFAULT_INJECTION=OFF Disable fault injection (default: ON)"
|
|
echo " -DRCCL_ROCPROFILER_REGISTER=OFF Disable rocprofiler-register support (default: ON)"
|
|
echo " -DTIMETRACE=ON Enable time-trace during compilation (default: OFF)"
|
|
echo ""
|
|
echo " Environment variables:"
|
|
echo " ONLY_FUNCS Build only specified collective functions (debug builds only)."
|
|
echo " Restricts GPU kernel generation to the listed collectives, significantly"
|
|
echo " reducing build time during development. Use '|' to separate multiple functions."
|
|
echo " Example: ONLY_FUNCS=\"AllReduce|SendRecv\" ./install.sh --debug -t"
|
|
echo " Available: AllReduce, Broadcast, Reduce, AllGather, ReduceScatter,"
|
|
echo " AlltoAllPivot, SendRecv, AlltoAllGda, AlltoAllvGda"
|
|
echo " Advanced: Specify algo, protocol, redop, and type per collective."
|
|
echo " ONLY_FUNCS=\"AllReduce RING SIMPLE Sum f32|SendRecv\""
|
|
echo " ROCSHMEM_INSTALL_DIR Path to a pre-built rocSHMEM installation (skips building from source)"
|
|
}
|
|
|
|
# #################################################
|
|
# Parameter parsing
|
|
# #################################################
|
|
|
|
# check if we have a modern version of getopt that can handle whitespace and long parameters
|
|
getopt -T
|
|
if [[ "$?" -eq 4 ]]; then
|
|
GETOPT_PARSE=$(getopt --name "${0}" --options cdfhij:lprtq --longoptions address-sanitizer,amdgpu_targets:,cmake-options:,debug,debug-fast,dependencies,device-linker,disable-roctx,disable-sym-kernels,disable-warp-speed,dump-asm,enable-code-coverage,enable_backtrace,enable-mpi-tests,fast,force-reduce-pipeline,generate-sym-kernels,help,install,jobs:,kernel-resource-use,local_gpu_only,log-trace,ninja,no_clean,no-device-linker,openmp-test-enable,package_build,prefix:,quiet-warnings,rm-legacy-include-dir,rocshmem,roctx-enable,run_tests_all,run_tests_quick,static,tests_build,time-trace,verbose -- "$@")
|
|
else
|
|
echo "Need a new version of getopt"
|
|
exit 1
|
|
fi
|
|
|
|
if [[ "$?" -ne 0 ]]; then
|
|
echo "getopt invocation failed; could not parse the command line";
|
|
exit 1
|
|
fi
|
|
|
|
eval set -- "${GETOPT_PARSE}"
|
|
|
|
while true; do
|
|
case "${1}" in
|
|
--address-sanitizer) build_address_sanitizer=true; shift ;;
|
|
--amdgpu_targets) build_amdgpu_targets=${2}; shift 2 ;;
|
|
--cmake-options) custom_cmake_options=${2}; shift 2 ;;
|
|
--debug) build_release=false; shift ;;
|
|
--debug-fast) build_release=false; debug_fast=true; shift ;;
|
|
-d | --dependencies) install_dependencies=true; shift ;;
|
|
--device-linker) device_linker=true; shift ;;
|
|
--disable-roctx) roctx_enabled=false; shift ;;
|
|
--disable-sym-kernels) generate_sym_kernels=false; shift ;;
|
|
--disable-warp-speed) warp_speed_enabled=false; shift ;;
|
|
--dump-asm) dump_asm=true; shift ;;
|
|
-c | --enable-code-coverage) enable_code_coverage=true; shift ;;
|
|
--enable_backtrace) build_bfd=true; shift ;;
|
|
--enable-mpi-tests) enable_mpi_tests=true; shift ;;
|
|
-f | --fast) build_local_gpu_only=true; shift ;;
|
|
--force-reduce-pipeline) force_reduce_pipeline=true; shift ;;
|
|
-h | --help) display_help; exit 0 ;;
|
|
-i | --install) install_library=true; shift ;;
|
|
-j | --jobs) num_parallel_jobs=${2}; shift 2 ;;
|
|
--kernel-resource-use) kernel_resource_use=true; shift ;;
|
|
-l | --local_gpu_only) build_local_gpu_only=true; shift ;;
|
|
--log-trace) log_trace=true; shift ;;
|
|
--ninja) use_ninja=true; shift ;;
|
|
--no_clean) clean_build=false; shift ;;
|
|
--no-device-linker) device_linker=false; shift ;;
|
|
--openmp-test-enable) openmp_test_enabled=true; shift ;;
|
|
-p | --package_build) build_package=true; shift ;;
|
|
--prefix) install_library=true; install_prefix=${2}; shift 2 ;;
|
|
-q | --quiet-warnings) quiet_warnings=true; shift ;;
|
|
--rocshmem) build_rocshmem_support=true; shift ;;
|
|
--run_tests_all) run_tests=true; run_tests_all=true; shift ;;
|
|
-r | --run_tests_quick) run_tests=true; shift ;;
|
|
--static) build_static=true; shift ;;
|
|
-t | --tests_build) build_tests=true; shift ;;
|
|
--time-trace) time_trace=true; shift ;;
|
|
--verbose) build_verbose=true; shift ;;
|
|
--) shift ; break ;;
|
|
*) echo "Unexpected command line parameter received; aborting";
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
# /etc/*-release files describe the system
|
|
if [[ -e "/etc/os-release" ]]; then
|
|
source /etc/os-release
|
|
elif [[ -e "/etc/centos-release" ]]; then
|
|
OS_ID=$(cat /etc/centos-release | awk '{print tolower($1)}')
|
|
VERSION_ID=$(cat /etc/centos-release | grep -oP '(?<=release )[^ ]*' | cut -d "." -f1)
|
|
else
|
|
echo "This script depends on the /etc/*-release files"
|
|
exit 2
|
|
fi
|
|
|
|
# CMake executable
|
|
cmake_executable=cmake
|
|
time_trace_ninja_msg="apt-get install ninja-build"
|
|
case "${OS_ID}" in
|
|
centos|rhel)
|
|
cmake_executable=cmake3
|
|
time_trace_ninja_msg="dnf install ninja-build"
|
|
;;
|
|
esac
|
|
|
|
# CMake build options; starts with toolchain info
|
|
cmake_common_options="--toolchain=toolchain-linux.cmake"
|
|
|
|
# throw error code after running a command in the install script
|
|
check_exit_code( )
|
|
{
|
|
if (( $1 != 0 )); then
|
|
exit "$1"
|
|
fi
|
|
}
|
|
|
|
# Set up a git worktree of the rocm-systems mono-repo so that
|
|
# projects/rocshmem is checked out at a pinned commit hash while the
|
|
# main working tree (which contains rccl at HEAD) stays untouched.
|
|
setup_rocshmem_worktree()
|
|
{
|
|
local mono_root
|
|
mono_root=$(git rev-parse --show-toplevel 2>/dev/null)
|
|
if [[ -z "$mono_root" ]]; then
|
|
echo "ERROR: Not inside a git repository. Cannot set up rocSHMEM worktree."
|
|
echo " Use ROCSHMEM_INSTALL_DIR to point to a pre-built rocSHMEM instead."
|
|
exit 1
|
|
fi
|
|
|
|
local pinned_hash="${rocshmem_mono_hash}"
|
|
local worktree_dir="${mono_root}/.rocshmem-worktree"
|
|
|
|
echo "=== Setting up rocSHMEM from mono-repo worktree ==="
|
|
echo " Pinned hash : ${pinned_hash:0:12}"
|
|
echo " Worktree dir : ${worktree_dir}"
|
|
|
|
if [[ -d "$worktree_dir" ]]; then
|
|
local current_hash
|
|
current_hash=$(git -C "$worktree_dir" rev-parse HEAD 2>/dev/null)
|
|
if [[ "${current_hash}" == "${pinned_hash}"* ]] || [[ "${pinned_hash}" == "${current_hash}"* ]]; then
|
|
echo " Worktree already at the correct hash — reusing."
|
|
rocshmem_source_dir="${worktree_dir}/projects/rocshmem"
|
|
return 0
|
|
fi
|
|
echo " Removing stale worktree..."
|
|
git -C "$mono_root" worktree remove "$worktree_dir" --force 2>/dev/null || rm -rf "$worktree_dir"
|
|
fi
|
|
|
|
git -C "$mono_root" worktree add --no-checkout "$worktree_dir" "$pinned_hash"
|
|
check_exit_code "$?"
|
|
|
|
git -C "$worktree_dir" sparse-checkout init --cone
|
|
git -C "$worktree_dir" sparse-checkout set projects/rocshmem
|
|
check_exit_code "$?"
|
|
|
|
git -C "$worktree_dir" checkout
|
|
check_exit_code "$?"
|
|
|
|
if [[ ! -d "${worktree_dir}/projects/rocshmem" ]]; then
|
|
echo "ERROR: projects/rocshmem not found in worktree."
|
|
exit 1
|
|
fi
|
|
|
|
rocshmem_source_dir="${worktree_dir}/projects/rocshmem"
|
|
echo " rocSHMEM source ready at: ${rocshmem_source_dir}"
|
|
echo "=================================================="
|
|
}
|
|
|
|
# set RCCL-UnitTests path
|
|
if [[ "${build_release}" == true ]]; then
|
|
unit_test_path="./build/release/test/rccl-UnitTests"
|
|
else
|
|
unit_test_path="./build/debug/test/rccl-UnitTests"
|
|
fi
|
|
|
|
if [[ "${run_tests}" == true ]] && [[ -f "${unit_test_path}" ]]; then
|
|
if [[ "${build_tests}" == false ]]; then
|
|
clean_build=false
|
|
fi
|
|
fi
|
|
|
|
# #################################################
|
|
# rocSHMEM worktree setup (must run before cd-ing into the build directory)
|
|
# #################################################
|
|
rocshmem_source_dir=""
|
|
if [[ "${build_rocshmem_support}" == true ]] && [[ -z "${ROCSHMEM_INSTALL_DIR}" ]]; then
|
|
setup_rocshmem_worktree
|
|
fi
|
|
|
|
# #################################################
|
|
# prep
|
|
# #################################################
|
|
# ensure a clean build environment
|
|
if [[ "${clean_build}" == true ]]; then
|
|
if [[ "${build_release}" == true ]]; then
|
|
rm -rf build/release
|
|
else
|
|
rm -rf build/debug
|
|
fi
|
|
fi
|
|
|
|
# Create and go to the build directory.
|
|
mkdir -p build; cd build
|
|
|
|
# Create and go to build type directory
|
|
if [[ "${build_release}" == true ]]; then
|
|
mkdir -p release; cd release
|
|
else
|
|
mkdir -p debug; cd debug
|
|
fi
|
|
|
|
# build type
|
|
if [[ "${build_release}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Release"
|
|
else
|
|
if [[ "${debug_fast}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_BUILD_SUBTYPE=DebugFast"
|
|
else
|
|
cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Debug"
|
|
fi
|
|
fi
|
|
|
|
# Address sanitizer
|
|
if [[ "${build_address_sanitizer}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DBUILD_ADDRESS_SANITIZER=ON"
|
|
fi
|
|
|
|
# Enable code coverage
|
|
if [[ "${enable_code_coverage}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DENABLE_CODE_COVERAGE=ON"
|
|
fi
|
|
|
|
# Backtrace support
|
|
if [[ "${build_bfd}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DBUILD_BFD=ON"
|
|
fi
|
|
|
|
# Build local GPU arch only
|
|
if [[ "${build_local_gpu_only}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DBUILD_LOCAL_GPU_TARGET_ONLY=ON"
|
|
fi
|
|
|
|
# Build for specified GPU target(s) only
|
|
if [[ ! -z "${build_amdgpu_targets}" ]]; then
|
|
cmake_common_options="${cmake_common_options} -DGPU_TARGETS=${build_amdgpu_targets}"
|
|
fi
|
|
|
|
# shared vs static
|
|
if [[ "${build_static}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DBUILD_SHARED_LIBS=OFF"
|
|
fi
|
|
|
|
# Install dependencies
|
|
if [[ "${install_dependencies}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DINSTALL_DEPENDENCIES=ON"
|
|
fi
|
|
|
|
# Install RCCL library
|
|
if [[ "${install_library}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DCMAKE_INSTALL_PREFIX=${install_prefix}"
|
|
fi
|
|
|
|
if [[ "${kernel_resource_use}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DREPORT_KERNEL_RESOURCE_USE=ON"
|
|
fi
|
|
|
|
# Enable trace debug level
|
|
if [[ "${log_trace}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DTRACE=ON"
|
|
fi
|
|
|
|
# Disable ROCTX
|
|
if [[ "${roctx_enabled}" == false ]]; then
|
|
cmake_common_options="${cmake_common_options} -DROCTX=OFF"
|
|
fi
|
|
|
|
# Dump ASM files from GPU compilation
|
|
if [[ "${dump_asm}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DDUMP_ASM=ON"
|
|
fi
|
|
|
|
# Enable OpenMP in unit tests
|
|
if [[ "${openmp_test_enabled}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DOPENMP_TESTS_ENABLED=ON"
|
|
fi
|
|
|
|
# Enable MPI tests (debug only)
|
|
if [[ "${enable_mpi_tests}" == true ]]; then
|
|
if [[ "${build_release}" == true ]]; then
|
|
echo "ERROR: --enable-mpi-tests requires --debug. Please re-run with --debug."
|
|
exit 1
|
|
fi
|
|
cmake_common_options="${cmake_common_options} -DENABLE_MPI_TESTS=ON"
|
|
fi
|
|
|
|
# Force Reduce pipeline
|
|
if [[ "${force_reduce_pipeline}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DFORCE_REDUCE_PIPELINING=ON"
|
|
fi
|
|
|
|
# Disable symmetric memory kernels
|
|
if [[ "${generate_sym_kernels}" == false ]]; then
|
|
cmake_common_options="${cmake_common_options} -DGENERATE_SYM_KERNELS=OFF"
|
|
fi
|
|
|
|
# Device linker (assembly-extract pipeline, no -fgpu-rdc)
|
|
# Enabled by default; pass -DENABLE_DEVICE_LINKER=OFF when explicitly disabled.
|
|
if [[ "${device_linker}" == false ]]; then
|
|
cmake_common_options="${cmake_common_options} -DENABLE_DEVICE_LINKER=OFF"
|
|
fi
|
|
|
|
# Enable WARP_SPEED only on MI350/MI300 platforms
|
|
if [[ "${warp_speed_enabled}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DENABLE_WARP_SPEED=ON"
|
|
fi
|
|
|
|
# Suppress Warnings
|
|
if [[ "${quiet_warnings}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DQUIET_WARNINGS=ON"
|
|
fi
|
|
|
|
|
|
# Enable rocSHMEM support
|
|
if [[ "${build_rocshmem_support}" == true ]]; then
|
|
cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=ON"
|
|
if [[ -n "${ROCSHMEM_INSTALL_DIR}" ]]; then
|
|
cmake_common_options="${cmake_common_options} -DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}"
|
|
elif [[ -n "${rocshmem_source_dir}" ]]; then
|
|
cmake_common_options="${cmake_common_options} -DROCSHMEM_SOURCE_DIR=${rocshmem_source_dir}"
|
|
fi
|
|
else
|
|
cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=OFF"
|
|
fi
|
|
|
|
check_exit_code "$?"
|
|
|
|
# Build system selection. Default is Make for broadest compatibility
|
|
# (Jenkins CI runs `make package` directly in the build directory).
|
|
# Ninja is enabled when explicitly requested via --ninja or --time-trace
|
|
# (the latter requires it). Ninja parallelizes the multi-arch device
|
|
# pipeline considerably better than Make, so it's the recommended generator
|
|
# for builds that target many GPU architectures at once.
|
|
if [[ "${time_trace}" == true || "${use_ninja}" == true ]]; then
|
|
if ! hash ninja &>/dev/null ; then
|
|
if [[ "${time_trace}" == true ]]; then
|
|
# Ninja is mandatory for --time-trace (the trace post-processing
|
|
# consumes Ninja's .ninja_log), so this is a hard error.
|
|
echo "ninja could not be found (required for --time-trace)"
|
|
echo "Use \"${time_trace_ninja_msg}\" to install ninja"
|
|
exit 1
|
|
fi
|
|
# --ninja is only a build-speed opt-in, so degrade gracefully to Make
|
|
# rather than aborting when ninja is unavailable.
|
|
echo "WARNING: ninja could not be found; falling back to the Make generator for --ninja"
|
|
echo " Use \"${time_trace_ninja_msg}\" to install ninja"
|
|
build_system="make"
|
|
else
|
|
build_system="ninja"
|
|
enable_ninja="-GNinja"
|
|
fi
|
|
else
|
|
build_system="make"
|
|
fi
|
|
|
|
# Add common CMake options
|
|
cmake_common_options="${cmake_common_options} -DROCM_PATH=${ROCM_PATH} ${enable_ninja}"
|
|
|
|
# Build RCCL-UnitTests, if enabled
|
|
if [[ "${build_tests}" == true ]] || ([[ "${run_tests}" == true ]] && [[ ! -x ./test/rccl-UnitTests ]]); then
|
|
cmake_common_options="${cmake_common_options} -DBUILD_TESTS=ON"
|
|
fi
|
|
|
|
# Add build directory to RPATH for packaging dependency resolution
|
|
cmake_common_options="${cmake_common_options} -DCMAKE_EXE_LINKER_FLAGS=\"-Wl,-rpath,${PWD}\""
|
|
|
|
# Append any custom CMake options passed via --cmake-options
|
|
if [[ ! -z "${custom_cmake_options}" ]]; then
|
|
cmake_common_options="${cmake_common_options} ${custom_cmake_options}"
|
|
fi
|
|
|
|
# Initiate RCCL CMake
|
|
# Passing ONLY_FUNCS separately (not as part of ${cmake_common_options}) as
|
|
# ${ONLY_FUNCS} is a debug-only feature
|
|
${cmake_executable} ${cmake_common_options} -DONLY_FUNCS="${ONLY_FUNCS}" ../../.
|
|
check_exit_code "$?"
|
|
|
|
# Enable verbose output from the build system
|
|
if [[ "${build_verbose}" == true ]]; then
|
|
if [[ "${build_system}" == "ninja" ]]; then
|
|
build_system="${build_system} -v"
|
|
else
|
|
build_system="${build_system} VERBOSE=1"
|
|
fi
|
|
fi
|
|
|
|
# Initiate RCCL build (and install)
|
|
if [[ "${install_library}" == true ]]; then
|
|
${build_system} -j ${num_parallel_jobs} install
|
|
else
|
|
${build_system} -j ${num_parallel_jobs}
|
|
fi
|
|
check_exit_code "$?"
|
|
|
|
# Initiate package build (`make package` or `ninja package`), if enabled
|
|
if [[ "${build_package}" == true ]]; then
|
|
if [[ "${build_system}" == "ninja"* ]]; then
|
|
ninja package
|
|
else
|
|
make package
|
|
fi
|
|
check_exit_code "$?"
|
|
fi
|
|
|
|
# For ASAN builds, enable XNACK for GPU address sanitizer support
|
|
if [[ "${build_address_sanitizer}" == true ]]; then
|
|
export HSA_XNACK=1
|
|
fi
|
|
|
|
# Optionally, run RCCL-UnitTests, if they're enabled.
|
|
if [[ "${run_tests}" == true ]]; then
|
|
if [[ ! -x "./test/rccl-UnitTests" ]]; then
|
|
echo "RCCL-UnitTests have not been built yet; Please re-run script with \"-t\" to build the binary."
|
|
exit 1
|
|
fi
|
|
if [[ "${build_release}" == false && ! -x "./test/rccl-UnitTestsFixturesDebug" ]]; then
|
|
echo "RCCL-UnitTestsFixturesDebug have not been built yet; Please re-run script with \"-t\" to build the binary."
|
|
exit 1
|
|
fi
|
|
if [[ "${run_tests_all}" == true ]]; then
|
|
if [[ -x "./test/rccl-UnitTests" ]]; then
|
|
./test/rccl-UnitTests
|
|
fi
|
|
if [[ -x "./test/rccl-UnitTestsFixtures" ]]; then
|
|
./test/rccl-UnitTestsFixtures
|
|
fi
|
|
if [[ "${build_release}" == false && -x "./test/rccl-UnitTestsFixturesDebug" ]]; then
|
|
./test/rccl-UnitTestsFixturesDebug
|
|
fi
|
|
else
|
|
if [[ -x "./test/rccl-UnitTests" ]]; then
|
|
./test/rccl-UnitTests --gtest_filter="AllReduce.*"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
# Generate time trace for RCCL build using tools/time-trace
|
|
if [[ "${time_trace}" == true ]]; then
|
|
search_dir="../../tools"
|
|
time_trace_dir=$(find "${search_dir}" -type d -name "time-trace" -print -quit)
|
|
|
|
if [[ -n "${time_trace_dir}" ]]; then
|
|
time_trace_script="${time_trace_dir}/rccl-TimeTrace.sh"
|
|
if [[ -x "${time_trace_script}" ]]; then
|
|
echo "Generating RCCL-compile-timeline.html..."
|
|
(cd "${time_trace_dir}" && ./rccl-TimeTrace.sh)
|
|
else
|
|
echo "Error: Unable to execute ${time_trace_script}. Make sure the file has the correct permissions."
|
|
fi
|
|
else
|
|
echo "Error: time-trace folder not found in ${search_dir}."
|
|
fi
|
|
fi
|