#!/bin/bash # Copyright (c) 2019-2023 Advanced Micro Devices, Inc. All rights reserved. # ################################################# # global variables # ################################################# ROCM_PATH=${ROCM_PATH:="/opt/rocm"} # Default values build_address_sanitizer=false build_bfd=false build_freorg_bkwdcomp=false build_local_gpu_only=false build_amdgpu_targets="" build_package=false build_release=true debug_fast=false build_static=false build_tests=false build_verbose=false clean_build=true dump_asm=false enable_code_coverage=false enable_ninja="" install_dependencies=false install_library=false install_prefix="${ROCM_PATH}" log_trace=false num_parallel_jobs=$(nproc) openmp_test_enabled=false enable_mpi_tests=false kernel_resource_use=false roctx_enabled=true run_tests=false run_tests_all=false time_trace=false use_ninja=false force_reduce_pipeline=false generate_sym_kernels=true device_linker=true warp_speed_enabled=true # note that this flag will be overridden to false for non MI350/MI300 platforms quiet_warnings=false build_rocshmem_support=false rocshmem_mono_hash="0e2998b11f99e8302c72f1ac2ce9f2b8c1816587" custom_cmake_options="" # ################################################# # helper functions # ################################################# function display_help() { echo "RCCL build & installation helper script" echo " Options:" echo " --address-sanitizer Build with address sanitizer enabled" echo " --amdgpu_targets Only compile for specified GPU architecture(s). For multiple targets, separate by ';' (builds for all supported GPU architectures by default)" echo " --cmake-options Pass additional CMake options (e.g. --cmake-options \"-DFOO=BAR -DBAZ=ON\")" echo " --debug Build debug library" echo " --debug-fast Build debug library with lto optimization disabled (fast build times)" echo " -d|--dependencies Install RCCL dependencies" echo " --device-linker Build with assembly-extract device linker (default)" echo " --disable-roctx Build without ROCTX logging" echo " --disable-sym-kernels Disable symmetric memory kernels" echo " --disable-warp-speed Disable WARP_SPEED kernel optimizations" echo " --dump-asm Disassemble code and dump assembly with inline code" echo " -c|--enable-code-coverage Enable code coverage" echo " --enable_backtrace Build with custom backtrace support" echo " --enable-mpi-tests Enable MPI-based tests (requires --debug and MPI installation; set MPI_PATH if not in /opt/ompi)" echo " -f|--fast Quick-build RCCL (local gpu arch only, no backtrace)" echo " --force-reduce-pipeline Force reduce_copy sw pipeline to be used for every reduce-based collectives and datatypes" echo " -h|--help Prints this help message" echo " -i|--install Install RCCL library (see --prefix argument below)" echo " -j|--jobs Specify how many parallel compilation jobs to run ($num_parallel_jobs by default)" echo " --kernel-resource-use Dump GPU kernel resource usage (e.g., VGPRs, scratch, spill) at link stage" echo " -l|--local_gpu_only Only compile for local GPU architecture" echo " --log-trace Build with log trace enabled (i.e. NCCL_DEBUG=TRACE)" echo " --no_clean Don't delete files if they already exist" echo " --no-device-linker Disable device linker, use standard -fgpu-rdc" echo " --openmp-test-enable Enable OpenMP in rccl unit tests" echo " -p|--package_build Build RCCL package" echo " --prefix Specify custom directory to install RCCL to (default: \`/opt/rocm\`)" echo " -q|--quiet-warnings Suppress majority of compiler warnings (not recommended)" echo " --rocshmem Build with rocSHMEM support" echo " --run_tests_all Run all rccl unit tests (must be built already)" echo " -r|--run_tests_quick Run small subset of rccl unit tests (must be built already)" echo " --static Build RCCL as a static library instead of shared library" echo " -t|--tests_build Build rccl unit tests, but do not run" echo " --time-trace Plot the build time of RCCL (requires \`ninja-build\` package installed on the system)" echo " --ninja Use the Ninja generator instead of Make (requires \`ninja-build\`; recommended for multi-arch builds)" echo " --verbose Show compile commands" echo "" echo " Available RCCL-specific CMake options for --cmake-options:" echo " -DBUILD_PLUGIN_EXAMPLES=ON Build plugin example libraries: net, tuner, profiler, env (default: OFF)" echo " -DDWORDX4_INTRINSICS=OFF Disable dwordx4 intrinsics (default: ON)" echo " -DENABLE_COMPRESS=OFF Disable GPU code compression (default: ON)" echo " -DENABLE_IFC=ON Enable indirect function call (default: OFF)" echo " -DFAULT_INJECTION=OFF Disable fault injection (default: ON)" echo " -DRCCL_ROCPROFILER_REGISTER=OFF Disable rocprofiler-register support (default: ON)" echo " -DTIMETRACE=ON Enable time-trace during compilation (default: OFF)" echo "" echo " Environment variables:" echo " ONLY_FUNCS Build only specified collective functions (debug builds only)." echo " Restricts GPU kernel generation to the listed collectives, significantly" echo " reducing build time during development. Use '|' to separate multiple functions." echo " Example: ONLY_FUNCS=\"AllReduce|SendRecv\" ./install.sh --debug -t" echo " Available: AllReduce, Broadcast, Reduce, AllGather, ReduceScatter," echo " AlltoAllPivot, SendRecv, AlltoAllGda, AlltoAllvGda" echo " Advanced: Specify algo, protocol, redop, and type per collective." echo " ONLY_FUNCS=\"AllReduce RING SIMPLE Sum f32|SendRecv\"" echo " ROCSHMEM_INSTALL_DIR Path to a pre-built rocSHMEM installation (skips building from source)" } # ################################################# # Parameter parsing # ################################################# # check if we have a modern version of getopt that can handle whitespace and long parameters getopt -T if [[ "$?" -eq 4 ]]; then GETOPT_PARSE=$(getopt --name "${0}" --options cdfhij:lprtq --longoptions address-sanitizer,amdgpu_targets:,cmake-options:,debug,debug-fast,dependencies,device-linker,disable-roctx,disable-sym-kernels,disable-warp-speed,dump-asm,enable-code-coverage,enable_backtrace,enable-mpi-tests,fast,force-reduce-pipeline,generate-sym-kernels,help,install,jobs:,kernel-resource-use,local_gpu_only,log-trace,ninja,no_clean,no-device-linker,openmp-test-enable,package_build,prefix:,quiet-warnings,rm-legacy-include-dir,rocshmem,roctx-enable,run_tests_all,run_tests_quick,static,tests_build,time-trace,verbose -- "$@") else echo "Need a new version of getopt" exit 1 fi if [[ "$?" -ne 0 ]]; then echo "getopt invocation failed; could not parse the command line"; exit 1 fi eval set -- "${GETOPT_PARSE}" while true; do case "${1}" in --address-sanitizer) build_address_sanitizer=true; shift ;; --amdgpu_targets) build_amdgpu_targets=${2}; shift 2 ;; --cmake-options) custom_cmake_options=${2}; shift 2 ;; --debug) build_release=false; shift ;; --debug-fast) build_release=false; debug_fast=true; shift ;; -d | --dependencies) install_dependencies=true; shift ;; --device-linker) device_linker=true; shift ;; --disable-roctx) roctx_enabled=false; shift ;; --disable-sym-kernels) generate_sym_kernels=false; shift ;; --disable-warp-speed) warp_speed_enabled=false; shift ;; --dump-asm) dump_asm=true; shift ;; -c | --enable-code-coverage) enable_code_coverage=true; shift ;; --enable_backtrace) build_bfd=true; shift ;; --enable-mpi-tests) enable_mpi_tests=true; shift ;; -f | --fast) build_local_gpu_only=true; shift ;; --force-reduce-pipeline) force_reduce_pipeline=true; shift ;; -h | --help) display_help; exit 0 ;; -i | --install) install_library=true; shift ;; -j | --jobs) num_parallel_jobs=${2}; shift 2 ;; --kernel-resource-use) kernel_resource_use=true; shift ;; -l | --local_gpu_only) build_local_gpu_only=true; shift ;; --log-trace) log_trace=true; shift ;; --ninja) use_ninja=true; shift ;; --no_clean) clean_build=false; shift ;; --no-device-linker) device_linker=false; shift ;; --openmp-test-enable) openmp_test_enabled=true; shift ;; -p | --package_build) build_package=true; shift ;; --prefix) install_library=true; install_prefix=${2}; shift 2 ;; -q | --quiet-warnings) quiet_warnings=true; shift ;; --rocshmem) build_rocshmem_support=true; shift ;; --run_tests_all) run_tests=true; run_tests_all=true; shift ;; -r | --run_tests_quick) run_tests=true; shift ;; --static) build_static=true; shift ;; -t | --tests_build) build_tests=true; shift ;; --time-trace) time_trace=true; shift ;; --verbose) build_verbose=true; shift ;; --) shift ; break ;; *) echo "Unexpected command line parameter received; aborting"; exit 1 ;; esac done # /etc/*-release files describe the system if [[ -e "/etc/os-release" ]]; then source /etc/os-release elif [[ -e "/etc/centos-release" ]]; then OS_ID=$(cat /etc/centos-release | awk '{print tolower($1)}') VERSION_ID=$(cat /etc/centos-release | grep -oP '(?<=release )[^ ]*' | cut -d "." -f1) else echo "This script depends on the /etc/*-release files" exit 2 fi # CMake executable cmake_executable=cmake time_trace_ninja_msg="apt-get install ninja-build" case "${OS_ID}" in centos|rhel) cmake_executable=cmake3 time_trace_ninja_msg="dnf install ninja-build" ;; esac # CMake build options; starts with toolchain info cmake_common_options="--toolchain=toolchain-linux.cmake" # throw error code after running a command in the install script check_exit_code( ) { if (( $1 != 0 )); then exit "$1" fi } # Set up a git worktree of the rocm-systems mono-repo so that # projects/rocshmem is checked out at a pinned commit hash while the # main working tree (which contains rccl at HEAD) stays untouched. setup_rocshmem_worktree() { local mono_root mono_root=$(git rev-parse --show-toplevel 2>/dev/null) if [[ -z "$mono_root" ]]; then echo "ERROR: Not inside a git repository. Cannot set up rocSHMEM worktree." echo " Use ROCSHMEM_INSTALL_DIR to point to a pre-built rocSHMEM instead." exit 1 fi local pinned_hash="${rocshmem_mono_hash}" local worktree_dir="${mono_root}/.rocshmem-worktree" echo "=== Setting up rocSHMEM from mono-repo worktree ===" echo " Pinned hash : ${pinned_hash:0:12}" echo " Worktree dir : ${worktree_dir}" if [[ -d "$worktree_dir" ]]; then local current_hash current_hash=$(git -C "$worktree_dir" rev-parse HEAD 2>/dev/null) if [[ "${current_hash}" == "${pinned_hash}"* ]] || [[ "${pinned_hash}" == "${current_hash}"* ]]; then echo " Worktree already at the correct hash — reusing." rocshmem_source_dir="${worktree_dir}/projects/rocshmem" return 0 fi echo " Removing stale worktree..." git -C "$mono_root" worktree remove "$worktree_dir" --force 2>/dev/null || rm -rf "$worktree_dir" fi git -C "$mono_root" worktree add --no-checkout "$worktree_dir" "$pinned_hash" check_exit_code "$?" git -C "$worktree_dir" sparse-checkout init --cone git -C "$worktree_dir" sparse-checkout set projects/rocshmem check_exit_code "$?" git -C "$worktree_dir" checkout check_exit_code "$?" if [[ ! -d "${worktree_dir}/projects/rocshmem" ]]; then echo "ERROR: projects/rocshmem not found in worktree." exit 1 fi rocshmem_source_dir="${worktree_dir}/projects/rocshmem" echo " rocSHMEM source ready at: ${rocshmem_source_dir}" echo "==================================================" } # set RCCL-UnitTests path if [[ "${build_release}" == true ]]; then unit_test_path="./build/release/test/rccl-UnitTests" else unit_test_path="./build/debug/test/rccl-UnitTests" fi if [[ "${run_tests}" == true ]] && [[ -f "${unit_test_path}" ]]; then if [[ "${build_tests}" == false ]]; then clean_build=false fi fi # ################################################# # rocSHMEM worktree setup (must run before cd-ing into the build directory) # ################################################# rocshmem_source_dir="" if [[ "${build_rocshmem_support}" == true ]] && [[ -z "${ROCSHMEM_INSTALL_DIR}" ]]; then setup_rocshmem_worktree fi # ################################################# # prep # ################################################# # ensure a clean build environment if [[ "${clean_build}" == true ]]; then if [[ "${build_release}" == true ]]; then rm -rf build/release else rm -rf build/debug fi fi # Create and go to the build directory. mkdir -p build; cd build # Create and go to build type directory if [[ "${build_release}" == true ]]; then mkdir -p release; cd release else mkdir -p debug; cd debug fi # build type if [[ "${build_release}" == true ]]; then cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Release" else if [[ "${debug_fast}" == true ]]; then cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_BUILD_SUBTYPE=DebugFast" else cmake_common_options="${cmake_common_options} -DCMAKE_BUILD_TYPE=Debug" fi fi # Address sanitizer if [[ "${build_address_sanitizer}" == true ]]; then cmake_common_options="${cmake_common_options} -DBUILD_ADDRESS_SANITIZER=ON" fi # Enable code coverage if [[ "${enable_code_coverage}" == true ]]; then cmake_common_options="${cmake_common_options} -DENABLE_CODE_COVERAGE=ON" fi # Backtrace support if [[ "${build_bfd}" == true ]]; then cmake_common_options="${cmake_common_options} -DBUILD_BFD=ON" fi # Build local GPU arch only if [[ "${build_local_gpu_only}" == true ]]; then cmake_common_options="${cmake_common_options} -DBUILD_LOCAL_GPU_TARGET_ONLY=ON" fi # Build for specified GPU target(s) only if [[ ! -z "${build_amdgpu_targets}" ]]; then cmake_common_options="${cmake_common_options} -DGPU_TARGETS=${build_amdgpu_targets}" fi # shared vs static if [[ "${build_static}" == true ]]; then cmake_common_options="${cmake_common_options} -DBUILD_SHARED_LIBS=OFF" fi # Install dependencies if [[ "${install_dependencies}" == true ]]; then cmake_common_options="${cmake_common_options} -DINSTALL_DEPENDENCIES=ON" fi # Install RCCL library if [[ "${install_library}" == true ]]; then cmake_common_options="${cmake_common_options} -DCMAKE_INSTALL_PREFIX=${install_prefix}" fi if [[ "${kernel_resource_use}" == true ]]; then cmake_common_options="${cmake_common_options} -DREPORT_KERNEL_RESOURCE_USE=ON" fi # Enable trace debug level if [[ "${log_trace}" == true ]]; then cmake_common_options="${cmake_common_options} -DTRACE=ON" fi # Disable ROCTX if [[ "${roctx_enabled}" == false ]]; then cmake_common_options="${cmake_common_options} -DROCTX=OFF" fi # Dump ASM files from GPU compilation if [[ "${dump_asm}" == true ]]; then cmake_common_options="${cmake_common_options} -DDUMP_ASM=ON" fi # Enable OpenMP in unit tests if [[ "${openmp_test_enabled}" == true ]]; then cmake_common_options="${cmake_common_options} -DOPENMP_TESTS_ENABLED=ON" fi # Enable MPI tests (debug only) if [[ "${enable_mpi_tests}" == true ]]; then if [[ "${build_release}" == true ]]; then echo "ERROR: --enable-mpi-tests requires --debug. Please re-run with --debug." exit 1 fi cmake_common_options="${cmake_common_options} -DENABLE_MPI_TESTS=ON" fi # Force Reduce pipeline if [[ "${force_reduce_pipeline}" == true ]]; then cmake_common_options="${cmake_common_options} -DFORCE_REDUCE_PIPELINING=ON" fi # Disable symmetric memory kernels if [[ "${generate_sym_kernels}" == false ]]; then cmake_common_options="${cmake_common_options} -DGENERATE_SYM_KERNELS=OFF" fi # Device linker (assembly-extract pipeline, no -fgpu-rdc) # Enabled by default; pass -DENABLE_DEVICE_LINKER=OFF when explicitly disabled. if [[ "${device_linker}" == false ]]; then cmake_common_options="${cmake_common_options} -DENABLE_DEVICE_LINKER=OFF" fi # Enable WARP_SPEED only on MI350/MI300 platforms if [[ "${warp_speed_enabled}" == true ]]; then cmake_common_options="${cmake_common_options} -DENABLE_WARP_SPEED=ON" fi # Suppress Warnings if [[ "${quiet_warnings}" == true ]]; then cmake_common_options="${cmake_common_options} -DQUIET_WARNINGS=ON" fi # Enable rocSHMEM support if [[ "${build_rocshmem_support}" == true ]]; then cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=ON" if [[ -n "${ROCSHMEM_INSTALL_DIR}" ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_INSTALL_DIR=${ROCSHMEM_INSTALL_DIR}" elif [[ -n "${rocshmem_source_dir}" ]]; then cmake_common_options="${cmake_common_options} -DROCSHMEM_SOURCE_DIR=${rocshmem_source_dir}" fi else cmake_common_options="${cmake_common_options} -DENABLE_ROCSHMEM=OFF" fi check_exit_code "$?" # Build system selection. Default is Make for broadest compatibility # (Jenkins CI runs `make package` directly in the build directory). # Ninja is enabled when explicitly requested via --ninja or --time-trace # (the latter requires it). Ninja parallelizes the multi-arch device # pipeline considerably better than Make, so it's the recommended generator # for builds that target many GPU architectures at once. if [[ "${time_trace}" == true || "${use_ninja}" == true ]]; then if ! hash ninja &>/dev/null ; then if [[ "${time_trace}" == true ]]; then # Ninja is mandatory for --time-trace (the trace post-processing # consumes Ninja's .ninja_log), so this is a hard error. echo "ninja could not be found (required for --time-trace)" echo "Use \"${time_trace_ninja_msg}\" to install ninja" exit 1 fi # --ninja is only a build-speed opt-in, so degrade gracefully to Make # rather than aborting when ninja is unavailable. echo "WARNING: ninja could not be found; falling back to the Make generator for --ninja" echo " Use \"${time_trace_ninja_msg}\" to install ninja" build_system="make" else build_system="ninja" enable_ninja="-GNinja" fi else build_system="make" fi # Add common CMake options cmake_common_options="${cmake_common_options} -DROCM_PATH=${ROCM_PATH} ${enable_ninja}" # Build RCCL-UnitTests, if enabled if [[ "${build_tests}" == true ]] || ([[ "${run_tests}" == true ]] && [[ ! -x ./test/rccl-UnitTests ]]); then cmake_common_options="${cmake_common_options} -DBUILD_TESTS=ON" fi # Add build directory to RPATH for packaging dependency resolution cmake_common_options="${cmake_common_options} -DCMAKE_EXE_LINKER_FLAGS=\"-Wl,-rpath,${PWD}\"" # Append any custom CMake options passed via --cmake-options if [[ ! -z "${custom_cmake_options}" ]]; then cmake_common_options="${cmake_common_options} ${custom_cmake_options}" fi # Initiate RCCL CMake # Passing ONLY_FUNCS separately (not as part of ${cmake_common_options}) as # ${ONLY_FUNCS} is a debug-only feature ${cmake_executable} ${cmake_common_options} -DONLY_FUNCS="${ONLY_FUNCS}" ../../. check_exit_code "$?" # Enable verbose output from the build system if [[ "${build_verbose}" == true ]]; then if [[ "${build_system}" == "ninja" ]]; then build_system="${build_system} -v" else build_system="${build_system} VERBOSE=1" fi fi # Initiate RCCL build (and install) if [[ "${install_library}" == true ]]; then ${build_system} -j ${num_parallel_jobs} install else ${build_system} -j ${num_parallel_jobs} fi check_exit_code "$?" # Initiate package build (`make package` or `ninja package`), if enabled if [[ "${build_package}" == true ]]; then if [[ "${build_system}" == "ninja"* ]]; then ninja package else make package fi check_exit_code "$?" fi # For ASAN builds, enable XNACK for GPU address sanitizer support if [[ "${build_address_sanitizer}" == true ]]; then export HSA_XNACK=1 fi # Optionally, run RCCL-UnitTests, if they're enabled. if [[ "${run_tests}" == true ]]; then if [[ ! -x "./test/rccl-UnitTests" ]]; then echo "RCCL-UnitTests have not been built yet; Please re-run script with \"-t\" to build the binary." exit 1 fi if [[ "${build_release}" == false && ! -x "./test/rccl-UnitTestsFixturesDebug" ]]; then echo "RCCL-UnitTestsFixturesDebug have not been built yet; Please re-run script with \"-t\" to build the binary." exit 1 fi if [[ "${run_tests_all}" == true ]]; then if [[ -x "./test/rccl-UnitTests" ]]; then ./test/rccl-UnitTests fi if [[ -x "./test/rccl-UnitTestsFixtures" ]]; then ./test/rccl-UnitTestsFixtures fi if [[ "${build_release}" == false && -x "./test/rccl-UnitTestsFixturesDebug" ]]; then ./test/rccl-UnitTestsFixturesDebug fi else if [[ -x "./test/rccl-UnitTests" ]]; then ./test/rccl-UnitTests --gtest_filter="AllReduce.*" fi fi fi # Generate time trace for RCCL build using tools/time-trace if [[ "${time_trace}" == true ]]; then search_dir="../../tools" time_trace_dir=$(find "${search_dir}" -type d -name "time-trace" -print -quit) if [[ -n "${time_trace_dir}" ]]; then time_trace_script="${time_trace_dir}/rccl-TimeTrace.sh" if [[ -x "${time_trace_script}" ]]; then echo "Generating RCCL-compile-timeline.html..." (cd "${time_trace_dir}" && ./rccl-TimeTrace.sh) else echo "Error: Unable to execute ${time_trace_script}. Make sure the file has the correct permissions." fi else echo "Error: time-trace folder not found in ${search_dir}." fi fi