# cmake/DeviceLinker.cmake # # Assembly-extract device linker pipeline, using the RCCLDEV custom language. # # The rccl-device-compile driver presents a compiler/linker interface to CMake. # Per-kernel compilation (cpp -> extract -> obj) is a native CMake compile step. # Per-arch linking (objects -> aggregate -> patch -> link -> elf) uses the driver # in --link mode via a custom command. # # Required variables (set by src/CMakeLists.txt before including this file): # HIPIFY_DIR, GEN_DIR, GPU_TARGETS, PROJECT_BINARY_DIR, PROJECT_SOURCE_DIR, # Python3_EXECUTABLE message(STATUS "Device Linker: assembly-extract pipeline enabled (RCCLDEV language)") # --------------------------------------------------------------------------- # Enable RCCLDEV custom language # --------------------------------------------------------------------------- list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake") enable_language(RCCLDEV) # Tell the driver where to find the real compiler. get_filename_component(_dl_compiler_dir "${CMAKE_CXX_COMPILER}" DIRECTORY) find_program(DL_CLANG NAMES amdclang++ clang++ HINTS "${_dl_compiler_dir}" "${ROCM_PATH}/bin" REQUIRED) find_program(DL_BUNDLER NAMES clang-offload-bundler HINTS "${_dl_compiler_dir}" "${_dl_compiler_dir}/../lib/llvm/bin" "${ROCM_PATH}/llvm/bin" REQUIRED) # Extract --hip-path and --hip-device-lib-path from CMAKE_CXX_FLAGS. # TheRock's amd-hip toolchain injects these so amdclang++ can locate HIP # headers and device bitcode in its split directory layout. Standard ROCm # installs don't set them (amdclang++ auto-discovers from its own location). # We must forward any that exist to every amdclang++ -x hip invocation we make. set(DL_HIP_COMPILER_FLAGS "") string(REGEX MATCHALL "--hip-path=[^ ]+" _hip_path_flags "${CMAKE_CXX_FLAGS}") list(APPEND DL_HIP_COMPILER_FLAGS ${_hip_path_flags}) string(REGEX MATCHALL "--hip-device-lib-path=[^ ]+" _hip_devlib_flags "${CMAKE_CXX_FLAGS}") list(APPEND DL_HIP_COMPILER_FLAGS ${_hip_devlib_flags}) if(DL_HIP_COMPILER_FLAGS) message(STATUS "Device Linker: forwarding HIP flags from toolchain: ${DL_HIP_COMPILER_FLAGS}") else() message(STATUS "Device Linker: no --hip-path/--hip-device-lib-path in CMAKE_CXX_FLAGS (standard ROCm install)") endif() set(DEVICE_BUILD_DIR "${PROJECT_BINARY_DIR}/device_build") set(SPECIALIZED_DIR "${GEN_DIR}/specialized") # --------------------------------------------------------------------------- # Parse GPU_TARGETS: strip target features, build offload-arch flag list # --------------------------------------------------------------------------- set(DL_GPU_TARGETS "") set(DL_OFFLOAD_ARCH_FLAGS "") foreach(_gpu_raw ${GPU_TARGETS}) string(REGEX REPLACE ":.*" "" _gpu "${_gpu_raw}") list(APPEND DL_GPU_TARGETS "${_gpu}") list(APPEND DL_OFFLOAD_ARCH_FLAGS "--offload-arch=${_gpu}") endforeach() message(STATUS "Device Linker: GPU targets = ${DL_GPU_TARGETS}") # --------------------------------------------------------------------------- # Optimization flags (passed to both compile and link modes of the driver) # --------------------------------------------------------------------------- if(CMAKE_BUILD_TYPE MATCHES "Debug") set(DL_OPT_FLAGS -O1 -g) else() set(DL_OPT_FLAGS -O3) endif() # --------------------------------------------------------------------------- # INTERFACE library: shared definitions and includes for device compilation. # Reads from the rccl target (already fully configured) and directory scope. # No manual lists — everything comes from what CMake already knows. # --------------------------------------------------------------------------- add_library(rccl_device_defs INTERFACE) # Target-scope definitions from the rccl target get_target_property(_rccl_defs rccl COMPILE_DEFINITIONS) if(_rccl_defs) target_compile_definitions(rccl_device_defs INTERFACE ${_rccl_defs}) endif() # Directory-scope definitions (add_compile_definitions / add_definitions in root CMakeLists.txt) get_directory_property(_dir_defs COMPILE_DEFINITIONS) if(_dir_defs) target_compile_definitions(rccl_device_defs INTERFACE ${_dir_defs}) endif() # __HIP_PLATFORM_AMD__ and FMT_HEADER_ONLY come from linked targets (hip::device) # and are not visible via get_target_property. Add them explicitly. target_compile_definitions(rccl_device_defs INTERFACE __HIP_PLATFORM_AMD__=1 FMT_HEADER_ONLY=1 ) # Include directories from the rccl target (only the device-relevant subset) get_target_property(_rccl_includes rccl INCLUDE_DIRECTORIES) if(_rccl_includes) target_include_directories(rccl_device_defs INTERFACE ${_rccl_includes}) endif() # System includes: HIP headers from hip::device (or hip::amdhip64, hip::host). # We query specific targets rather than iterating all LINK_LIBRARIES because # some targets use generator expressions in INTERFACE_INCLUDE_DIRECTORIES that # can't be resolved by get_target_property in manual flag construction. set(_hip_includes "") foreach(_hip_tgt hip::device hip::amdhip64 hip::host) if(TARGET ${_hip_tgt} AND NOT _hip_includes) get_target_property(_hip_includes ${_hip_tgt} INTERFACE_INCLUDE_DIRECTORIES) endif() endforeach() if(_hip_includes) target_include_directories(rccl_device_defs SYSTEM INTERFACE ${_hip_includes}) elseif(ROCM_PATH) target_include_directories(rccl_device_defs SYSTEM INTERFACE "${ROCM_PATH}/include") endif() # fmt headers: FetchContent provides fmt_SOURCE_DIR; find_package provides the target. if(fmt_SOURCE_DIR) target_include_directories(rccl_device_defs SYSTEM INTERFACE "${fmt_SOURCE_DIR}/include") elseif(TARGET fmt::fmt-header-only) get_target_property(_fmt_inc fmt::fmt-header-only INTERFACE_INCLUDE_DIRECTORIES) if(_fmt_inc) foreach(_p ${_fmt_inc}) if(NOT _p MATCHES "^\\$<") target_include_directories(rccl_device_defs SYSTEM INTERFACE "${_p}") endif() endforeach() endif() endif() # --------------------------------------------------------------------------- # Read specialized file list # --------------------------------------------------------------------------- set(SPECIALIZED_FILES_TXT "${GEN_DIR}/specialized_files.txt") if(NOT EXISTS "${SPECIALIZED_FILES_TXT}") message(FATAL_ERROR "Device Linker: ${SPECIALIZED_FILES_TXT} not found. generate.py must run first.") endif() file(STRINGS "${SPECIALIZED_FILES_TXT}" SPECIALIZED_ENTRIES) list(LENGTH SPECIALIZED_ENTRIES DL_KERNEL_COUNT) message(STATUS "Device Linker: ${DL_KERNEL_COUNT} specialized kernels") # --------------------------------------------------------------------------- # Guard evaluation: skip kernels whose #if guard excludes a GPU target. # --------------------------------------------------------------------------- function(dl_evaluate_guard GUARD GPU_TARGET RESULT_VAR) if("${GUARD}" STREQUAL "") set(${RESULT_VAR} TRUE PARENT_SCOPE) return() endif() if("${GUARD}" MATCHES "ENABLE_LL128" AND NOT LL128_ENABLED) set(${RESULT_VAR} FALSE PARENT_SCOPE) return() endif() string(REGEX MATCHALL "__gfx[0-9a-z]+__" _guard_archs "${GUARD}") if(NOT _guard_archs) set(${RESULT_VAR} TRUE PARENT_SCOPE) return() endif() foreach(_ga ${_guard_archs}) string(REGEX REPLACE "^__(.+)__$" "\\1" _arch "${_ga}") if("${_arch}" STREQUAL "${GPU_TARGET}") set(${RESULT_VAR} TRUE PARENT_SCOPE) return() endif() endforeach() set(${RESULT_VAR} FALSE PARENT_SCOPE) endfunction() # --------------------------------------------------------------------------- # Derive host triple for the offload bundler # --------------------------------------------------------------------------- string(TOLOWER "${CMAKE_SYSTEM_NAME}" _dl_sys_name) if(NOT _dl_sys_name) set(_dl_sys_name "linux") endif() set(_dl_host_triple "${CMAKE_SYSTEM_PROCESSOR}-unknown-${_dl_sys_name}-gnu") # =========================================================================== # Per-GPU-target: OBJECT library (compile) + link custom command # =========================================================================== set(ALL_DEVICE_ELFS "") set(DL_BUNDLER_TARGETS "host-${_dl_host_triple}-") set(DL_BUNDLER_INPUTS "--input=/dev/null") set(ALL_IR_FILES "") foreach(DL_GPU_TARGET ${DL_GPU_TARGETS}) # Sort CDNA targets first for better build scheduling (see original rationale) if(DL_GPU_TARGET MATCHES "^gfx9") set(DL_ARCH_DIR "${DEVICE_BUILD_DIR}/-${DL_GPU_TARGET}") else() set(DL_ARCH_DIR "${DEVICE_BUILD_DIR}/${DL_GPU_TARGET}") endif() file(MAKE_DIRECTORY ${DL_ARCH_DIR}) # ========================================================================= # Filter specialized sources for this arch # ========================================================================= set(ARCH_SOURCES "") set(_dl_skipped 0) foreach(ENTRY ${SPECIALIZED_ENTRIES}) if(NOT ENTRY MATCHES "^([^ ]+) +([^ ]+) *(.*)") continue() endif() set(CPP_FILE "${CMAKE_MATCH_1}") set(_entry_guard "${CMAKE_MATCH_3}") dl_evaluate_guard("${_entry_guard}" "${DL_GPU_TARGET}" _guard_ok) if(NOT _guard_ok) math(EXPR _dl_skipped "${_dl_skipped} + 1") continue() endif() list(APPEND ARCH_SOURCES "${SPECIALIZED_DIR}/${CPP_FILE}") endforeach() list(LENGTH ARCH_SOURCES _dl_built) if(_dl_skipped GREATER 0) message(STATUS "Device Linker [${DL_GPU_TARGET}]: ${_dl_built} kernels to build, ${_dl_skipped} skipped (arch guard)") endif() # ========================================================================= # OBJECT library: per-kernel device compilation via RCCLDEV language # ========================================================================= set(_dev_target "rccl_device_${DL_GPU_TARGET}") add_library(${_dev_target} OBJECT ${ARCH_SOURCES}) set_source_files_properties(${ARCH_SOURCES} PROPERTIES LANGUAGE RCCLDEV) set_target_properties(${_dev_target} PROPERTIES LINKER_LANGUAGE RCCLDEV ) target_compile_options(${_dev_target} PRIVATE --arch=${DL_GPU_TARGET} --clang=${DL_CLANG} ${DL_OPT_FLAGS} -std=c++17 ${DL_HIP_COMPILER_FLAGS} ) target_compile_definitions(${_dev_target} PRIVATE RCCL_DEVICE_LINKER) target_link_libraries(${_dev_target} PRIVATE rccl_device_defs) add_dependencies(${_dev_target} hipify_all copy_nccl_device_headers) if(ENABLE_ROCSHMEM AND TARGET rocshmem_static) # rocSHMEM headers land in ext/rocshmem/include only after ExternalProject # completes; ensure they are installed before device kernels start compiling. add_dependencies(${_dev_target} rocshmem_static) endif() # ========================================================================= # Link step: driver --link mode produces device.elf # ========================================================================= set(ARCH_DEVICE_ELF "${DL_ARCH_DIR}/device.elf") # Gather definitions and includes for the dispatcher compilation inside --link. # The driver forwards these to amdclang++ when compiling common.cu.cpp. get_target_property(_dev_defs ${_dev_target} COMPILE_DEFINITIONS) set(_link_def_flags "") if(_dev_defs) foreach(_d ${_dev_defs}) list(APPEND _link_def_flags "-D${_d}") endforeach() endif() # Also add interface definitions from rccl_device_defs get_target_property(_iface_defs rccl_device_defs INTERFACE_COMPILE_DEFINITIONS) if(_iface_defs) foreach(_d ${_iface_defs}) list(APPEND _link_def_flags "-D${_d}") endforeach() endif() list(REMOVE_DUPLICATES _link_def_flags) get_target_property(_dev_includes rccl_device_defs INTERFACE_INCLUDE_DIRECTORIES) set(_link_inc_flags "") if(_dev_includes) foreach(_inc ${_dev_includes}) list(APPEND _link_inc_flags "-I${_inc}") endforeach() endif() get_target_property(_dev_sys_includes rccl_device_defs INTERFACE_SYSTEM_INCLUDE_DIRECTORIES) if(_dev_sys_includes) foreach(_inc ${_dev_sys_includes}) if(NOT _inc IN_LIST CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES) list(APPEND _link_inc_flags "-isystem${_inc}") endif() endforeach() endif() set(_link_rsp "${DL_ARCH_DIR}/link_objects.rsp") file(GENERATE OUTPUT "${_link_rsp}" CONTENT "$,\n>\n") # When rocSHMEM is enabled, pass the per-arch device bitcode to the driver. # rocSHMEM device API symbols have hidden visibility and must be statically # present in the device ELF — they cannot be imported from a shared library. set(_rocshmem_bitcode_arg "") set(_rocshmem_link_depends "") if(ENABLE_ROCSHMEM AND ROCSHMEM_INSTALL_DIR) set(_rocshmem_bc "${ROCSHMEM_INSTALL_DIR}/lib/librocshmem_device_${DL_GPU_TARGET}.bc") set(_rocshmem_bitcode_arg "--rocshmem-bitcode=${_rocshmem_bc}") # Do NOT add _rocshmem_bc to DEPENDS: rocSHMEM only supports a subset of # GPU_TARGETS (e.g. gfx90a, gfx942, gfx950) and the bitcode files don't # exist at cmake configure time (ExternalProject). The Python driver checks # existence at build time and skips silently for unsupported arches. if(TARGET rocshmem_static) list(APPEND _rocshmem_link_depends rocshmem_static) endif() endif() add_custom_command( OUTPUT ${ARCH_DEVICE_ELF} COMMAND ${CMAKE_RCCLDEV_COMPILER} --link --arch=${DL_GPU_TARGET} --clang=${DL_CLANG} ${DL_HIP_COMPILER_FLAGS} --dispatcher=${HIPIFY_DIR}/src/device/common.cu.cpp ${_rocshmem_bitcode_arg} ${_link_def_flags} ${_link_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -o ${ARCH_DEVICE_ELF} @${_link_rsp} DEPENDS ${_dev_target} ${HIPIFY_DIR}/src/device/common.cu.cpp ${_rocshmem_link_depends} COMMENT "DL [${DL_GPU_TARGET}] link: device.elf" VERBATIM COMMAND_EXPAND_LISTS ) list(APPEND ALL_DEVICE_ELFS "${ARCH_DEVICE_ELF}") list(APPEND DL_BUNDLER_TARGETS "hip-amdgcn-amd-amdhsa--${DL_GPU_TARGET}") list(APPEND DL_BUNDLER_INPUTS "--input=${ARCH_DEVICE_ELF}") # ========================================================================= # Optional: emit LLVM IR for specialized kernels (ninja device_ir) # ========================================================================= set(DL_ARCH_IR_DIR "${DL_ARCH_DIR}/device_ir") file(MAKE_DIRECTORY ${DL_ARCH_IR_DIR}) foreach(ENTRY ${SPECIALIZED_ENTRIES}) if(NOT ENTRY MATCHES "^([^ ]+) +([^ ]+) *(.*)") continue() endif() set(CPP_FILE "${CMAKE_MATCH_1}") set(_entry_guard "${CMAKE_MATCH_3}") string(REGEX REPLACE "\\.cpp$" "" BASE "${CPP_FILE}") dl_evaluate_guard("${_entry_guard}" "${DL_GPU_TARGET}" _guard_ok) if(NOT _guard_ok) continue() endif() set(SRC "${SPECIALIZED_DIR}/${CPP_FILE}") set(IR_OUT "${DL_ARCH_IR_DIR}/${BASE}.ll") add_custom_command( OUTPUT ${IR_OUT} COMMAND ${DL_CLANG} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_link_inc_flags} -x hip --offload-device-only --offload-arch=${DL_GPU_TARGET} ${DL_HIP_COMPILER_FLAGS} -gline-tables-only -std=c++17 ${DL_OPT_FLAGS} -emit-llvm -S -o ${IR_OUT} ${SRC} DEPENDS ${SRC} COMMENT "DL [${DL_GPU_TARGET}] IR: ${CPP_FILE}" VERBATIM ) list(APPEND ALL_IR_FILES ${IR_OUT}) endforeach() endforeach() # end per-GPU-target loop # =========================================================================== # Bundle all per-arch device.elf files into a single .hipfb fat binary # =========================================================================== set(DEVICE_HIPFB "${DEVICE_BUILD_DIR}/device.hipfb") list(JOIN DL_BUNDLER_TARGETS "," _bundler_targets_str) set(DL_BUNDLER_COMPRESS "") if(ENABLE_COMPRESS) set(DL_BUNDLER_COMPRESS "--compress") endif() add_custom_command( OUTPUT ${DEVICE_HIPFB} COMMAND ${DL_BUNDLER} --type=bc --targets=${_bundler_targets_str} ${DL_BUNDLER_INPUTS} --output=${DEVICE_HIPFB} ${DL_BUNDLER_COMPRESS} DEPENDS ${ALL_DEVICE_ELFS} COMMENT "DL bundle: device.elf(s) -> device.hipfb [${DL_GPU_TARGETS}]" VERBATIM ) # =========================================================================== # Host compile common.cu.cpp with embedded device binary # =========================================================================== set(COMMON_FAT_OBJ "${DEVICE_BUILD_DIR}/common.o") set(DL_HOST_COMPRESS "") if(ENABLE_COMPRESS) set(DL_HOST_COMPRESS "--offload-compress") endif() # Gather include flags for host compile (same paths as device) set(_host_inc_flags "") if(_dev_includes) foreach(_inc ${_dev_includes}) list(APPEND _host_inc_flags "-I${_inc}") endforeach() endif() if(_dev_sys_includes) foreach(_inc ${_dev_sys_includes}) if(NOT _inc IN_LIST CMAKE_CXX_IMPLICIT_INCLUDE_DIRECTORIES) list(APPEND _host_inc_flags "-isystem${_inc}") endif() endforeach() endif() add_custom_command( OUTPUT ${COMMON_FAT_OBJ} COMMAND ${DL_CLANG} -x hip --offload-host-only ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -Xclang -fcuda-include-gpubinary -Xclang ${DEVICE_HIPFB} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC ${DL_HOST_COMPRESS} -c -o ${COMMON_FAT_OBJ} ${HIPIFY_DIR}/src/device/common.cu.cpp DEPENDS ${DEVICE_HIPFB} ${HIPIFY_DIR}/src/device/common.cu.cpp COMMENT "DL host compile: common.cu.cpp with embedded device binary" VERBATIM ) # =========================================================================== # Onerank: normal HIP compilation (host+device, no RDC) # =========================================================================== set(ONERANK_FAT_OBJ "${DEVICE_BUILD_DIR}/onerank.o") add_custom_command( OUTPUT ${ONERANK_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${ONERANK_FAT_OBJ} ${HIPIFY_DIR}/src/device/onerank.cu.cpp DEPENDS ${HIPIFY_DIR}/src/device/onerank.cu.cpp COMMENT "DL compile: onerank.cu.cpp (normal fat object)" VERBATIM ) # =========================================================================== # collectives.cc: contains a __global__ kernel launch (hierarchicalAGShuffle) # so it needs full HIP compilation, not --offload-host-only. # =========================================================================== # Dependency tracking note (CMake >= 3.20 vs < 3.20): # add_custom_command only rebuilds when files listed in DEPENDS change. It # does NOT automatically track transitive headers (e.g. ce_coll.h), so a # struct layout change in a header would not invalidate collectives.o. # CMake 3.20 DEPFILE support fixes this by reading the compiler-generated .d # file; on older CMake the workaround is: touch hipify/src/collectives.cc. # =========================================================================== set(COLLECTIVES_FAT_OBJ "${DEVICE_BUILD_DIR}/collectives.o") if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.20") set(COLLECTIVES_DEPFILE "${DEVICE_BUILD_DIR}/collectives.d") add_custom_command( OUTPUT ${COLLECTIVES_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -MD -MF ${COLLECTIVES_DEPFILE} -c -o ${COLLECTIVES_FAT_OBJ} ${HIPIFY_DIR}/src/collectives.cc DEPENDS ${HIPIFY_DIR}/src/collectives.cc DEPFILE ${COLLECTIVES_DEPFILE} COMMENT "DL compile: collectives.cc (has __global__ kernel, header-tracking via DEPFILE)" VERBATIM ) else() add_custom_command( OUTPUT ${COLLECTIVES_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${COLLECTIVES_FAT_OBJ} ${HIPIFY_DIR}/src/collectives.cc DEPENDS ${HIPIFY_DIR}/src/collectives.cc COMMENT "DL compile: collectives.cc (has __global__ kernel)" VERBATIM ) endif() # =========================================================================== # dda_all_reduce_ipc.cu.cpp: contains device kernels and kernel launches. # Like collectives.cc, it cannot be built with --offload-host-only on the # main rccl target or __hip_fatbin_* stays undefined in librccl.so. # =========================================================================== set(DDA_ALL_REDUCE_IPC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_all_reduce_ipc.o") set(DDA_REDUCE_SCATTER_IPC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_reduce_scatter_ipc.o") set(DDA_ALL_GATHER_IPC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_all_gather_ipc.o") set(DDA_ALLTOALL_IPC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_alltoall_ipc.o") add_custom_command( OUTPUT ${DDA_ALL_REDUCE_IPC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_all_reduce_ipc.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_all_reduce_ipc.cu.cpp COMMENT "DL compile: dda_all_reduce_ipc.cu.cpp (has device kernels)" VERBATIM ) add_custom_command( OUTPUT ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_reduce_scatter_ipc.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_reduce_scatter_ipc.cu.cpp COMMENT "DL compile: dda_reduce_scatter_ipc.cu.cpp (has device kernels)" VERBATIM ) add_custom_command( OUTPUT ${DDA_ALL_GATHER_IPC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_all_gather_ipc.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_all_gather_ipc.cu.cpp COMMENT "DL compile: dda_all_gather_ipc.cu.cpp (has device kernels)" VERBATIM ) add_custom_command( OUTPUT ${DDA_ALLTOALL_IPC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${DDA_ALLTOALL_IPC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_alltoall_ipc.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_alltoall_ipc.cu.cpp COMMENT "DL compile: dda_alltoall_ipc.cu.cpp (has device kernels)" VERBATIM ) # =========================================================================== # dda_all_reduce_fabric.cu.cpp: fabric/VMM counterpart of the IPC file above. # =========================================================================== set(DDA_ALL_REDUCE_FABRIC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_all_reduce_fabric.o") add_custom_command( OUTPUT ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -w -c -o ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_all_reduce_fabric.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_all_reduce_fabric.cu.cpp COMMENT "DL compile: dda_all_reduce_fabric.cu.cpp (has device kernels)" VERBATIM ) set(DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_reduce_scatter_fabric.o") set(DDA_ALL_GATHER_FABRIC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_all_gather_fabric.o") set(DDA_ALLTOALL_FABRIC_FAT_OBJ "${DEVICE_BUILD_DIR}/dda_alltoall_fabric.o") add_custom_command( OUTPUT ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -w -c -o ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_reduce_scatter_fabric.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_reduce_scatter_fabric.cu.cpp COMMENT "DL compile: dda_reduce_scatter_fabric.cu.cpp (has device kernels)" VERBATIM ) add_custom_command( OUTPUT ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -w -c -o ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_all_gather_fabric.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_all_gather_fabric.cu.cpp COMMENT "DL compile: dda_all_gather_fabric.cu.cpp (has device kernels)" VERBATIM ) add_custom_command( OUTPUT ${DDA_ALLTOALL_FABRIC_FAT_OBJ} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -w -c -o ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${HIPIFY_DIR}/src/dda_alltoall_fabric.cu.cpp DEPENDS ${HIPIFY_DIR}/src/dda_alltoall_fabric.cu.cpp COMMENT "DL compile: dda_alltoall_fabric.cu.cpp (has device kernels)" VERBATIM ) # =========================================================================== # Symmetric kernels: per-instantiation device TUs from gensrc/symmetric/. # Each instantiation file defines a handful of __global__ ncclSymkDevKernel_* # entries. Compiled standalone as multi-arch fat objects, mirroring onerank.o. # # SYM_FAT_OBJS is plural (vs the singular COMMON/ONERANK/COLLECTIVES_FAT_OBJ # siblings) because the symmetric generator emits one TU per instantiation. # =========================================================================== set(SYM_FAT_OBJS "") if(GENERATE_SYM_KERNELS) file(GLOB _sym_srcs CONFIGURE_DEPENDS "${HIPIFY_DIR}/gensrc/symmetric/*.cpp") foreach(_sym_src IN LISTS _sym_srcs) get_filename_component(_sym_name "${_sym_src}" NAME_WE) set(_sym_obj "${DEVICE_BUILD_DIR}/sym_${_sym_name}.o") add_custom_command( OUTPUT ${_sym_obj} COMMAND ${DL_CLANG} -x hip ${DL_OFFLOAD_ARCH_FLAGS} ${DL_HIP_COMPILER_FLAGS} -DRCCL_DEVICE_LINKER ${_link_def_flags} ${_host_inc_flags} ${DL_OPT_FLAGS} -std=c++17 -fPIC -c -o ${_sym_obj} ${_sym_src} DEPENDS ${_sym_src} COMMENT "DL compile: sym ${_sym_name} (multi-arch fat object)" VERBATIM ) list(APPEND SYM_FAT_OBJS ${_sym_obj}) endforeach() endif() # =========================================================================== # Top-level target # =========================================================================== add_custom_target(device_linker_build ALL DEPENDS ${COMMON_FAT_OBJ} ${ONERANK_FAT_OBJ} ${COLLECTIVES_FAT_OBJ} ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${DDA_ALLTOALL_IPC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${SYM_FAT_OBJS} ) add_dependencies(device_linker_build hipify_all copy_nccl_device_headers) set(DEVICE_LINKER_OBJECTS ${COMMON_FAT_OBJ} ${ONERANK_FAT_OBJ} ${COLLECTIVES_FAT_OBJ} ${DDA_ALL_REDUCE_IPC_FAT_OBJ} ${DDA_REDUCE_SCATTER_IPC_FAT_OBJ} ${DDA_ALL_GATHER_IPC_FAT_OBJ} ${DDA_ALLTOALL_IPC_FAT_OBJ} ${DDA_ALL_REDUCE_FABRIC_FAT_OBJ} ${DDA_REDUCE_SCATTER_FABRIC_FAT_OBJ} ${DDA_ALL_GATHER_FABRIC_FAT_OBJ} ${DDA_ALLTOALL_FABRIC_FAT_OBJ} ${SYM_FAT_OBJS} ) # =========================================================================== # Optional: emit LLVM IR (ninja device_ir) # =========================================================================== add_custom_target(device_ir DEPENDS ${ALL_IR_FILES}) add_dependencies(device_ir hipify_all copy_nccl_device_headers)