# Ensure the compiler is a valid clang when building the GPU target.
set(req_ver "${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}")
if(LLVM_VERSION_MAJOR AND NOT (CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang" AND
   ${CMAKE_CXX_COMPILER_VERSION} VERSION_EQUAL "${req_ver}"))
   message(WARNING "openmp for GPU requires an up-to-date clang. CMake compiler "
                      "'${CMAKE_CXX_COMPILER_ID} ${CMAKE_CXX_COMPILER_VERSION}' "
                      " is not 'Clang ${req_ver}'.")
endif()

option(OFFLOAD_ENABLE_EMISSARY_APIS "Enable build of GPU Emissary APIs" ON)
if(OFFLOAD_ENABLE_EMISSARY_APIS)
  add_definitions(-DOFFLOAD_ENABLE_EMISSARY_APIS)
  set(emissary_sources
    src/EmissaryFortrt.cpp
    src/EmissaryPrint.cpp
  )
endif()

set(src_files
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Allocator.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Configuration.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Debug.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Kernel.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/LibC.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Mapping.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Misc.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Parallelism.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Reduction.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/State.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Synchronization.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Tasking.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/DeviceUtils.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Workshare.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/ExtraMapping.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Xteamr.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Memory.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/Xteams.cpp
  ${CMAKE_CURRENT_SOURCE_DIR}/src/DeviceMemInit.cpp
  ${emissary_sources}

)
# Only link AMD ockl/ocml device libraries when actually building for AMDGCN.
# The previous condition checked LLVM_TARGETS_TO_BUILD which includes all backends
# (e.g. X86;AMDGPU;NVPTX), causing ockl.bc/ocml.bc to be linked into NVPTX builds
# and contaminating the NVPTX DeviceRTL bitcode with AMDGCN intrinsics.
if("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn" OR
   "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^amdgcn")
  find_package(AMDDeviceLibs REQUIRED CONFIG
               HINTS ${CMAKE_BINARY_DIR}/../../tools/rocm-device-libs
                     ${CMAKE_BINARY_DIR}/../rocm-device-libs-prefix/src/rocm-device-libs-build
                     ${CMAKE_INSTALL_PREFIX}
  )
  get_target_property(_ocml_bc ocml IMPORTED_LOCATION)
  get_target_property(_ockl_bc ockl IMPORTED_LOCATION)
  get_target_property(_asanrtl_bc asanrtl IMPORTED_LOCATION)

  if(NOT _ockl_bc)
    message(FATAL_ERROR "Could not find ockl.bc")
  endif()
  if(NOT _ocml_bc)
    message(FATAL_ERROR "Could not find ocml.bc")
  endif()
  if(NOT _asanrtl_bc)
    message(FATAL_ERROR "Could not find asanrtl.bc")
  endif()
  if(SANITIZER_AMDGPU)
    list(APPEND compile_flags "SHELL: -DSANITIZER_AMDGPU=1")
    # For ASan: Don't link asanrtl.bc and ockl.bc at compile time of libompdevice.a
    # They will be linked once in POST_BUILD step to avoid duplicates symbols
    # Note: Oclc control constants are provided using Platform.h
    #
  else()
    list(APPEND compile_flags "SHELL: -Xclang -mlink-builtin-bitcode -Xclang ${_ockl_bc}")
  endif()
  list(APPEND compile_flags "SHELL: -Xclang -mlink-builtin-bitcode -Xclang ${_ocml_bc}")
endif()

list(APPEND compile_options -flto)
list(APPEND compile_options -fvisibility=hidden)
list(APPEND compile_options -nogpulib)
list(APPEND compile_options -nostdlibinc)
list(APPEND compile_options -fno-rtti)
list(APPEND compile_options -fno-exceptions)
list(APPEND compile_options -fconvergent-functions)
list(APPEND compile_options -Wno-unknown-cuda-version)
list(APPEND compile_options -fno-sanitize=address)
list(APPEND compile_options -O3)
if(LLVM_DEFAULT_TARGET_TRIPLE)
  list(APPEND compile_options --target=${LLVM_DEFAULT_TARGET_TRIPLE})
endif()

# We disable the slp vectorizer during the runtime optimization to avoid
# vectorized accesses to the shared state. Generally, those are "good" but
# the optimizer pipeline (esp. Attributor) does not fully support vectorized
# instructions yet and we end up missing out on way more important constant
# propagation. That said, we will run the vectorizer again after the runtime
# has been linked into the user program.
list(APPEND compile_flags "SHELL: -mllvm -vectorize-slp=false")
if("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^amdgcn" OR
   "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^amdgcn")
  set(target_name "amdgpu")
  list(APPEND compile_flags "SHELL:-Xclang -mcode-object-version=none")
elseif("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^nvptx" OR
       "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^nvptx")
  set(target_name "nvptx")
  list(APPEND compile_options --cuda-feature=+ptx63)
elseif("${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^spirv" OR
       "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^spirv")
  set(target_name "spirv") 
endif()

# Trick to combine these into a bitcode file via the linker's LTO pass.
add_executable(libompdevice ${src_files})
set_target_properties(libompdevice PROPERTIES
   RUNTIME_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
  LINKER_LANGUAGE CXX
  BUILD_RPATH ""
  INSTALL_RPATH ""
  RUNTIME_OUTPUT_NAME libomptarget-${target_name}.bc)

# If the user built with the GPU C library enabled we will use that instead.
if(TARGET libc)
  target_compile_definitions(libompdevice PRIVATE OMPTARGET_HAS_LIBC)
endif()
target_compile_definitions(libompdevice PRIVATE SHARED_SCRATCHPAD_SIZE=512)

target_include_directories(libompdevice PRIVATE
                           ${CMAKE_CURRENT_SOURCE_DIR}/include
                           ${CMAKE_CURRENT_SOURCE_DIR}/../../libc
                           ${CMAKE_CURRENT_SOURCE_DIR}/../../offload/include)
target_compile_options(libompdevice PRIVATE ${compile_options} ${compile_flags})
if(NOT "${LLVM_DEFAULT_TARGET_TRIPLE}" MATCHES "^spirv" AND
   NOT "${CMAKE_CXX_COMPILER_TARGET}" MATCHES "^spirv")
  target_link_options(libompdevice PRIVATE
                  "-flto" "-r" "-nostdlib" "-Wl,--lto-emit-llvm")
else()
  target_link_options(libompdevice PRIVATE
                  "-nostdlib" "-emit-llvm")
endif()

if(LLVM_DEFAULT_TARGET_TRIPLE)
  target_link_options(libompdevice PRIVATE "--target=${LLVM_DEFAULT_TARGET_TRIPLE}")
endif()

# For SANITIZER_AMDGPU: Use llvm-link POST_BUILD to merge ASAN runtime and OCKL
# into the device library bitcode to resolve all and __asan_* __ockl_* symbols
if(SANITIZER_AMDGPU)
  find_program(LLVM_LINK_EXECUTABLE NAMES llvm-link HINTS ${LLVM_TOOLS_BINARY_DIR})
  if(NOT LLVM_LINK_EXECUTABLE)
    message(FATAL_ERROR "llvm-link not found")
  endif()
  add_custom_command(TARGET libompdevice POST_BUILD
    COMMAND ${CMAKE_COMMAND} -E copy 
      ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc
      ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc.tmp
    COMMAND ${LLVM_LINK_EXECUTABLE}
      ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc.tmp
      ${_asanrtl_bc}
      ${_ockl_bc}
      --only-needed
      -o ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc
    COMMAND ${CMAKE_COMMAND} -E remove
      ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc.tmp
    COMMENT "Merging asanrtl.bc and ockl.bc into libomptarget-${target_name}.bc"
  )
endif()

install(TARGETS libompdevice
        PERMISSIONS OWNER_WRITE OWNER_READ GROUP_READ WORLD_READ
        DESTINATION ${OPENMP_INSTALL_LIBDIR}
        COMPONENT openmp)

add_library(ompdevice.all_objs OBJECT IMPORTED)
set_property(TARGET ompdevice.all_objs APPEND PROPERTY IMPORTED_OBJECTS
              ${CMAKE_CURRENT_BINARY_DIR}/libomptarget-${target_name}.bc)

# Archive all the object files generated above into a static library
add_library(ompdevice STATIC)
add_dependencies(ompdevice libompdevice)
set_target_properties(ompdevice PROPERTIES
   ARCHIVE_OUTPUT_DIRECTORY "${OPENMP_INSTALL_LIBDIR}"
   ARCHIVE_OUTPUT_NAME ompdevice
  LINKER_LANGUAGE CXX
)
target_link_libraries(ompdevice PRIVATE ompdevice.all_objs)
install(TARGETS ompdevice ARCHIVE DESTINATION "${OPENMP_INSTALL_LIBDIR}"
        COMPONENT openmp)

openmp_register_meta_dep(ompdevice)