# Copyright 2021-2024 Free Software Foundation, Inc.
# Copyright (C) 2021-2024 Advanced Micro Devices, Inc. All rights reserved.

# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

# This file is a dejagnu "board file" and is used to run the testsuite
# against C/C++ device code compiled with amdclang++ (HIP).
#
# Example usage:
#  bash$ make check RUNTESTFLAGS='--target_board=hip'

# Running tests against the GPU device is accomplished with linking
# lib/hip/hip-driver.cc into every testcase.  That file contains both
# the host's main() entry point, and a kernel entry point.  The kernel
# calls the testcase's main, in GPU context.
#
# Then the board overrides gdb_compile to compile the C/C++ testcase
# files targeting the device, and link in the driver.
#
# Ideally we would not have to modify any of the testcases sources.
# Unfortunately that isn't possible, but we can avoid a number of
# classes of changes, effectively reducing required absolute changes
# significantly.
#
# For example, to force the compiler to emit __device__ functions
# without sprinkling __device__ all over the place in all functions in
# all testcases, we use:
#
#  #pragma clang force_cuda_host_device begin
#
# And to force the compiler to emit __device__ globals, we use:
#
#  #pragma clang attribute (__device__, apply_to = variable (is_global))
#
# The board file tweaks the compilation routine to inject that in
# translation units by compiling a wrapper hip-test-wrapper.cc instead
# of the desired file.  The wrapper file does the magic mentioned
# above and includes the file we wanted to compile.
#
# The hip-test-wrapper.cc file is also responsible for renaming the
# testcase's main function, so that it doesn't conflict with the
# host's main:
#
#  #define main gdb_hip_test_main
#
# The testsuite, for each testcase, normally compiles the test
# program, and then runs it to main (like "b main; run").  Because we
# rename the device/testcase "main" function, "b main" sets a
# breakpoint on the host "main" function, not the testcase's entry
# point.  That is addressed by having GDB itself rename the
# "gdb_hip_test_main" function at symbol read time, pretending that
# it's really called "main".  The host "main" function is also renamed
# to get it out of the way.
#
# Parallel mode testing is supported by making use of the file locking
# implemented in lib/rocm.exp and also used by the gdb.rocm/
# testcases.  That means that only one GDB at a time runs, but, most
# compilations run in parallel, which still provides a speed
# improvement, since compiling for the GPU is slow.
#
# Note that HIP is C++ based, so we're running all C testcases in C++
# mode.  E.g., most gdb.base/ testcases.
#
# The multi-threading testcases are disabled. The device kernel runs a
# single work-item.

load_lib rocm.exp

load_generic_config "unix"
process_multilib_options ""

set_board_info compiler [find_hip_compiler]
set_board_info c++compiler [find_hip_compiler]

set_board_info gdb,cannot_call_functions 1

set_board_info gdb,can_reverse 0
set_board_info gdb,use_precord 0

# Override the target set in build/gdb/testsuite/site.exp.
set target_alias amdgcn-amd-amdhsa
set target_triplet amdgcn-amd-amdhsa

# This is needed otherwise dejagnu tries to rsh to host "hip".
load_board_description "local-board"

# So we can override skip_gdbserver_tests.
load_lib gdbserver-support.exp

# Tell GDB to enable the HIP test mode.  This renames the main and
# gdb_hip_test_name functions to provide the illusion that the
# kernel's main is the real main.
set ::env(GDB_HIP_TEST_MODE) 1

proc skip_gdbserver_tests {} {
    return 1
}

proc skip_fortran_tests {} {
    return 1
}

proc skip_ada_tests {} {
    return 1
}

proc skip_go_tests {} {
    return 1
}

proc skip_d_tests {} {
    return 1
}

proc skip_rust_tests {} {
    return 1
}

proc skip_shlib_tests {} {
    return 1
}

proc skip_btrace_tests {} {
    return 1
}

proc skip_compile_feature_tests {} {
    return 1
}

# So we can override skip_opencl_tests.
load_lib "opencl.exp"

proc skip_opencl_tests {} {
    return 1
}

# Yup, we don't want to run the gdb.rocm/ tests against this board.
# That would be kind of recursive.  Those tests really expect that
# "main()" is host code and start kernels themselves.
proc allow_hip_tests {} {
    return 0
}

# Override this to fail faster.  If we let the default run, tests
# compile successfully, but fail at link time.
proc gdb_compile_pthreads {source dest type options} {
    unsupported "skipped compiling [file tail $source]: pthreads tests disabled on HIP"
    return -1
}

proc do_gpu_lock_release {} {
    global gpu_lock_rc
    if {[info exists gpu_lock_rc]} {
	lock_file_release $gpu_lock_rc
	unset gpu_lock_rc
    }
}

# Because building the testcases is so slow, it's quite a speed boost
# to run the testsuite in parallel mode, so that multiple programs are
# compiled at the same time.  However, only one GDB may be using the
# GPU at a time.  So we use a lock file to synchronize access to the
# GPU -- we acquire the GPU lock when GDB is started, and release it
# when GDB exits.  Generally testcases compile their programs before
# starting GDB.  We hook into gdb_start and gdb_exit here to do just
# that.

if {[info exists GDB_PARALLEL]} {
    if { [info procs hip_org_gdb_spawn] == "" } {
	rename gdb_spawn hip_org_gdb_spawn
    }
    proc gdb_spawn {} {
	global gdb_spawn_id

	if ![info exists gdb_spawn_id] {
	    global gpu_lock_filename
	    global gpu_lock_rc
	    set gpu_lock_rc [lock_file_acquire $gpu_lock_filename]
	}

	set res [hip_org_gdb_spawn]
	if {$res != 0} {
	    lock_file_release $gpu_lock_rc
	    unset gpu_lock_rc
	}
	return $res
    }

    if { [info procs hip_org_gdb_exit] == "" } {
	rename gdb_exit hip_org_gdb_exit
    }
    proc gdb_exit {} {
	verbose -log "hip: gdb_exit"

	set res [hip_org_gdb_exit]

	do_gpu_lock_release

	return $res
    }

    if { [info procs hip_org_mi_gdb_exit] == "" } {
	rename mi_gdb_exit hip_org_mi_gdb_exit
    }
    proc mi_gdb_exit {} {
	verbose -log "hip: mi_gdb_exit"

	set res [hip_org_mi_gdb_exit]

	do_gpu_lock_release

	return $res
    }
}

# As a speed optimization, avoid recompiling the driver over and over
# for each testcase program.
global gdb_saved_hip_driver_obj
set gdb_saved_hip_driver_obj ""

# True if compiling lib/hip/hip-driver.cc.  If true, we don't want to
# use the hip-test-wrapper.cc wrapper.
global compiling_hip_driver
set compiling_hip_driver 0

# True if calling rocm_assemble.  If true, we don't want to force
# compilation targeting the device.
global calling_rocm_assemble
set calling_rocm_assemble 0

proc hip_get_compile_language {options} {
    foreach lang {c++ rust ada f77 f90 go hip} {
	if {[lsearch -exact $options $lang] != -1} {
	    return $lang
	}
    }

    return c
}

# Override gdb_compile, to inject our driver, which has a host main()
# function that starts a device kernel that calls the testcase's entry
# point (main renamed to gdb_hip_test_main).

if { [info procs hip_org_gdb_compile] == "" } {
    rename gdb_compile hip_org_gdb_compile
}
proc gdb_compile {source dest type options} {
    global srcdir objdir
    global gdb_saved_hip_driver_obj
    global compiling_hip_driver
    global calling_rocm_assemble

    set lang [hip_get_compile_language {$options}]

    if {[lsearch -exact $options getting_compiler_info] == -1
	&& !$calling_rocm_assemble
	&& ($lang == "c" || $lang == "c++")} {

	# We need:
	#
	# . -std=gnu++11 because some testcases rely on GNU extensions.
	#
	# . -fgpu-rdc enables separate compilation mode, so we can
	#   compile the kernel from multiple translation units.
	#
	# . -amdgpu-spill-cfi-saved-regs makes backtraces actually
	#   work.
	#
	# . -Wno-unused-command-line-argument is needed otherwise we
	#   get this warning during linking, even though the option is
	#   really needed when linking:
	#
	#    clang: warning: argument unused during compilation:
	#    '-mllvm -amdgpu-spill-cfi-saved-regs'
	#    [-Wunused-command-line-argument]
	#
	# When ROCM_PATH is set in the environment, pass --rocm-path
	# so amdclang++ uses that specific ROCm install for HIP.
	# When unset, let amdclang++ auto-discover (parent of its own
	# LLVM dir, falling back on /opt/rocm).
	set rocm_path_arg ""
	if {[info exists ::env(ROCM_PATH)] && $::env(ROCM_PATH) ne ""} {
	    set rocm_path_arg "--rocm-path=$::env(ROCM_PATH)"
	}
	set hip_options "early_flags=$rocm_path_arg\
			 -std=gnu++11\
			 -fgpu-rdc\
			 -mllvm -amdgpu-spill-cfi-saved-regs\
			 -Wno-unused-command-line-argument"

	global CC_FOR_TARGET
	if {[board_info target exists compiler]} {
	    set compiler [target_info compiler]
	} elseif {[info exists CC_FOR_TARGET]} {
	    set compiler $CC_FOR_TARGET
	}
	if {[string first "amdclang++" $compiler] != -1} {

	    # Remove "c++" from the options when linking, otherwise we
	    # would get:
	    #
	    #  build/gdb/testsuite/hip-driver.o:1:1: error: expected unqualified-id
	    #  <U+007F>ELF<U+0002><U+0001><U+0001><U+0000><U+0000><U+0000><U+0000><U+0000>
	    #
	    # ... because -x c++ would make Clang blindly treat the
	    # input .o files as C++.
	    #
	    set idx [lsearch -exact $options "c++"]
	    set options [lreplace $options $idx $idx]

	    # Since we wrap translation units with hip-test-wrapper.cc
	    # when compiling C/C++ source files (see below), we can't
	    # mix source and object input files in the same
	    # compilation.  Fix that by forcing separate compilation
	    # for each input file that doesn't look like an object
	    # file, and replacing the corresponding SOURCE array
	    # element with the corresponding built object file.
	    if { $type == "executable" } {
		set new_source {}
		foreach src $source {
		    if {[file extension $src] != ".o"} {
			set out [standard_temp_file [file tail $src].o]
			set res [gdb_compile $src $out object $options]
			if {$res != ""} {
			    return $res
			}
			lappend new_source $out
		    } else {
			lappend new_source $src
		    }
		}
		set source $new_source

		# Compile the driver.
		if { $gdb_saved_hip_driver_obj == "" } {
		    verbose "compiling hip-driver"
		    set hip_driver_src ${srcdir}/lib/hip/hip-driver.cc
		    set hip_driver_obj ${objdir}/hip-driver.o

		    set compiling_hip_driver 1
		    set result [gdb_compile "${hip_driver_src}" "${hip_driver_obj}" object {debug}]
		    set compiling_hip_driver 0

		    if { $result != "" } {
			return $result
		    }
		    if {[is_remote host]} {
			set gdb_saved_hip_driver_obj hip-driver.o
		    } else {
			set gdb_saved_hip_driver_obj ${objdir}/hip-driver.o
		    }
		    # Link a copy of the output object, because the
		    # original may be automatically deleted.
		    remote_download host $hip_driver_obj $gdb_saved_hip_driver_obj
		}
		# Prefix "-x none" so amdclang++ does not treat the
		# driver .o as HIP source (the preceding inputs on
		# the command line are .cpp tagged "-x hip").
		lappend options "ldflags=-x none $gdb_saved_hip_driver_obj"

		lappend options $hip_options
		lappend options "early_flags=--hip-link"

		# All compiled.  Now link, and we're done.
		return [hip_org_gdb_compile $source $dest $type $options]
	    }

	    # Compiling some source file to an object file.

	    # If we're not linking, then we're guaranteed to only have
	    # one source file, due to the recursion in the
	    # "executable" case above.
	    if {[llength $source] != 1} {
		error "more than one source file?"
	    }

	    # See if we're actually compiling an assembly file.
	    set src [lindex $source 0]
	    if {[file extension $src] == ".S" || [file extension $src] == ".s"} {
		set calling_rocm_assemble 1
		set res [rocm_assemble $src $dest $options]
		set calling_rocm_assemble 0
		return $res
	    }

	    # Make HIP/C++ look a little bit more like C.
	    if {$lang == "c"} {
		append hip_options "\
				    -Wno-writable-strings\
				    -Wno-c++11-narrowing"
	    }

	    # Clang automatically includes
	    # __clang_hip_runtime_wrapper.h via -include on the
	    # command line.  We need to override the malloc definition
	    # so we need to wrap the wrapper header...  Our wrapper
	    # lives in the lib/hip/ dir.
	    append hip_options " -I ${srcdir}/lib/hip/"

	    if {!$compiling_hip_driver} {
		# Wrap the file we actually want to compile with
		# hip-test-wrapper.cc.
		append hip_options " -DINC_TEST_FILE=\"$source\""
		set source "${srcdir}/lib/hip/hip-test-wrapper.cc"
	    }

	    lappend options $hip_options
	    lappend options "early_flags=-x hip"
	}
    }
    hip_org_gdb_compile $source $dest $type $options
}