// RUN: fir-opt --split-input-file --cuf-add-constructor %s | FileCheck %s // RUN: fir-opt --split-input-file --cuf-add-constructor="cuda-unified=true" %s | FileCheck %s --check-prefixes=CHECK,UNIFIED module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry, dense<64> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git cae351f3453a0a26ec8eb2ddaf773c24a29d929e)", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda} : !fir.array<5xi32> fir.global @_QMmtestsEndev {data_attr = #cuf.cuda} : !fir.box>> { %c0 = arith.constant 0 : index %0 = fir.zero_bits !fir.heap> %1 = fircg.ext_embox %0(%c0) {allocator_idx = 2 : i32} : (!fir.heap>, index) -> !fir.box>> fir.has_value %1 : !fir.box>> } // A kernel and the device globals must be present in the gpu.module // for the constructor to register them (it skips modules without kernels // and globals not in the gpu.module's symbol table). gpu.module @cuda_device_mod { gpu.func @_QMmtestsPkernel() kernel { gpu.return } fir.global @_QMmtestsEn(dense<[3, 4, 5, 6, 7]> : tensor<5xi32>) {data_attr = #cuf.cuda} : !fir.array<5xi32> fir.global @_QMmtestsEndev {data_attr = #cuf.cuda} : !fir.box>> { %c0 = arith.constant 0 : index %0 = fir.zero_bits !fir.heap> %1 = fircg.ext_embox %0(%c0) {allocator_idx = 2 : i32} : (!fir.heap>, index) -> !fir.box>> fir.has_value %1 : !fir.box>> } } } // CHECK: gpu.module @cuda_device_mod // CHECK: llvm.func internal @__cudaFortranConstructor() { // CHECK-DAG: %[[MODULE:.*]] = cuf.register_module @cuda_device_mod -> !llvm.ptr // CHECK-DAG: %[[VAR_NAME:.*]] = fir.address_of(@_QQ{{.*}}) : !fir.ref> // CHECK-DAG: %[[VAR_ADDR:.*]] = fir.address_of(@_QMmtestsEn) : !fir.ref> // CHECK-DAG: %[[MODULE2:.*]] = fir.convert %[[MODULE]] : (!llvm.ptr) -> !fir.ref> // CHECK-DAG: %[[VAR_ADDR2:.*]] = fir.convert %[[VAR_ADDR]] : (!fir.ref>) -> !fir.ref // CHECK-DAG: %[[VAR_NAME2:.*]] = fir.convert %[[VAR_NAME]] : (!fir.ref>) -> !fir.ref // CHECK-DAG: %[[CST:.*]] = arith.constant 20 : index // CHECK-DAG: %[[CST2:.*]] = fir.convert %[[CST]] : (index) -> i64 // CHECK-DAG: fir.call @_FortranACUFRegisterVariable(%[[MODULE2]], %[[VAR_ADDR2]], %[[VAR_NAME2]], %[[CST2]]) : (!fir.ref>, !fir.ref, !fir.ref, i64) -> () // CHECK-DAG: %[[BOX:.*]] = fir.address_of(@_QMmtestsEndev) : !fir.ref>>> // CHECK-DAG: %[[BOXREF:.*]] = fir.convert %[[BOX]] : (!fir.ref>>>) -> !fir.ref // CHECK-DAG: fir.call @_FortranACUFRegisterVariable(%[[MODULE:.*]], %[[BOXREF]], %{{.*}}, %{{.*}}) // CHECK-NOT: fir.call @_FortranACUFInitModule // ----- // Checking that constant global variables are not registered // CHECK: @_FortranACUFRegisterAllocator // CHECK-NOT: fir.call @_FortranACUFRegisterVariable module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git 3372303188df0f7f8ac26e7ab610cf8b0f716d42)", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMiso_c_bindingECc_int constant : i32 fir.type_info @_QM__fortran_builtinsT__builtin_c_ptr noinit nodestroy nofinal : !fir.type<_QM__fortran_builtinsT__builtin_c_ptr{__address:i64}> gpu.module @cuda_device_mod { fir.global @_QMiso_c_bindingECc_int {data_attr = #cuf.cuda} constant : i32 gpu.func @_QMdevmodPdevsub(%arg0: !fir.ref>>>) -> i32 { %c0 = arith.constant 0 : index %c4_i32 = arith.constant 4 : i32 %0 = fir.alloca i32 {bindc_name = "devsub", uniq_name = "_QMdevmodFdevsubEdevsub"} %1 = fir.alloca i32 {bindc_name = "__builtin_warpsize", uniq_name = "_QM__fortran_builtinsEC__builtin_warpsize"} %2 = fir.load %arg0 : !fir.ref>>> %3:3 = fir.box_dims %2, %c0 : (!fir.box>>, index) -> (index, index, index) %4 = fir.convert %3#1 : (index) -> i32 %5 = arith.muli %4, %c4_i32 : i32 fir.store %5 to %0 : !fir.ref %6 = fir.load %0 : !fir.ref gpu.return %6 : i32 } } } // ----- module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.ident = "flang version 20.0.0 (https://github.com/llvm/llvm-project.git 3372303188df0f7f8ac26e7ab610cf8b0f716d42)", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMmEa00 {data_attr = #cuf.cuda} : !fir.box>> { %c0 = arith.constant 0 : index %0 = fir.zero_bits !fir.heap> %1 = fir.shape %c0, %c0, %c0, %c0, %c0 : (index, index, index, index, index) -> !fir.shape<5> %2 = fir.embox %0(%1) {allocator_idx = 3 : i32} : (!fir.heap>, !fir.shape<5>) -> !fir.box>> fir.has_value %2 : !fir.box>> } // A kernel and the managed global must be in the gpu.module for // the constructor to register them. gpu.module @cuda_device_mod { gpu.func @_QMmPkernel() kernel { gpu.return } fir.global @_QMmEa00 {data_attr = #cuf.cuda} : !fir.box>> { %c0 = arith.constant 0 : index %0 = fir.zero_bits !fir.heap> %1 = fir.shape %c0, %c0, %c0, %c0, %c0 : (index, index, index, index, index) -> !fir.shape<5> %2 = fir.embox %0(%1) {allocator_idx = 3 : i32} : (!fir.heap>, !fir.shape<5>) -> !fir.box>> fir.has_value %2 : !fir.box>> } } } // CHECK: llvm.func internal @__cudaFortranConstructor() // CHECK: fir.address_of(@_QMmEa00) // CHECK: fir.call @_FortranACUFRegisterVariable // CHECK-NOT: fir.call @_FortranACUFInitModule // ----- // Non-allocatable managed global: should create pointer global in // __nv_managed_data__ and register with CUFRegisterManagedVariable. // // Fortran source: // module test // integer*4, managed :: manx(100) // contains // attributes(global) subroutine kernel() // end subroutine // end module module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMtestEmanx {data_attr = #cuf.cuda} : !fir.array<100xi32> { %0 = fir.zero_bits !fir.array<100xi32> fir.has_value %0 : !fir.array<100xi32> } // A kernel and the managed global must be in the gpu.module for // the constructor to register them. gpu.module @cuda_device_mod { gpu.func @_QMtestPkernel() kernel { gpu.return } fir.global @_QMtestEmanx {data_attr = #cuf.cuda} : !fir.array<100xi32> { %0 = fir.zero_bits !fir.array<100xi32> fir.has_value %0 : !fir.array<100xi32> } } } // Pointer global should be created with section attribute. // CHECK: fir.global internal @_QMtestEmanx.managed.ptr {section = "__nv_managed_data__"} : !fir.llvm_ptr // CHECK: fir.zero_bits !fir.llvm_ptr // Constructor should register with CUFRegisterManagedVariable then init module. // CHECK: llvm.func internal @__cudaFortranConstructor() // CHECK: fir.address_of(@_QMtestEmanx.managed.ptr) : !fir.ref> // CHECK: fir.call @_FortranACUFRegisterManagedVariable // CHECK: fir.call @_FortranACUFInitModule // ----- // Test that when the gpu.module has no kernels (e.g., host TU that USEs // a kernel module), the constructor registers allocators but skips module // registration. module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry, dense<64> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMkernels_mEdev_var {data_attr = #cuf.cuda} : f32 { %0 = arith.constant 0.0 : f32 fir.has_value %0 : f32 } gpu.module @cuda_device_mod { gpu.func @_QMkernels_mPnot_a_kernel() { gpu.return } } } // CHECK: llvm.func internal @__cudaFortranConstructor() // CHECK-NEXT: llvm.call @_FortranACUFRegisterAllocator() // CHECK-NEXT: llvm.return // CHECK: llvm.mlir.global_ctors ctors = [@__cudaFortranConstructor] // ----- // Test that when the gpu.module has no kernels but a device global, the // constructor registers the global. module attributes {dlti.dl_spec = #dlti.dl_spec<#dlti.dl_entry : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry, dense<32> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry, dense<64> : vector<4xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry : vector<2xi64>>, #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.stack_alignment", 128 : i64>>, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMkernels_mEdev_var {data_attr = #cuf.cuda} : f32 { %0 = arith.constant 0.0 : f32 fir.has_value %0 : f32 } gpu.module @cuda_device_mod { gpu.func @_QMkernels_mPnot_a_kernel() { gpu.return } fir.global @_QMkernels_mEdev_var {data_attr = #cuf.cuda} : f32 { %0 = arith.constant 0.0 : f32 fir.has_value %0 : f32 } } } // CHECK: llvm.func internal @__cudaFortranConstructor() // CHECK: llvm.call @_FortranACUFRegisterAllocator() // CHECK: cuf.register_module @cuda_device_mod -> !llvm.ptr // CHECK: fir.address_of(@_QMkernels_mEdev_var) : !fir.ref // CHECK: fir.call @_FortranACUFRegisterVariable(%3, %4, %5, %6) : (!fir.ref>, !fir.ref, !fir.ref, i64) -> () // ----- // Under -gpu=mem:unified, a non-allocatable host module global referenced from // device code (mirrored as an external-linkage clone in the GPU module by the // CUFDeviceGlobal pass) must be registered with // _FortranACUFRegisterExternalVariable so the device-side `.extern` symbol // resolves to the host pointer at module-load time. module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMmtestsEm(dense<[1, 2, 3, 4, 5]> : tensor<5xi32>) : !fir.array<5xi32> gpu.module @cuda_device_mod { gpu.func @_QMmtestsPg1() kernel { gpu.return } // External declaration: no body, no init value -- lowers to PTX // `.extern .global ...`. CUFAddConstructor must register it so the // CUDA runtime maps it to the host pointer at module-load time. fir.global @_QMmtestsEm : !fir.array<5xi32> } } // UNIFIED: llvm.func internal @__cudaFortranConstructor() // UNIFIED: cuf.register_module @cuda_device_mod -> !llvm.ptr // UNIFIED: fir.address_of(@_QMmtestsEm) : !fir.ref> // UNIFIED: fir.call @_FortranACUFRegisterExternalVariable // UNIFIED-NOT: fir.call @_FortranACUFInitModule // ----- // Under -gpu=mem:unified, an allocatable host module global also gets // registered via _FortranACUFRegisterExternalVariable; the registered symbol // is the descriptor (fir.box>). The host runtime allocates the // data buffer in HMM/ATS-accessible memory. module attributes {dlti.dl_spec = #dlti.dl_spec : vector<2xi64>, i16 = dense<16> : vector<2xi64>, i1 = dense<8> : vector<2xi64>, !llvm.ptr = dense<64> : vector<4xi64>, f80 = dense<128> : vector<2xi64>, i128 = dense<128> : vector<2xi64>, i64 = dense<64> : vector<2xi64>, !llvm.ptr<271> = dense<32> : vector<4xi64>, !llvm.ptr<272> = dense<64> : vector<4xi64>, f128 = dense<128> : vector<2xi64>, !llvm.ptr<270> = dense<32> : vector<4xi64>, f16 = dense<16> : vector<2xi64>, f64 = dense<64> : vector<2xi64>, i32 = dense<32> : vector<2xi64>, "dlti.stack_alignment" = 128 : i64, "dlti.endianness" = "little">, fir.defaultkind = "a1c4d8i4l4r4", fir.kindmap = "", gpu.container_module, llvm.data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", llvm.target_triple = "x86_64-unknown-linux-gnu"} { fir.global @_QMmtestsEma : !fir.box>> { %c0 = arith.constant 0 : index %0 = fir.zero_bits !fir.heap> %1 = fircg.ext_embox %0(%c0) {allocator_idx = 2 : i32} : (!fir.heap>, index) -> !fir.box>> fir.has_value %1 : !fir.box>> } gpu.module @cuda_device_mod { gpu.func @_QMmtestsPg1() kernel { gpu.return } fir.global @_QMmtestsEma : !fir.box>> } } // UNIFIED: llvm.func internal @__cudaFortranConstructor() // UNIFIED: cuf.register_module @cuda_device_mod -> !llvm.ptr // UNIFIED: fir.address_of(@_QMmtestsEma) : !fir.ref>>> // UNIFIED: fir.call @_FortranACUFRegisterExternalVariable // UNIFIED-NOT: fir.call @_FortranACUFInitModule