TheRock/compiler/amd-llvm/mlir/test/Dialect/GPU/canonicalize.mlir

// RUN: mlir-opt %s -canonicalize="test-convergence" --split-input-file -allow-unregistered-dialect | FileCheck %s

// Fold all the gpu.wait ops as they are redundant.
// CHECK-LABEL: func @fold_wait_op_test1
func.func @fold_wait_op_test1() {
  %1 = gpu.wait async
  gpu.wait []
  %3 = gpu.wait async
  gpu.wait [%3]
  return
}
// CHECK-NOT: gpu.wait

// -----

// CHECK-LABEL: func @erase_barriers
//       CHECK-NEXT: gpu.barrier{{$}}
//       CHECK-NEXT: return
func.func @erase_barriers() {
  gpu.barrier
  gpu.barrier
  return
}

// CHECK-LABEL: func @erase_barriers_first_full_fence
//       CHECK-NEXT: gpu.barrier{{$}}
//       CHECK-NEXT: return
func.func @erase_barriers_first_full_fence() {
  gpu.barrier
  gpu.barrier memfence [#gpu.address_space<workgroup>]
  return
}

// CHECK-LABEL: func @erase_barriers_second_full_fence
//       CHECK-NEXT: gpu.barrier{{$}}
//       CHECK-NEXT: return
func.func @erase_barriers_second_full_fence() {
  gpu.barrier memfence [#gpu.address_space<workgroup>]
  gpu.barrier
  return
}

// CHECK-LABEL: func @erase_barriers_merge_memfence
//       CHECK-NEXT: gpu.barrier memfence [#gpu.address_space<workgroup>, #gpu.address_space<global>]
//       CHECK-NEXT: return
func.func @erase_barriers_merge_memfence() {
  gpu.barrier memfence [#gpu.address_space<workgroup>]
  gpu.barrier memfence [#gpu.address_space<global>]
  return
}

// CHECK-LABEL: func @erase_barriers_merge_memfence_same
//       CHECK-NEXT: gpu.barrier memfence [#gpu.address_space<workgroup>]
//       CHECK-NEXT: return
func.func @erase_barriers_merge_memfence_same() {
  gpu.barrier memfence [#gpu.address_space<workgroup>]
  gpu.barrier memfence [#gpu.address_space<workgroup>]
  return
}

// CHECK-LABEL: func @erase_barriers_empty_memfence
//       CHECK-NEXT: gpu.barrier memfence []
//       CHECK-NEXT: return
func.func @erase_barriers_empty_memfence() {
  gpu.barrier memfence []
  gpu.barrier memfence []
  return
}

// CHECK-LABEL: func @erase_barriers_same_scope
//       CHECK-NEXT: gpu.barrier scope <subgroup>
//       CHECK-NEXT: return
func.func @erase_barriers_same_scope() {
  gpu.barrier scope <subgroup>
  gpu.barrier scope <subgroup>
  return
}

// CHECK-LABEL: func @no_fold_different_scope
//       CHECK-NEXT: gpu.barrier scope <subgroup>
//       CHECK-NEXT: gpu.barrier scope <cluster>
//       CHECK-NEXT: return
func.func @no_fold_different_scope() {
  gpu.barrier scope <subgroup>
  gpu.barrier scope <cluster>
  return
}

// CHECK-LABEL: func @no_fold_different_named_barriers
//       CHECK-NEXT: gpu.initialize_named_barrier
//       CHECK-NEXT: gpu.initialize_named_barrier
//       CHECK-NEXT: gpu.barrier named
//       CHECK-NEXT: gpu.barrier named
//       CHECK-NEXT: return
func.func @no_fold_different_named_barriers(%member_count : i32) {
  %nb1 = gpu.initialize_named_barrier %member_count : i32 -> !gpu.named_barrier
  %nb2 = gpu.initialize_named_barrier %member_count : i32 -> !gpu.named_barrier
  gpu.barrier named(%nb1 : !gpu.named_barrier)
  gpu.barrier named(%nb2 : !gpu.named_barrier)
  return
}

// CHECK-LABEL: func @fold_same_named_barrier
//       CHECK-NEXT: gpu.initialize_named_barrier
//       CHECK-NEXT: gpu.barrier named
//       CHECK-NOT: gpu.barrier
//       CHECK-NEXT: return
func.func @fold_same_named_barrier(%member_count : i32) {
  %nb = gpu.initialize_named_barrier %member_count : i32 -> !gpu.named_barrier
  gpu.barrier named(%nb : !gpu.named_barrier)
  gpu.barrier named(%nb : !gpu.named_barrier)
  return
}

// -----

// Replace uses of gpu.wait op with its async dependency.
// CHECK-LABEL: func @fold_wait_op_test2
func.func @fold_wait_op_test2(%arg0: i1) -> (memref<5xf16>, memref<5xf16>) {
  %0 = gpu.wait async
  %memref, %asyncToken = gpu.alloc async [%0] () : memref<5xf16>
  gpu.wait [%0]
  %1 = gpu.wait async [%0]
  %memref_0, %asyncToken_0 = gpu.alloc async [%1] () : memref<5xf16>
  gpu.wait [%1]
  return %memref, %memref_0 : memref<5xf16>, memref<5xf16>
}
// CHECK-NEXT: %[[TOKEN0:.*]] = gpu.wait async
// CHECK-NEXT: gpu.alloc async [%[[TOKEN0]]] ()
// CHECK-NEXT: %[[TOKEN1:.*]] = gpu.wait async
// CHECK-NEXT: gpu.alloc async [%[[TOKEN1]]] ()
// CHECK-NEXT: return

// -----

// CHECK-LABEL: func @fold_memcpy_op
func.func @fold_memcpy_op(%arg0: i1) {
    %cst = arith.constant 0.000000e+00 : f16
    %1 = memref.alloc() : memref<2xf16>
    %2 = gpu.wait async
    %memref, %asyncToken = gpu.alloc async [%2] () : memref<2xf16>
    gpu.wait [%2]
    affine.store %cst, %memref[0] : memref<2xf16>
    %3 = gpu.wait async
    %4 = gpu.memcpy async [%3] %1, %memref : memref<2xf16>, memref<2xf16>
    gpu.wait [%3]
    %5 = scf.if %arg0 -> (i1) {
      memref.dealloc %1 : memref<2xf16>
      scf.yield %arg0 : i1
    } else {
      memref.dealloc %1 : memref<2xf16>
      scf.yield %arg0 : i1
    }
    return
}
// CHECK-NOT: gpu.memcpy

// -----

// We cannot fold memcpy here as dest is a block argument.
// CHECK-LABEL: func @do_not_fold_memcpy_op1
func.func @do_not_fold_memcpy_op1(%arg0: i1, %arg1: memref<2xf16>) {
    %cst = arith.constant 0.000000e+00 : f16
    %2 = gpu.wait async
    %memref, %asyncToken = gpu.alloc async [%2] () : memref<2xf16>
    gpu.wait [%2]
    affine.store %cst, %memref[0] : memref<2xf16>
    %3 = gpu.wait async
    %4 = gpu.memcpy async [%3] %arg1, %memref : memref<2xf16>, memref<2xf16>
    gpu.wait [%3]
    return
}
// CHECK: gpu.memcpy

// -----

// We cannot fold gpu.memcpy as it is used by an op having read effect on dest.
// CHECK-LABEL: func @do_not_fold_memcpy_op2
func.func @do_not_fold_memcpy_op2(%arg0: i1, %arg1: index) -> f16 {
    %cst = arith.constant 0.000000e+00 : f16
    %1 = memref.alloc() : memref<2xf16>
    %2 = gpu.wait async
    %memref, %asyncToken = gpu.alloc async [%2] () : memref<2xf16>
    gpu.wait [%2]
    affine.store %cst, %memref[0] : memref<2xf16>
    %3 = gpu.wait async
    %4 = gpu.memcpy async [%3] %1, %memref : memref<2xf16>, memref<2xf16>
    gpu.wait [%3]
    %5 = memref.load %1[%arg1] : memref<2xf16>
    return %5 : f16
}
// CHECK: gpu.memcpy

// -----

// We cannot fold gpu.memcpy, as the defining op if dest is not a alloc like op.
// CHECK-LABEL: func @do_not_fold_memcpy_op3
func.func @do_not_fold_memcpy_op3(%arg0: memref<1xi8>, %arg1: memref<i1>) {
  %0 = arith.constant 0 : index
  %1 = memref.view %arg0[%0][] : memref<1xi8> to memref<i1>
  gpu.memcpy  %1, %arg1 : memref<i1>, memref<i1>
  func.return
}
// CHECK: gpu.memcpy

// -----

// CHECK-LABEL: @memcpy_after_cast
func.func @memcpy_after_cast(%arg0: memref<10xf32>, %arg1: memref<10xf32>) {
  // CHECK-NOT: memref.cast
  // CHECK: gpu.memcpy
  %0 = memref.cast %arg0 : memref<10xf32> to memref<?xf32>
  %1 = memref.cast %arg1 : memref<10xf32> to memref<?xf32>
  gpu.memcpy %0, %1 : memref<?xf32>, memref<?xf32>
  return
}

// -----

// CHECK-LABEL: @memset_after_cast
func.func @memset_after_cast(%arg0: memref<10xf32>, %arg1: f32) {
  // CHECK-NOT: memref.cast
  // CHECK: gpu.memset
  %0 = memref.cast %arg0 : memref<10xf32> to memref<?xf32>
  gpu.memset %0, %arg1 : memref<?xf32>, f32
  return
}

// -----

// Test case: Folding of memref.dim(gpu.alloc(%size), %idx) -> %size
// CHECK-LABEL: func @gpu_dim_of_alloc(
//  CHECK-SAME:     %[[SIZE:[0-9a-z]+]]: index
//  CHECK-NEXT:   return %[[SIZE]] : index
func.func @gpu_dim_of_alloc(%size: index) -> index {
  %0 = gpu.alloc(%size) : memref<?xindex>
  %c0 = arith.constant 0 : index
  %1 = memref.dim %0, %c0 : memref<?xindex>
  return %1 : index
}

// -----

// CHECK-LABEL: func @out_of_bound_memref.dim
//  CHECK:   %[[MEMREF:.[a-z0-9A-Z_]+]] = memref.dim
//  CHECK:   return %[[MEMREF]] : index
func.func @out_of_bound_memref.dim(%arg : memref<?xi8>, %size: index) -> index {
  %c2 = arith.constant 2 : index
  %1 = memref.dim %arg, %c2 : memref<?xi8>
  return %1 : index
}

// -----

// CHECK-LABEL: func @simplify_gpu_launch
func.func @simplify_gpu_launch() attributes {llvm.emit_c_interface} {
  %cst = arith.constant 0.000000e+00 : f32
  %c1 = arith.constant 1 : index
  %c32 = arith.constant 32 : index
  %c16 = arith.constant 16 : index
  %c2 = arith.constant 2 : index
  %c0 = arith.constant 0 : index
  %0 = memref.alloc() : memref<2x16x16xf32>
  scf.for %arg0 = %c0 to %c2 step %c1 {
    scf.for %arg1 = %c0 to %c16 step %c1 {
      scf.for %arg2 = %c0 to %c16 step %c1 {
        memref.store %cst, %0[%arg0, %arg1, %arg2] : memref<2x16x16xf32>
      }
    }
  }
  %1 = gpu.wait async
  %memref, %asyncToken = gpu.alloc async [%1] () : memref<2x16x16xf32>
  %2 = gpu.memcpy async [%1] %memref, %0 : memref<2x16x16xf32>, memref<2x16x16xf32>
  gpu.wait [%1]
  gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %c1, %arg7 = %c1, %arg8 = %c1)
    threads(%arg3, %arg4, %arg5) in (%arg9 = %c32, %arg10 = %c1, %arg11 = %c1) {
    %3 = arith.muli %arg5, %c32 : index
    %4 = arith.muli %arg4, %c32 : index
    %5 = arith.addi %3, %4 : index
    %6 = arith.addi %5, %arg3 : index
    %7 = arith.divui %6, %c32 : index
    %8 = arith.muli %arg0, %c16 : index
    %9 = arith.muli %arg1, %c2 : index
    %10 = arith.muli %7, %c2 : index
    %11 = arith.addi %9, %10 : index
    %12 = memref.load %memref[%11, %c0, %8] : memref<2x16x16xf32>
    %13 = arith.addi %11, %c1 : index
    %14 = memref.load %memref[%13, %c0, %8] : memref<2x16x16xf32>
    memref.store %12, %memref[%11, %c0, %8] : memref<2x16x16xf32>
    memref.store %14, %memref[%13, %c0, %8] : memref<2x16x16xf32>
    gpu.terminator
  }
  return
}

// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index
// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index
// CHECK: gpu.launch blocks(%{{.*}}, %{{.*}}, %{{.*}}) in (%{{.*}} = %[[C1]], %{{.*}} = %[[C1]], %{{.*}} = %[[C1]]) threads(%[[TIDX:.*]], %{{.*}}, %{{.*}}) in (%{{.*}} = %c32, %{{.*}} = %[[C1]], %{{.*}} = %[[C1]]) {
// CHECK-NEXT:    arith.divui %[[TIDX]], %c32 : index
// CHECK-NEXT:    arith.muli %{{.*}}, %c2 : index
// CHECK-NEXT:    memref.load %memref[%{{.*}}, %[[C0]], %[[C0]]] : memref<2x16x16xf32>
// CHECK-NEXT:    arith.addi %{{.*}}, %[[C1]] : index
// CHECK-NEXT:    memref.load %memref[%{{.*}}, %[[C0]], %[[C0]]] : memref<2x16x16xf32>
// CHECK-NEXT:    memref.store %{{.*}}, %memref[%{{.*}}, %[[C0]], %[[C0]]] : memref<2x16x16xf32>
// CHECK-NEXT:    memref.store %{{.*}}, %memref[%{{.*}}, %[[C0]], %[[C0]]] : memref<2x16x16xf32>
// CHECK-NEXT:    gpu.terminator
// CHECK-NEXT:  }

// -----

// CHECK-LABEL: func @make_reduce_uniform
//       CHECK: gpu.launch blocks
//       CHECK: %[[V1:.*]] = "test.test2"() : () -> i32
//       CHECK: %[[V2:.*]] = gpu.all_reduce add %[[V1]] uniform {
//       CHECK: "test.test3"(%[[V2]]) : (i32) -> ()
func.func @make_reduce_uniform() {
  %0:6 = "test.test1"() : () -> (index, index, index, index, index, index)
  gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %0#0, %arg7 = %0#1, %arg8 = %0#2)
    threads(%arg3, %arg4, %arg5) in (%arg9 = %0#3, %arg10 = %0#4, %arg11 = %0#5) {
    %1 = "test.test2"() : () -> i32
    %2 = gpu.all_reduce add %1 {} : (i32) -> (i32)
    "test.test3"(%2) : (i32) -> ()
    gpu.terminator
  }
  return
}

// -----

// CHECK-LABEL: func @make_subgroup_reduce_uniform
//       CHECK: gpu.launch blocks
//       CHECK: %[[V1:.*]] = "test.test2"() : () -> i32
//       CHECK: %[[V2:.*]] = gpu.subgroup_reduce add %[[V1]] uniform
//       CHECK: "test.test3"(%[[V2]]) : (i32) -> ()
func.func @make_subgroup_reduce_uniform() {
  %0:6 = "test.test1"() : () -> (index, index, index, index, index, index)
  gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %0#0, %arg7 = %0#1, %arg8 = %0#2)
    threads(%arg3, %arg4, %arg5) in (%arg9 = %0#3, %arg10 = %0#4, %arg11 = %0#5) {
    %1 = "test.test2"() : () -> i32
    %2 = gpu.subgroup_reduce add %1 : (i32) -> (i32)
    "test.test3"(%2) : (i32) -> ()
    gpu.terminator
  }
  return
}

// -----

// CHECK-LABEL: func @subgroup_reduce_cluster_size_1
//       CHECK: gpu.launch blocks
//       CHECK: %[[V1:.*]] = "test.test2"() : () -> i32
//       CHECK: "test.test3"(%[[V1]]) : (i32) -> ()
func.func @subgroup_reduce_cluster_size_1() {
  %0:6 = "test.test1"() : () -> (index, index, index, index, index, index)
  gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %0#0, %arg7 = %0#1, %arg8 = %0#2)
    threads(%arg3, %arg4, %arg5) in (%arg9 = %0#3, %arg10 = %0#4, %arg11 = %0#5) {
    %1 = "test.test2"() : () -> i32
    %2 = gpu.subgroup_reduce add %1 cluster(size=1) : (i32) -> (i32)
    "test.test3"(%2) : (i32) -> ()
    gpu.terminator
  }
  return
}

// -----

// The GPU kernel does not have any side effecting ops, so the entire
// gpu.launch op can fold away.

// CHECK-LABEL: func @gpu_launch_without_side_effects
//   CHECK-NOT:   gpu.launch
func.func @gpu_launch_without_side_effects() {
  %0:6 = "test.test1"() : () -> (index, index, index, index, index, index)
  gpu.launch blocks(%arg0, %arg1, %arg2) in (%arg6 = %0#0, %arg7 = %0#1, %arg8 = %0#2)
    threads(%arg3, %arg4, %arg5) in (%arg9 = %0#3, %arg10 = %0#4, %arg11 = %0#5) {
    %1 = arith.addi %arg0, %arg1 : index
    gpu.terminator
  }
  return
}

// -----


// CHECK-LABEL: func @broadcast_of_broadcast1
//  CHECK-SAME:   (%[[VALUE:.*]]: f32, %[[LANE:.*]]: i32)
//       CHECK:   %[[RES:.*]] = gpu.subgroup_broadcast %[[VALUE]], first_active_lane : f32
//       CHECK:   return %[[RES:.*]]
func.func @broadcast_of_broadcast1(%value : f32, %lane : i32) -> f32 {
  %0 = gpu.subgroup_broadcast %value, first_active_lane : f32
  %1 = gpu.subgroup_broadcast %0, specific_lane %lane : f32
  return %1 : f32
}

// CHECK-LABEL: func @broadcast_of_broadcast2
//  CHECK-SAME:   (%[[VALUE:.*]]: f32, %[[LANE:.*]]: i32)
//       CHECK:   %[[RES:.*]] = gpu.subgroup_broadcast %[[VALUE]], specific_lane %[[LANE]] : f32
//       CHECK:   return %[[RES:.*]]
func.func @broadcast_of_broadcast2(%value : f32, %lane : i32) -> f32 {
  %0 = gpu.subgroup_broadcast %value, specific_lane %lane : f32
  %1 = gpu.subgroup_broadcast %0, first_active_lane : f32
  return %1 : f32
}