TheRock/compiler/amd-llvm/mlir/test/Dialect/SCF/one-shot-bufferize-tensor-copy-insertion.mlir

// RUN: mlir-opt %s -test-tensor-copy-insertion=allow-return-allocs-from-loops -allow-unregistered-dialect -split-input-file | FileCheck %s
// RUN: mlir-opt %s -test-tensor-copy-insertion="allow-return-allocs-from-loops bufferize-function-boundaries" -split-input-file | FileCheck %s --check-prefix=CHECK-FUNC

// CHECK-LABEL: func @scf_for(
//  CHECK-SAME:     %[[A:.*]]: tensor<?xf32>, %[[B:.*]]: tensor<?xf32>
func.func @scf_for(%A : tensor<?xf32>, %B : tensor<?xf32>,
                   %lb : index, %ub : index, %step : index)
  -> (tensor<?xf32>, tensor<?xf32>)
{
  // CHECK: %[[A_copy:.*]] = bufferization.alloc_tensor() copy(%[[A]]) : tensor<?xf32>
  // CHECK: %[[B_copy:.*]] = bufferization.alloc_tensor() copy(%[[B]]) : tensor<?xf32>
  // CHECK:   %[[for:.*]]:2 = scf.for {{.*}} iter_args(%[[iter1:.*]] = %[[A_copy]], %[[iter2:.*]] = %[[B_copy]])
  %r0:2 = scf.for %i = %lb to %ub step %step iter_args(%tA = %A, %tB = %B)
      -> (tensor<?xf32>, tensor<?xf32>)
  {
    // CHECK: scf.yield %[[iter1]], %[[iter2]]
    scf.yield %tA, %tB : tensor<?xf32>, tensor<?xf32>
  }

  return %r0#0, %r0#1 : tensor<?xf32>, tensor<?xf32>
}

// -----

// CHECK-LABEL: func @scf_for_swapping_yields(
//  CHECK-SAME:     %[[A:.*]]: tensor<?xf32>, %[[B:.*]]: tensor<?xf32>
func.func @scf_for_swapping_yields(%A : tensor<?xf32>, %B : tensor<?xf32>,
                                   %lb : index, %ub : index, %step : index)
  -> (tensor<?xf32>, tensor<?xf32>)
{
  // CHECK: %[[A_copy:.*]] = bufferization.alloc_tensor() copy(%[[A]]) : tensor<?xf32>
  // CHECK: %[[B_copy:.*]] = bufferization.alloc_tensor() copy(%[[B]]) : tensor<?xf32>
  // CHECK:   %[[for:.*]]:2 = scf.for {{.*}} iter_args(%[[iter1:.*]] = %[[A_copy]], %[[iter2:.*]] = %[[B_copy]])
  %r0:2 = scf.for %i = %lb to %ub step %step iter_args(%tA = %A, %tB = %B)
      -> (tensor<?xf32>, tensor<?xf32>)
  {
    // Yield tensors in different order.
    // CHECK-DAG: %[[yield1:.*]] = bufferization.alloc_tensor() copy(%[[iter2]]) : tensor<?xf32>
    // CHECK-DAG: %[[yield2:.*]] = bufferization.alloc_tensor() copy(%[[iter1]]) : tensor<?xf32>
    // CHECK: scf.yield %[[yield1]], %[[yield2]]
    scf.yield %tB, %tA : tensor<?xf32>, tensor<?xf32>
  }

  return %r0#0, %r0#1 : tensor<?xf32>, tensor<?xf32>
}

// -----

// CHECK-LABEL: func @scf_while(
//  CHECK-SAME:     %[[A:.*]]: tensor<5xi1>, %[[B:.*]]: tensor<5xi1>
func.func @scf_while(%A: tensor<5xi1>, %B: tensor<5xi1>, %idx: index)
  -> (tensor<5xi1>, tensor<5xi1>)
{
  // CHECK: %[[A_copy:.*]] = bufferization.alloc_tensor() copy(%[[A]]) : tensor<5xi1>
  // CHECK: %[[B_copy:.*]] = bufferization.alloc_tensor() copy(%[[B]]) : tensor<5xi1>
  // CHECK: %[[loop:.*]]:2 = scf.while (%[[w0:.*]] = %[[A_copy]], %[[w1:.*]] = %[[B_copy]]) {{.*}} {
  %r0, %r1 = scf.while (%w0 = %A, %w1 = %B)
      : (tensor<5xi1>, tensor<5xi1>) -> (tensor<5xi1>, tensor<5xi1>) {
    // CHECK: %[[condition:.*]] = tensor.extract %[[w0]]
    %condition = tensor.extract %w0[%idx] : tensor<5xi1>
    // Yield tensors in different order.
    // CHECK: scf.condition(%[[condition]]) %[[w0]], %[[w1]]
    scf.condition(%condition) %w0, %w1 : tensor<5xi1>, tensor<5xi1>
  } do {
  ^bb0(%b0: tensor<5xi1>, %b1: tensor<5xi1>):
    // CHECK: } do {
    // CHECK: ^bb0(%[[b0:.*]]: tensor<5xi1>, %[[b1:.*]]: tensor<5xi1>):
    // CHECK: scf.yield %[[b0]], %[[b1]]
    // CHECK: }
    scf.yield %b0, %b1 : tensor<5xi1>, tensor<5xi1>
  }

  return %r0, %r1 : tensor<5xi1>, tensor<5xi1>
}

// -----

// CHECK-LABEL: func @scf_while_non_equiv_condition_and_body(
//  CHECK-SAME:     %[[A:.*]]: tensor<5xi1>, %[[B:.*]]: tensor<5xi1>
func.func @scf_while_non_equiv_condition_and_body(%A: tensor<5xi1>,
                                                  %B: tensor<5xi1>,
                                                  %idx: index)
  -> (tensor<5xi1>, tensor<5xi1>)
{
  // CHECK: %[[A_copy:.*]] = bufferization.alloc_tensor() copy(%[[A]]) : tensor<5xi1>
  // CHECK: %[[B_copy:.*]] = bufferization.alloc_tensor() copy(%[[B]]) : tensor<5xi1>
  // CHECK: %[[loop:.*]]:2 = scf.while (%[[w0:.*]] = %[[A_copy]], %[[w1:.*]] = %[[B_copy]]) {{.*}} {
  %r0, %r1 = scf.while (%w0 = %A, %w1 = %B)
      : (tensor<5xi1>, tensor<5xi1>) -> (tensor<5xi1>, tensor<5xi1>) {
    // CHECK: %[[condition:.*]] = tensor.extract %[[w0]]
    %condition = tensor.extract %w0[%idx] : tensor<5xi1>
    // Yield tensors in different order.
    // CHECK-DAG: %[[yield0:.*]] = bufferization.alloc_tensor() copy(%[[w1]]) : tensor<5xi1>
    // CHECK-DAG: %[[yield1:.*]] = bufferization.alloc_tensor() copy(%[[w0]]) : tensor<5xi1>
    // CHECK: scf.condition(%[[condition]]) %[[yield0]], %[[yield1]]
    scf.condition(%condition) %w1, %w0 : tensor<5xi1>, tensor<5xi1>
  } do {
  ^bb0(%b0: tensor<5xi1>, %b1: tensor<5xi1>):
    // CHECK: } do {
    // CHECK: ^bb0(%[[b0:.*]]: tensor<5xi1>, %[[b1:.*]]: tensor<5xi1>):
    // CHECK: scf.yield %[[b1]], %[[b0]]
    // CHECK: }
    scf.yield %b1, %b0 : tensor<5xi1>, tensor<5xi1>
  }

  return %r0, %r1 : tensor<5xi1>, tensor<5xi1>
}

// -----

// CHECK-LABEL: func @scf_forall_out_of_place(
//  CHECK-SAME:     %[[arg0:.*]]: tensor<100xf32>, %[[arg1:.*]]: tensor<100xf32>
// CHECK-FUNC-LABEL: func @scf_forall_out_of_place(
func.func @scf_forall_out_of_place(%in: tensor<100xf32>,
                                   %out: tensor<100xf32>) {
  %c1 = arith.constant 1 : index
  %num_threads = arith.constant 100 : index

  // CHECK-FUNC-NOT: alloc_tensor
  // CHECK: %[[alloc:.*]] = bufferization.alloc_tensor() copy(%[[arg1]]) : tensor<100xf32>
  // CHECK: scf.forall {{.*}} shared_outs(%[[o:.*]] = %[[alloc]])
  %result = scf.forall (%thread_idx) in (%num_threads) shared_outs(%o = %out) -> tensor<100xf32> {
      // CHECK: tensor.extract_slice
      // CHECK: scf.forall.in_parallel
      // CHECK: tensor.parallel_insert_slice %{{.*}} into %[[o]]
      %1 = tensor.extract_slice %in[%thread_idx][1][1] : tensor<100xf32> to tensor<1xf32>
      scf.forall.in_parallel {
        tensor.parallel_insert_slice %1 into %o[%thread_idx][1][1] :
          tensor<1xf32> into tensor<100xf32>
      }
  // CHECK: } {mapping = [#gpu.thread<x>]}
  } {mapping = [#gpu.thread<x>]}
  return
}

// -----

// CHECK-LABEL: func @in_order_multiple_parallel_writes
func.func @in_order_multiple_parallel_writes(%2: tensor<320xf32>,
                                             %3: tensor<320xf32>)
  -> (tensor<320xf32>, tensor<320xf32>)
{
  %c0 = arith.constant 0 : index
  %cst = arith.constant -0.000000e+00 : f32
  %c320 = arith.constant 320 : index
  %4:2 = scf.forall (%arg0) in (%c320) shared_outs(%arg1 = %2, %arg2 = %3) -> (tensor<320xf32>, tensor<320xf32>) {
    // CHECK: tensor.extract_slice {{.*}}
    %6 = tensor.extract_slice %arg1[%arg0] [1] [1] : tensor<320xf32> to tensor<1xf32>
    // CHECK: tensor.extract_slice {{.*}}
    %7 = tensor.extract_slice %arg2[%arg0] [1] [1] : tensor<320xf32> to tensor<1xf32>
    // CHECK: linalg.fill {{.*}}
    %8 = linalg.fill ins(%cst : f32) outs(%7 : tensor<1xf32>) -> tensor<1xf32>

    // CHECK: tensor.parallel_insert_slice {{.*}}
    // CHECK: tensor.parallel_insert_slice {{.*}}
    scf.forall.in_parallel {
      tensor.parallel_insert_slice %6 into %arg2[%arg0] [1] [1] : tensor<1xf32> into tensor<320xf32>
      tensor.parallel_insert_slice %8 into %arg1[%arg0] [1] [1] : tensor<1xf32> into tensor<320xf32>
    }
  }
  return %4#0, %4#1 : tensor<320xf32>, tensor<320xf32>
}