Files
derek.holloway 7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
Flatten subprojects into a single base monorepo
2026-09-02 16:34:47 -07:00

231 lines
15 KiB
MLIR

// RUN: mlir-opt %s -split-input-file | FileCheck %s
// This file contains tests for sparse MMA (mma.sp.sync) operations with KIND variants.
// The kind::f8f6f4 variant was introduced in PTX ISA 8.7 for sm_90+ architectures.
//
// Based on PTX ISA documentation:
// https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-instructions-for-sparse-mma
//
// KIND::F8F6F4 enables:
// - Additional FP8 types: e3m2, e2m3, e2m1
// - F16 accumulator for m16n8k64 FP8 operations
// - Mixed-precision FP8 computations
//
// Requirements:
// - ONLY works with ordered metadata (sp::ordered_metadata)
// - ONLY for shape m16n8k64
// - ONLY for FP8 types (not integers or other floats)
// =============================================================================
// FP8 e4m3 Sparse MMA with KIND (m16n8k64)
// =============================================================================
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e4m3_f16
func.func @nvvm_mma_sp_kind_m16n8k64_e4m3_f16(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : vector<2xf16>, %c1 : vector<2xf16>,
%meta : i32) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e4m3>, multiplicandBPtxType = #nvvm.mma_type<e4m3>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e4m3>,
multiplicandBPtxType = #nvvm.mma_type<e4m3>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
return %0 : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
}
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e4m3_f32
func.func @nvvm_mma_sp_kind_m16n8k64_e4m3_f32(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : f32, %c1 : f32, %c2 : f32, %c3 : f32,
%meta : i32) -> !llvm.struct<(f32, f32, f32, f32)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}, {{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e4m3>, multiplicandBPtxType = #nvvm.mma_type<e4m3>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1, %c2, %c3]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e4m3>,
multiplicandBPtxType = #nvvm.mma_type<e4m3>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
return %0 : !llvm.struct<(f32, f32, f32, f32)>
}
// =============================================================================
// FP8 e5m2 Sparse MMA with KIND (m16n8k64)
// =============================================================================
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e5m2_f16
func.func @nvvm_mma_sp_kind_m16n8k64_e5m2_f16(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : vector<2xf16>, %c1 : vector<2xf16>,
%meta : i32) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e5m2>, multiplicandBPtxType = #nvvm.mma_type<e5m2>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e5m2>,
multiplicandBPtxType = #nvvm.mma_type<e5m2>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
return %0 : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
}
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e5m2_f32
func.func @nvvm_mma_sp_kind_m16n8k64_e5m2_f32(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : f32, %c1 : f32, %c2 : f32, %c3 : f32,
%meta : i32) -> !llvm.struct<(f32, f32, f32, f32)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}, {{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e5m2>, multiplicandBPtxType = #nvvm.mma_type<e5m2>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1, %c2, %c3]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e5m2>,
multiplicandBPtxType = #nvvm.mma_type<e5m2>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
return %0 : !llvm.struct<(f32, f32, f32, f32)>
}
// =============================================================================
// FP8 e3m2 Sparse MMA with KIND (m16n8k64)
// NOTE: e3m2 is ONLY available with kind::f8f6f4
// =============================================================================
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e3m2_f16
func.func @nvvm_mma_sp_kind_m16n8k64_e3m2_f16(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : vector<2xf16>, %c1 : vector<2xf16>,
%meta : i32) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e3m2>, multiplicandBPtxType = #nvvm.mma_type<e3m2>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e3m2>,
multiplicandBPtxType = #nvvm.mma_type<e3m2>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
return %0 : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
}
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e3m2_f32
func.func @nvvm_mma_sp_kind_m16n8k64_e3m2_f32(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : f32, %c1 : f32, %c2 : f32, %c3 : f32,
%meta : i32) -> !llvm.struct<(f32, f32, f32, f32)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}, {{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e3m2>, multiplicandBPtxType = #nvvm.mma_type<e3m2>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1, %c2, %c3]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e3m2>,
multiplicandBPtxType = #nvvm.mma_type<e3m2>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
return %0 : !llvm.struct<(f32, f32, f32, f32)>
}
// =============================================================================
// FP8 e2m3 Sparse MMA with KIND (m16n8k64)
// NOTE: e2m3 is ONLY available with kind::f8f6f4
// =============================================================================
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e2m3_f16
func.func @nvvm_mma_sp_kind_m16n8k64_e2m3_f16(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : vector<2xf16>, %c1 : vector<2xf16>,
%meta : i32) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e2m3>, multiplicandBPtxType = #nvvm.mma_type<e2m3>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e2m3>,
multiplicandBPtxType = #nvvm.mma_type<e2m3>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
return %0 : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
}
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e2m3_f32
func.func @nvvm_mma_sp_kind_m16n8k64_e2m3_f32(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : f32, %c1 : f32, %c2 : f32, %c3 : f32,
%meta : i32) -> !llvm.struct<(f32, f32, f32, f32)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}, {{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e2m3>, multiplicandBPtxType = #nvvm.mma_type<e2m3>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1, %c2, %c3]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e2m3>,
multiplicandBPtxType = #nvvm.mma_type<e2m3>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
return %0 : !llvm.struct<(f32, f32, f32, f32)>
}
// =============================================================================
// FP8 e2m1 Sparse MMA with KIND (m16n8k64)
// NOTE: e2m1 is ONLY available with kind::f8f6f4
// =============================================================================
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e2m1_f16
func.func @nvvm_mma_sp_kind_m16n8k64_e2m1_f16(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : vector<2xf16>, %c1 : vector<2xf16>,
%meta : i32) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e2m1>, multiplicandBPtxType = #nvvm.mma_type<e2m1>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e2m1>,
multiplicandBPtxType = #nvvm.mma_type<e2m1>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)>
return %0 : !llvm.struct<(vector<2xf16>, vector<2xf16>)>
}
// CHECK-LABEL: @nvvm_mma_sp_kind_m16n8k64_e2m1_f32
func.func @nvvm_mma_sp_kind_m16n8k64_e2m1_f32(
%a0 : i32, %a1 : i32, %a2 : i32, %a3 : i32,
%b0 : i32, %b1 : i32, %b2 : i32, %b3 : i32,
%c0 : f32, %c1 : f32, %c2 : f32, %c3 : f32,
%meta : i32) -> !llvm.struct<(f32, f32, f32, f32)> {
%sel = llvm.mlir.constant(0 : i32) : i32
// CHECK: nvvm.mma.sp.sync A[{{.*}}, {{.*}}, {{.*}}, {{.*}}] B[{{.*}}, {{.*}}, {{.*}}, {{.*}}] C[{{.*}}, {{.*}}, {{.*}}, {{.*}}] sparseMetadata[{{.*}}] selector[{{.*}}] {kind = #nvvm.mma_kind<f8f6f4>, multiplicandAPtxType = #nvvm.mma_type<e2m1>, multiplicandBPtxType = #nvvm.mma_type<e2m1>, orderedMetadata, shape = #nvvm.shape<m = 16, n = 8, k = 64>} : (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
%0 = nvvm.mma.sp.sync A[%a0, %a1, %a2, %a3] B[%b0, %b1, %b2, %b3] C[%c0, %c1, %c2, %c3]
sparseMetadata[%meta] selector[%sel]
{kind = #nvvm.mma_kind<f8f6f4>,
orderedMetadata,
multiplicandAPtxType = #nvvm.mma_type<e2m1>,
multiplicandBPtxType = #nvvm.mma_type<e2m1>,
shape = #nvvm.shape<m = 16, n = 8, k = 64>}
: (i32, i32, f32) -> !llvm.struct<(f32, f32, f32, f32)>
return %0 : !llvm.struct<(f32, f32, f32, f32)>
}