derek.holloway
7502bf64a4
CodeQL / Analyze (actions) (push) Canceled after 0s
CodeQL / Analyze (python) (push) Canceled after 0s
pre-commit / pre-commit (push) Canceled after 0s
Multi-Arch CI / setup (push) Canceled after 0s
Publish build_manylinux_x86_64 images / publish_build_manylinux_x86_64 (push) Canceled after 0s
Setup (Multi-Arch) / setup (push) Canceled after 0s
Multi-Arch CI / Linux::${{ fromJSON(needs.setup.outputs.linux_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubi10 image / publish_no_rocm_image_ubi10 (push) Canceled after 0s
Multi-Arch CI / Windows::${{ fromJSON(needs.setup.outputs.windows_build_config || '{}').build_variant_label || 'skip' }} (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Multi-Arch CI / Manifest Diff (push) Canceled after 0s
TheRock Manifest Diff Report / Generate Manifest Diff Report (push) Canceled after 0s
Multi-Arch CI / CI Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_ocl_rt images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04_rocgdb images / publish_no_rocm_image_ubuntu24_04_rocgdb (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_ocl_rt (push) Canceled after 0s
Publish TheRock Docker image / build-and-push-image (push) Canceled after 0s
Unit Tests / Unit Tests :: ubuntu-24.04 (push) Canceled after 0s
Unit Tests / Unit Tests :: windows-2022 (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_media (push) Canceled after 0s
Unit Tests / Unit Tests Summary (push) Canceled after 0s
Publish no_rocm_image_ubuntu24_04 images / publish_no_rocm_image_ubuntu24_04_openmpi (push) Canceled after 0s
1371 lines
72 KiB
Python
1371 lines
72 KiB
Python
################################################################################
|
|
#
|
|
# Copyright (C) 2025-2026 Advanced Micro Devices, Inc. All rights reserved.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
#
|
|
################################################################################
|
|
|
|
import math
|
|
from functools import lru_cache
|
|
from typing import Any, Union
|
|
|
|
from .Architectures import SUPPORTED_ISA
|
|
from .Types import IsaVersion
|
|
|
|
################################################################################
|
|
# Enumerate Valid Solution Parameters
|
|
################################################################################
|
|
|
|
validThreadTileSides = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] + list(
|
|
range(20, 256, 4)
|
|
)
|
|
validThreadTiles = []
|
|
for i in validThreadTileSides:
|
|
for j in validThreadTileSides:
|
|
validThreadTiles.append([i, j])
|
|
|
|
validActivationFormats = ("NCHW", "NHWC", "CNHW", "NCDHW", "NDHWC", "CNDHW")
|
|
validWeightFormats = ("KCYX", "KYXC", "CKYX", "CYXK", "KCZYX", "CKZYX", "CZYXK")
|
|
validMacroTileSides = [
|
|
1,
|
|
2,
|
|
4,
|
|
8,
|
|
16,
|
|
32,
|
|
64,
|
|
128,
|
|
256,
|
|
512,
|
|
1024,
|
|
6,
|
|
12,
|
|
24,
|
|
48,
|
|
96,
|
|
192,
|
|
384,
|
|
768,
|
|
]
|
|
validMacroTiles = []
|
|
validISA = [IsaVersion(0, 0, 0)]
|
|
validISA.extend(SUPPORTED_ISA)
|
|
for i in validMacroTileSides:
|
|
for j in validMacroTileSides:
|
|
validMacroTiles.append([i, j])
|
|
validTT = 32
|
|
|
|
maxWGsInCluster = 16
|
|
# 2D [x, y] only — z is always 1 (hardcoded in HipSolutionAdapter); C++ uses dim3 for HIP API compat.
|
|
validClusterDimensions = []
|
|
for i in range(1, maxWGsInCluster + 1):
|
|
for j in range(1, maxWGsInCluster + 1):
|
|
if i * j <= maxWGsInCluster:
|
|
validClusterDimensions.append([i, j])
|
|
|
|
# Shared valid values for LdsBlockSizePerPadA and LdsBlockSizePerPadB so they stay in sync.
|
|
validLdsBlockSizePerPad = [-1, 0, 16, 32, 64, 96, 128, 192, 256, 320, 384, 448, 512, 576, 640, 704, 768,
|
|
832, 896, 960, 1024, 1088, 1152, 1216, 1280, 1344, 1408, 1472, 1536, 1600, 1664,
|
|
1728, 1792, 1856, 1920, 1984, 2048, 2176, 2304, 2432, 2560, 2688, 2816, 2944,
|
|
3072, 3200, 3328, 3456, 3584, 3712, 3840, 3968, 4096, 4352, 4608, 4864, 5120,
|
|
5376, 5632, 6144, 6656, 7168, 7680, 8192]
|
|
|
|
@lru_cache
|
|
def makeValidWorkGroups():
|
|
validWorkGroups = []
|
|
for numThreads in range(32, 1025, 32):
|
|
for nsg in [1, 2, 4, 8, 16, 32, 64, 96, 128, 256]:
|
|
for sg0 in range(1, numThreads // nsg + 1):
|
|
sg1 = numThreads // nsg // sg0
|
|
if sg0 * sg1 * nsg == numThreads:
|
|
workGroup = [sg0, sg1, nsg]
|
|
validWorkGroups.append(workGroup)
|
|
return validWorkGroups
|
|
|
|
def makeValidWMMA():
|
|
return [[16, 16, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1], [16, 16, 32, 1], [16, 16, 64, 1], [16, 16, 128, 1], [32, 16, 128, 1]]
|
|
|
|
def makeValidSWMMAC():
|
|
return [[16, 16, 32, 1], [16, 16, 64, 1], [16, 16, 128, 1]]
|
|
|
|
@lru_cache
|
|
def makeValidMFMA():
|
|
validMFMA = {}
|
|
validMFMA["HH"] = [[32, 32, 16, 1], [32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 32, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]]
|
|
validMFMA["SS"] = [[32, 32, 1, 2], [32, 32, 2, 1], [16, 16, 1, 4], [16, 16, 4, 1], [4, 4, 1, 16], [16, 16, 32, 1], [32, 32, 16, 1]]
|
|
validMFMA["BB"] = [[32, 32, 16, 1], [32, 32, 2, 2], [32, 32, 4, 1], [16, 16, 32, 1], [16, 16, 2, 4], [16, 16, 8, 1], [4, 4, 2, 16]]
|
|
validMFMA["4xi84xi8"] = [
|
|
[32, 32, 4, 2],
|
|
[32, 32, 8, 1],
|
|
[16, 16, 4, 4],
|
|
[16, 16, 16, 1],
|
|
[4, 4, 4, 16],
|
|
[32, 32, 16, 1],
|
|
[16, 16, 32, 1],
|
|
]
|
|
validMFMA["DD"] = [[16, 16, 4, 1], [4, 4, 4, 4]]
|
|
validMFMA["B1k"] = [[32, 32, 4, 2], [32, 32, 8, 1], [16, 16, 4, 4], [16, 16, 16, 1], [4, 4, 4, 16]]
|
|
validMFMA["CC"] = validMFMA["SS"]
|
|
validMFMA["ZZ"] = validMFMA["DD"]
|
|
validMFMA["I8I8"] = [
|
|
[32, 32, 4, 2],
|
|
[32, 32, 8, 1],
|
|
[16, 16, 4, 4],
|
|
[16, 16, 16, 1],
|
|
[4, 4, 4, 16],
|
|
] + [[32, 32, 16, 1], [16, 16, 32, 1],] + [[16, 16, 64, 1],[32, 32, 32, 1]]
|
|
validMFMA["XX"] = [[32, 32, 4, 1], [16, 16, 8, 1], [16, 16, 16, 1], [16, 16, 32, 1], [32, 32, 16, 1]]
|
|
validMFMA["F8F8"] = [[32, 32, 16, 1], [16, 16, 32, 1], [32, 32, 64, 1], [16, 16, 128, 1]]
|
|
validMFMA["B8B8"] = validMFMA["F8F8"]
|
|
validMFMA["F8B8"] = validMFMA["F8F8"]
|
|
validMFMA["B8F8"] = validMFMA["F8F8"]
|
|
validMFMA["F8NF8N"] = [[32, 32, 16, 1], [16, 16, 32, 1]]
|
|
validMFMA["B8NB8N"] = validMFMA["F8NF8N"]
|
|
validMFMA["F8NB8N"] = validMFMA["F8NF8N"]
|
|
validMFMA["B8NF8N"] = validMFMA["F8NF8N"]
|
|
# fnuz (NANOO) OCP pairings: DataType *_fnuz uses *N suffix; same MFMA sets as F8B8 / B8F8.
|
|
validMFMA["F8B8NF8B8N"] = validMFMA["F8B8"]
|
|
validMFMA["B8F8NB8F8N"] = validMFMA["B8F8"]
|
|
validMFMA["F8B8NB8F8N"] = validMFMA["F8B8"]
|
|
validMFMA["B8F8NF8B8N"] = validMFMA["F8B8"]
|
|
validMFMA["F6F6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["B6B6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F6B6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["B6F6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F8F6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F6F8"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F8F4"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F4F8"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F6F4"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F4F6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["B6F4"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F4B6"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["F4F4"] = [[16,16,128,1], [32,32,64,1]]
|
|
validMFMA["_format9"] = []
|
|
|
|
for MFMA in [
|
|
validMFMA["HH"],
|
|
validMFMA["SS"],
|
|
validMFMA["BB"],
|
|
validMFMA["DD"],
|
|
validMFMA["XX"],
|
|
validMFMA["F8F8"],
|
|
makeValidWMMA(),
|
|
]:
|
|
for MI in MFMA:
|
|
for bm in range(int(math.log(MI[3], 2)) + 1):
|
|
for tt0 in range(1, validTT + 1):
|
|
for tt1 in range(1, validTT + 1):
|
|
for wave_m in range(3):
|
|
for wave_n in range(3):
|
|
validMFMA["_format9"].append(
|
|
[MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n]
|
|
)
|
|
return validMFMA
|
|
|
|
@lru_cache
|
|
def makeValidSMFMA():
|
|
validSMFMA = {}
|
|
validSMFMA["HH"] = [[32, 32, 16, 1], [16, 16, 32, 1], [16, 16, 64, 1], [32, 32, 32, 1]]
|
|
validSMFMA["BB"] = [[32, 32, 16, 1], [16, 16, 32, 1], [16, 16, 64, 1], [32, 32, 32, 1]]
|
|
validSMFMA["4xi84xi8"] = [[32, 32, 32, 1], [16, 16, 64, 1], [16, 16, 128, 1], [32, 32, 64, 1]]
|
|
validSMFMA["I8I8"] = validSMFMA["4xi84xi8"]
|
|
validSMFMA["F8F8"] = [[32, 32, 32, 1], [16, 16, 64, 1], [16, 16, 128, 1], [32, 32, 64, 1]]
|
|
validSMFMA["B8B8"] = validSMFMA["F8F8"]
|
|
validSMFMA["F8B8"] = validSMFMA["F8F8"]
|
|
validSMFMA["B8F8"] = validSMFMA["F8F8"]
|
|
validSMFMA["F8NF8N"] = validSMFMA["F8F8"]
|
|
validSMFMA["B8NB8N"] = validSMFMA["F8F8"]
|
|
validSMFMA["F8NB8N"] = validSMFMA["F8NF8N"]
|
|
validSMFMA["B8NF8N"] = validSMFMA["F8NF8N"]
|
|
validSMFMA["F8B8NF8B8N"] = validSMFMA["F8B8"]
|
|
validSMFMA["B8F8NB8F8N"] = validSMFMA["B8F8"]
|
|
validSMFMA["F8B8NB8F8N"] = validSMFMA["F8B8"]
|
|
validSMFMA["B8F8NF8B8N"] = validSMFMA["F8B8"]
|
|
validSMFMA["_format9"] = []
|
|
for SMFMA in [validSMFMA["HH"], validSMFMA["BB"], validSMFMA["4xi84xi8"], validSMFMA["F8NF8N"], makeValidSWMMAC()]:
|
|
for MI in SMFMA:
|
|
for bm in range(int(math.log(MI[3], 2)) + 1):
|
|
for tt0 in range(1, validTT + 1):
|
|
for tt1 in range(1, validTT + 1):
|
|
for wave_m in range(3):
|
|
for wave_n in range(3):
|
|
validSMFMA["_format9"].append(
|
|
[MI[0], MI[1], MI[2], MI[3], 2**bm, tt0, tt1, 2**wave_m, 2**wave_n]
|
|
)
|
|
return validSMFMA
|
|
|
|
@lru_cache
|
|
def makeValidMatrixInstructions():
|
|
mfma = makeValidMFMA()
|
|
smfma = makeValidSMFMA()
|
|
wmma = makeValidWMMA()
|
|
validMatrixInstructions = (
|
|
[[], [-1]]
|
|
+ mfma["HH"]
|
|
+ mfma["SS"]
|
|
+ mfma["BB"]
|
|
+ mfma["DD"]
|
|
+ mfma["B1k"]
|
|
+ mfma["XX"]
|
|
+ smfma["HH"]
|
|
+ smfma["BB"]
|
|
+ smfma["4xi84xi8"]
|
|
+ wmma
|
|
)
|
|
return validMatrixInstructions + mfma["_format9"] + smfma["_format9"]
|
|
|
|
|
|
validParameters = { # we need to make sure this matches develop
|
|
# 0: Global read is along parallel direction in thread level,
|
|
# each load instruction stride whole threads.
|
|
# ----> perp
|
|
# | [w0, w0, w1,w1,w2,w2,w3,w3, w0, w0, w1,w1,w2,w2,w3,w3]
|
|
# | [ t0,t32] [ ] [ t0,t32] [ ]
|
|
# para | [ t1,t33] [ wave 1,2,3 ] [ t1,t33] [ wave 1,2,3 ]
|
|
# | [ .., ..] [ ] [ .., ..] [ ]
|
|
# | [t31,t63] [ ] [t31,t63] [ ]
|
|
# V [-load_1] [-load_2]
|
|
#
|
|
# 1: Each wave load a block of memory,
|
|
# each load instruction stride 64 threads.
|
|
# ----> perp
|
|
# [ w0, w0, w0, w0, w1,w1,w1,w1, w2,w2,w2,w2, w3,w3,w3,w3]
|
|
# | [ t0,t32][ t0,t32]
|
|
# para | [ t1,t33][ t1,t33]
|
|
# | [ .., ..][ .., ..]
|
|
# | [t31,t63][t31,t63]
|
|
# V [-load_1][-load_2]
|
|
#
|
|
#
|
|
# 2: Each load instruction spread threads evenly in the perp direction
|
|
# ----> perp
|
|
# | [w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3, w0, w1, w2, w3]
|
|
# | [t0 ] [t0 ] [t32] [t32]
|
|
# para | [t1 ] [t1 ] [t33] [t33]
|
|
# | [.. ] [.. ] [.. ] [.. ]
|
|
# | [t31] [t31] [t63] [t63]
|
|
# V [load_1] [load_2] [load_1] [load_2]
|
|
#
|
|
"WaveSeparateGlobalReadA": [0, 1, 2],
|
|
"WaveSeparateGlobalReadB": [0, 1, 2],
|
|
# Add an unrolled loop and NGLL loop with swapped GRA and GRB order.
|
|
# which may change the tlb thrashing behavior.
|
|
"UnrollLoopSwapGlobalReadOrder": [0, 1],
|
|
# swap global read order
|
|
# 0: A->B->A->B->repeat... (or swap depending on DTL and/or DTV)
|
|
# 1: B->A->B->A->repeat...
|
|
# normal/DTL/DTV should be same for A and B to swap GR order
|
|
# (normalA + normalB) or (DTLA + DTLB) or (DTVA + DTVB)
|
|
"SwapGlobalReadOrder": [0, 1],
|
|
# PrefetchGlobalRead = 1:
|
|
# Requires 2X LDS space, and VGPRs for buffering data on way into LDS
|
|
# prefetch / double-buffer reads from global memory -> vgprs -> lds.
|
|
#
|
|
# PrefetchGlobalRead = 2:
|
|
# Do another prefetch while writing data from vgpr to lds.
|
|
# prefetch / double-buffer reads from global memory -> vgprs --> lds.
|
|
# |-> prefetch reads
|
|
# PrefetchGlobalRead >= 3:
|
|
# DirectToLds only. Do PGR times prefetch global read before main loop.
|
|
# Need to allocate PGR+1 or PGR LDS buffer
|
|
# Allocating PGR+1 LDS buffer is better for instruction scheduling.
|
|
"PrefetchGlobalRead": [0, 1, 2] + list(range(3,16 + 1)),
|
|
# number of iteration prefetch local reads from lds to VGPRs buffer = PLR
|
|
"PrefetchLocalRead": list(range(128 + 1)),
|
|
# Enable global memory to GL2 cache prefetch using global_prefetch_b8 instruction (gfx1250 only).
|
|
# So when global reads are issued, the data is likely to be in GL2 cache.
|
|
# 0: disable
|
|
# 1: prefetch one load tile (MTxDepthU) ahead of PrefetchGlobalRead
|
|
# 2: prefetch two load tiles (MTxDepthU) ahead of PrefetchGlobalRead
|
|
# Currently we do not support GSU, StaggerU, StreamK and general batch. May remove these limitations in the future.
|
|
"PrefetchGL2": [0, 1, 2],
|
|
# MatrixInstruction Only
|
|
# If set ClusterLocalRead, each iteration dedicated vgprBuffer for localRead
|
|
# So we can schedule these localReads to the front of the loop
|
|
"ClusterLocalRead": [0, 1],
|
|
# Allocating PGR+1 LDS buffer if we have enough LDS memory size
|
|
# Only for DirectToLdsA+B + PGR>=2
|
|
# -1: auto (enable this for PGR>=3)
|
|
# 0: disable
|
|
# 1: enable (force to 0 if not applicable)
|
|
# IF LDS size exceeds maxLDS,
|
|
# PGR>=3: disable (set 0) and continue
|
|
# PGR==2: reject (use -1 or 0 in that case)
|
|
# 1LDSBuffer will be 0 if DtlPlusLdsBuf if enabled
|
|
"DtlPlusLdsBuf": [-1,0,1],
|
|
# We use double LDS buffer when PrefetchGlobalRead.
|
|
# While it reads data from LDS[0]/[1], it prefetch global data and writes to LDS[1]/[0]
|
|
# If we can make sure all data are read from LDS to register before writing data to LDS, we can use 1 LDS buffer to save LDS memory.
|
|
# this can help to generate Kernel that LDS usage originally exceed MaxLDS if using double LDS buffer,
|
|
# or help to increase Occupancy.
|
|
# 1 means: Force to use 1 LDS Buffer even with PrefetchGlobalRead
|
|
# -1 means: generator will use 1 LDS buffer only when LDS exceed MaxLDS
|
|
# Use case:
|
|
# SIA2: 1LDSBuffer is set to 1 natively
|
|
# SIA3: 1LDSBuffer works only when PGR=True
|
|
# TODO: optimize scheduling to support more cases.
|
|
"1LDSBuffer": [-1, 0, 1],
|
|
# gfx1250 LDS segment interleave: raises LDS read bandwidth by putting operand A's
|
|
# two halves in different 64KiB LDS segments so its two MFMA read ports stop conflicting.
|
|
# Supported: TDMInst=3 (TDM load for A and B), gfx1250, MIWaveGroup [2,2], dtype bf16 / fp16 /
|
|
# fp8 (incl. MXFP8). Not applied for 1LDSBuffer, subtile, sparse, or TDMSplit kernels.
|
|
# Mechanism: reorder LDS from the baseline [A0][A1][B0][B1] to [A0][B0][A1][B1]. Only operand A
|
|
# is helped -- B's halves move too, but both ports can still hit the same B segment.
|
|
# Two cases:
|
|
# tight (one A-half + one B-half >= 64KiB): [A0][B0] fills a segment, so [A1][B1] land in
|
|
# the next one -- no extra LDS.
|
|
# aligned (< 64KiB): pad [A0][B0] up to the segment boundary to push [A1][B1] over -- uses
|
|
# more LDS, and needs PrefetchGlobalRead=2.
|
|
# Values:
|
|
# -1 = auto: apply "tight" only (skip "aligned").
|
|
# 0 = off (default): baseline layout.
|
|
# 1 = force on: apply both "tight" and "aligned" wherever valid.
|
|
# Recommended: set [0, 1] when tuning, so both baseline and interleaved kernels are benchmarked.
|
|
"LDSSegmentInterleave": [-1, 0, 1],
|
|
# StreamK persistent loop: use the current tile's no-load-loop window to
|
|
# issue the first global-read group for the next persistent tile. The
|
|
# generated code keeps that first-PGR data durable and restores borrowed
|
|
# current-tile state before current tail/NLL code resumes.
|
|
"PrefetchAcrossPersistent": [0, 1],
|
|
# Split the unroll summation into multiple sections and combine the sections
|
|
# GSU applies only to the unroll summation dimension
|
|
# Set to 0 to disable GSU, kernel code will be generated without GSU support
|
|
# Set to -1 to choose GSU automatically in runtime, determined by function calculateAutoGSU
|
|
"GlobalSplitU": list(range(-1, 1024 + 1)),
|
|
# choose how to do GlobalSplitU
|
|
# 1: use atomic operation to accumulate on one buffer
|
|
# 2: each GSU group write to each own buffer and accumulate by another kernel
|
|
# 3: each GSU group write to each own buffer and accumulate by same kernel
|
|
"GlobalSplitUAlgorithm": ["SingleBuffer", "MultipleBuffer", "MultipleBufferSingleKernel"],
|
|
# don't create a whole copy of the Unroll loop with loads removed - instead
|
|
# use buffer limits to suppress global loads and ignore unnecessary ds_reads
|
|
"SuppressNoLoadLoop": [False, True],
|
|
# StinkyTofu: whether SwPrefetchInsertionPass may insert software instruction prefetch.
|
|
# True: Turn on StinkyTofu instruction prefetch for this solution (extra SGPR on supported ISAs only).
|
|
# False: no prefetch SGPR, prefetch pass disabled for this kernel.
|
|
#
|
|
# Purpose: command-processor (CP) prefetch only covers a bounded amount of code. When
|
|
# the kernel's assembly footprint is large enough to exceed that window, the front of the
|
|
# kernel can fall out of the I-cache before execution reaches it, causing misses. Software
|
|
# prefetch instructions bring hot code back under software control so execution stays ahead of
|
|
# the fetch pointer and avoids those misses.
|
|
"SwInstructionPrefetch": [False, True],
|
|
# For PrefetchGlobalRead=1, create a second copy of the unroll loop with
|
|
# the LDS pointer swaps expanded into inline constants for LDS read and write instructions
|
|
# This eliminates 4 vector XOR instructions used for pointer swap
|
|
"ExpandPointerSwap": [False, True],
|
|
# Schedule global reads and global read increments into LocalRead iterations
|
|
# Can reduce pressure on local read instruction dispatch queue
|
|
# 0=perform global reads at start of instruction loop
|
|
# 1=schedule into the local read instruction iterations
|
|
"ScheduleGlobalRead": [0, 1],
|
|
# Schedule local writes into LocalRead iterations.
|
|
# Can reduce pressure on local read instruction dispatch queue
|
|
"ScheduleLocalWrite": [0, 1],
|
|
# Scheduling algorithm to use for each iteration:
|
|
# 0 = minimal/no scheduling. Global Read and increments, followed by local reads,
|
|
# followed by local writes, followed by MACs
|
|
"ScheduleIterAlg": [0, 1, 2, 3, 4],
|
|
# For MatrixInstruction and SIA3, number of GlobalReadInstruction between mfma
|
|
# the purpose of this parameter is to control density of global read instruction scheduling
|
|
# Scheduling global read back to back can have better memory efficiency
|
|
# However, when full of vmem FIFO, it will block other instruction to be issued
|
|
# Range from 0.01 to 32
|
|
# 0.1 means 1 GR per 10 mfma
|
|
# 5 means 5 GR per 1 mfma
|
|
"GlobalReadPerMfma": [i / 100 for i in range(1, 3200)],
|
|
#
|
|
# For MatrixInstruction and SIA3, number of LocalWriteInstruction between mfma
|
|
# the purpose of this parameter is to control density of local write instruction scheduling
|
|
# In PGR1, we want to schedule local write more denser, so we can have more
|
|
# latency to hide global read
|
|
# In PGR2, since LW is followed by GR, every LW has same whole loop latency
|
|
# to hide global read. We want to schedule LW less denser, can
|
|
# avoid full of vmem FIFO.
|
|
# Range from 0.01 to 32
|
|
# 0.1 means 1 LW per 10 mfma
|
|
# 5 means 5 LW per 1 mfma
|
|
# -1 will derived an optimized value internally
|
|
# -2 will derived an optimized value and override LWPM silently (debug only, not recommended)
|
|
"LocalWritePerMfma": [i / 100 for i in range(1, 3200)] + [-1],
|
|
# This is to specify minimum number of GR inc instructions per MFMA
|
|
# PGR>=2 and LocalWritePerMfma==-1 case, lwStart is decided by (number of GRInc) / max(roundup(miLatencyLeft/2), 1)
|
|
# gfx950 MI16 case, miLatencyLeft is 2 and we can schedule only 1 GRInc per MFMA.
|
|
# This pushes lwStart below and ends up with very less room for GR scheduling.
|
|
# For mid/small MT size case, we have chance to improve Global Read scheduling by putting more GRInc instructions
|
|
# regardless of miLatencyLeft (overhead of GR inst is often more than the latency of GR inc inst)
|
|
"MinGRIncPerMfma": [-1] + list(range(1,10)),
|
|
# Interleave alpha scale calculation with beta loads and address calcs - rather
|
|
# than as a separate block of instructions
|
|
"InterleaveAlpha": [0, 1],
|
|
# Create a copy of NoLoadLoop which interleaves the stores with the final mac
|
|
# calculation and may perform other optimizations
|
|
# 0 = no interleave
|
|
# 1 = interleave one stores after required macs have completed execution
|
|
# 2 = interleave two stores after required macs have completed execution
|
|
"OptNoLoadLoop": [0, 1, 2],
|
|
"BufferLoad": [False, True],
|
|
"BufferStore": [False, True],
|
|
# CompactLoopStore default (opt-in, off by default). When enabled, the
|
|
# per-batch global write body is wrapped in a CLS countdown loop and
|
|
"CompactLoopStore": [False, True],
|
|
# Attempt to load directly from global memory into Vgpr.
|
|
# Assembly only
|
|
"DirectToVgprA": [False, True],
|
|
"DirectToVgprB": [False, True],
|
|
"DirectToVgprMXSA": [False, True],
|
|
"DirectToVgprMXSB": [False, True],
|
|
"DirectToVgprSparseMetadata": [False, True],
|
|
# Attempt to load directly from global memory into LDS.
|
|
# Assembly only
|
|
# Requires BufferLoad, assembler support for lds modifier on buffer
|
|
# loads (checked automatically), GlobalVectorWidth=1 (this is hw
|
|
# requirement) and A/B must not require any transpose.
|
|
# DirectToLds reduces load latency and eliminates the
|
|
# G2L registers used to stage data. Also replaces the
|
|
# local write offset with an SGPR.
|
|
# For an 8x8 TT with PrefetchGlobalRead=1 this can save 33 VGPRs.
|
|
# - Requirements for DirectToLds=1:
|
|
# GlobalReadVectorWidth = 1/2/4 (GRVW * bpe must be 4 for now)
|
|
# TransposeLDS = 1 for TLU=0 case
|
|
# DirectToLds support for x1 only for now
|
|
# 0: no DirectToLds
|
|
# 1: DirectToLds A and B
|
|
# 2: DirectToLds A only (no DTLB)
|
|
# 3: DirectToLds B only (no DTLA)
|
|
"DirectToLds": [0, 1, 2, 3],
|
|
# DirectToLds for sparse metadata.
|
|
# Requires DirectToLds on the dense side (B if Sparse==2 else A),
|
|
# and GlobalReadVectorWidthMetadata ∈ {4, 16} (16 needs HasDirectToLdsx4).
|
|
"DirectToLdsMetadata": [0, 1],
|
|
# Enable subtile-based kernel implementation for MX FP4 (gfx950 only).
|
|
# When True, uses a subtile scheduling strategy with DTL global reads and
|
|
# an optimized storeD path. Automatically forced False on non-gfx950.
|
|
"UseSubtileImpl": [False, True],
|
|
# Load options:
|
|
# (GRO = Global Read Offset)
|
|
# BufferLoad=0:
|
|
# = Use flat instructions with 64 bit GRO for each load
|
|
# + supports sizes up to 2^64
|
|
# - uses many VGPR for addressing
|
|
# - uses execmask+compares for edge detection
|
|
# - generates extra LDS traffic (could convert flat->global load)
|
|
# BufferLoad=1:
|
|
# = Use buffer load instructions with 32-bit offset
|
|
# + Less VGPRS (32b offset vs 64-bit) needed for addressing
|
|
# + Uses hardware buffer limit for edge detection
|
|
# - Limited range - the bot-right corner of macro-tile (plus padding=GRVW
|
|
# for shift-pointer, if ShiftPtr is required) must be within 2^32.
|
|
# ShiftPtrPad = MayShift ? GRWV*BPE : 0
|
|
# For TLU=1: Unroll*StrideA1 + ShiftPtrPad <= 2^32
|
|
# For TLU=0: MT*StrideA1 + ShiftPtrPad <= 2^32
|
|
# These conditions should be checked using Assert - TODO
|
|
# = UseSgprForGRO=1:
|
|
# + Attempt to use SGPR for Global Read Offsets.
|
|
# + Use one VGPR base GRO + many SGPR GRO rather than many VGPR GRO.
|
|
# + Each SGPR stores an offset from base GlobalReadOffset+0.
|
|
# - Requirements for UseSgprForGRO=1:
|
|
# - BufferLoad=1
|
|
# - Use appropriate Assert*ElementMultiple or GRVW=1 to eliminate need for ShifPtr
|
|
# (UseSgprForGRO does not support ShiftPtr since ShiftPtr needs to potentially shift GRO)
|
|
# = KernelWriterAssembly also supports 64-bit 2D buffer size (see use64bPbcLimit)
|
|
# - Requires 4 instructions to move scalar limit and a couple SGPR
|
|
# - Enabled by default. If the overhead matters we can add asserts/YAML parm to specialize
|
|
# = UseInstOffsetForGRO=1:
|
|
# + Attempt to use Instruction offset for Global Read Offsets.
|
|
# + This feature avoid updating m0 for subsequent GRO(s) for directToLds feature
|
|
# - Requirements for UseInstOffsetForGRO=1:
|
|
# - BufferLoad=1
|
|
# - DirectToLds=1
|
|
# converting m0 update from LocalWriteAddrSGpr using is usually win
|
|
# -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR
|
|
"UseInstOffsetForGRO": [-1, 0, 1],
|
|
# Converting VGPR GRO into SGPR GRO is usually a win
|
|
# However, the mode may exhaust all available SGPR, in particular for large unroll
|
|
# -1 attempt to use a heuristic to determine when the tile size will use too many SGPR and fall back to VGPR
|
|
"UseSgprForGRO": [-1, 0, 1],
|
|
# Use a 64-bit shadow limit register to allow buffers larger than 2^32 bytes
|
|
"Use64bShadowLimit": [True, False],
|
|
# Use a 64-bit shadow limit register for MXSA/B to allow buffers larger than 2^32 bytes
|
|
"Use64bShadowLimitMX": [True, False],
|
|
# Assertion properties
|
|
# These provide information or assertions that the problem size meets certain requirements
|
|
# for sizes or alignments. The kernel generator can use this information to produce
|
|
# a kernel which uses those assertions to produce a faster kernel.
|
|
#
|
|
# If modifying or adding Assertions also change ProblemProperties class in TensileTypes.h
|
|
# Kernel generator will assume that the summation size is some multiple of the element size
|
|
# and uses this to optimize the kernel.
|
|
# This can result in more efficient kernels, but requires runtime checking to ensure the specified
|
|
# summation value meets the requirements.
|
|
# (Recommended AF1EM value is 8 for half, 4 for single, 2 for double)
|
|
#
|
|
# Optimizations enabled by AssertSummationElementMultiple>1:
|
|
# - If >=2 for half:
|
|
# - Tail loop loads can be vectorized 2X to use dword
|
|
# - Enables asm kernels on V20
|
|
# - Can use DirectToLds for both unroll and tail loops
|
|
# - Tail loop can be unrolled up to InnerUnroll amount if AssertSummationElementMultiple%InnerUnroll==0
|
|
#
|
|
# 1 indicates no assertion (since all sizes are multiples of 1)
|
|
"AssertSummationElementMultiple": [1, 2, 4, 8, 16, 32, 64, 128, 256],
|
|
# Kernel generator will assume that the FreeIndex[0] size is some multiple of the element size
|
|
# and uses this to optimize the kernel.
|
|
# FreeIndex[0] is usually letter "I"
|
|
# (Recommended AF0EM value is 8 for half, 4 for single, 2 for double)
|
|
#
|
|
# Optimizations enabled by AssertFree0ElementMultiple>1:
|
|
# Load optimizations:
|
|
# - For TLU=1 matrix, if AF1WM>=GLVW then can enable UseSgprForGRO
|
|
# - Reduces registers used for address calculations
|
|
# - Removes address shift/unshift code
|
|
# - UseSgprForGRO will only be enabled if all matrices meet assertion requirements.
|
|
#
|
|
# Store Optimizations:
|
|
# - Can vectorize stores in edge tiles. Vector width can be up to AF0EM.
|
|
# (since C matrix is always coalesced in Free0 index direction and this assertion guarantees the index element multiple)
|
|
#
|
|
# 1 indicates no assertion (since all sizes are multiples of 1)
|
|
"AssertFree0ElementMultiple": [1, 2, 4, 8, 16, 32],
|
|
# Kernel generator will assume that the FreeIndex[1] size is some multiple of the element size
|
|
# and uses this to optimize the kernel.
|
|
# FreeIndex[1] is usually letter "J"
|
|
# (Recommended AF1EM value is 8 for half, 4 for single, 2 for double)
|
|
# Optimizations enabled by AssertFree1ElementMultiple>1:
|
|
# - See above AssertFree0ElementMultiple "Load optimizations"
|
|
# 1 indicates no assertion (since all sizes are multiples of 1)
|
|
"AssertFree1ElementMultiple": [1, 2, 4, 8, 16, 32],
|
|
# Assertions that require arithmetic intensity to be specified value.
|
|
# Arithmetic intensity measures the ratio of computation to memory bandwidth required for a problem.
|
|
# These predicates can be used to adjust solution selection compute-bound or memory-bound problems.
|
|
"AssertAIGreaterThanEqual": -1,
|
|
"AssertAILessThanEqual": -1,
|
|
# Stagger the start summation position of the tiles.
|
|
# Elements from the summation dimension are loaded at offsets rather than all starting at 0.
|
|
# StaggerU is the max 'clicks' of StaggerUStride bytes where each wg starts ; see StaggerUMapping
|
|
# for how the specific stagger for a given wg is determined.
|
|
#
|
|
# The tile assignment C are same as with StaggerOffset=0 ; the difference is the
|
|
# order that the summation elements are added.
|
|
# GRO will wrap back to the row start when the edge is reached.
|
|
#
|
|
# This can be effective for TLU=0 style matrices where the K dimension is a large power-of-2.
|
|
# In this case the start of each row of the tile is separated by an exact power-of-2
|
|
# which causes poor dram, cache, and tlb behavior. V20 has 16 channels each 256 bytes wide.
|
|
# StaggerU adjusts the start position in the summation (aka 'U') dimension
|
|
# to avoid these conflicts. Both A and B matrix start at the adjusted position.
|
|
# If >0 specifies the offset in multiples of the macro-tile "unroll" dim
|
|
# - Higher values will spread traffic to more channels but provide less L2 re-use.
|
|
# - StaggerU and WorkGroupMapping interact and should be tuned together -
|
|
# The WGM controls how tiles are assigned in C matrix, while StaggerU controls where those
|
|
# tiles start reading their summation dim parms.
|
|
# - StaggerU requires BufferLoad==1 and is silently ignored if BufferLoad==0
|
|
"StaggerU": [0, 2, 4, 8, 16, 32, 64],
|
|
# Stride in bytes for each staggeru 'click'.
|
|
# 256 is recommended since this is the width of memory channel (on gfx803,gfx900,gf906) - so
|
|
# each click will start in a new memory channel and spread traffic among the 16 available channels.
|
|
# For example StaggerUStride=256 and StaggerU=8 will use 8 unique starting points
|
|
# in summation dimension, each offset by 256-bytes - provided the tensor dims are large
|
|
# enough to support this.
|
|
# StaggerUStride will be internally increased so it is an integer multiple of DepthU*BpeAB.
|
|
# (the implementation requires this - the unroll iteration accesses data in steps of
|
|
# DepthU*BPE
|
|
# StaggerUStride = 0 is valid only when StaggerU = 0
|
|
"StaggerUStride": [-1, 0, 16, 32, 64, 128, 256, 512, 1024, 2048],
|
|
# How the tile assignment (wg0, wg1, wg2) controls the initial StaggerU offset:
|
|
# 0: Use wg0
|
|
# 1: Use wg1
|
|
# 2: Use wg2
|
|
# 3: Use wgSerial, wgSerial = wg0 + wg1 * nwg0 + wg2 * (nwg0 * nwg1)
|
|
# 4: Debug mode, offset each tile max allowed StaggerU. This just moves hotspot
|
|
# to a different bank since all workgroups still start at same point.
|
|
"StaggerUMapping": [0, 1, 2, 3, 4],
|
|
# GSU Workgroup Coalesced Ordering
|
|
# False: {(wg0,wg1,wg2,wgn)|(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)}
|
|
# True: {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)}
|
|
"GlobalSplitUCoalesced": [False, True],
|
|
# GSU Workgroup Mapping
|
|
# False: wg issued order = {(wg0,wg1,wg2,wgn),(wg0,wg1,wg2,wgn)|...|(wg0,wg1,wg2,wgn)}
|
|
# -> workgroups do the summation by tile -> slower GR but faster GW
|
|
# True: wg issused oder = {(wg0,wg0,wg0)|(wg1,wg1,wg1)|(wg2,wg2,wg2)|...|(wgn,wgn,wgn)}
|
|
# -> workgroups split up the summation -> faster GR but slower GW
|
|
"GlobalSplitUWorkGroupMappingRoundRobin": [False, True],
|
|
# 0=don't use magic div (source only)
|
|
# 1=magic div alg #1. Slightly faster but limited range (if magic number is 2^32)
|
|
# 2=magic div alg#2. Slightly slower but handles all unsigned ints up to 2^32
|
|
"MagicDivAlg": [0, 1, 2],
|
|
# For Block Mapping type:
|
|
# 0 : Use hardware-assigned wg number with no remapping.
|
|
# N : WG block width. "Wrap" to a new wg1 "row" assignment after N WGs assigned in that row.
|
|
# Tensor C always mapped with first free coord as fastest moving
|
|
# (Elements in this dimension are sequential in memory.
|
|
#
|
|
# For 2D nonbatched Matrix this means index order is I, then J
|
|
# For 2D batched Matrix this means index order is I, then J, then K.
|
|
#
|
|
# Then for 2D case:
|
|
# - If drawn in row-major format, I is the width and J is the height.
|
|
# - WGM determines dimensions of the box used to assign tiles from C
|
|
# - WGM is the height of the box (in the J dimension)
|
|
# - Given WGM, the box width (in I dim) is determined by number of CUs
|
|
# - The box always moves across matrixC in the fastest-moving "I" dim, then
|
|
# wraps to next J. TODO - might be useful to change this?
|
|
#
|
|
# Examples for 2D matrix:
|
|
# WGM=8: on CU64 machine this is a square box
|
|
# WGM=1: Short/Fat - this will cover maximum width in I dimension of C. This matches hardware assigned mapping.
|
|
# WGM=64: Tall/Skinny - this will cover maximum width in J dimension of C.
|
|
#
|
|
# Formula for wgSerial:
|
|
# wgSerial = wg0 + (wg1 % WorkGroupMapping) * nwg0
|
|
"WorkGroupMapping": list(
|
|
range(-1024, 1024 + 1)
|
|
), # change a workgroup's id so that the all the workgroups on the gpu at a time are hitting L2 cache the best
|
|
# 0: WorkGroupMapping is predicted at runtime.
|
|
# 1: No mapping
|
|
"WorkGroupMappingXCC": [
|
|
-1,
|
|
1,
|
|
2,
|
|
4,
|
|
8,
|
|
16,
|
|
32,
|
|
], # change a workgroup's id so that contiguous workgroup can map on same XCC
|
|
# -1: WorkGroupMappingXCCGroup will be set dynamically at runtime. Note that this is consistent with what StreamKXCCMapping
|
|
# does, but the value for the mapping will be set at runtime (=num_xcc).
|
|
# Please ensure that (CU_count % WGMXCC == 0).
|
|
"WorkGroupMappingXCCGroup": list(
|
|
range(-1, 1024)
|
|
), # change a workgroup's id so that contiguous workgroup can map on same XCC, remap workgroup in a group of WGMXCCG.
|
|
# Use Space-filling-algorithm to determine workgroup mapping
|
|
# Parameter is provided as a list of integers. The length of the list indicates the level depth.
|
|
# Each integer represents an ordering ID to denote which ordering to use at each level.
|
|
# An empty array means use default (old) WGM algo
|
|
#
|
|
# With multi-level orderings the WGM values contains grid dims for each level encoded as 8bit values
|
|
# WGM: (msb) [GridDimN_L2, GridDimM_L2, GridDimN_L1, GridDimM_L1] (lsb)
|
|
#
|
|
# Order IDS:
|
|
# 0 : map workgroup IDs to C based on col-major order
|
|
# 1 : map workgroup IDs to C based on row-major order
|
|
# 2 : map workgroup IDs to C tiles based on Hilbert curve
|
|
# 3 : map workgroup IDs to C tiles based on Morton Z-curve
|
|
# 4 : map workgroup IDs to C tiles based on Morton (reverseN)-curve
|
|
# 5 : map workgroup IDs to C tiles based on Morton U-curve
|
|
#
|
|
# Examples:
|
|
# [0]: Single level ordering of C tiles using column major ordering
|
|
# [0,1]: Double level ordering of C tiles using column major ordering, then row-major
|
|
# [1,1,1]: Triple level ordering of C tiles using row major ordering at each level
|
|
"SpaceFillingAlgo" : -1,
|
|
# Used to specify the grid dims for each level when using SpaceFillingAlgo. This value is directly passed into the
|
|
# WGM parameter. The format is
|
|
# WGM: (msb) [GridDimN_L2, GridDimM_L2, GridDimN_L1, GridDimM_L1] (lsb)
|
|
#
|
|
# Examples:
|
|
# 0x01020304
|
|
# - Level1 grid dim 4x3
|
|
# - Level2 grid dim 2x1 (if enabled, otherwise last 16 bit values are ignored)
|
|
# 0x10010820
|
|
# - Level1 grid dim 32x8
|
|
# - Level2 grid dim 1x16 (if enabled, otherwise last 16 bit values are ignored)
|
|
"SFCWGM" : -1,
|
|
"MaxOccupancy": list(
|
|
range(1, 40 + 1)
|
|
), # wg / CU; if cache thrashing is hurting performance, this allocates extra lds to artificially limit occupancy
|
|
"MaxLDS": [-1, 65536, 163840, 327680],
|
|
"WorkGroup": makeValidWorkGroups(), # ( wg0 x wg1 x LocalSplitU ) dimensions of the workgroup which will operate on a tile and share lds
|
|
# ThreadTile: ( tt0 x tt1 ) dimensions of the C tile that each thread works on,
|
|
# TT=4 and VW=4 means a thread will work on a tight 4x4 tile of C, where VW=1 means the tile will work on 16 spread out values
|
|
# Generally, the VW determines the consecutive a WI will work on, then it will skip ahead SG0*VW elements to get to the next row of VGPR inputs
|
|
"ThreadTile": validThreadTiles,
|
|
"MacroTile": validMacroTiles, # MT0 = wg0*tt0, MT1 = wg1*tt1
|
|
"WavefrontSize": [-1, 32, 64], # -1 = auto-detect from ISA archCaps["HasWave32"]
|
|
# MatrixInstruction: (M x N x K x B)
|
|
# XDLOPS tile definition, only valid for gfx908, gfx90a
|
|
# MxNxKxB specifies matrix instruction variants
|
|
# MxNxB determines the shape of the C tile each instruction worked on
|
|
# K determines the unroll depth
|
|
# If empty, do not use these instructions
|
|
#
|
|
# Alternative format: (M x N x K x B x MIBlockM x WaveTileM x WaveTileN x WaveM x WaveN)
|
|
# (Note: MxN means M-by-N in the following comments)
|
|
# MIBlockM determines how many blocks along M dimension for multi-block MI variants. Concrete examples:
|
|
# - MI 16x16x1x4 (4-block variant) with MIBlockM=4 -> (16x16)*(4x1)=64x16 tile per instruction executed
|
|
# - MI 32x32x1x2 (2-block variant) with MIBlockM=1 -> (32x32)*(1x2)=32x64 tile per instruction executed
|
|
# WaveTileM/N are dimensions of the C tile each wave works on, and is close to the concept of ThreadTile in classic VALU kernels
|
|
# - WT 4x1 -> each wave executes 4x1 matrix instructions on the C tile of total area (4*MITileM)x(1*MITileN)
|
|
# WaveM/N are dimensions of waves spawned for one workgroup where each wave consists of 64 threads
|
|
# - Wave2x2 -> a total of 4 waves in one workgroup of shape 2x2
|
|
# Putting it all together:
|
|
# - [32, 32, 1, 2, 1, 4, 1, 2, 2]
|
|
# ^^^^^^^^^^^^ ^ ^^^^ ^^^^
|
|
# MatrixInst BlkM WT Wave
|
|
# - means (32x64) per MI * (4x1) per wave * (2x2) per workgroup = (32*4*2)x(64*1*2) = 256x128 macro tile
|
|
# Tensile will ignore the parameters ThreadTile and WorkGroup when the alternative format is used
|
|
# NOTE: MatrixInstruction is no longer validated through this structure, but is instead validated via the
|
|
# ``TensileLogic`` program.
|
|
"MatrixInstruction": -1,
|
|
# StoreRemap: Optimize MatrixInstruction store patterns to enhance performance.
|
|
# MI output data between each threads are along N dims.
|
|
# But global memory is along M dim continuous.
|
|
# That mean global write between each threads are not continuous.
|
|
# Therefore, store performance for MI instruction is poor.
|
|
# How StoreRemap works in final store stage:
|
|
# 1. Put all thread output data into LDS.
|
|
# 2. All thread read data from LDS along M dims.
|
|
# (match global Memory continuous direction)
|
|
# 3. All thread write out data into global memory.
|
|
# 0: Disable StoreRemap (default)
|
|
# 1~8: Enable StoreRemap and set the global write vector width
|
|
# Suggest optimum value: fp32 = [2,4], fp16 or bf16 = [4,8] (dwordx2 and dowrdx4)
|
|
# -1: Use dwordx2 if support SRVW, or set SRVW to 0
|
|
"StoreRemapVectorWidth": [-1, 0, 1, 2, 4, 8],
|
|
# SourceSwap: Optimizes MatrixInstruction store pattern by swapping mfma input order.
|
|
"SourceSwap": [False, True],
|
|
# UseDualFMAC: emit RDNA3/3.5/4 VOPD v_dual_fmac_f32 pairs in the f32 source/MAC inner
|
|
# loop (2x FMA issue rate). Source (non-MFMA) f32 kernels on gfx11/gfx12 only; auto-
|
|
# disabled elsewhere (see SolutionStructs.Solution.assignProblemIndependentDerivedParameters).
|
|
"UseDualFMAC": [False, True],
|
|
# Following parameters are designed for store scheduling.
|
|
# (store stands for load from C (with beta) and store to C/D)
|
|
#
|
|
# we want to hide store behind unroll loop
|
|
# 1. if we can launch 2 WorkGroups per CU (occupancy >= 2, large M/N)
|
|
# 2. if there are remaining global memory bandwidth in unroll loop (compute bound kernel)
|
|
#
|
|
# we can hide store behind the other WG's loop by lowering priority of store
|
|
# priority of loop is the same as priority of store
|
|
# WG0: ???????????????\__
|
|
# |<-- loop --->|<-- store -->|end
|
|
#
|
|
# WG1: ___________________________/????????????\__
|
|
# |<--------- loop ------------------->|<-- store -->|end
|
|
#
|
|
# priority of loop is higher than priority of store
|
|
# WG0: ???????\____________________
|
|
# |<-- loop --->|<------ store ----->|end
|
|
#
|
|
# WG1: _____________/?????\__________________
|
|
# |<------- loop -------->|<----- store ---->|end
|
|
"StorePriorityOpt": [False, True],
|
|
#
|
|
# If we issue store in short period of time, kernel will become from compute bound to memory bound
|
|
# 0 means issue instructions as many as possible if VGPR available
|
|
"NumElementsPerBatchStore": list(range(-1, 256)),
|
|
#
|
|
# add sync after per batch store in order to store contiguous elements
|
|
# add sleep after per batch store in order to distribute store over whole loops
|
|
# NOTE: this parameter is highly depends on size_k
|
|
# 0 means no sync and sleep
|
|
"StoreSyncOpt": list(range(0, 256)),
|
|
#
|
|
# There are index or address calculation between global instructions.
|
|
# issue global instruction b2b has better performance
|
|
"GroupLoadStore": [False, True],
|
|
# In order to remove the copying from Acc vgpr to Arch vgpr, only use Arch vgprs for v_mfma_xxx.
|
|
# Only support for kernel whose totalVgpr counts less than 256 and gcn that has control bit ACC_CD.
|
|
"MIArchVgpr": [False, True],
|
|
# StreamK (SK) kernels divide work evenly among CUs by splitting along MT and K dimensions.
|
|
# Total work units are calculated as (#MTs x #LoopIters) and divided among workgroups.
|
|
# In most cases each workgroup will calculate a partial tile that are accumulated in a fixup step in the same kernel
|
|
# 0 : Standard data-parallel kernel
|
|
# 3 : Two-Tile StreamK with DP before SK tiles
|
|
# 4 : Dynamic StreamK using per-XCD work queues
|
|
# 5 : Hybrid SK3 + SK4 in one kernel; mode bit 30 of MagicShiftItersPerTile
|
|
# selects the active sub-path (see StreamKHybrid in StreamK.py).
|
|
# StreamK kernels can adjust the number of CUs being used.
|
|
# Using fewer sometimes increases overall throughput by allowing other kernels to run in parallel.
|
|
# StreamK grid is controlled by setting these enviornment variables:
|
|
# TENSILE_STREAMK_FIXED_GRID lets you override the default grid size with a specific number
|
|
# 0 = override disabled (default)
|
|
# TENSILE_STREAMK_FULL_TILES sets the number of full tiles to be included in stream-k work
|
|
# -1 = use prediction model for best performance (not yet implemented)
|
|
# 0 = only remainder tiles run in stream-k
|
|
# 1+ = remainder + 1 (or more) full grids of tiles run in stream-k (default=1)
|
|
# TENSILE_STREAMK_DYNAMIC_GRID selects dynamic grid mode, which automatically limits the number of CUs used:
|
|
# 0 = Off, always use all CUs.
|
|
# 1 = Only reduce CUs for small problems to number of output tiles when num_tiles < CU count.
|
|
# 2 = Also reduce CUs used for large sizes to improve data-parallel portion and reduce power.
|
|
# 3 = Analytically predict the best grid-size by weighing the cost of the fix-up step and the cost of processing MACs (default).
|
|
# Note: dynamic grid coefficients currently apply to gfx942 variants
|
|
# 4 = StreamK algorithm will behave as data parallel (Launch WGs = #CUs)
|
|
# 5 = StreamK Algorithm will use Origami's "select_best_grid_size" function
|
|
# TENSILE_STREAMK_DYNAMIC_WGM Enables Origami's analytical model-based WGM selection
|
|
# TENSILE_STREAMK_MAX_CUS allows the user to manually set maximum number of CUs used, which could free up some CUs for
|
|
# other operations to run in parallel with gemm.
|
|
# TENSILE_STREAMK_GRID_MULTIPLIER lets you set how many workgroups are created per CU being used.
|
|
# 1 = 1 WG per CU (default), for example. 2 will launch WGs = 2 x CU count.
|
|
# The priority of these environment variables is defined as follows:
|
|
# TENSILE_STREAMK_FIXED_GRID > TENSILE_STREAMK_DYNAMIC_GRID > TENSILE_STREAMK_MAX_CUS > TENSILE_STREAMK_GRID_MULTIPLIER
|
|
"StreamK": [0, 3, 4, 5],
|
|
# Force StreamK=3 to run all output tiles through the persistent DP path.
|
|
# When enabled, dispatch uses the single-kernel StreamK path, sets skTiles=0
|
|
# to skip the SK region, and keeps the normal StreamK grid selection policy.
|
|
# The invariant is no partial output tile fixup and no SK-region processing.
|
|
# Valid only with DP-first, non-atomic StreamK mode 3.
|
|
"StreamKForceDPOnly": [0, 1],
|
|
# Determines if StreamK kernel uses atomics
|
|
# 0: uses workspace to store partial tiles, accumulate in deterministic fix-up step
|
|
# 1: uses atomics to accumulate partial tiles
|
|
"StreamKAtomic": [0, 1],
|
|
# Codegen-time toggle for single-hop next-neighbor work stealing in the
|
|
# dynamic-queue StreamK fetch (SK4 / SK5-dynamic). Queue count =
|
|
# archCaps['NumXCD'] (8 on gfx942/gfx950). When a workgroup's home queue
|
|
# empties, it makes one atomic attempt on its next-neighbor per-XCD queue.
|
|
# Valid only for StreamK in (4, 5).
|
|
# 0: off
|
|
# 1: on
|
|
"StreamKWorkStealing": [0, 1],
|
|
# Enables XCC-based remapping of workgroups, set the value to the number of XCCs
|
|
# for the device/configuration being used
|
|
# 0: uses default workgroup assignment
|
|
# 2+: remaps workgroups to be contiguous within an XCC for a given number of XCCs
|
|
"StreamKXCCMapping": [0] + list(range(2, 9)),
|
|
# Enables using a Tree-reduction for the fixup step of StreamK algorithm
|
|
# 0: use linear reduction
|
|
# 1: use tree reduction
|
|
"StreamKFixupTreeReduction": [0, 1],
|
|
# Debug settings for stream-k kernels to disable parts of the kernel
|
|
# Bit 0: Don't generate fixup code
|
|
# Bit 1: Don't generate write to partials code
|
|
# Both parts can be disabled together
|
|
# 0 = Debug mode off, generate full kernel
|
|
# 1 = No fixup
|
|
# 2 = No partials
|
|
# 3 = Nofixup and no partials
|
|
"DebugStreamK": [0, 1, 2, 3],
|
|
# Persistent-kernel debug: when True, the persistent loop never exits.
|
|
# Used as a co-tenant load kernel for contended-perf benchmarking.
|
|
# Termination is via process death. Requires StreamK = 3.
|
|
"DebugPersistentKernelLoopForever": [False, True],
|
|
# Controls desired width (#elements) for loads from global memory -> LDS.
|
|
# and eliminates the pointer unshift logic
|
|
# -1 : Set GlobalReadVectorWidth = VectorWidth
|
|
# NOTE: for input bpe=32, max GRVW is 4 (to fit dwordx4) (FP32), min GRVW is 1 (dword)
|
|
# bpe=16, max GRVW is 8 (to fit dwordx4) (FP16), min GRVW is 2 (dword)
|
|
# bpe=8, max GRVW is 16 (to fit dwordx4) (INT8), min GRVW is 4 (dword)
|
|
"GlobalReadVectorWidthA": [-2, -1, 1, 2, 3, 4, 6, 8, 16, 32],
|
|
"GlobalReadVectorWidthB": [-2, -1, 1, 2, 3, 4, 6, 8, 16, 32],
|
|
# Controls desired width (#elements) for loads from LDS -> VGPR.
|
|
# -1 : Set LocalReadVectorWidth = VectorWidth
|
|
# 1 cannot be used for half type.
|
|
# used in combination with TransposeLDS=True
|
|
# in TransposeLDS=1 case, use wider load to fetch elements in summation dimension from LDS
|
|
# helps optimizing instruction scheduling between MFMA and nonMFMA instructions
|
|
# NOTE: for input bpe=32, max LRVW is 4 (to fit ds_read_b128) (FP32)
|
|
# bpe=16, max LRVW is 8 (to fit ds_read_b128) (FP16)
|
|
# bpe=8, max LRVW is 16 (to fit ds_read_b128) (INT8)
|
|
"LocalReadVectorWidth": [-1, 1, 2, 4, 8, 16, 32],
|
|
"LocalReadVectorWidthA": [-1, 1, 2, 4, 8, 16, 32],
|
|
"LocalReadVectorWidthB": [-1, 1, 2, 4, 8, 16, 32],
|
|
# threads should read/write/operate on this many contiguous elements from the C matrix.
|
|
# If VW=4 then thread0 will process 4 consec C elements, then thread1 next 4, etc.
|
|
# If the ThreadTile is > VectorWidth then thread0 will next operate on the 4 elements in C at (4*NumThreads)
|
|
# Typically the load vector width and store vector width are directly related to the VW.
|
|
# The global load width is closely related to the width of local stores so
|
|
# GlobalReadVectorWidth also controls local write width.
|
|
# Local read width also matches since VectorWidth consec elements must be read
|
|
# Typically matching 16 bytes is good choice since the stores will be optimally coalesced with 16 bytes/WI.
|
|
# Using a VW too large which results in >16bytes/thread isn't supported
|
|
# For MFMA non SourceSwap: this parameter didn't take effect
|
|
# -1 means set vw to largest localReadWidth according to MIWaveTile, LDS padding and LDS capacity
|
|
"VectorWidthA": [-1, 1, 2, 3, 4, 6, 8],
|
|
"VectorWidthB": [-1, 1, 2, 3, 4, 6, 8],
|
|
# If 0, store 1 element per instruction.
|
|
# If 1, store vector-width elements per instruction.
|
|
# if -1, store vector-wide elements per instruction unless PBD would not generate a valid kernel
|
|
"VectorStore": [-1, 0, 1],
|
|
# Controls desired width (#elements) for stores from reg to global memory.
|
|
# When MatrixInstruciton == None, derived parameter gwvw takes precedence.
|
|
# -1 : Set StoreVectorWidth = VectorWidth
|
|
"StoreVectorWidth": [-1, 1, 2, 3, 4, 6, 8],
|
|
# when loading all the data from global into lds requires multiple load instructions, these parameters govern which
|
|
# loads will pull which rectangle of data from global into lds
|
|
# NLC=1 means one load along the coalesced dimension, which results in the most coalescing possible
|
|
# NLC=-1 looks for the largest number of reads along the coalesced dimension which results in the least ammount of coalescing;
|
|
# however in this case the stride between one load and another is a static value, therefore buffer loads only need one set of registers
|
|
# whereas the =1 case has a stride which is a multiple of a kernel argument and therefore needs one address per load in the perpendicular dimension
|
|
"NumLoadsCoalescedA": list(range(-1, 64 + 1)),
|
|
"NumLoadsCoalescedB": list(range(-1, 64 + 1)),
|
|
# DepthU, LocalSplitU (which is the 3rd number in WorkGroup), and LoopUnroll are closely related
|
|
# LoopUnroll=4 means there are 4 subiterations within the loop, 4 actual iterations written in the code.
|
|
# LocalSplit=2 means the workgroup is split up into 2 subgroups, and each subgroup is doing different parts of the summation.
|
|
# subgroup0 does k=0-3, 8-11... and subgroup1 does k=4-7, 12-15...
|
|
# So, each iteration through the summation loop, which has 4 actual subiterations, does 8 summation iterations, because each subgroup did 4;
|
|
# and when data is read from global memory the threads read 8 elements along the summation dimension.
|
|
# DepthU = LoopUnroll * LocalSplitU = 4*2 in this case
|
|
# it made more sense for the user to directly control LocalSplitU and DepthU, then derrive afterwards LoopUnroll=DepthU/LocalSplitU
|
|
# -1 : Only allow GLVW=1
|
|
# -2 : Only allow max(GLVWA,GLVWB) < VW ?
|
|
# -3 : Only allow min(GLVWA,GLVWB) < VW ?
|
|
"DepthU": [-1] + list(range(2, 1024 + 1, 1)),
|
|
# integer amount of padding to put into LDS, in 2016 this didn't seem to help performance, profilers were showing that channel conflicts weren't really hurting
|
|
# performance so this has been deprecated and probably doesn't work
|
|
# -1 means use same padding as the VectorWidth if TLU=0 else 0. (Padding only helps when transpose is required)
|
|
# With MatrixInstruciton: -1 means max(GRVW,MIInput) if TLU=0
|
|
"LdsPadA": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64],
|
|
"LdsPadMXSA": [ -1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64],
|
|
"LdsPadB": [-1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64],
|
|
"LdsPadMXSB": [ -1, 0, 1, 2, 3, 4, 8, 16, 32, 48, 64],
|
|
"LdsPadMetadata": [-1, 0, 1, 2, 3, 4, 8],
|
|
# Padding boundary for LDS. defines block-size for pad insertion. for every 'LdsBlockSizePerPad' bytes, LDS padding (pad value from LdsPad parameter)
|
|
# is added (readOffset aware of the pad and adjusts offset value based on this parameter value).
|
|
# Only support LdsBlockSizePerPad >= unrollDepth * BPE
|
|
# 0 means disable LdsBlockSizePerPad
|
|
"LdsBlockSizePerPadA": validLdsBlockSizePerPad,
|
|
"LdsBlockSizePerPadMXSA": [-1, 0, 64, 128, 256, 512, 1024, 2048],
|
|
"LdsBlockSizePerPadB": validLdsBlockSizePerPad,
|
|
"LdsBlockSizePerPadMXSB": [-1, 0, 64, 128, 256, 512, 1024, 2048],
|
|
"LdsBlockSizePerPadMetadata": [-1, 0, 64, 128, 256, 512, 1024, 2048],
|
|
# Transpose LDS format. Local store in coalesced dimension , same as optimized global fetch dimension . applicable only in TLU=0 case for miSIMD(s)
|
|
# -1 : keep LDS layout same as global fetch dimension for both A and B
|
|
# set TLDS = 1 for NN,TN,TT
|
|
# set TLDS = 0 for NT
|
|
# 0 : coalesced dimension of lds is tile dimension
|
|
# 1 : keep LDS layout same as global fetch dimension for both A and B for NN,TN,TT, but NT would be rejected
|
|
# 2 : coalesced dimension of lds is unroll dimension for both A and B
|
|
"TransposeLDS": [-1, 1, 0, 2],
|
|
"TransposeLDSMetadata": [-1, 1, 0],
|
|
# add gls or slc after global memory read/writes to change caching, not caching the writes is promising and improved performance a tiny bit
|
|
# 0: none, 1: glc, 2: slc, 3: glc slc
|
|
# For gfx942, sets sc0/sc1/nt bits
|
|
# 0: none, 1: sc0, 2: sc1, 3: sc0 sc1, 4: nt, 5: nt sc0, 6: nt sc1, 7: nt sc0 sc1
|
|
"NonTemporalE": list(range(0, 8)),
|
|
"NonTemporalGate": list(range(0, 8)),
|
|
"NonTemporalD": list(range(0, 8)),
|
|
"NonTemporalC": list(range(0, 8)),
|
|
"NonTemporalA": list(range(0, 8)),
|
|
"NonTemporalMXSA": list(range(0,8)),
|
|
"NonTemporalB": list(range(0, 8)),
|
|
"NonTemporalMXSB": list(range(0,8)),
|
|
"NonTemporalWS": list(range(0, 8)),
|
|
"NonTemporalMetadata": list(range(0, 8)),
|
|
"NonTemporal": list(range(-1, 8)),
|
|
# gfx1250-only temporal-hint modifier.
|
|
"TemporalHint": list(range(-1, 8)),
|
|
"TemporalHintE": list(range(0, 8)),
|
|
"TemporalHintD": list(range(0, 8)),
|
|
"TemporalHintC": list(range(0, 8)),
|
|
"TemporalHintA": list(range(0, 8)),
|
|
"TemporalHintMXSA": list(range(0, 8)),
|
|
"TemporalHintB": list(range(0, 8)),
|
|
"TemporalHintMXSB": list(range(0, 8)),
|
|
"TemporalHintWS": list(range(0, 8)),
|
|
"TemporalHintMetadata": list(range(0, 8)),
|
|
# gfx1250-only non-volatile memory modifier.
|
|
"NonVolatile": [-1, 0, 1],
|
|
"NonVolatileE": [0, 1],
|
|
"NonVolatileD": [0, 1],
|
|
"NonVolatileC": [0, 1],
|
|
"NonVolatileA": [0, 1],
|
|
"NonVolatileMXSA": [0, 1],
|
|
"NonVolatileB": [0, 1],
|
|
"NonVolatileMXSB": [0, 1],
|
|
"NonVolatileWS": [0, 1],
|
|
"NonVolatileMetadata": [0, 1],
|
|
# Group together unroll iterations inside the unroll loop.
|
|
# For example, InnerUnroll=2 will fetch LDS for two unroll iterations
|
|
"InnerUnroll": [1, 2, 4, 8, 16, 32, 64],
|
|
# Enable CP preload kernel arguments feature
|
|
# It can reduce time of loading kernel arguments by s_load.
|
|
# It needs new complier and vbios to support this feature.
|
|
"PreloadKernArgs": [False, True],
|
|
# Kernels should be written in assembly or source
|
|
# if assembly, ISA will determine architecture
|
|
# if source, Runtime will determine language
|
|
# later on, we'll relax this to inner kernel languages and outer kernel languages, such as inline asm embedded in ocl or in llvm
|
|
"KernelLanguage": ["Assembly"],
|
|
# We set validParams["ISA"] in multiple places
|
|
"ISA": validISA, # arch for assembly kernels
|
|
# Name of the custom kernel located at `CUSTOM_KERNEL_PATH`.
|
|
# a custom kernel is a user written assembly kernel with its associated configuration parameters included in a custom.config section
|
|
# inside the yaml block between the --- and ... markers. These parameters are only used for information purposes, not kernel generation.
|
|
# Ex:
|
|
# custom.config:
|
|
# ProblemType:
|
|
# OperationType: GEMM
|
|
# etc...
|
|
# ThreadTile: [8, 8]
|
|
# etc...
|
|
#
|
|
# Custom kernels can be included in a BenchmarkProblemSizeGroup by having their name (without file extension) listed under the "CustomKernels"
|
|
# category alongside InitialSolutionParameters, BenchmarkCommonParameters, etc...
|
|
"CustomKernelName": -1,
|
|
# Will allow a kernel to be accepted even when checks determine it's not viable.
|
|
# Intended for use with custom kernels which have confirmed to be correct
|
|
"NoReject": [False, True],
|
|
# Debug use only.
|
|
"ActivationFused": [True],
|
|
# True- function call
|
|
# False- inline
|
|
"ActivationFuncCall": [False, True],
|
|
# Alternative implementation for activation function
|
|
# Currently only supports GSU == 1
|
|
"ActivationAlt": [False, True],
|
|
# Do workgroup reduction. Currently for DBias
|
|
"WorkGroupReduction": [False],
|
|
# 4:2 Structured Sparse A Matrix, 0=Non Sparse, 1=Sparse Matrix A, 2=Sparse Matrix B
|
|
"Sparse": [0, 1, 2],
|
|
# in mix mode F8 need to convert to F16, do this before(0) ds or after(1) ds
|
|
"ConvertAfterDS": [False, True],
|
|
# Force disable shadow init to release more sgpr in preloop
|
|
"ForceDisableShadowInit": [False, True],
|
|
# Use WMMA/MFMA with src C=0 to initialize C accumulators (skipping v_mov initC).
|
|
# -1: auto-detect
|
|
# 0: force disable
|
|
# 1: force enable (rejected if the auto-disable conditions are met)
|
|
"InitCIterWmma": [-1, 0, 1],
|
|
# Enable LDS Transpose Instruction
|
|
"LDSTrInst": [False, True],
|
|
# False: Use LocalSplitU. Number of WorkGroup[2] WorkItems (wave or thread) will compute the same output elements (matrix D) along different
|
|
# unroll indices. The local sum from those WorkItems are reduced through LDS.
|
|
# True: Use WaveSplitK. Number of WorkGroup[2] threads in the same wave compute the same output elements (matrix D) along different unroll indices.
|
|
# The local sum from those threads are reduced through suffling using VALU instructions. Currently only support dot2 kernel.
|
|
"WaveSplitK": [False, True],
|
|
# Control mbsk reduction prefetch order
|
|
# -1 : Select between 0/1 based on # store elements.
|
|
# 0 : Fetch from workgroup dim -> elements dim. (default)
|
|
# 1 : Fetch from elements dim -> workgroup dim. Has better prefetch pattern when # store elements is large.
|
|
"MbskPrefetchMethod": [-1, 0, 1],
|
|
# Select whether to enable CMS
|
|
# CMS is supported for a given solution if there exists a CMS schedule in Components/CustomSchedule.py
|
|
#
|
|
# -1 : 0 if CMS is not supported, 1 if CMS is supported
|
|
# 0 : disable CMS even if supported
|
|
# 1 : enable CMS, is set to 0 if not supported
|
|
"UseCustomMainLoopSchedule" : [-1, 0, 1],
|
|
# 0 : Generate original Store blocks: NonEdgeN, ThenN, and Then1 for StoreVectorWidth N
|
|
# 1 : Generate adaptive Store blocks: NonEdgeN, ThenN, ThenN/2, ..., Then1 and select by runtime problem size
|
|
"AdaptiveGemm": [0, 1],
|
|
# 0 : disable
|
|
# 1 : merge MB and MBSK assembly code and select best GW path in runtime
|
|
"AdaptiveGemmGSUA": [0, 1],
|
|
# 0 : NonTemporalA and NonTemporalB use fixed values from kernel parameters
|
|
# 1 : NonTemporalA and NonTemporalB are determined at runtime based on problem size and stride alignment
|
|
# Adaptive selection applies to the main loop only; prefetch (prolog) and tail loop still use the fixed NonTemporalA/B
|
|
"AdaptiveGemmNTAB": [0, 1],
|
|
# Add extra latency to calculate number of MFMA to insert between local read and wait
|
|
# Negative value means reduce interval between local read and wait (for DirectToVgpr only)
|
|
"ExtraLatencyForLR": list(range(0,17,2)) + list(range(-80,0,10)),
|
|
# Add extra miLatencyLeft to improve local read scheduling
|
|
# Adding more room for scheduling local read instructions
|
|
# Tentative: setting >=0 value will invalidate miLatency/miIssueLatency adjustment for gfx950 + 2Byte
|
|
"ExtraMiLatencyLeft": [-1,0,1,2,3,4,6,8],
|
|
# TailLoop in NoLoadLoop optimization
|
|
# generate TailLoop code in NoLoadLoop to take advantage of prefetch and wider globalLoad plus instruction scheduling
|
|
# Need certain conditions to use TailloopInNll optimization
|
|
# - NT transpose or AssertSummationElementMultiple * bpeGR is multiple of 4 (with BufferLoad + ShiftPtr)
|
|
"TailloopInNll": [False, True],
|
|
# Schedule global read instructions over barrier sync.
|
|
# Only for DirectToLdsA+B + PGR>=2.
|
|
# -1: auto (enable this for PGR>=3)
|
|
# 0: disable
|
|
# 1: enable (force to 0 if not applicable)
|
|
"ScheduleGROverBarrier": [-1,0,1],
|
|
# This is to improve scheduling for packing code (so far, X32F emulation code only)
|
|
# 0: Do prefetch for local read only
|
|
# 1: Do prefetch for pack code
|
|
# CMS (UseCustomMainLoopSchedule) case, this is internally set and setting it from yaml will not change CMS config
|
|
"UsePLRPack": [0,1],
|
|
# Enable tensor data mover for VM -> LDS
|
|
# 0: Disable
|
|
# 1: Use TDM for A
|
|
# 2: Use TDM for B
|
|
# 3: Use TDM for both A and B
|
|
"TDMInst": [0, 1, 2, 3],
|
|
# Split each TDM data tensor (A or B) load across two tensor_load_to_lds instructions,
|
|
# each covering half the macro-tile in the M/N dimension. MX scale tensors (MXSA/MXSB)
|
|
# are not split regardless of this flag. When True, two extra SGPRs are allocated to
|
|
# hold the per-iteration LDS and global address increments for the split loads.
|
|
"TDMSplit": [False, True],
|
|
# In-device layout of the MX scale tensors (MXSA/MXSB).
|
|
# User-facing values:
|
|
# "NoSwizzle": no swizzling; plain row/column layout (this is the default
|
|
# unless TDMInst != 0, in which case InMemorySwizzle is the default).
|
|
# "HostPreSwizzle": scales pre-swizzled on the host (gfx950 subtile pipeline);
|
|
# requires MXLoadInst="BufferLoad" and gfx950 host pipeline.
|
|
# "InMemorySwizzle": scales swizzled in device memory by TDM;
|
|
# requires MXLoadInst="TDM".
|
|
# Internal sentinel (do NOT use in yamls):
|
|
# "Auto": triggers conditional defaulting in Solution.assignDerivedParameters.
|
|
# The defaultBenchmarkCommonParameters entry is "Auto" so the
|
|
# derivation fires; users should write one of the concrete values above.
|
|
# Codegen + rejection logic combine MXScaleFormat with MXLoadInst and
|
|
# archCaps["HasMXScaleSwizzle"].
|
|
"MXScaleFormat": ["Auto", "NoSwizzle", "HostPreSwizzle", "InMemorySwizzle"],
|
|
# How A/MXSA and B/MXSB are loaded from VRAM to LDS.
|
|
# User-facing values:
|
|
# "TDM": tensor_load_to_lds (requires asmCaps["HasTDM"]; today always implies
|
|
# InMemorySwizzle MX scale layout).
|
|
# "BufferLoad": buffer_load_* (compatible with NoSwizzle or HostPreSwizzle MX scale layouts).
|
|
# "GlobalLoad": reserved for a future flat/global_load MX path; rejected for now.
|
|
# Internal sentinel (do NOT use in yamls):
|
|
# "Auto": triggers defaulting in Solution.assignDerivedParameters. The default is
|
|
# TDM iff TDMInst != 0, otherwise BufferLoad.
|
|
"MXLoadInst": ["Auto", "TDM", "BufferLoad", "GlobalLoad"],
|
|
# Cluster dimension. Clusters have up to 16 work-groups in a cluster, but each work-group in a
|
|
# cluster runs on a separate WGP.
|
|
"ClusterDim": validClusterDimensions,
|
|
# Enable PLR 0.5 to save vgprs
|
|
# 0: Disabled
|
|
# 1: Use PLR 0.5 for A
|
|
# 2: Use PLR 0.5 for B
|
|
# 3: Use PLR 0.5 for both A and B
|
|
"HalfPLR": [0, 1, 2, 3],
|
|
# Enable iterate-mode TDM
|
|
# -1: Auto. Enable per-tensor when LBSPP > 1024 B (exceeds pad_interval encoding).
|
|
# 0: Disabled
|
|
# 1: Use iterate-mode for A
|
|
# 2: Use iterate-mode for B
|
|
# 3: Use iterate-mode for both A and B
|
|
"TDMIterateMode": [-1, 0, 1, 2, 3]
|
|
}
|
|
|
|
newMIValidParameters = {
|
|
"EnableF32XdlMathOp": [False, True],
|
|
"UseF32XEmulation": [False, True],
|
|
'EnableMatrixInstruction': [False, True],
|
|
'ISA': -1,
|
|
'MFMA_BF16_1K': [False, True],
|
|
'MIBlock': -1,
|
|
'MIInputPerThread': -1,
|
|
'MIInputPerThreadA': -1,
|
|
'MIInputPerThreadB': -1,
|
|
'MIInputPerThreadMetadata': -1,
|
|
'MIWaveGroup': -1,
|
|
'MIWaveTile': -1,
|
|
'MatrixInstM': -1,
|
|
'MatrixInstN': -1,
|
|
'MatrixInstK': -1,
|
|
'MatrixInstB': -1,
|
|
'MatrixInstBM': -1,
|
|
'MatrixInstBN': -1,
|
|
'MatrixInstruction': -1,
|
|
'Sparse': -1,
|
|
'ThreadTile': -1,
|
|
'WavefrontSize': -1,
|
|
'WorkGroup': -1,
|
|
}
|
|
|
|
|
|
# Expected-type derivation for solution parameters --------------------------
|
|
#
|
|
# These helpers live here, next to `validParameters`, because that registry
|
|
# is the source of truth for "what types this parameter is allowed to take".
|
|
# Consumers (Solution.validateParameterTypes, and the input-YAML strict gate
|
|
# in checkParametersAreValid) import them from this module. Keeping the
|
|
# registry and its derived expected-types map co-located preserves the
|
|
# Common -> Solution import direction; the prior layout (defined inside
|
|
# Solution.py and imported back by ValidParameters extensions) would have
|
|
# introduced a Common -> Solution reverse import.
|
|
|
|
def _getExpectedTypes(validParams: dict[str, Union[int, list[Any], Any]]) -> dict[str, set[type]]:
|
|
"""Build a map from parameter name to the set of allowed Python types.
|
|
|
|
Uses the validParameters registry as the source of truth. For each
|
|
parameter whose allowed-value list is not the sentinel ``-1``, we
|
|
collect the concrete ``type()`` of every allowed value. Because
|
|
Python ``bool`` is a subclass of ``int``, we use ``type()`` (not
|
|
``isinstance``) so that ``bool`` and ``int`` are kept distinct.
|
|
|
|
Returns:
|
|
dict[str, set[type]]: e.g. {"UseCustomMainLoopSchedule": {int},
|
|
"BufferLoad": {bool}, ...}
|
|
"""
|
|
typeMap = {}
|
|
for name, allowedValues in validParams.items():
|
|
if isinstance(allowedValues, list):
|
|
if len(allowedValues) == 0:
|
|
raise ValueError(f"Invalid parameter value: {name} = {allowedValues}")
|
|
else: # Sentinel value -1 is allowed for all parameters
|
|
if allowedValues != -1:
|
|
raise ValueError(f"Invalid parameter value: {name} = {allowedValues}")
|
|
continue
|
|
|
|
typeMap[name] = set(type(v) for v in allowedValues)
|
|
return typeMap
|
|
|
|
# Pre-compute once at import time so the per-Solution cost is a dict lookup.
|
|
_expectedParamTypes = _getExpectedTypes(validParameters)
|
|
|
|
# Parameters to skip during type validation because YAML serialization
|
|
# inherently produces a different type (e.g. [9, 0, 10] -> list) and the
|
|
# conversion to the canonical type happens downstream in the pipeline.
|
|
# Also skip DataType* parameters as they are converted from strings/ints to
|
|
# DataType objects.
|
|
_skipTypeCheck = {
|
|
"ISA",
|
|
"DataType", "DataTypeA", "DataTypeB", "DataTypeC", "DataTypeD", "DataTypeE",
|
|
"MacDataTypeA", "MacDataTypeB",
|
|
"DataTypeAmaxD", "DataTypeAmaxC", "DataTypeAmaxA", "DataTypeAmaxB",
|
|
"DataTypeMXSA", "DataTypeMXSB",
|
|
"DestDataType", "ComputeDataType",
|
|
"F32XdlMathOp", # Also converted to DataType
|
|
}
|
|
|
|
|
|
def checkSpaceFillAlgoIsValid(name, value):
|
|
if type(value) != list:
|
|
msgBase = "Invalid parameter value: {} = {}\nMust be a list of values"
|
|
raise Exception(msgBase.format(name, value))
|
|
elif len(value) > 3:
|
|
msgBase = "Invalid parameter value: {} = {}\nOnly 3 level ordering supported"
|
|
raise Exception(msgBase.format(name, value))
|
|
else:
|
|
maxOrderID = 5
|
|
for orderId in value:
|
|
if orderId not in range(0,maxOrderID + 1): # pragma: no mutate
|
|
msgBase = "Invalid parameter value: {} = {}\nOrderID out of range"
|
|
raise Exception(msgBase.format(name, value))
|
|
|
|
def checkSpaceFillAlgoWGMIsValid(name, value):
|
|
if type(value) != list:
|
|
msgBase = "Invalid parameter value: {} = {}\nMust be a nested list of values"
|
|
raise Exception(msgBase.format(name, value))
|
|
elif len(value) > 2:
|
|
msgBase = "Invalid parameter value: {} = {}\nOnly 3 level ordering supported"
|
|
raise Exception(msgBase.format(name, value))
|
|
else:
|
|
for pair in value:
|
|
if len(pair) != 2:
|
|
msgBase = "Invalid parameter value: {} = {}\nMust be exactly 2 values per level"
|
|
raise Exception(msgBase.format(name, value))
|
|
for dim in pair:
|
|
if dim not in range(0,256): # pragma: no mutate
|
|
msgBase = "Invalid parameter value: {} = {}\nGridDim {} out of range [0,256)"
|
|
raise Exception(msgBase.format(name, value, dim))
|
|
|
|
|
|
def checkParametersAreValid(
|
|
param,
|
|
validParams,
|
|
*,
|
|
keyPathPrefix: str = "",
|
|
srcFile: str = "",
|
|
):
|
|
"""Ensures parameters in params exist and have valid values as specified by validParams.
|
|
|
|
In addition to the legacy name+value-membership checks, when the
|
|
parameter name has an entry in the derived ``_expectedParamTypes`` map
|
|
and is not in ``_skipTypeCheck``, each element of ``values`` is also
|
|
type-checked against the union of allowed-value types. ``type(value)``
|
|
is used (not ``isinstance``) so ``bool`` and ``int`` are distinguished.
|
|
Raises :class:`ConfigTypeError` immediately on the first type mismatch.
|
|
|
|
Args:
|
|
param: ``(name, values)`` tuple; ``values`` is the list of
|
|
candidate values.
|
|
validParams: registry to validate ``name``/``values`` against.
|
|
keyPathPrefix: dotted/bracketed prefix for the keypath in error
|
|
messages.
|
|
srcFile: YAML file path, used for src:line in messages.
|
|
"""
|
|
# Defer imports of the shared error machinery so this module stays
|
|
# importable in any context that doesn't already pull in Common.
|
|
from .TypeValidationErrors import (
|
|
ConfigTypeError,
|
|
formatMismatch,
|
|
)
|
|
|
|
(name, values) = param
|
|
if name == "ProblemSizes":
|
|
return
|
|
|
|
if name not in validParams:
|
|
raise Exception(
|
|
"Invalid parameter name: {}\nValid parameters are {}.".format(
|
|
name, sorted(validParameters.keys())
|
|
)
|
|
)
|
|
|
|
runTypeCheck = (
|
|
name in _expectedParamTypes
|
|
and name not in _skipTypeCheck
|
|
)
|
|
|
|
for idx, value in enumerate(values):
|
|
if validParams[name] != -1 and value not in validParams[name]:
|
|
msgBase = "Invalid parameter value: {} = {}\nValid values for {} are {}{}."
|
|
msgExt = (
|
|
" (only first 32 combos printed)\nRefer to Common.py for more info"
|
|
if len(validParams[name]) > 32
|
|
else ""
|
|
)
|
|
raise Exception(msgBase.format(name, value, name, validParams[name][:32], msgExt))
|
|
elif name == "SpaceFillingAlgo":
|
|
checkSpaceFillAlgoIsValid(name, value)
|
|
elif name == "SFCWGM":
|
|
checkSpaceFillAlgoWGMIsValid(name, value)
|
|
|
|
if runTypeCheck:
|
|
expectedTypes = _expectedParamTypes[name]
|
|
actualType = type(value)
|
|
if actualType not in expectedTypes:
|
|
# Build keypath. Lists of candidate values use [idx]; a
|
|
# single-element list (the common case for one value per
|
|
# key) still gets [0] so the location is unambiguous.
|
|
base = f"{keyPathPrefix}.{name}" if keyPathPrefix else name
|
|
keyPath = f"{base}[{idx}]" if len(values) > 1 else base
|
|
msg = formatMismatch(srcFile, keyPath, value, expectedTypes)
|
|
raise ConfigTypeError(msg)
|
|
|
|
|
|
def validateInternalSupportParams(
|
|
d,
|
|
srcFile: str = "",
|
|
keyPathPrefix: str = "InternalSupportParams",
|
|
):
|
|
"""Validate an InternalSupportParams dict against defaultInternalSupportParams.
|
|
|
|
Sibling to ``checkParametersAreValid`` rather than a fold-in (per the
|
|
plan's B3): ``checkParametersAreValid`` has a ``(name, list)``
|
|
contract and ``InternalSupportParams`` arrives as a dict, so folding
|
|
would break the function's signature.
|
|
|
|
Each key in ``d`` must exist in ``defaultInternalSupportParams`` and
|
|
have ``type(default)``. Mismatches and unknown-key errors are
|
|
Raises :class:`ConfigTypeError` on the first bad key encountered. Strict is
|
|
the only behaviour.
|
|
"""
|
|
if not d:
|
|
return
|
|
|
|
from .TypeValidationErrors import (
|
|
ConfigTypeError, formatMismatch,
|
|
)
|
|
|
|
# defaultInternalSupportParams lives in Common/GlobalParameters; import
|
|
# lazily because that module pulls in a lot.
|
|
from .GlobalParameters import defaultInternalSupportParams
|
|
|
|
for key, value in d.items():
|
|
if key not in defaultInternalSupportParams:
|
|
raise ConfigTypeError(
|
|
f"{keyPathPrefix}.{key}: unknown key. "
|
|
f"Valid keys are {sorted(defaultInternalSupportParams.keys())}."
|
|
)
|
|
default = defaultInternalSupportParams[key]
|
|
expectedTypes = {type(default)}
|
|
if type(value) not in expectedTypes:
|
|
raise ConfigTypeError(formatMismatch(srcFile, f"{keyPathPrefix}.{key}", value, expectedTypes))
|