# Copyright Advanced Micro Devices, Inc., or its affiliates. # SPDX-License-Identifier: MIT """gfx1250 LDS segment-conflict interleave oracle. Pure function of `state` that decides whether a wave-separated TDM kernel should split its A/B halves across LDS segments (so the two MFMA read ports hit different segments), and returns the offsets the emit sites consume. """ # gfx1250 LDS segment size (5 x 64 KiB segments). SEG = 65536 def _bpe(state): # DataType is a DataType object, not a string; numBytes() is 2.0 for bf16. return int(state["ProblemType"]["DataType"].numBytes()) def _pad(x, blk, padElems, bpe): if blk == 0 or padElems == 0: return 0 return (x // blk) * (padElems * bpe) def _data_bytes(state, tc): numComp = state["NumWaves"] // 2 mt = state["MacroTile0"] if tc == "A" else state["MacroTile1"] return (mt // numComp) * state["DepthU"] * _bpe(state) def _footprint(state, tc): d = _data_bytes(state, tc) blk = state["LdsBlockSizePerPad%s" % tc] padElems = state["LdsPad%s" % tc] return d + _pad(d, blk, padElems, _bpe(state)) def _mx_scale_bases(state, mxsaStart): """MX scale-block LDS bases, placed after the interleaved A/B region. Returns (ldsBaseMXSA, ldsBaseMXSB, end); a base is None when that scale is not LDS-resident.""" pt = state["ProblemType"] hasA = bool(pt.get("MXBlockA")) and not state.get("DirectToVgprMXSA") hasB = bool(pt.get("MXBlockB")) and not state.get("DirectToVgprMXSB") if not (hasA or hasB): return None, None, mxsaStart baseA = mxsaStart szA = int(state.get("LdsNumElementsAlignedMXSA", 0)) if hasA else 0 baseB = baseA + szA szB = int(state.get("LdsNumElementsAlignedMXSB", 0)) if hasB else 0 return (baseA if hasA else None), (baseB if hasB else None), baseB + szB def _coarse_vw(state): # A must cover a full component (never crosses one). B may be narrower, as long as its column # span (vIdxColsB below) divides compColsB evenly so B reads split on component boundaries. numComp = state["NumWaves"] // 2 mi_threads = min(state["MatrixInstM"], state["MatrixInstN"]) coarseA = mi_threads * state["VectorWidthA"] >= state["MacroTile0"] // numComp if not coarseA: return False coarseB = mi_threads * state["VectorWidthB"] >= state["MacroTile1"] // numComp if coarseB: return True compColsB = state["MacroTile1"] // numComp vIdxColsB = state["MatrixInstN"] * state.get("MatrixInstBN", 1) * state["MIWaveGroup"][1] * state["VectorWidthB"] return vIdxColsB > 0 and compColsB % vIdxColsB == 0 def _no(reason): return {"applicable": False, "aligned": False, "offsets": None, "blockSpan": 0, "reason": reason, "segmentMap": ""} def _ceil_seg(x): return ((x + SEG - 1) // SEG) * SEG def aligned_budget_ok(blockSpan, numLdsBlk, naturalOffsetBlk, maxLDS): """Return (ok, per-buffer block) for the aligned branch: the block is the next power of two >= max(naturalOffsetBlk, blockSpan), valid only if double-buffering it fits MaxLDS.""" if numLdsBlk != 2: return (False, None) offsetBlk = max(naturalOffsetBlk, blockSpan) if offsetBlk <= 0: return (False, None) roundup = 1 << (offsetBlk - 1).bit_length() # next power of two if roundup * 2 > maxLDS: return (False, None) # total = roundup + blockSpan <= roundup*2 return (True, roundup) def evaluate(state): pt = state["ProblemType"] # Tri-state knob: -1 = auto (default), 0 = force baseline, 1 = force on where applicable. # Auto takes only the no-trade-off tight branch; the LDS-growing aligned branch needs 1. mode = state.get("LDSSegmentInterleave", -1) if mode == 0: return _no("parameter off") if tuple(state.get("ISA", ()))[:2] != (12, 5): return _no("not gfx1250") if not (state.get("enableTDMA") and state.get("enableTDMB") and state["NumWaves"] > 1): return _no("not wave-separated TDM") if state.get("LocalSplitU", 1) > 1: return _no("LocalSplitU>1") if not state.get("UnrollMajorLDSA") or not state.get("UnrollMajorLDSB"): return _no("not unrollMajor") if state["NumWaves"] // 2 != 2: return _no("numComp!=2") # Both write (WaveIdx//2 -> 2 comps) and read (wtid0*stride, num1DWaves=MIWaveGroup dim) # assume exactly 2 waves per MFMA dim. MIWaveGroup!=[2,2] (e.g. [4,1]) loses the component # jump on the dim==1 tensor and reads OOB on the dim==4 one. if list(state.get("MIWaveGroup", [])) != [2, 2]: return _no("MIWaveGroup!=[2,2]") if state.get("TDMSplit") or pt.get("Sparse"): return _no("split/sparse") # Subtile uses a separate codegen body; the emit path these offsets target runs only for # non-subtile kernels. if state.get("UseSubtileImpl"): return _no("subtile") # Needs double-buffering; 1LDSBuffer==1 breaks the assumed layout. Unresolved -1 is rejected too # (Solution.py resolves it later, then re-evaluates). if state.get("1LDSBuffer", 0) != 0: return _no("needs 1LDSBuffer==0") _dt = pt["DataType"] # fp8 covers mxf8; its MX scales are relocated as a trailing block (see _mx_scale_bases). if not (_dt.isBFloat16() or _dt.isHalf() or _dt.is8bitFloat()): return _no("bf16/fp16/fp8 only") if not _coarse_vw(state): return _no("fine VW") fA, fB = _footprint(state, "A"), _footprint(state, "B") base = state["LdsOffsetA"] bpe = _bpe(state) if (base % SEG) + fA + fB < SEG: # Small MacroTile: A0,B0 fit one segment, so push component 1 to the next segment boundary # with a segment-aligned stride. Grows LDS (Solution.py budget-checks); PGR2 double-buffer only. if state.get("PrefetchGlobalRead") != 2: return _no("small MT: PGR!=2") if mode == -1: return _no("auto: skip aligned (LDS growth)") pre = _ceil_seg(base + fA + fB) - base # segment-aligned stride (== SEG for base A1/B1 land exactly at the previous tile's end). No LDS growth. offsets = { "ldsBaseB": base + fA, # B0 right after A0 "writeStrideBytes": fA + fB, # footprint stride (post-pad), no re-pad on the jump "readWaveStride": (fA + fB) // bpe, # same, in elements "footprintPacked": True, } # mxf8: put the scale block after B1. Uses no more LDS than the non-interleaved layout. bMXSA, bMXSB, _ = _mx_scale_bases(state, base + 2 * (fA + fB)) if bMXSA is not None: offsets["ldsBaseMXSA"] = bMXSA if bMXSB is not None: offsets["ldsBaseMXSB"] = bMXSB a0 = base // SEG a1 = (base + fA + fB) // SEG # tight branch guarantees a1 > a0 seg_map = "CLEAN seg%d={A0,B0} seg%d={A1,B1}" % (a0, a1) return {"applicable": True, "aligned": False, "offsets": offsets, "blockSpan": 0, "reason": "tight", "segmentMap": seg_map}