// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #pragma once #include #include #include #include #include #include #include #include #include namespace rocRoller::Scheduling::LDSModel { enum class LdsDirection { Read, Write }; struct MemoryOpLDS { LdsDirection direction; }; using MemoryOp = std::variant; struct Summary { struct Banks { uint bankIndex; size_t workitemsAccessed; bool imbalanced; }; struct Access { int ldsTag; std::vector accessedBanks; std::vector> banksToWorkitems; }; std::map tagToAccess; std::set imbalancedTags; std::string toString() const; }; std::ostream& operator<<(std::ostream& stream, Summary const& summary); struct RuntimeLDSInstruction { MemoryOpLDS memoryOp; int dwords; std::vector baseAddresses; std::string toString() const; }; std::ostream& operator<<(std::ostream& stream, RuntimeLDSInstruction const& instruction); struct OperationAccesses { int operationTag; int ldsTag; std::vector instructions; }; struct OperationsAnalysis { std::map tagToAccess; GPUArchitectureGFX gfx; std::string toString() const; }; std::ostream& operator<<(std::ostream& stream, OperationsAnalysis const& operationAnalysis); struct LDSBankAccess { int operationTag; int ldsTag; LdsDirection direction; uint workitem; uint bankIndex; }; /** * @brief Calculate the data cycles for an LDS instruction * * I.e. how long does the instruction take once it is at the front of the LDS queue? * * @param instr The LDS instruction * @param gfx The GPU architecture */ uint getInstructionDataCycles(const RuntimeLDSInstruction& instr, GPUArchitectureGFX gfx); /** * @brief Get the number of cycles to issue an instruction (stalls excluded) * * I.e. how long does instruction take to enter the queue (issue) when there's sufficient space in the LDS queue? * * @param memoryOp The memory operation (read or write) * @param dwords Number of dwords being accessed (1 for b32, 2 for b64, 3 for b96, 4 for b128) * @return Number of cycles to issue the instruction */ uint getInstructionIssueCycles(const MemoryOpLDS& memoryOp, uint dwords); /** * @brief Get the equilibrium cycles when continuously issuing the same instruction * * @param instr The LDS instruction with memory operation details and addresses * @param gfx The GPU architecture * @return Total number of cycles for the instruction */ uint getInstructionCycles(const RuntimeLDSInstruction& instr, GPUArchitectureGFX gfx); /** * @brief Get the number of LDS banks for a given GPU architecture and memory operation * * @param gfx The GPU architecture * @param memoryOp The memory operation (read or write) * @param dwords Number of dwords (1 for b32, 2 for b64, 3 for b96, 4 for b128) * @return Number of LDS banks (64 for ds_read_b64/b128 on GFX950, otherwise 32) */ uint getNumLDSBanks(GPUArchitectureGFX gfx, const MemoryOpLDS& memoryOp, uint dwords); /** * @brief Returns the number of threads that can operate per clock for a given memory operation * * @param memoryOp Read/write * @param dwords Number of dwords (1 for b32, 2 for b64, 3 for b96, 4 for b128) * @param gfx The GPU architecture */ uint getThreadsPerClock(const MemoryOpLDS& memoryOp, uint dwords, GPUArchitectureGFX gfx); /** * @brief Divide addresses into thread groups based on threadsPerClock limit * * @param addresses Vector of base addresses to divide into groups * @param threadsPerClock Maximum number of threads that can operate per clock * @return Vector of thread groups, each containing addresses for that group */ std::vector> divideIntoThreadGroups(const std::vector& addresses, uint threadsPerClock); /** * @brief Determines how many addresses accesses each bank * * For multi-dword accesses, includes addresses at starting at the baseAddress and * extending for dwords number of dwords. * * @param baseAddresses Vector of base addresses. * @param dwords Number of dwords accessed starting from each base address * @param gfx The GPU architecture * @param memoryOp The memory operation (read or write) * @return A map, where the key is the bank index and the value is the count of addresses accessing that bank */ std::map createBankToAddressCounts(const std::vector& baseAddresses, uint dwords, GPUArchitectureGFX gfx, const MemoryOpLDS& memoryOp); /** * @brief Calculate the number of clock cycles needed to resolve bank conflicts * * Simulates the bank conflict resolution process where only one address per bank * can be serviced per clock cycle. * * @param bankToAddressCounts Map from bank index to count of addresses accessing that bank * @return Number of clock cycles */ uint calculateBankConflictCycles(const std::map& bankToAddressCounts); /** * @brief Compute bank-to-address-count mappings for each thread group * * This function divides the instruction's base addresses into thread groups * and computes the bank-to-address-count mapping for each group. * * @param instr The LDS instruction containing memory operation details and addresses * @param gfx The GPU architecture * @return Vector of bank-to-address count mappings, one per thread group */ std::vector> computeThreadGroupBankMappings(const RuntimeLDSInstruction& instr, GPUArchitectureGFX gfx); /** * @brief Calculate total cycles from bank mappings * * Takes the per-thread-group bank mappings and calculates the total * number of cycles needed, including bank conflict resolution * * @param threadGroupBankMappings Vector of bank to address count mappings * @return Total number of clock cycles */ uint calculateTotalCyclesFromBankMappings( const std::vector>& threadGroupBankMappings); /** * @brief Check if an instruction's wave accesses non-overlapping banks across cycles * * The hardware will optimize this by having the waves access out-of-phase, * resulting in no penalty from bank conflicts in cases with more than 1 active SIMD. * * @param instr The LDS instruction containing memory operation details and addresses * @param gfx The GPU architecture * @return true if each cycle accesses different banks (no overlap), false otherwise */ bool hasNonOverlappingBankAccess(const RuntimeLDSInstruction& instr, GPUArchitectureGFX gfx); /** * @brief Parse LDS instruction information from opcode * * @param opCode The opcode string (e.g., "ds_read_b64", "ds_write_b128") * @return Optional pair of (LdsDirection, dwords) if opcode is supported, nullopt otherwise */ std::optional> getLdsInfoFromOpcodeIfSupported(const std::string& opCode); /** * @brief Calculate queue slots required for an LDS operation * * @param direction Read or Write operation * @param dwords Number of dwords (1-4) * @return Number of queue slots required */ int getQueueSlotsRequired(LdsDirection direction, int dwords); /** * @brief LDS Scheduler for managing LDS instruction scheduling and queue state * * This class encapsulates the scheduling logic for LDS instructions, * managing command and data queues, tracking conflicts, and predicting stalls. * It can be used independently of the observer system. * * Note: This scheduler works in hardware cycles, not quadcycles. */ class LDSModule { public: LDSModule(GPUArchitectureGFX gfx, int waveCount); void incrementProgramCycleBy(int cycles); int getProgramCycle() const { return m_programCycle; } void reset(); // Returns tuple of (stall_cycles, additional_cycles) std::tuple predictCycles(const RuntimeLDSInstruction& instr) const; void scheduleInstruction(const RuntimeLDSInstruction& instr); int predictWaitcntStallCycles(int waitcnt) const; void updateQueues(); private: // gfx942 to gfx950 // actually 40 deep 16 entries wide -> simplify to 10 deep 64 entries wide static constexpr int dataQueueSize = 10; static constexpr int commandQueueSize = 8; // Before gfx942, data queue should be 64 deep 16 entries wide, command queue remains same GPUArchitectureGFX m_gfx; int m_programCycle; int m_multiplierQueueSlots; int m_multiplierWaveCount; std::deque m_commandQueue; std::deque m_waitcntQueue; // Includes round-trip delay std::deque m_dataQueue; int getRemainingDataSlots() const; }; }