// Copyright Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT #pragma once #include #include #include #include namespace rocRoller { namespace Scheduling { template using WaitQueueMap = std::unordered_map; using WaitQueueRegisters = std::array; using WaitCntQueues = WaitQueueMap>; /** * @brief This struct is used to store the _unallocated_ state of the waitcnt queues. * */ struct WaitcntState { public: WaitcntState(); WaitcntState(WaitQueueMap const& needsWaitZero, WaitQueueMap const& typeInQueue, WaitCntQueues const& instruction_queues_with_alloc); bool operator==(const WaitcntState& rhs) const = default; /** * @brief Returns true if this WaitcntState and the provided WaitcntState are safe * to see at a label and a branch that goes to that label. * * @param labelState */ void assertSafeToBranchTo(const WaitcntState& labelState, std::string const& label) const; private: // These members are duplicates of the waitcntobserver members, except we're storing a // std::vector for the registers instead of // std::array // so that we don't maintain the allocations. WaitQueueMap>> m_instructionQueues; WaitQueueMap m_needsWaitZero; WaitQueueMap m_typeInQueue; }; class WaitcntObserver { public: WaitcntObserver(); WaitcntObserver(ContextPtr context); InstructionStatus peek(Instruction const& inst) const; void modify(Instruction& inst) const; /** * This function handles updating the waitqueues and flags when an instruction is scheduled. * 1. If there is a wait included in the instruction, apply it to the queues. * 2. Add n copies of the instruction's registers to all the queues that it affects, where n is its waitcnt. * 3. If the wait is zero, or the queue already has an instruction of a different type, set the m_needsWaitZero flag. * 4. Set the instruction type flag for each queue the instruction affects. **/ void observe(Instruction const& inst); constexpr static bool required(GPUArchitectureTarget const& target); private: std::weak_ptr m_context; WaitCntQueues m_instructionQueues; // This member tracks a flag for each queue which indicates that a waitcnt 0 is needed. WaitQueueMap m_needsWaitZero; // This member tracks the instruction type that is currently in a given queue. // If there are ever multiple instruction types in a queue, and a register intersection occurs, // a waitcnt 0 is required. WaitQueueMap m_typeInQueue; std::string m_barrierOpcode; bool m_includeExplanation; bool m_displayState; // This member tracks, for every label, what the waitcnt state was when that label was encountered. std::unordered_map m_labelStates; // This member tracks, for every label, what the waitcnt state is everywhere a branch instruction targets that label. std::unordered_map> m_branchStates; /** * This function updates the given wait queue by applying the given waitcnt. **/ void applyWaitToQueue(int waitCnt, GPUWaitQueue queue); /** * Determines if there's a waitcount that's needed due to a barrier if the * alwaysWaitZeroBeforeBarrier kernelOption is set. * * @param inst * @param[out] explanation is an output parameter for an explanation of the wait count required. * @return WaitCount */ WaitCount computeZeroBarrierWaitCount(Instruction const& inst, std::string* explanation = nullptr) const; /** * Determines if there's a waitcount that's needed due to the instruction having a * WaitCount::SyncQueue() wait attached. * * @param inst * @param[out] explanation is an output parameter for an explanation of the wait count required. * @return WaitCount */ WaitCount computeSyncQueueWaitCount(Instruction const& inst, std::string* explanation = nullptr) const; /** * Determines if there's a waitcount that's needed due to a register dependency * intersection. * * It searches backwards through each wait queue looking for registers that * intersect with the new instruction. * If an intersection is found a wait is inserted for the intersection location * or 0 if the wait_zero flag is set for the queue. * * @param inst * @param[out] explanation is an output parameter for an explanation of the wait count required. * @return WaitCount */ WaitCount computeRegisterWaitCount(Instruction const& inst, std::string* explanation = nullptr) const; /** * @brief This function determines if an instruction needs a wait count inserted before * it and provides an explanation as to why it's needed. * * @param inst * @param[out] explanation is an output parameter for an explanation of the wait count required. * @return WaitCount */ WaitCount computeWaitCount(Instruction const& inst, std::string* explanation = nullptr) const; /** * @brief Get a string representation of the state of the Wait Queues * * @return std::string */ std::string getWaitQueueState() const; /** * @brief Add the current waitcnt state to the label state tracker. * * @param label the currently encountered label. */ void addLabelState(std::string const& label); /** * @brief Add the current waitcnt state to the branch state tracker. * * @param label the label currently being branched to. */ void addBranchState(std::string const& label); /** * @brief Assert that all branch and label waitcnt states are consistent. * * This function checks that the waitcnt state at every branch to a label is the same * as the waitcnt state when that label was encountered. */ void assertLabelConsistency(); }; static_assert(CObserverConst); } } #include