/******************************************************************************* * * MIT License * * Copyright (C) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * *******************************************************************************/ #pragma once #include #include #include #include #include #include #include #include #include #include namespace TensileLite { /** * \addtogroup SolutionLibrary * @{ */ template using LibraryEntry = std::shared_ptr>; template using LibraryRow = std::pair>; /** * Represents a set of sub-libraries, each with associated predicates. It * should be placed in order of best to worst solutions. We assume the best * solution is the first one where we match the predicates. * * Examples: Picking solutions written for a particular GPU, solutions that * assume that a particular size is a multiple of something, etc. */ template struct ExactLogicLibrary : public SolutionLibrary { using Row = LibraryRow; std::vector rows; mutable std::atomic lastFindTopRetAll = false; ExactLogicLibrary() = default; ExactLogicLibrary(std::initializer_list init) : rows(init) { } ExactLogicLibrary(std::vector const& init) : rows(init) { } virtual std::shared_ptr getSolutionByIndex(MyProblem const& problem, Hardware const& hardware, const int index) const override { std::shared_ptr rv; const bool streamK = Debug::Instance().useExperimentalSelection() == 2; for(auto const& row : rows) { if(row.first.value->type() == "ExperimentalStreamK" && !streamK) continue; if(row.first(problem, hardware)) { rv = row.second->getSolutionByIndex(problem, hardware, index); if(rv) return rv; } } return rv; } virtual std::shared_ptr findBestSolution(MyProblem const& problem, Hardware const& hardware, double* fitness = nullptr) const override { std::shared_ptr fallbackRv; const bool streamK = Debug::Instance().useExperimentalSelection() == 2; // Return exact match immediately; otherwise keep first successful fallback. for(auto const& row : rows) { if(row.first.value->type() == "ExperimentalStreamK" && !streamK) continue; if(!row.first(problem, hardware)) continue; if(!row.first.isFallbackMatch(hardware)) { auto rv = row.second->findBestSolution(problem, hardware, fitness); if(rv && dynamic_cast( row.first.value.get())) rv->tag = MySolution::MatchingTag::Equal; if(rv) { if(Debug::Instance().printDeviceSelection()) { std::cout << " Solution found (exact): " << rv->name() << " [MatchingTag: " << rv->matchingTag() << "]" << std::endl; } return rv; } } else if(!fallbackRv) { fallbackRv = row.second->findBestSolution(problem, hardware, fitness); if(fallbackRv && dynamic_cast( row.first.value.get())) fallbackRv->tag = MySolution::MatchingTag::Equal; } } if(fallbackRv && Debug::Instance().printDeviceSelection()) { std::cout << " Solution found (fallback): " << fallbackRv->name() << " [MatchingTag: " << fallbackRv->matchingTag() << "]" << std::endl; } return fallbackRv; } public: virtual SolutionSet findAllSolutions(MyProblem const& problem, Hardware const& hardware, SolutionLibrarySearchType searchType = SolutionLibrarySearchType::DEFAULT) const override { SolutionSet rv; const bool debug = Debug::Instance().printPropertyEvaluation(); const bool streamK = Debug::Instance().useExperimentalSelection() == 2; const auto& excludedLib = Debug::Instance().excludedLibFromGetAll(); auto amdGPU = static_cast(&hardware); bool isStandardCUDevice = amdGPU->isStandardCU(); for(auto const& row : rows) { // we want to exclude this lib from getAll. If the Set is not empty, // it means we want to skip searched GridBased, Prediction. // But if this is a non-standard-CU GPU, we still need to search all for CU-Fallback solutions. // In this case, we don't skip the excludedLib. if(!excludedLib.empty() && isStandardCUDevice) { if(excludedLib.count(row.first.value->type())) continue; } if(row.first.value->type() == "ExperimentalStreamK" && !streamK) continue; if(row.first.value->type() == "AMDGPU" && !row.first(problem, hardware)) continue; auto rowSolutions = row.second->findAllSolutions(problem, hardware, searchType); // hipblaslt_ext::matmulIsTuned() -> rocblaslt_matmul_is_tuned() needs this Equal test if(dynamic_cast(row.first.value.get())) { for(auto& sol : rowSolutions) sol->tag = MySolution::MatchingTag::Equal; } // except for Equal, we test others only when debug mode. else if(debug) { if(dynamic_cast( row.first.value.get())) { for(auto& sol : rowSolutions) sol->tag = MySolution::MatchingTag::GridBased; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : rowSolutions) sol->tag = MySolution::MatchingTag::Range; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : rowSolutions) sol->tag = MySolution::MatchingTag::FreeSize; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : rowSolutions) sol->tag = MySolution::MatchingTag::Prediction; } // TODO- Experimental? } rv.insert(rowSolutions.begin(), rowSolutions.end()); } return rv; } virtual SolutionSet findAllSolutionsGroupedGemm(std::vector const& problems, Hardware const& hardware, SolutionLibrarySearchType searchType = SolutionLibrarySearchType::DEFAULT) const override { SolutionSet rv; for(auto const& row : rows) { if(row.first.value->type() == "AMDGPU" && !row.first(problems[0], hardware)) continue; auto rowSolutions = row.second->findAllSolutionsGroupedGemm(problems, hardware, searchType); rv.insert(rowSolutions.begin(), rowSolutions.end()); } return rv; } virtual std::string description() const override { return concatenate(this->type(), " library (", rows.size(), " rows)"); } virtual SolutionVector findTopSolutions(MyProblem const& problem, Hardware const& hardware, int numSolutions) const override { SolutionVector rv, solutions; const bool debug = Debug::Instance().printPropertyEvaluation(); const bool streamK = Debug::Instance().useExperimentalSelection() == 2; const auto forceDynamic = Debug::Instance().streamK5ForceMode(); const bool effectiveDynamic = (forceDynamic == 1) || (forceDynamic != 0 && problem.getParams().streamKTileSchedulingMode() != 0); const bool predictionLib = Debug::Instance().usePredictionLibrary() || effectiveDynamic; // false in case of early return; lastFindTopRetAll = false; for(auto const& row : rows) { if(row.first.value->type() == "ExperimentalStreamK" && !streamK) continue; if(predictionLib && ((row.first.value->type() == "EqualityMatching") || (row.first.value->type() == "RangeMatching"))) continue; if(row.first(problem, hardware)) { solutions = row.second->findTopSolutions(problem, hardware, numSolutions - rv.size()); // hipblaslt_ext::matmulIsTuned() -> rocblaslt_matmul_is_tuned() needs this Equal test if(dynamic_cast( row.first.value.get())) { for(auto& sol : solutions) sol->tag = MySolution::MatchingTag::Equal; } // except for Equal, we test others only when debug mode. else if(debug) { if(dynamic_cast( row.first.value.get())) { for(auto& sol : solutions) sol->tag = MySolution::MatchingTag::GridBased; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : solutions) sol->tag = MySolution::MatchingTag::Range; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : solutions) sol->tag = MySolution::MatchingTag::FreeSize; } else if(dynamic_cast( row.first.value.get())) { for(auto& sol : solutions) sol->tag = MySolution::MatchingTag::Prediction; } // TODO- Experimental } rv.insert(std::end(rv), std::begin(solutions), std::end(solutions)); if(rv.size() == numSolutions) return rv; } } // can't reach the requested number, means findTop already done its best lastFindTopRetAll = (rv.size() < numSolutions); return rv; } virtual bool lastFindTopAlreadyRetAll() const override { return lastFindTopRetAll; } virtual SolutionVector findTopSolutionsGroupedGemm(std::vector const& problems, Hardware const& hardware, int numSolutions) const override { SolutionVector rv, solutions; for(auto const& row : rows) { if(row.first(problems[0], hardware)) { solutions = row.second->findTopSolutionsGroupedGemm( problems, hardware, numSolutions - rv.size()); rv.insert(std::end(rv), std::begin(solutions), std::end(solutions)); if(rv.size() == numSolutions) return rv; } } return rv; } }; struct HardwarePredicate { std::shared_ptr> value; // The chip IDs this predicate targets, if any. Set by callers that // know the chip IDs (e.g. makeHwPred, deserialization). When empty // the predicate has no chip-ID constraint and every match is exact. std::set targetPciChipIds; HardwarePredicate() = default; HardwarePredicate(std::shared_ptr> init, std::optional chipId = std::nullopt) : value(init) { if(chipId) targetPciChipIds.insert(chipId.value()); } HardwarePredicate(std::shared_ptr> init, std::set chipIds) : value(init) , targetPciChipIds(chipIds) { } template bool operator()(Any const& problem, Hardware const& hardware) const { bool debug = Debug::Instance().printDeviceSelection(); bool rv = (*value)(hardware); if(debug) { PredicateDebugger::printHeader(std::cout, "ExactLogic: Hardware"); value->debugEval(hardware, std::cout); PredicateDebugger::printFooter(std::cout, rv); } return rv; } // A match is a fallback when the predicate targets a specific chip ID // and the GPU's chip ID is different (operator() already confirmed // compatibility via ChipIdRegistry::canUseSolution). bool isFallbackMatch(Hardware const& hardware) const { if(targetPciChipIds.empty()) return false; auto gpuChipId = hardware.pciChipId(); if(!gpuChipId) return false; return targetPciChipIds.count(gpuChipId.value()) == 0; } }; template struct HardwareSelectionLibrary : public ExactLogicLibrary { using Base = ExactLogicLibrary; HardwareSelectionLibrary() = default; HardwareSelectionLibrary(std::initializer_list init) : Base(init) { } HardwareSelectionLibrary(std::vector const& init) : Base(init) { } static std::string Type() { return "Hardware"; } virtual std::string type() const override { return Type(); } }; template struct ProblemPredicate { std::shared_ptr> value; ProblemPredicate() = default; ProblemPredicate(std::shared_ptr> init) : value(init) { } bool operator()(MyProblem const& problem, Hardware const& hardware) const { bool debug = Debug::Instance().printPredicateEvaluation(); bool rv = (*value)(problem); if(debug) { PredicateDebugger::printHeader(std::cout, "ExactLogic: Problem"); value->debugEval(problem, std::cout); PredicateDebugger::printFooter(std::cout, rv); } return rv; } // Problem predicates never involve hardware fallback. bool isFallbackMatch(Hardware const&) const { return false; } }; template struct ProblemSelectionLibrary : public ExactLogicLibrary> { using Base = ExactLogicLibrary>; ProblemSelectionLibrary() = default; ProblemSelectionLibrary(std::initializer_list init) : Base(init) { } ProblemSelectionLibrary(std::vector const& init) : Base(init) { } static std::string Type() { return "Problem"; } virtual std::string type() const { return Type(); } }; /** * @} */ } // namespace TensileLite