/******************************************************************************* * * MIT License * * Copyright (C) 2022-2026 Advanced Micro Devices, Inc. All rights reserved. * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE * SOFTWARE. * *******************************************************************************/ #pragma once #include #include #include #include #include #include #include #include #include #include namespace fs = std::filesystem; namespace TensileLite { /** * \ingroup SolutionLibrary */ template using SolutionMap = std::map>; template struct LibraryIOContext { std::string filename; std::vector preloaded; // If lazy loading is used, this may be updated in const functions SolutionMap* solutions; std::mutex* solutionsGuard; std::set* loadedFiles; void* indexLoadedLibraries; }; /** * \ingroup SolutionLibrary * * Root level library object. Contains all individual solutions in a map * for serialization purposes. */ template struct MasterSolutionLibrary : public SolutionLibrary { static std::string Type() { return "Master"; } std::string type() const override { return Type(); } std::string description() const override { if(library == nullptr) return concatenate( type(), " (", solutions.size(), " solutions, next level: nullptr)"); else return concatenate(type(), " (", solutions.size(), " solutions, next level: ", library->type(), ")"); } std::string libraryDirectory; std::string suffix; std::map libraryMapping; mutable std::set loadedFiles; mutable std::map>> indexLoadedLibraries; std::shared_ptr> library; mutable SolutionMap solutions; std::string version; mutable std::mutex solutionsGuard; mutable std::atomic lastFindTopRetAll = false; MasterSolutionLibrary() = default; bool initLibraryMapping(const std::string& tensileLibPath) { // Invariant: tensileLibPath is the logical library name ending in a // single extension (".dat" or ".yaml"), never the compressed // ".dat.zlib" name. fileToMsgObject() resolves the ".zlib" variant // internally by appending it as a probe suffix, so callers always // pass the bare ".dat" name. extension()/stem() below rely on this: // a double-extension name would yield suffix ".zlib" and an arch of // ".dat", producing a wrong per-arch mapping path. fs::path path(tensileLibPath); libraryDirectory = path.parent_path().string(); suffix = path.extension().string(); // Derive the arch from the master library filename, e.g. // "TensileLibrary_lazy_gfx1100.dat" -> "gfx1100", and load the // matching per-arch mapping file. Per-arch mapping files are // required so single-arch shard installs don't collide on a // shared "TensileLiteLibrary_lazy_Mapping.dat". std::string stem = path.stem().string(); const std::string prefix = "TensileLibrary_lazy_"; if(stem.compare(0, prefix.size(), prefix) != 0) { std::cout << "Cannot derive arch from " << tensileLibPath << std::endl; return false; } const std::string arch = stem.substr(prefix.size()); path = fs::path(libraryDirectory) / ("TensileLiteLibrary_lazy_" + arch + "_Mapping.dat"); libraryMapping = LoadLibraryMapping(path.string()); if(libraryMapping.empty()) { std::cout << "No library mapping found at " << path.string() << std::endl; return false; } return true; } void loadLibrary(const int index) const { // TODO(#7080): point-key + upper_bound misses on above-largest and // gap-between-keys; switch to range-encoded mapping. auto it = libraryMapping.upper_bound(index); if(it == libraryMapping.begin()) { if(Debug::Instance().printDataInit()) { std::cout << "Index " << index << " not in this arch's mapping range" << std::endl; } return; } --it; std::string filePrefix = it->second; // load the file here directly and push the library for later use. { std::lock_guard lock(solutionsGuard); if(loadedFiles.find(filePrefix) != loadedFiles.end()) { return; } } if(Debug::Instance().printDataInit()) { std::cout << "Loading library for index " << index << " from file: " << filePrefix << std::endl; } fs::path path(libraryDirectory); path = path / (filePrefix + suffix); auto newLibrary = LoadLibraryFile(path.string()); auto mLibrary = static_cast*>(newLibrary.get()); using std::begin; using std::end; std::lock_guard lock(solutionsGuard); if(loadedFiles.find(filePrefix) != loadedFiles.end()) { return; } // Push to cache indexLoadedLibraries[filePrefix] = mLibrary->library; std::transform(begin(mLibrary->solutions), end(mLibrary->solutions), std::inserter(solutions, end(solutions)), [this, filePrefix](auto& i) { i.second->codeObjectFilename = filePrefix + ".co"; return i; }); loadedFiles.insert(filePrefix); if(Debug::Instance().printCodeObjectInfo()) { std::cout << "load placeholder library " << path << std::endl << mLibrary->solutions.size() << " solutions loaded" << std::endl; } } virtual std::shared_ptr getSolutionByIndex(MyProblem const& problem, Hardware const& hardware, const int index) const override { std::lock_guard lock(solutionsGuard); if(solutions.find(index) == solutions.end()) { return std::shared_ptr(); } auto solution = solutions.at(index); if(solution->requiredHostWorkspaceSizePerProblem == static_cast(-1)) { solution->requiredHostWorkspaceSizePerProblem = solution->requiredHostSizeGroupedGemmSingle(problem, hardware); } return solution; } virtual std::shared_ptr getSolutionByIndex(Hardware const& hardware, const int index) const override { loadLibrary(index); std::lock_guard lock(solutionsGuard); if(solutions.find(index) == solutions.end()) { return std::shared_ptr(); } auto solution = solutions.at(index); TensileLite::TensorOps nop; if(solution->requiredHostWorkspaceSizePerProblem == static_cast(-1)) { const auto& pt = solution->problemType; bool isComplexInput = (pt.aType == rocisa::DataType::ComplexFloat || pt.aType == rocisa::DataType::ComplexDouble); rocisa::DataType alphaBetaType = isComplexInput ? pt.aType : pt.computeType; auto problem = MyProblem::createDefaultProblem(solution->problemType.transA, solution->problemType.transB, solution->problemType.aType, solution->problemType.bType, solution->problemType.cType, solution->problemType.dType, alphaBetaType, alphaBetaType, solution->problemType.computeInputTypeA, solution->problemType.computeInputTypeB, solution->problemType.computeType, 1.0, 1.0, solution->problemType.useBias, solution->problemType.useGradient, solution->problemType.biasDataTypeWhiteList, solution->problemType.biasSrcWhiteList, solution->problemType.groupedGemm, std::numeric_limits::max(), nop, nop, nop, nop, solution->problemType.useGateResidual, solution->problemType.gateResidualDataTypeWhiteList); solution->requiredHostWorkspaceSizePerProblem = solution->requiredHostSizeGroupedGemmSingle(problem, hardware); } return solution; } virtual std::shared_ptr findBestSolution(MyProblem const& problem, Hardware const& hardware, double* fitness = nullptr) const override { if(Debug::Instance().printSolutionSelectionTime()) { auto start = std::chrono::steady_clock::now(); auto result = findBestSolution_runner(problem, hardware, fitness); auto end = std::chrono::steady_clock::now(); double time = std::chrono::duration(end - start).count(); std::cout << "Solution selection time: " << time << " us" << std::endl; return result; } else { return findBestSolution_runner(problem, hardware, fitness); } } std::shared_ptr findBestSolution_runner(MyProblem const& problem, Hardware const& hardware, double* fitness = nullptr) const { const int solution_index = Debug::Instance().getSolutionIndex(); std::shared_ptr rv; if(solution_index >= 0) { std::cout << "Tensile will use solution index: " << solution_index << std::endl; std::cout << "Warning: Tensile will only work for a particular transpose and data type." << std::endl; std::cout << "Set TENSILE_SOLUTION_INDEX to a negative number to restore the " "default behavior." << std::endl; { std::lock_guard guard(solutionsGuard); auto selected_solution = solutions.at(solution_index); Task task(hardware, problem, *(selected_solution)); if((*selected_solution->problemPredicate)(problem) && (*selected_solution->taskPredicate)(task) && (*selected_solution->hardwarePredicate)(hardware)) rv = selected_solution; else return nullptr; } } else rv = library->findBestSolution(problem, hardware, fitness); if(Debug::Instance().printLibraryLogicIndex()) { if(rv) std::cout << "Library logic solution index of winning solution: " << rv->libraryLogicIndex << std::endl; else std::cout << "No solution found" << std::endl; } return rv; } virtual SolutionSet findAllSolutions(MyProblem const& problem, Hardware const& hardware, SolutionLibrarySearchType searchType = SolutionLibrarySearchType::DEFAULT) const override { return library->findAllSolutions(problem, hardware, searchType); } virtual SolutionSet findAllSolutionsGroupedGemm(std::vector const& problems, Hardware const& hardware, SolutionLibrarySearchType searchType = SolutionLibrarySearchType::DEFAULT) const override { return library->findAllSolutionsGroupedGemm(problems, hardware, searchType); } virtual SolutionVector findTopSolutions(MyProblem const& problem, Hardware const& hardware, int numSolutions) const override { if(Debug::Instance().printSolutionSelectionTime()) { auto start = std::chrono::steady_clock::now(); auto result = library->findTopSolutions(problem, hardware, numSolutions); auto end = std::chrono::steady_clock::now(); double time = std::chrono::duration(end - start).count(); std::cout << "Solution selection time: " << time << " us" << std::endl; lastFindTopRetAll = library->lastFindTopAlreadyRetAll(); return result; } else { const auto& result = library->findTopSolutions(problem, hardware, numSolutions); lastFindTopRetAll = library->lastFindTopAlreadyRetAll(); return result; } } virtual bool lastFindTopAlreadyRetAll() const override { return lastFindTopRetAll; } virtual SolutionVector findTopSolutionsGroupedGemm(std::vector const& problems, Hardware const& hardware, int numSolutions) const override { return library->findTopSolutionsGroupedGemm(problems, hardware, numSolutions); } }; } // namespace TensileLite