// Copyright (c) Advanced Micro Devices, Inc. // SPDX-License-Identifier: MIT #include "common/tool_runner.hpp" #include "common/argument_registration.hpp" #include "common/common_utils.hpp" #include "common/defines.h" #include "common/env_vars.hpp" #include "common/environment.hpp" #include "common/path.hpp" #include "core/argparse.hpp" #include "core/mproc.hpp" #include "core/timemory.hpp" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include namespace argparse = ::tim::argparse; namespace signals = ::tim::signals; namespace path = rocprofsys::common::path; namespace env_vars = rocprofsys::env_vars; namespace utils = rocprofsys::common_utils; using settings = ::rocprofsys::settings; using parser_data_t = rocprofsys::argparse::parser_data; namespace { using rocprofsys::common::update_mode; using parser_t = argparse::argument_parser; using parser_err_t = typename parser_t::result_type; constexpr int HELP_PADDING = 8; constexpr int MAX_DESC_WIDTH = 120; constexpr size_t LAUNCHER_INJECT_SLOTS = 2; constexpr int EXEC_FAILURE_STATUS = 127; constexpr std::string_view ANSI_FATAL = "\033[31m"; // red constexpr std::string_view ANSI_RESET = "\033[0m"; std::atomic& monochrome_flag() { static std::atomic flag{ false }; return flag; } std::string_view fatal_color() { return monochrome_flag().load(std::memory_order_relaxed) ? std::string_view{} : ANSI_FATAL; } std::string_view reset_color() { return monochrome_flag().load(std::memory_order_relaxed) ? std::string_view{} : ANSI_RESET; } std::string_view basename_of(std::string_view path) { const auto slash = path.rfind('/'); return (slash == std::string_view::npos) ? path : path.substr(slash + 1); } int terminal_columns() { struct winsize ws {}; if(ioctl(STDOUT_FILENO, TIOCGWINSZ, &ws) == 0 && ws.ws_col > 0) return ws.ws_col; return 0; // unknown / not a tty } std::string getenv_string(std::string_view key, std::string_view default_value) { const auto* val = std::getenv(std::string{ key }.c_str()); return (val != nullptr) ? std::string{ val } : std::string{ default_value }; } // All string_view fields point at static-storage data (string literals from the // make_*_config factories). Do not assign from temporaries. struct tool_config { rocprofsys::common_utils::tool_mode mode = rocprofsys::common_utils::tool_mode::run; std::string_view tool_name; std::string_view version_name; std::string_view summary; std::string_view workflow; std::string_view output_prefix = {}; bool force_sampling = false; bool enable_fork = false; bool enable_launcher = false; bool show_sample_flag = false; bool disable_cputime_on_realtime_only = false; std::unordered_map deprecated_flags = {}; }; tool_config make_run_config() { tool_config cfg{}; cfg.mode = rocprofsys::common_utils::tool_mode::run; cfg.tool_name = "run"; cfg.version_name = "rocprof-sys-run"; cfg.summary = "Execute instrumented binaries with ROCm Systems Profiler " "configuration."; cfg.workflow = R"(INSTRUMENTATION WORKFLOW: 1. Instrument: rocprof-sys-instrument -o app.inst -- ./app 2. Run: rocprof-sys-run --preset=balanced -- ./app.inst 3. Analyze: cat rocprof-sys-output/wall_clock.txt)"; cfg.output_prefix = "ROCPROFSYS: "; cfg.enable_fork = true; cfg.enable_launcher = true; cfg.show_sample_flag = true; return cfg; } tool_config make_sample_config() { tool_config cfg{}; cfg.mode = rocprofsys::common_utils::tool_mode::sample; cfg.tool_name = "sample"; cfg.version_name = "rocprof-sys-sample"; cfg.summary = "Call-stack sampling profiler for applications without binary " "instrumentation."; cfg.workflow = R"(PROFILING WORKFLOW: 1. Profile: rocprof-sys-sample --preset=balanced -- ./app 2. Analyze: cat rocprof-sys-output/wall_clock.txt 3. Visualize: Open rocprof-sys-output/perfetto-trace.proto in ui.perfetto.dev)"; cfg.force_sampling = true; cfg.disable_cputime_on_realtime_only = true; cfg.deprecated_flags = { { "--cputime", "--sample-cputime" }, { "--realtime", "--sample-realtime" }, { "--freq", "--sampling-freq" }, }; return cfg; } std::string replace_all(std::string str, std::string_view from, std::string_view replacement) { auto pos = str.find(from); while(pos != std::string::npos) { str.replace(pos, from.size(), replacement); pos = str.find(from, pos + replacement.size()); } return str; } auto toggle_suppression(std::tuple incoming) { auto previous = std::make_tuple(settings::suppress_config(), settings::suppress_parsing()); std::tie(settings::suppress_config(), settings::suppress_parsing()) = incoming; return previous; } // Suppresses timemory env-parsing during static init; restored in parse_args. // // Why a global: timemory reads ROCPROFSYS_CONFIG_FILE / settings during its // own translation-unit init, which fires before main(). We must beat that // init by setting suppress_{config,parsing} as early as possible. Moving this // into run() is too late. // // Static-init order safety: tim::settings::suppress_{config,parsing}() are // function-local statics inside timemory, so accessing them here triggers // their construction — no order-of-init UB across translation units. // // Captured value holds the prior settings so they can be restored once we // have control inside main() and can decide when parsing should run. const auto pre_main_suppression_guard = toggle_suppression({ true, true }); bool needs_full_parse(int argc, char** argv) { for(int arg_idx = 1; arg_idx < argc; ++arg_idx) { if(argv[arg_idx] == nullptr) continue; auto arg = std::string_view{ argv[arg_idx] }; if(arg == "--" || arg == "-?" || arg == "-h" || arg == "--help" || arg == "--version" || arg == "--export-config" || arg.find("--export-config=") == 0 || arg == "--list-presets" || arg == "--explain" || arg.find("--explain=") == 0) { return true; } } return argc > 1 && argv[1] != nullptr && std::string_view{ argv[1] }.find('-') == 0; } bool help_requested(const parser_t& parser, int argc, char** argv) { constexpr std::array help_args{ "-h", "--help", "-?" }; if(parser.exists("help") || argc == 1) return true; if(argc <= 1 || argv[1] == nullptr) return false; return std::find(help_args.begin(), help_args.end(), std::string_view{ argv[1] }) != help_args.end(); } class tool_runner { public: tool_runner(int argc_, char** argv_, tool_config cfg) : argc{ argc_ } , argv{ argv_ } , config{ std::move(cfg) } {} [[nodiscard]] int run(); private: void print_usage() const; void update_verbose_from_env(); void get_initial_environment(); void prepare_command(const char* exe); void prepare_environment(); void parse_command_fast_path(); void configure_parser(parser_t& parser); std::optional apply_post_parse(parser_t& parser); std::optional do_full_parse(); std::optional parse_args(); std::string build_description() const; int argc; char** argv; tool_config config; parser_data_t data{}; rocprofsys::common_utils::domain_flag_state domain_state{}; }; std::string tool_runner::build_description() const { auto cmd = std::string{ "rocprof-sys-" }.append(config.tool_name); auto desc = replace_all( R"( @SUMMARY@ QUICK REFERENCE: Presets: --preset=balanced (default), --preset=profile-only, --preset=trace-hpc, --preset=workload-trace Domains: --gpu, --rocm, --cpu, --parallel (composable with presets) Output: Results saved to rocprof-sys-output/ directory Visualize: Open perfetto-trace.proto in https://ui.perfetto.dev EXAMPLES: Quick Start: @CMD@ --preset=balanced -- ./myapp Workload-Specific Presets: @CMD@ --preset=trace-hpc -- ./hpc_app # HPC/MPI/OpenMP @CMD@ --preset=workload-trace -- python train.py # AI/ML/GPU workloads @CMD@ --preset=profile-only -- ./myapp # Minimal overhead Domain Flags (composable): @CMD@ --gpu -- ./myapp # GPU metrics @CMD@ --preset=balanced --gpu=temp,power -- ./app # Preset + specific GPU metrics @CMD@ --rocm=hip,kernel --cpu=100 -- ./app # ROCm APIs + CPU sampling Custom Configuration File: @CMD@ --preset=./my-config.json -- ./myapp Export Configuration: @CMD@ --preset=balanced --gpu --export-config > my-config.json @WORKFLOW@ )", "@CMD@", cmd); desc = replace_all(std::move(desc), "@SUMMARY@", config.summary); desc = replace_all(std::move(desc), "@WORKFLOW@", config.workflow); return desc; } void tool_runner::print_usage() const { std::cerr << fatal_color() << "Usage: " << argv[0] << " [OPTIONS] -- " << reset_color() << '\n'; } void tool_runner::update_verbose_from_env() { const auto* log_level = std::getenv(env_vars::LOG_LEVEL); if(log_level != nullptr) data.out.verbose = env_vars::log_level_to_verbose(log_level); } void tool_runner::get_initial_environment() { // Single-threaded assumption: this runs from main() before any worker thread // is spawned, so iterating `environ` and calling getenv/setenv is safe. if(environ != nullptr) { for(int idx = 0; environ[idx] != nullptr; ++idx) { data.env.initial.emplace(environ[idx]); data.env.current.emplace_back(environ[idx]); } } auto libexec_path = path::realpath(path::get_internal_script_path()); if(!libexec_path.empty()) data.env.set(env_vars::SCRIPT_PATH, libexec_path); update_verbose_from_env(); if(auto llvm_dir = rocprofsys::common::discover_llvm_libdir_for_ompt(); !llvm_dir.empty()) { data.env.set("LD_LIBRARY_PATH", llvm_dir, update_mode::APPEND); // Also mutate the live process env: any dlopen() that happens before // execvpe (e.g. OMPT runtime discovery) reads the real environ, not // data.env.current. const auto* current_ld = std::getenv("LD_LIBRARY_PATH"); const auto new_ld = (current_ld != nullptr) ? (llvm_dir + ":" + current_ld) : llvm_dir; setenv("LD_LIBRARY_PATH", new_ld.c_str(), 1); } if(config.force_sampling) { auto mode = getenv_string(env_vars::MODE, "sampling"); // Bool value flows through update_env's to_env_string(bool) overload, // becoming "true"/"false" in the env string. data.env.set(env_vars::USE_SAMPLING, (mode != "causal")); } } void tool_runner::prepare_command(const char* exe) { if(data.out.launcher.empty()) return; bool injected = false; std::vector new_argv; new_argv.reserve(data.out.command.size() + LAUNCHER_INJECT_SLOTS); for(const auto& arg : data.out.command) { if(!injected && basename_of(arg).find(data.out.launcher) != std::string_view::npos) { new_argv.emplace_back(exe); new_argv.emplace_back("--"); injected = true; } new_argv.emplace_back(arg); } if(!injected) { throw std::runtime_error(fmt::format( R"(Unable to match launcher "{}" to any arguments on the command line: "{}")", data.out.launcher, fmt::join(data.out.command, " "))); } data.out.command = std::move(new_argv); } void tool_runner::prepare_environment() { // launcher mode re-injects LD_PRELOAD itself, so skip it here if(!config.enable_launcher || data.out.launcher.empty()) { rocprofsys::argparse::add_ld_preload(data); rocprofsys::argparse::add_ld_library_path(data); } rocprofsys::argparse::add_torch_library_path(data); rocprofsys::common::consolidate_env_entries(data.env.current); } void tool_runner::parse_command_fast_path() { toggle_suppression(pre_main_suppression_guard); rocprofsys::argparse::init_parser(data); signals::disable_signal_detection(signals::signal_settings::get_enabled()); bool past_separator = false; for(int arg_idx = 1; arg_idx < argc; ++arg_idx) { if(argv[arg_idx] == nullptr) continue; if(past_separator) data.out.command.emplace_back(argv[arg_idx]); else if(std::string_view{ argv[arg_idx] } == "--") past_separator = true; } } void tool_runner::configure_parser(parser_t& parser) { parser.on_error([](parser_t&, const parser_err_t& err) { // Throw rather than exit() so RAII destructors run (parse_data, parser, args). // Caught at the top of run() and converted to EXIT_FAILURE. throw std::runtime_error(err.what()); }); parser.enable_help().count(-1).min_count(0).max_count(1).dtype("topic"); parser.enable_version(std::string{ config.version_name }, ROCPROFSYS_ARGPARSE_VERSION_INFO); if(auto cols = terminal_columns(); cols > parser.get_help_width() + HELP_PADDING) parser.set_description_width( std::min(cols - parser.get_help_width() - HELP_PADDING, MAX_DESC_WIDTH)); data.reg.processed_groups.emplace("causal"); if(!config.show_sample_flag) data.reg.processed_environs.emplace("sampling"); if(!config.enable_launcher) data.reg.processed_environs.emplace("launcher"); rocprofsys::argparse::add_core_arguments(parser, data); rocprofsys::argparse::add_extended_arguments(parser, data); rocprofsys::common_utils::register_preset_and_domain_arguments( parser, config.tool_name, domain_state, [this](std::string_view key, std::string_view val) { data.env.set(key, std::string{ val }); }); if(config.enable_fork) { parser.start_group("EXECUTION OPTIONS", ""); parser.add_argument({ "--fork" }, "Execute via fork + execvpe instead of execvpe") .min_count(0) .max_count(1) .dtype("boolean") .action([this](parser_t& parser_ref) { data.out.fork_exec = parser_ref.get("fork"); }); } } std::optional tool_runner::apply_post_parse(parser_t& parser) { if(config.disable_cputime_on_realtime_only) { if(parser.exists("sample-realtime") && !parser.exists("sample-cputime")) data.env.set(env_vars::SAMPLING_CPUTIME, false); } if(parser.exists("profile") && parser.exists("flat-profile")) throw std::runtime_error( "Error! '--profile' argument conflicts with '--flat-profile' argument"); if(domain_state.export_config_requested) { rocprofsys::common_utils::export_config( data.env.current, data.env.initial, domain_state.active_preset_name, config.tool_name, domain_state.export_config_file); return EXIT_SUCCESS; } rocprofsys::common_utils::run_post_parse_validation(config.tool_name, domain_state, data.out.verbose); return std::nullopt; } std::optional tool_runner::do_full_parse() { toggle_suppression(pre_main_suppression_guard); rocprofsys::argparse::init_parser(data); signals::disable_signal_detection(signals::signal_settings::get_enabled()); auto parser = parser_t{ std::string{ basename_of(argv[0]) }, build_description() }; configure_parser(parser); auto args = rocprofsys::common_utils::translate_arguments( argc, argv, domain_state.registry, config.deprecated_flags); data.out.command = std::move(args.command); auto parse_err = parser.parse_args(static_cast(args.argv_ptrs.size()), args.argv_ptrs.data()); if(help_requested(parser, argc, argv)) return rocprofsys::common_utils::dispatch_help(parser, config.tool_name, EXIT_SUCCESS); if(parse_err) throw std::runtime_error(parse_err.what()); if(domain_state.early_exit) return domain_state.early_exit; monochrome_flag().store(data.out.monochrome, std::memory_order_relaxed); // Keep timemory's own logger in sync — its background paths still use its color. tim::log::monochrome() = data.out.monochrome; return apply_post_parse(parser); } std::optional tool_runner::parse_args() { get_initial_environment(); if(!needs_full_parse(argc, argv)) { parse_command_fast_path(); return std::nullopt; } return do_full_parse(); } int tool_runner::run() try { if(argc == 1) { print_usage(); return EXIT_FAILURE; } if(auto exit_code = parse_args()) return *exit_code; if(config.enable_launcher) prepare_command(argv[0]); prepare_environment(); if(data.out.command.empty()) { print_usage(); return EXIT_FAILURE; } update_verbose_from_env(); const auto verbose = data.out.verbose; if(verbose >= 0) utils::print_environment(data.env.current, data.env.updated, verbose >= 1, config.output_prefix); if(verbose >= 1) utils::print_command(data.out.command, config.output_prefix); auto argv_ptrs = utils::to_c_argv(data.out.command); auto envp_ptrs = utils::to_c_argv(data.env.current); if(data.out.fork_exec) { auto pid = fork(); if(pid < 0) { std::perror("fork"); return EXIT_FAILURE; } if(pid == 0) { execvpe(argv_ptrs.front(), argv_ptrs.data(), envp_ptrs.data()); // execvpe only returns on failure; use _exit to skip parent atexit handlers std::perror("execvpe"); _exit(EXEC_FAILURE_STATUS); } auto status = rocprofsys::mproc::wait_pid(pid); auto exit_code = rocprofsys::mproc::diagnose_status(pid, status); if(exit_code != 0 && data.out.verbose >= 0) { std::fprintf(stderr, "[rocprof-sys][fatal] process %i exiting with non-zero exit " "code: %i\n", pid, exit_code); } else if(data.out.verbose >= 2) { std::fprintf(stderr, "[rocprof-sys][fatal] rocprof-sys run in process %i completed. " "exit code: %i\n", pid, exit_code); } return exit_code; } execvpe(argv_ptrs.front(), argv_ptrs.data(), envp_ptrs.data()); std::perror("execvpe"); return EXEC_FAILURE_STATUS; } catch(const std::exception& ex) { std::cerr << "[rocprof-sys] error: " << ex.what() << '\n'; return EXIT_FAILURE; } } // namespace namespace rocprofsys::common_utils { int run_tool(int argc, char** argv, tool_mode mode) { auto config = (mode == tool_mode::sample) ? make_sample_config() : make_run_config(); return tool_runner{ argc, argv, std::move(config) }.run(); } } // namespace rocprofsys::common_utils