TheRock/rocm-systems/projects/rocprofiler-systems/source/lib/rocprof-sys/library.cpp

// Copyright (c) Advanced Micro Devices, Inc.
// SPDX-License-Identifier: MIT

#include <cstdint>
#include <timemory/log/color.hpp>
//
//  above should always be included first
//
#include "api.hpp"
#include "common/defines.h"
#include "common/delimit.hpp"
#include "common/env_vars.hpp"
#include "common/setup.hpp"
#include "common/static_object.hpp"
#include "core/agent.hpp"
#include "core/agent_manager.hpp"
#include "core/categories.hpp"
#include "core/components/fwd.hpp"
#include "core/concepts.hpp"
#include "core/config.hpp"
#include "core/constraint.hpp"
#include "core/cpu.hpp"
#include "core/gpu.hpp"
#include "core/locking.hpp"
#include "core/node_info.hpp"
#include "core/output_file_registry.hpp"
#include "core/perfetto_fwd.hpp"
#include "core/progress/bar.hpp"
#include "core/progress/callback.hpp"
#include "core/rocpd/data_processor.hpp"
#include "core/timemory.hpp"
#include "core/trace_cache/cache_manager.hpp"
#include "core/trace_cache/cacheable.hpp"
#include "core/trace_cache/metadata_registry.hpp"
#include "core/utility.hpp"
#include "library/causal/data.hpp"
#include "library/causal/experiment.hpp"
#include "library/causal/sampling.hpp"
#include "library/components/exit_gotcha.hpp"
#include "library/components/fork_gotcha.hpp"
#include "library/components/mpi_gotcha.hpp"
#include "library/components/numa_gotcha.hpp"
#include "library/components/pthread_gotcha.hpp"
#include "library/components/shmem_gotcha_policy.hpp"
#include "library/components/ucx_gotcha_policy.hpp"
#include "library/components/vaapi_gotcha.hpp"
#include "library/coverage.hpp"
#include "library/kokkosp.hpp"
#include "library/process_sampler.hpp"
#include "library/rocprofiler-sdk.hpp"
#include "library/rocprofiler-sdk/roctx_client.hpp"
#include "library/rocprofiler-sdk/trace_control.hpp"
#include "library/runtime.hpp"
#include "library/sampling.hpp"
#include "library/thread_data.hpp"
#include "library/thread_info.hpp"
#include "library/tracing.hpp"
#include "rocprofiler-systems/categories.h"  // in rocprof-sys-user

#include <timemory/hash/types.hpp>
#include <timemory/log/logger.hpp>
#include <timemory/manager/manager.hpp>
#include <timemory/mpl/type_traits.hpp>
#include <timemory/operations/types/file_output_message.hpp>
#include <timemory/process/process.hpp>
#include <timemory/process/threading.hpp>
#include <timemory/settings/types.hpp>
#include <timemory/signals/signal_handlers.hpp>
#include <timemory/signals/signal_mask.hpp>
#include <timemory/signals/types.hpp>
#include <timemory/utility/backtrace.hpp>
#include <timemory/utility/procfs/maps.hpp>

#include <rocprofiler-sdk/agent.h>
#include <rocprofiler-sdk/registration.h>

#include <nlohmann/json.hpp>

#include "logger/debug.hpp"

#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdio>
#include <cstdlib>
#include <mutex>
#include <pthread.h>
#include <sstream>
#include <stdexcept>
#include <string_view>
#include <unistd.h>
#include <utility>

using namespace rocprofsys;

//======================================================================================//

namespace rocprofsys
{
namespace timeout
{
void
setup() ROCPROFSYS_INTERNAL_API;
}
}  // namespace rocprofsys

namespace
{
std::atomic<bool>  rocprofsys_init_library_done{ false };
std::atomic<pid_t> rocprofsys_init_tooling_done{ 0 };
std::atomic<bool>  rocprofsys_finalization_done{ false };
auto               _timemory_manager  = tim::manager::instance();
auto               _timemory_settings = tim::settings::shared_instance();

void
set_metadata_process_start_timestamp(std::int64_t _ts)
{
    auto process_info  = trace_cache::get_metadata_registry().get_process_info();
    process_info.start = _ts;
    trace_cache::get_metadata_registry().set_process(process_info);
}

void
set_metadata_process_end_timestamp(std::int64_t _ts)
{
    auto process_info = trace_cache::get_metadata_registry().get_process_info();
    process_info.end  = _ts;
    trace_cache::get_metadata_registry().set_process(process_info);
}

void
set_metadata_environment_json(const std::string& _environment_json)
{
    auto process_info        = trace_cache::get_metadata_registry().get_process_info();
    process_info.environment = _environment_json;
    trace_cache::get_metadata_registry().set_process(process_info);
}

std::string
escape_quotes(std::string str)
{
    std::string::size_type pos = 0;
    while((pos = str.find('"', pos)) != std::string::npos)
    {
        str.replace(pos, 1, "\"\"");
        pos += 2;
    }
    return str;
}

bool
ensure_initialization(bool _offset, std::int64_t _glob_n, std::int64_t _offset_n)
{
    auto _exit_info = component::exit_gotcha::get_exit_info();
    if(_exit_info.is_known && _exit_info.exit_code != EXIT_SUCCESS) return _offset;

    auto _tid              = utility::get_thread_index();
    auto _peak_num_threads = grow_data(_tid + 1);

    if(_tid > 0 && _tid < _peak_num_threads)
    {
        const auto& _info = thread_info::get();
        LOG_DEBUG("thread info: {}, offset: {}, global counter: {}, offset counter: {}, "
                  "max threads: {}",
                  static_cast<bool>(_info), _offset, _glob_n, _offset_n,
                  _peak_num_threads);
    }

    return _offset;
}

void
finalization_handler()
{
    if(get_state() == State::Active) rocprofsys_finalize();
}

auto
ensure_finalization(bool _static_init = false)
{
    if(config::set_signal_handler(nullptr) == nullptr)
        config::set_signal_handler(&finalization_handler);

    if(_static_init)
    {
        auto _idx = threading::add_callback(&ensure_initialization);
        if(_idx < 0)
            throw exception<std::runtime_error>("failure adding threading callback");
    }

    if(config::set_signal_handler(nullptr) != &finalization_handler)

        throw std::runtime_error(fmt::format(
            "Assignment of signal handler failed. signal handler is {}, expected "
            "{}",
            fmt::format("0x{:X}",
                        reinterpret_cast<uintptr_t>(config::set_signal_handler(nullptr))),
            fmt::format("0x{:X}", reinterpret_cast<uintptr_t>(&finalization_handler))));

    const auto& _info = thread_info::init();
    const auto& _tid  = _info->index_data;
    if(_tid)
    {
        if(_tid->sequent_value != threading::get_id())
        {
            throw std::runtime_error(fmt::format("Error! internal tid != {} :: {}",
                                                 threading::get_id(),
                                                 _tid->sequent_value));
        }
        if(_tid->system_value != threading::get_sys_tid())
        {
            throw std::runtime_error(fmt::format("Error! system tid != {} :: {}",
                                                 threading::get_sys_tid(),
                                                 _tid->system_value));
        }
    }

    if(common::get_env(env_vars::MONOCHROME, false)) tim::log::monochrome() = true;

    timeout::setup();

    (void) tim::manager::instance();
    (void) tim::settings::shared_instance();

    if(!tim::get_shared_ptr_pair_callback())
    {
        tim::get_shared_ptr_pair_callback() =
            new tim::shared_ptr_pair_callback_t{ [](std::int64_t _n) {
                if(_n == 0) rocprofsys_finalize_hidden();
            } };
    }

    if(_static_init)
    {
        auto _verbose =
            get_verbose_env() + ((get_debug_env() || get_debug_init()) ? 16 : 0);
        auto _search_paths = fmt::format(
            "{}:{}:{}:{}:{}", rocprofsys::get_env<std::string>(env_vars::PATH, ""),
            rocprofsys::get_env<std::string>("PWD"), ".",
            rocprofsys::get_env<std::string>("LD_LIBRARY_PATH", ""),
            rocprofsys::get_env<std::string>("LIBRARY_PATH", ""),
            rocprofsys::get_env<std::string>("PATH", ""));
        common::setup_environ(_verbose, _search_paths);
    }

    if(_timemory_manager) _timemory_manager->set_write_metadata(-1);

    return scope::destructor{ []() { rocprofsys_finalize_hidden(); } };
}

template <typename... Tp>
struct fini_bundle
{
    using data_type = std::tuple<Tp...>;

    fini_bundle()                                  = default;
    fini_bundle(const fini_bundle&)                = default;
    fini_bundle(fini_bundle&&) noexcept            = default;
    fini_bundle& operator=(const fini_bundle&)     = default;
    fini_bundle& operator=(fini_bundle&&) noexcept = default;

    fini_bundle(std::string_view _label)
    : m_label{ _label }
    {}

    template <typename... Args>
    void start(Args&&... _args)
    {
        ROCPROFSYS_FOLD_EXPRESSION(tim::operation::start<Tp>{}(
            std::get<Tp>(m_data), std::forward<Args>(_args)...));
    }

    template <typename... Args>
    void stop(Args&&... _args)
    {
        ROCPROFSYS_FOLD_EXPRESSION(tim::operation::stop<Tp>{}(
            std::get<Tp>(m_data), std::forward<Args>(_args)...));
    }

    std::string as_string(bool _print_prefix = true) const
    {
        std::stringstream _ss;
        if(_print_prefix && m_label.length() > 0) _ss << m_label << " : ";
        size_t _idx = 0;
        ((_ss << (_idx++ > 0 ? ", " : "") << std::get<Tp>(m_data)), ...);
        return _ss.str();
    }

    std::string_view m_label = {};
    data_type        m_data  = {};
};

template <typename... Tp>
struct fini_bundle<tim::lightweight_tuple<Tp...>>
{
    using base_type = fini_bundle<Tp...>;
};

using fini_bundle_t = typename fini_bundle<main_bundle_t>::base_type;
}  // namespace

//======================================================================================//
///
///
///
//======================================================================================//

namespace
{
struct set_env_s  // NOLINT
{};
}  // namespace

extern "C" void
rocprofsys_set_env_hidden(const char* env_name, const char* env_val)
{
    tim::auto_lock_t _lk{ tim::type_mutex<set_env_s>() };

    static auto _set_envs = std::set<std::string_view>{};
    bool        _success  = _set_envs.emplace(env_name).second;

    // just search env to avoid initializing the settings
    if(get_debug_init())
    {
        LOG_DEBUG("Setting env: {} = {}", env_name, env_val);
    }

    rocprofsys::set_env(env_name, env_val, 0);

    if(_success && get_state() >= State::Init)
    {
        LOG_WARNING(
            "rocprofsys_set_env(\"{}\", \"{}\") called after rocprof-sys was "
            "initialized. state = {}. This environment variable will have no effect",
            env_name, env_val, static_cast<int>(get_state()));
    }
}

//======================================================================================//
///
///
///
//======================================================================================//

namespace
{
bool                  _set_mpi_called   = false;
std::function<void()> _preinit_callback = []() { get_preinit_bundle()->start(); };

std::vector<std::string>
read_command_line(pid_t _pid)
{
    auto _cmdline = std::vector<std::string>{};
    auto fcmdline = std::stringstream{};
    fcmdline << "/proc/" << _pid << "/cmdline";
    auto ifs = std::ifstream{ fcmdline.str().c_str() };
    if(ifs)
    {
        std::string sarg;
        while(std::getline(ifs, sarg, '\0'))
        {
            _cmdline.push_back(sarg);
        }
        ifs.close();
    }

    return _cmdline;
}

void
rocprofsys_preinit_cache()
{
    const auto        _cmd_line = read_command_line(getpid());
    const std::string _command  = _cmd_line.empty()
                                      ? "rocprofiler-systems"
                                      : fmt::format("{}", fmt::join(_cmd_line, " "));

    std::stringstream _extdata_stream;
    config::print_settings_json(_extdata_stream);

    trace_cache::get_metadata_registry().set_process(
        { getpid(), getppid(), _command, "", escape_quotes(_extdata_stream.str()), 0,
          0 });
}

void
rocprofsys_preinit_cpu_agents()
{
    cpu::query_cpu_agents();
}

void
rocprofsys_preinit_hidden()
{
    // run once and discard
    _preinit_callback();
    _preinit_callback = []() {};
}

using callback_t = void (*)();
std::mutex              external_pause_resume_callbacks_mutex;
std::vector<callback_t> external_pause_callbacks;
std::vector<callback_t> external_resume_callbacks;

void
invoke_external_pause_callbacks()
{
    std::lock_guard<std::mutex> _lk{ external_pause_resume_callbacks_mutex };
    for(auto* _fn : external_pause_callbacks)
        _fn();
}

void
invoke_external_resume_callbacks()
{
    std::lock_guard<std::mutex> _lk{ external_pause_resume_callbacks_mutex };
    for(auto* _fn : external_resume_callbacks)
        _fn();
}

}  // namespace

extern "C" void
rocprofsys_external_register_pause_callbacks(void (*pause_fn)(), void (*resume_fn)())
{
    std::lock_guard<std::mutex> _lk{ external_pause_resume_callbacks_mutex };

    if(pause_fn)
    {
        external_pause_callbacks.emplace_back(pause_fn);
    }

    if(resume_fn)
    {
        external_resume_callbacks.emplace_back(resume_fn);
    }
}

extern "C" void
rocprofsys_set_mpi_hidden(bool use)
{
    static bool _once = false;
    static bool _arg  = use;

    // this function may be called multiple times if multiple libraries are instrumented
    // we want to guard against multiple calls which with different arguments
    if(_once && _arg == use) return;
    _once = true;

    // just search env to avoid initializing the settings
    if(get_debug_init())
    {
        LOG_DEBUG("use: {}", (use) ? "y" : "n");
    }

    _set_mpi_called = true;

    if(use && get_state() == State::PreInit)
    {
        rocprofsys::set_env(env_vars::USE_PID, "ON", 1);
    }
    else if(!use)
    {
        trait::runtime_enabled<mpi_gotcha_t>::set(false);
    }

    if(get_state() >= State::Init)
    {
        LOG_WARNING(
            "rocprofsys_set_mpi(use={}) called after rocprof-sys was "
            "initialized. state = {}. MPI support may not be properly initialized. Use "
            "ROCPROFSYS_USE_MPIP=ON and ROCPROFSYS_USE_PID=ON to ensure full support",
            use, static_cast<int>(get_state()));
    }

    rocprofsys_preinit_hidden();
}

//======================================================================================//

extern "C" void
rocprofsys_init_library_hidden()
{
    auto _tid = threading::get_id();
    (void) _tid;

    auto _debug_init = get_debug_init();

    int _selinux_mode = 0;
    {
        std::ifstream _fenforcing{ "/sys/fs/selinux/enforce" };
        if(!(_fenforcing >> _selinux_mode)) _selinux_mode = 0;
        _fenforcing.close();
    }

    if(_selinux_mode == 1)
    {
        LOG_DEBUG("/sys/fs/selinux/enforce has a value of {}.", _selinux_mode);
        LOG_CRITICAL("SELinux enforcing mode detected. Consider disabling SELinux "
                     "or configure permissive mode with 'sudo setenforce 0'. Aborting.");
        std::exit(EXIT_FAILURE);
    }

    if(_debug_init)
    {
        LOG_DEBUG("State is {}...", std::to_string(get_state()));
    }

    if(get_state() != State::PreInit)
    {
        throw std::runtime_error(
            fmt::format("State is not PreInit :: {}", std::to_string(get_state())));
    }

    if(get_state() != State::PreInit) return;
    if(rocprofsys_init_library_done.exchange(true)) return;

    ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);

    if(_debug_init)
    {
        LOG_DEBUG("State is {}. Setting to {}...", std::to_string(get_state()),
                  std::to_string(State::Init));
        LOG_DEBUG("Calling backtrace once so that the one-time call of malloc in "
                  "glibc's backtrace() occurs...");
    }

    {
        std::stringstream _ss{};
        timemory_print_backtrace<16>(_ss);
        (void) _ss;
    }

    set_state(State::Init);

    if(get_state() != State::Init)
    {
        throw std::runtime_error(fmt::format("set_state(State::Init) failed. state is {}",
                                             std::to_string(get_state())));
    }

    if(_debug_init)
    {
        LOG_DEBUG("Configuring settings...");
    }

    // configure the settings
    configure_settings();

    // Disable Timemory console output for specified ranks
    if(!config::output_filtering::is_log_output_enabled_for_current_mpi_rank())
    {
        auto* _settings = tim::settings::instance();
        if(_settings)
        {
            _settings->cout_output() = false;
            _settings->verbose()     = -1;
            _settings->banner()      = false;
        }
    }

    auto _debug_value = get_debug();
    if(_debug_init) config::set_setting_value(std::string{ env_vars::DEBUG_MODE }, true);
    scope::destructor _debug_dtor{ [_debug_value, _debug_init]() {
        if(_debug_init)
            config::set_setting_value(std::string{ env_vars::DEBUG_MODE }, _debug_value);
    } };
}

//======================================================================================//

extern "C" bool
rocprofsys_init_tooling_hidden(void)
{
    if(get_env(env_vars::MONOCHROME, false)) tim::log::monochrome() = true;

    if(!rocprofsys::get_env(env_vars::INIT_TOOLING, true))
    {
        rocprofsys_init_library_hidden();
        return false;
    }

    auto _debug_init = get_debug_init();

    if(_debug_init)
    {
        LOG_DEBUG("State is {}...", std::to_string(get_state()));
    }

    if(get_state() != State::PreInit) return false;

    pid_t expected = 0;
    if(!rocprofsys_init_tooling_done.compare_exchange_strong(expected, getpid()))
        return false;

    ROCPROFSYS_SCOPED_THREAD_STATE(ThreadState::Internal);

    if(get_state() == State::Init)
    {
        throw std::runtime_error(
            fmt::format("{} called after rocprofsys_init_library() was explicitly called",
                        __FUNCTION__));
    }

    LOG_DEBUG("Instrumentation mode: {}", std::to_string(config::get_mode()));

    if(_debug_init)
    {
        LOG_DEBUG("Calling rocprofsys_init_library()...");
    }

    rocprofsys_init_library_hidden();

    if(_debug_init)
    {
        LOG_DEBUG("Printing banner...");
    }

    print_banner();

    auto _dtor = scope::destructor{ []() {
        // if set to finalized, don't continue
        if(get_state() > State::Active) return;

        rocprofsys_preinit_cache();

#if(defined(ROCPROFSYS_USE_MPI_HEADERS) && ROCPROFSYS_USE_MPI_HEADERS > 0) ||            \
    (defined(ROCPROFSYS_USE_MPI) && ROCPROFSYS_USE_MPI > 0)

        component::mpi_gotcha::subscribe_to_init_event([](int rank, int size) {
            nlohmann::json _environment_json;
            _environment_json["MPI_COMM_WORLD_SIZE"] = size;
            _environment_json["MPI_COMM_WORLD_RANK"] = rank;

            set_metadata_environment_json(escape_quotes(_environment_json.dump()));
        });
#endif

        if(get_use_process_sampling())
        {
            ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
            process_sampler::setup();
        }
        if(get_use_causal())
        {
            {
                ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
                causal::sampling::setup();
            }
            push_enable_sampling_on_child_threads(get_use_causal());
            sampling::unblock_signals();
        }
        else if(get_use_sampling())
        {
            {
                ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
                sampling::setup();
            }
            push_enable_sampling_on_child_threads(get_use_sampling());
            sampling::unblock_signals();
        }
        get_main_bundle()->start();
        LOG_DEBUG("State: {} -> State::Active", std::to_string(get_state()));

        {
            ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);
            trace_cache::get_buffer_storage().start(getpid());
        }

        auto trace_controller = rocprofiler_sdk::get_trace_controller();
        if(trace_controller)
        {
            auto pause_callback = [](void) {
                LOG_DEBUG("Pause callback...");
                rocprofiler_sdk::pause();
                sampling::pause();
                component::mpi_gotcha::pause();
                component::ucx_gotcha<rocprofsys::DefaultUCXPolicy>::pause();
                component::shmem_gotcha<rocprofsys::DefaultSHMEMPolicy>::pause();
                component::vaapi_gotcha::pause();
                ::rocprofsys::pthread_gotcha::pause();
                component::numa_gotcha::pause();
                rocprofsys::kokkosp::pause();
                process_sampler::pause();
                invoke_external_pause_callbacks();
            };
            auto resume_callback = [](void) {
                LOG_DEBUG("Resume callback...");
                rocprofiler_sdk::resume();
                sampling::resume();
                component::mpi_gotcha::resume();
                component::ucx_gotcha<rocprofsys::DefaultUCXPolicy>::resume();
                component::shmem_gotcha<rocprofsys::DefaultSHMEMPolicy>::resume();
                component::vaapi_gotcha::resume();
                ::rocprofsys::pthread_gotcha::resume();
                component::numa_gotcha::resume();
                rocprofsys::kokkosp::resume();
                process_sampler::resume();
                invoke_external_resume_callbacks();
            };
            trace_controller->register_region_pause_resume_callbacks(resume_callback,
                                                                     pause_callback);

            trace_controller->force_initial_pause();
        }

        set_state(State::Active);  // set to active as very last operation
    } };

    ROCPROFSYS_SCOPED_SAMPLING_ON_CHILD_THREADS(false);

    // ideally these have already been started
    rocprofsys_preinit_hidden();

    // start these gotchas once settings have been initialized
    if(get_init_bundle()) get_init_bundle()->start();

    if(get_use_ucx())
    {
        LOG_DEBUG("Setting up UCX traces...\n");
        component::ucx_gotcha<rocprofsys::DefaultUCXPolicy>::start();
    }

    if(get_use_shmem())
    {
        LOG_DEBUG("Setting up OpenSHMEM traces...\n");
        component::shmem_gotcha<rocprofsys::DefaultSHMEMPolicy>::start();
    }

    if(get_use_vaapi_tracing())
    {
        LOG_DEBUG("Setting up VA-API traces...");
        component::vaapi_gotcha::start();
    }

    if(get_use_sampling()) sampling::block_signals();

    // perfetto initialization
    if(get_use_perfetto())
    {
        LOG_DEBUG("Setting up Perfetto...");
        rocprofsys::perfetto::setup();
    }

    if(get_use_causal()) causal::start_experimenting();

    if(get_use_timemory())
    {
        comp::user_global_bundle::global_init();
        std::set<int> _comps{};
        // convert string into set of enumerations
        for(auto&& itr : rocprofsys::delimit(tim::settings::global_components()))
            _comps.emplace(tim::runtime::enumerate(itr));
        if(_comps.size() == 1 && _comps.find(TIMEMORY_WALL_CLOCK) != _comps.end())
        {
            // using wall_clock directly is lower overhead than using it via user_bundle
            instrumentation_bundle_t::get_initializer() =
                [](instrumentation_bundle_t& _bundle) {
                    _bundle.initialize<comp::wall_clock>();
                };
        }
        else if(!_comps.empty())
        {
            // use user_bundle for other than wall-clock
            instrumentation_bundle_t::get_initializer() =
                [](instrumentation_bundle_t& _bundle) {
                    _bundle.initialize<comp::user_global_bundle>();
                };
        }
        else
        {
            tim::trait::runtime_enabled<project::rocprofsys>::set(false);
        }
    }

    if(get_use_perfetto())
    {
        LOG_DEBUG("Starting Perfetto...");
        rocprofsys::perfetto::start();
    }

    categories::setup();

    // if static objects are destroyed in the inverse order of when they are
    // created this should ensure that finalization is called before perfetto
    // ends the tracing session
    static auto _ensure_finalization = ensure_finalization();

    return true;
}

//======================================================================================//

extern "C" void
rocprofsys_init_hidden(const char* _mode, bool _is_binary_rewrite, const char* _argv0_c)
{
    static int  _total_count = 0;
    static auto _args = std::make_pair(std::string_view{ _mode }, _is_binary_rewrite);

    auto _count   = _total_count++;
    auto _mode_sv = std::string_view{ _mode };
    auto _argv0   = (_argv0_c) ? std::string{ _argv0_c } : config::get_exe_name();
    // this function may be called multiple times if multiple libraries are instrumented
    // we want to guard against multiple calls which with different arguments
    if(_count > 0 &&
       std::tie(_args.first, _args.second) == std::tie(_mode_sv, _is_binary_rewrite))
        return;

    if(_count > 0 &&
       std::tie(_args.first, _args.second) != std::tie(_mode_sv, _is_binary_rewrite))
    {
        throw std::runtime_error(fmt::format(
            "rocprofsys_init(...) called multiple times with different arguments for "
            "mode and/or is_binary_rewrite:"
            "\n    Invocation #1: rocprofsys_init(mode={}, is_binary_rewrite={}, ...)"
            "\n    Invocation #%i: rocprofsys_init(mode={}, is_binary_rewrite={}, ...)",
            _args.first.data(), std::to_string(_args.second).c_str(), _count + 1, _mode,
            std::to_string(_is_binary_rewrite)));
    }

    // always the first
    (void) get_state();
    (void) tracing::push_count();
    (void) tracing::pop_count();

    if(get_state() >= State::Init)
    {
        if(std::string_view{ _mode } != "trace" && std::string_view{ _mode } != "Trace")
        {
            LOG_WARNING(
                "rocprofsys_init(mode={}, is_binary_rewrite={}, argv0={}) "
                "called after rocprof-sys was initialized. state = {}. Mode-based "
                "settings (via -M <MODE> passed to rocprof-sys exe) may not be "
                "properly configured.",
                _mode, std::to_string(_is_binary_rewrite), _argv0,
                std::to_string(get_state()));
        }
    }

    tracing::get_finalization_functions().emplace_back([_argv0_c]() {
        if(get_state() != State::Active)
        {
            throw std::runtime_error(
                fmt::format("Finalizer function for popping main invoked in non-active "
                            "state :: state = {}",
                            std::to_string(get_state())));
        }
        if(get_state() == State::Active)
        {
            auto _name = (_argv0_c) ? std::string{ _argv0_c } : config::get_exe_name();
            // if main hasn't been popped yet, pop it
            LOG_DEBUG("Running rocprofsys_pop_trace({})...", _name);
            rocprofsys_pop_trace_hidden(_name.c_str());
        }
    });

    std::atexit([]() {
        // if active (not already finalized) then we should finalize
        if(get_state() == State::Active) rocprofsys_finalize_hidden();
    });

    set_metadata_process_start_timestamp(comp::wall_clock::record());

    if(get_debug_env() || get_verbose_env() > 2)
    {
        LOG_DEBUG("mode: {} | is binary rewrite: {} | command: {}", _mode,
                  (_is_binary_rewrite) ? "y" : "n", _argv0);
    }

    rocprofsys::set_env(env_vars::MODE, _mode, 0);
    config::is_binary_rewrite() = _is_binary_rewrite;

    if(_set_mpi_called)
    {
        rocprofsys_preinit_hidden();
    }
}

//======================================================================================//

extern "C" void
rocprofsys_reset_preload_hidden(void)
{
    rocprofsys::set_env(env_vars::PRELOAD, "0", 1);
    auto&& _preload_libs = common::get_env("LD_PRELOAD", std::string{});
    if(_preload_libs.find("librocprof-sys") != std::string::npos)
    {
        auto _modified_preload = std::string{};
        for(const auto& itr : delimit(_preload_libs, ":"))
        {
            if(itr.find("librocprof-sys") != std::string::npos) continue;
            _modified_preload += fmt::format(":{}", itr);
        }
        if(!_modified_preload.empty() && _modified_preload.find(':') == 0)
            _modified_preload = _modified_preload.substr(1);

        rocprofsys::set_env("LD_PRELOAD", _modified_preload, 1);
    }
}

//======================================================================================//

extern "C" void
rocprofsys_finalize_hidden(void)
{
    // Prevent multiple finalization calls (e.g., from atexit handlers after reset_state)
    if(rocprofsys_finalization_done.exchange(true))
    {
        LOG_DEBUG("Finalization already completed. Skipping.");
        return;
    }

    // disable thread id recycling during finalization
    threading::recycle_ids() = false;
    // disable initialization callback
    threading::remove_callback(&ensure_initialization);

    bool _is_child = is_child_process();
    set_thread_state(ThreadState::Completed);

    // return if not active
    if(get_state() != State::Active)
    {
        LOG_DEBUG("State = {}. Finalization skipped", std::to_string(get_state()));
        return;
    }

    set_metadata_process_end_timestamp(comp::wall_clock::record());

    if(_is_child)
    {
        set_state(State::Finalized);

        // Flush buffered traces in case of child process

        LOG_DEBUG("Shutting down ROCm...");
        rocprofiler_sdk::shutdown();

        auto&      _manager = rocprofsys::trace_cache::cache_manager::get_instance();
        const auto _agents  = get_agent_manager_instance().get_agents();
        _manager.shutdown();
        const auto metadata_filepath =
            trace_cache::utility::get_metadata_filepath(get_root_process_id(), getpid());
        _manager.get_metadata_registry().save_to_file(metadata_filepath, _agents);

        std::quick_exit(EXIT_SUCCESS);
        return;
    }

    LOG_INFO("Finalizing rocprof-sys...");

    sampling::block_samples();

    thread_info::set_stop(comp::wall_clock::record());

    tim::signals::block_signals(get_sampling_signals(),
                                tim::signals::sigmask_scope::process);

    rocprofsys_reset_preload_hidden();

    // some functions called during finalization may alter the push/pop count so we need
    // to save them here
    auto _push_count = tracing::push_count().load();
    auto _pop_count  = tracing::pop_count().load();

    // e.g. rocprofsys_pop_trace("main");
    if(_push_count > _pop_count)
    {
        for(auto& itr : tracing::get_finalization_functions())
        {
            itr();
            ++_pop_count;
        }
    }

    set_state(State::Finalized);

    push_enable_sampling_on_child_threads(false);
    set_sampling_on_all_future_threads(false);

    // if the categories are not enabled, it can/will suppress generating output for data
    // in category
    categories::enable_categories();

    auto _debug_init  = get_debug_finalize();
    auto _debug_value = get_debug();
    if(_debug_init) config::set_setting_value(std::string{ env_vars::DEBUG_MODE }, true);
    scope::destructor _debug_dtor{ [_debug_value, _debug_init]() {
        if(_debug_init)
            config::set_setting_value(std::string{ env_vars::DEBUG_MODE }, _debug_value);
    } };

    auto& _thread_bundle = thread_data<thread_bundle_t>::instance();
    if(_thread_bundle) _thread_bundle->stop();

    if(get_verbose() >= 1 || get_debug())
    {
        if(dmp::rank() == 0)
        {
            config::print_settings(
                rocprofsys::get_env<bool>(env_vars::PRINT_ENV, get_debug()));
        }
    }

    LOG_DEBUG("rocprofsys_push_trace/rocprofsys_push_trace_with_args :: called {}",
              _push_count);
    LOG_DEBUG("rocprofsys_pop_trace  :: called {}", _pop_count);

    tim::signals::enable_signal_detection({ tim::signals::sys_signal::Interrupt },
                                          [](int) {});

    LOG_DEBUG("Copying over all timemory hash information to main thread...");
    tracing::copy_timemory_hash_ids();

    // stop the main bundle which has stats for run
    if(get_main_bundle())
    {
        LOG_DEBUG("Stopping main bundle...");
        get_main_bundle()->stop();
    }

    fini_bundle_t _finalization{};
    _finalization.start();

    if(get_use_ucx())
    {
        LOG_DEBUG("Shutting down UCX tracing...\n");
        component::ucx_gotcha<rocprofsys::DefaultUCXPolicy>::shutdown();
    }

    if(get_use_shmem())
    {
        LOG_DEBUG("Shutting down OpenSHMEM tracing...\n");
        component::shmem_gotcha<rocprofsys::DefaultSHMEMPolicy>::shutdown();
    }

    if(get_use_vaapi_tracing())
    {
        LOG_DEBUG("Shutting down VA-API tracing...");
        component::vaapi_gotcha::shutdown();
    }

    if(get_use_process_sampling())
    {
        LOG_DEBUG("Shutting down background sampler...");
        process_sampler::shutdown();
    }

    LOG_DEBUG("Shutting down ROCm...");
    rocprofiler_sdk::shutdown();

    LOG_DEBUG("Stopping and destroying instrumentation bundles...");
    auto* _bundles = instrumentation_bundles::get();
    for(size_t i = 0; _bundles && i < thread_info::get_peak_num_threads(); ++i)
    {
        if(i >= _bundles->size()) continue;
        const auto& _info = thread_info::get(i, SequentTID);
        auto&       itr   = _bundles->at(i);
        while(itr != nullptr && !itr->empty())
        {
            if(_info->is_offset)
            {
                ++_pop_count;
            }
            LOG_WARNING("Instrumentation bundle on thread {} (TID={}) "
                        "with label '{}' was not stopped.",
                        i, itr->back()->tid(), itr->back()->key());

            itr->back()->stop();
            itr->back()->pop();
            itr->pop_back();
        }
    }

    // stop the main gotcha which shuts down the pthread gotchas
    if(get_init_bundle())
    {
        LOG_DEBUG("Stopping main gotcha...");
        get_init_bundle()->stop();

        pthread_gotcha::shutdown();
        component::numa_gotcha::shutdown();
    }

    // stop the gotcha bundle
    if(get_preinit_bundle())
    {
        LOG_DEBUG("Shutting down miscellaneous gotchas...");
        get_preinit_bundle()->stop();
        component::mpi_gotcha::shutdown();
    }

    if(get_use_causal())
    {
        LOG_DEBUG("Shutting down causal sampling...");
        causal::sampling::shutdown();
    }

    if(get_use_sampling())
    {
        LOG_DEBUG("Shutting down sampling...");
        sampling::shutdown();
    }

    LOG_TRACE("Reporting the process- and thread-level metrics...");
    // report the high-level metrics for the process
    if(get_main_bundle())
    {
        std::string _msg = get_main_bundle()->as_string();
        auto        _pos = _msg.find(">>>  ");
        if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
        LOG_INFO("{}", _msg);
        LOG_DEBUG("Resetting main bundle...");
        get_main_bundle()->reset();
    }

    // print out thread-data if they are not still running
    // if they are still running (e.g. thread-pool still alive), the
    // thread-specific data will be wrong if try to stop them from
    // the main thread.
    auto _thr_verbose = (config::get_use_causal()) ? 1 : 0;
    if(thread_data<thread_bundle_t>::get())
    {
        for(auto& itr : *thread_data<thread_bundle_t>::get())
        {
            if(itr && itr->get<comp::wall_clock>() &&
               !itr->get<comp::wall_clock>()->get_is_running())
            {
                std::string _msg = itr->as_string();
                auto        _pos = _msg.find(">>>  ");
                if(_pos != std::string::npos) _msg = _msg.substr(_pos + 5);
                if(_thr_verbose >= 0)
                {
                    LOG_INFO("{}", _msg);
                }
            }
        }
    }

    // ensure that all the MT instances are flushed
    if(get_use_sampling())
    {
        LOG_DEBUG("Post-processing the sampling backtraces...");
        sampling::post_process();
    }

    auto _output_registry = output_file_registry{};

    if(get_use_causal())
    {
        LOG_DEBUG("Finishing the causal experiments...");
        causal::finish_experimenting();

        auto _base = config::get_causal_output_filename();
        _output_registry.register_file(fmt::format("{}.json", _base),
                                       output_format::causal_json);
        _output_registry.register_file(fmt::format("{}.txt", _base),
                                       output_format::causal_text);
    }

    if(get_use_process_sampling())
    {
        LOG_DEBUG("Post-processing the system-level samples...");
        process_sampler::post_process();
    }

    if(get_use_code_coverage())
    {
        LOG_DEBUG("Post-processing the code coverage...");
        coverage::post_process();
    }

    tracing::copy_timemory_hash_ids();

    // Flush any pending region cache entries (e.g., main entry point that wasn't
    // explicitly stopped before finalization)
    LOG_DEBUG("Flushing pending region cache entries...");
    rocprofsys_flush_pending_region_cache_hidden();

    bool _perfetto_output_error = false;
    if(get_use_perfetto())
    {
        LOG_DEBUG("Finalizing perfetto...");
        rocprofsys::perfetto::post_process(_timemory_manager.get(),
                                           _perfetto_output_error, _output_registry);
    }

    {
        auto& _manager = rocprofsys::trace_cache::cache_manager::get_instance();
        _manager.shutdown();

        rocprofsys::progress::bar_options _bar_opts;
        _bar_opts.verbose = config::get_verbose();

        rocprofsys::progress::tracker _tracker{ [_bar_opts](std::string   _label,
                                                            std::uint64_t _total) {
            auto                      _bar = std::make_shared<rocprofsys::progress::bar>(std::move(_label),
                                                                                         _total, _bar_opts);
            return rocprofsys::progress::progress_callback{ [_bar](std::uint64_t _delta) {
                _bar->on_advance(_delta);
            } };
        } };

        _manager.post_process_bulk(_output_registry, _tracker);
    }

    if(_timemory_manager && _timemory_manager != nullptr)
    {
        _timemory_manager->add_metadata([](auto& ar) {
            auto _maps = tim::procfs::read_maps(process::get_id());
            auto _libs = std::set<std::string>{};
            for(auto& itr : _maps)
            {
                auto&& _path = itr.pathname;
                if(!_path.empty() && _path.at(0) != '[' && filepath::exists(_path))
                    _libs.emplace(_path);
            }
            ar(tim::cereal::make_nvp("memory_maps_files", _libs),
               tim::cereal::make_nvp("memory_maps", _maps));
        });

        static auto* attach_add_session_id = getenv(env_vars::REATTACH_ADD_SESSION_ID);
        static auto  session_id            = 0;

        if(attach_add_session_id)
            settings::default_process_suffix() = fmt::format("%pid%-{}", session_id++);

        // Disable Timemory file output for disabled ranks
        if(!config::output_filtering::is_file_output_enabled_for_current_mpi_rank())
        {
            auto* _settings = tim::settings::instance();
            if(_settings)
            {
                _settings->file_output() = false;
                _settings->text_output() = false;
                _settings->json_output() = false;
            }
        }

        LOG_DEBUG("Finalizing timemory...");
        tim::timemory_finalize(_timemory_manager.get());

        auto _cfg       = settings::compose_filename_config{};
        _cfg.use_suffix = config::get_use_pid();
        _cfg.suffix     = settings::default_process_suffix();
        _timemory_manager->write_metadata(settings::get_global_output_prefix(),
                                          "rocprofsys", _cfg);

        if(config::get_use_timemory())
        {
            auto _components = config::get_setting_value<std::string>(
                                   std::string{ env_vars::TIMEMORY_COMPONENTS })
                                   .value_or("wall_clock");

            for(auto&& _comp_name : rocprofsys::delimit(_components, ",; "))
            {
                if(_comp_name.empty()) continue;

                _output_registry.register_file(
                    settings::compose_output_filename(_comp_name, "txt", _cfg),
                    output_format::text, _comp_name);
                _output_registry.register_file(
                    settings::compose_output_filename(_comp_name, "json", _cfg),
                    output_format::json, _comp_name);
            }
        }
    }

    if(config::output_filtering::is_log_output_enabled_for_current_mpi_rank())
    {
        _output_registry.print_summary();
    }

    categories::shutdown();

    _finalization.stop();

    if(_perfetto_output_error)
    {
        throw std::runtime_error(fmt::format("Error opening perfetto output file: {}",
                                             get_perfetto_output_filename()));
    }

    if(_push_count > _pop_count)
    {
        LOG_WARNING("rocprofsys_push_trace/rocprofsys_push_trace_with_args was called "
                    "more times than rocprofsys_pop_trace. This is not fatal, but trace "
                    "output will not include regions that were still open during "
                    "finalization :: pushed: {} vs. popped: {}.",
                    _push_count, _pop_count);
    }

    // debug::close_file();
    config::finalize();

    LOG_DEBUG("Finalized: {}", _finalization.as_string());

    tim::signals::enable_signal_detection(
        { tim::signals::sys_signal::SegFault, tim::signals::sys_signal::Stop },
        [](int) {});

    common::destroy_static_objects();

    // Note: rocprofsys_init_library_done, rocprofsys_init_tooling_done, and state are NOT
    // reset here. They are only reset during re-attach (in rocprofiler-sdk.cpp) when
    // explicitly preparing for a new session. Resetting them during normal exit
    // can cause crashes if cleanup code triggers reinitialization.
}

extern "C" void
rocprofsys_set_finalization_done_hidden(void)
{
    rocprofsys_finalization_done.store(true);
}

extern "C" void
rocprofsys_reset_for_reattach_hidden(void)
{
    rocprofsys_finalization_done.store(false);
    rocprofsys_init_library_done.store(false);
    rocprofsys_init_tooling_done.store(0);
    ::rocprofsys::reset_database_path_memo();
    ::rocprofsys::reset_state();
}

//======================================================================================//

namespace
{
// if static objects are destroyed randomly (relatively uncommon behavior)
// this might call finalization before perfetto ends the tracing session
// but static variable in rocprofsys_init_tooling_hidden is more likely
auto _ensure_finalization = ensure_finalization(true);
auto _manager             = tim::manager::instance();
auto _settings            = tim::settings::shared_instance();
}  // namespace