TheRock/rocm-libraries/projects/hipblaslt/utilities/QuickTune/gemm_tuning.py

# Copyright (C) Advanced Micro Devices, Inc., or its affiliates.
# SPDX-License-Identifier:  MIT

import os
import argparse
import subprocess
import time

from remove_duplicate import parse_input_log
from utils import parse_hipblaslt_output, export_csv, dynamic_iters, convert_command

def run_baseline(input_file, args, tuning_info):
    # Delete Tuning File Environment Variable to avoid tuning
    hipblaslt_tuning_file = ""
    if "HIPBLASLT_TUNING_FILE" in os.environ:
        hipblaslt_tuning_file = os.environ["HIPBLASLT_TUNING_FILE"]
        del os.environ["HIPBLASLT_TUNING_FILE"]

    with open(input_file, 'r') as f, open(args.output_path + '/baseline_reproduce_commands.log', 'w+') as f_out:
        for line in f:
            command = convert_command(line, args, tuning_info, "baseline")
            f_out.write(command + '\n')
            result = subprocess.run(command, shell=True, capture_output=True, text=True)
            parse_hipblaslt_output(result.stdout, line, tuning_info, "baseline")

    # Restore HIPBLASLT_TUNING_FILE
    if hipblaslt_tuning_file != "":
        os.environ["HIPBLASLT_TUNING_FILE"] = hipblaslt_tuning_file

def run_tuning(input_file, args, tuning_info):
    # set default tuning file if it is not in the environment
    default_tuning_file = False
    if "HIPBLASLT_TUNING_FILE" not in os.environ:
        default_tuning_file = True
        os.environ["HIPBLASLT_TUNING_FILE"] = args.output_path + "/tuning.txt"

    with open(input_file, 'r') as f, open(args.output_path + '/tuning_reproduce_commands.log', 'w+') as f_out:
        for line in f:
            command = convert_command(line, args, tuning_info, "tuning")
            f_out.write(command + '\n')
            result = subprocess.run(command, shell=True, capture_output=True, text=True)
            parse_hipblaslt_output(result.stdout, line, tuning_info, "tuning")

    # Remove HIPBLASLT_TUNING_FILE
    if default_tuning_file:
        del os.environ["HIPBLASLT_TUNING_FILE"]

def gpu_setup(device):
    os.environ["HIP_FORCE_DEV_KERNARG"] = "1"
    # amd-smi set requires elevated privileges; enable performance determinism
    # with a GFXCLK soft-max of 1900 MHz to stabilize the GPU frequency.
    subprocess.run(["sudo", "-n", "amd-smi", "set", "-g", str(device), "--perf-determinism", "1900"], check=True)

def gpu_reset(device):
    del os.environ["HIP_FORCE_DEV_KERNARG"]
    # Disable performance determinism (inverse of gpu_setup).
    subprocess.run(["sudo", "-n", "amd-smi", "reset", "-g", str(device), "--perf-determinism"], check=True)

def main():
    parser = argparse.ArgumentParser(description="Execute Gemm Tuning")
    parser.add_argument("--input_file", type=str, help="Path to the list of gemm ops")
    parser.add_argument("--output_path", type=str, default='./tuning_result', help="Path to output file")
    parser.add_argument("--swizzleA", action='store_true', help="Whether to enable swizzleA tuning")
    parser.add_argument("--requested_solution", type=int, default=128, help="Searching space for gemm tuning")
    parser.add_argument("--cold_iters", type=int, default=-1, help="warm-up iteration to measure kernel performance")
    parser.add_argument("--iters", type=int, default=-1, help="iteration to measure kernel performance")
    parser.add_argument("--gpu_id", type=int, default=0, help="select gpu devices")
    parser.add_argument("--stablize_gpu", action='store_true', help="whether to stablelize GPU frequency")
    args = parser.parse_args()

    # set_up gpu status
    os.environ["HIPBLASLT_LOG_MASK"] = "32"
    os.environ["HIP_VISIBLE_DEVICES"] = str(args.gpu_id)
    if args.stablize_gpu:
        gpu_setup(args.gpu_id)

    tuning_info = dict()
    unique_log_name = parse_input_log(args, tuning_info)

    start_time = time.time()
    run_baseline(unique_log_name, args, tuning_info)
    end_time = time.time()
    print("baseline time elapsed = {}".format(end_time - start_time))

    start_time = time.time()
    run_tuning(unique_log_name, args, tuning_info)
    end_time = time.time()
    print("tuning time elapsed = {}".format(end_time - start_time))

    # reset gpu status
    if args.stablize_gpu:
        gpu_reset(args.gpu_id)

    success_count, total_count = export_csv(unique_log_name, tuning_info, args)

    # Print summary report
    failed_count = total_count - success_count
    print("\n" + "="*60)
    print("GEMM TUNING SUMMARY")
    print("="*60)
    print(f"Total GEMMs processed: {total_count}")
    print(f"  - Successful: {success_count}")
    print(f"  - Failed: {failed_count}")
    print("="*60)
    print(f"Results saved to: {args.output_path}/tuning_result.csv")
    print()


if __name__ == "__main__":
    main()