TheRock/rocm-libraries/projects/rocalution/scripts/rocalution-bench-regression.py

#!/usr/bin/env python3

# ########################################################################
# Copyright (C) 2022 Advanced Micro Devices, Inc. All rights Reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#
# ########################################################################

import argparse
import subprocess
import os
import re # regexp package
import sys
import tempfile
import json

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-v', '--verbose',         required=False, default = False, action = "store_true")
    parser.add_argument('-t', '--tol',         required=True, default = 2.0,type=float)
    user_args, unknown_args = parser.parse_known_args()

    verbose=user_args.verbose
    percentage_tol = user_args.tol
    data = []
    num_files = len(unknown_args)

    titles = []
    for file_index in range(num_files):
        titles.append(os.path.basename(os.path.splitext(unknown_args[file_index])[0]))

    for file_index in range(num_files):
        with open(unknown_args[file_index],"r") as f:
            data.append(json.load(f))

    cmd = [d['cmdline'] for d in data]
    xargs = [d['xargs'] for d in data]
    yargs = [d['yargs'] for d in data]
    samples = [d['results'] for d in data]
    num_samples = len(samples[0])
    len_xargs = len(xargs[0])

    if verbose:
        print('//rocalution-bench-regression CONFIG')
        for i in range(num_files):
            print('//rocalution-bench-regression file'+str(i) +'      : \'' + unknown_args[i] + '\'')
        print('//rocalution-bench-regression COMPARISON')

####
    for i in range(1,num_files):
        if xargs[0] != xargs[i]:
            print('xargs\'s must be equal, xargs from file \''+unknown_args[i]+'\' is not equal to xargs from file \''+unknown_args[0]+'\'')
            exit(1)

    if verbose:
        print('//rocalution-bench-regression  -  xargs checked.')
####
    for i in range(1,num_files):
        if yargs[0] != yargs[i]:
            print('yargs\'s must be equal, yargs from file \''+unknown_args[i]+'\' is not equal to yargs from file \''+unknown_args[0]+'\'')
            exit(1)
    if verbose:
        print('//rocalution-bench-regression  -  yargs checked.')
####
    for i in range(1,num_files):
        if num_samples != len(samples[i]):
            print('num_samples\'s must be equal, num_samples from file \''+unknown_args[i]+'\' is not equal to num_samples from file \''+unknown_args[0]+'\'')
            exit(1)
    if verbose:
        print('//rocalution-bench-regression  -  num samples checked.')
####
    if verbose:
        print('//rocalution-bench-regression percentage_tol: ' + str(percentage_tol) + '%')

    global_regression=False
    global_improvement=False
    for file_index in range(1,num_files):
        print("//rocalution-bench-regression - '"+ unknown_args[file_index])
        for iplot in range(len(yargs[0])):
            print('//rocalution-bench-regression plot index ' + str(iplot) + ': \'' + yargs[0][iplot] + '\'',end='')
            mx_rel_time_analyze=0
            mx_rel_time_solve=0
            mx_rel_iter=0
            mx_rel_norm_residual=0
            mn_rel_time_analyze=0
            mn_rel_time_solve=0
            mn_rel_iter=0
            mn_rel_norm_residual=0
            regression=False
            improvement=False
            regression_time_solve=0
            regression_time_analyze=0
            regression_iter=0
            regression_norm_residual=0
            for ixarg  in range(len_xargs):
                isample = iplot * len_xargs + ixarg
                tg = samples[file_index][isample]["timing"]
                tg0=samples[0][isample]["timing"]
                rel_time_analyze = 100*(float(tg["median"]["time_analyze"])-float(tg0["median"]["time_analyze"]))/float(tg0["median"]["time_analyze"])
                rel_time_solve = 100*(float(tg["median"]["time_solve"])-float(tg0["median"]["time_solve"]))/float(tg0["median"]["time_solve"])
                rel_iter = 100*(float(tg["median"]["iter"])-float(tg0["median"]["iter"]))/float(tg0["median"]["iter"])
                rel_norm_residual = 100*(float(tg["median"]["norm_residual"])-float(tg0["median"]["norm_residual"]))/float(tg0["median"]["norm_residual"])

                if ixarg > 0:
                    mx_rel_time_solve=max(mx_rel_time_solve,rel_time_solve)
                    mn_rel_time_solve=min(mn_rel_time_solve,rel_time_solve)

                    mx_rel_time_analyze=max(mx_rel_time_analyze,rel_time_analyze)
                    mn_rel_time_analyze=min(mn_rel_time_analyze,rel_time_analyze)

                    mx_rel_iter=max(mx_rel_iter,rel_iter)
                    mn_rel_iter=min(mn_rel_iter,rel_iter)

                    mx_rel_norm_residual=max(mx_rel_norm_residual,rel_norm_residual)
                    mn_rel_norm_residual=min(mn_rel_norm_residual,rel_norm_residual)
                else:

                    mx_rel_norm_residual=rel_norm_residual
                    mn_rel_norm_residual=rel_norm_residual

                    mx_rel_iter=rel_iter
                    mn_rel_iter=rel_iter

                    mx_rel_time_analyze=rel_time_analyze
                    mn_rel_time_analyze=rel_time_analyze

                    mx_rel_time_solve=rel_time_solve
                    mn_rel_time_solve=rel_time_solve

                if (rel_time_solve > percentage_tol):
                    regression_time_solve = -1
                elif (rel_time_solve < -percentage_tol):
                    regression_time_solve = 1

                if (rel_time_analyze > percentage_tol):
                    regression_time_analyze = -1
                elif (rel_time_analyze < -percentage_tol):
                    regression_time_analyze = 1

                if (rel_iter > percentage_tol):
                    regression_iter = -1
                elif (rel_iter < -percentage_tol):
                    regression_iter = 1

                if (rel_norm_residual > percentage_tol):
                    regression_norm_residual = -1
                elif (rel_norm_residual < -percentage_tol):
                    regression_norm_residual = 1

                if (regression_time_solve == -1) or (regression_time_analyze == -1) or (regression_iter == -1) or (regression_norm_residual == -1):
                    print("")
                if (regression_time_solve == 1):
                    improvement=True
                if (regression_time_analyze == 1):
                    improvement=True
                if (regression_iter == 1):
                    improvement=True
                if (regression_norm_residual == 1):
                    improvement=True

                if (regression_time_solve == -1):
                    improvement=False
                    regression=True
                    print("//rocalution-bench-regression   FAIL time_solve exceeds tolerance of  "  +  str(percentage_tol) + "%, [" +"{:.2f}".format(mn_rel_time_solve) + "," + "{:.2f}".format(mx_rel_time_solve) + "] from '" + xargs[file_index][ixarg] + "'")

                if (regression_time_analyze == -1):
                    improvement=False
                    regression=True
                    print("//rocalution-bench-regression   FAIL time_analyze exceeds tolerance of  "  +  str(percentage_tol) + "%, [" +"{:.2f}".format(mn_rel_time_analyze) + "," + "{:.2f}".format(mx_rel_time_analyze) + "] from '" + xargs[file_index][ixarg] + "'")

                if (regression_iter == -1):
                    improvement=False
                    regression=True
                    print("//rocalution-bench-regression   FAIL iter exceeds tolerance of  "  +  str(percentage_tol) + "%, [" +"{:.2f}".format(mn_rel_iter) + "," + "{:.2f}".format(mx_rel_iter) + "] from '" + xargs[file_index][ixarg] + "'")

                if (regression_norm_residual == -1):
                    improvement=False
                    regression=True
                    print("//rocalution-bench-regression   FAIL norm_residual exceeds tolerance of  "  +  str(percentage_tol) + "%, [" +"{:.2f}".format(mn_rel_norm_residual) + "," + "{:.2f}".format(mx_rel_norm_residual) + "] from '" + xargs[file_index][ixarg] + "'")

            if not improvement:
                global_improvement=False

            if regression:
                global_regression=True
            if regression:
                print("   FAILED")
                print('//rocalution-bench-regression plot index ' + str(iplot) + ': \'' + yargs[0][iplot] + '\' FAILED')
            else:
                if improvement:
                    print("   IMPROVED")
                else:
                    print("   PASSED")
        if verbose:
            print("//rocalution-bench-regression    time_solve [" +"{:.2f}".format(mn_rel_time_solve) + "," + "{:.2f}".format(mx_rel_time_solve) + "], " + "time_analyze [" +"{:.2f}".format(mn_rel_time_analyze) + "," + "{:.2f}".format(mx_rel_time_analyze) + "], " + "iter [" +"{:.2f}".format(mn_rel_iter) + "," + "{:.2f}".format(mx_rel_iter) + "], " + "norm_residual [" +"{:.2f}".format(mn_rel_norm_residual) + "," + "{:.2f}".format(mx_rel_norm_residual) )
    if global_regression:
        exit(1)
    else:
        exit(0)
if __name__ == "__main__":
    main()