TheRock/rocm-libraries/projects/rocfft/scripts/perf/rocfft-perf

#!/usr/bin/env python3

# Copyright (C) 2021 - 2025 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""Performance utilities for rocFFT.

Overview
========

General workflow:

- run: runs a suite of FFTs to collect timing information
- post: post processes timing information to compute various statistics
- plot: generate pdf or html plots of the results
- autoperf: clones, builds, runs, posts, and plots two rocFFT commits

Multiple runs can be compared at the post processing and plotting
stages.  Multiple runs may:

- be from different benchmark programs (eg, rocFFT, cuFFT, vkFFT, etc)
- be from dyna-rocfft-bench.

Usually:

- a single bench (rocFFT) would be used to track performance over
  time;
- multiple benchess (rocFFT, cuFFT) would be used to compare different
  FFT libraries;
- a dyna-bench with multiple libraries (rocFFT) would be used to
  compare two different rocFFT commits.

Runs/subprocesses are logged to `rocfft-perf.log`.


Run
===

The 'run' command drives FFT benchmarkers (if they accept the same
command line arguments as `rocfft-bench`).  The benchmark program to
use is specified by the `--bench/-w` switch.

Test problems are generated using a `ProblemGenerator` and a filter.
The default generator is a simple radix based generator.

See

  $ rocfft-perf run -h

for more details.  To see which problems will be run without running
them, use `--list/-l`.

Using the `--suite/-S` option, problems are loaded from a "suites"
file.  The default suites file is `suites.py`.  Alternatively, you can
load the suite named "qa1" from a file called "mysuites.py" like this:

  $ rocfft-perf run -S mysuites:qa1 ...

That is, FILENAME:SUITENAME.

By default, output files are stored in the `out0` directory.  This can
be changed with the `--output/-o` agrument.


Dynamic testing
===============

Dynamic testing is enabled by specifying more than one `--lib/-i`
option.  These are passed down to the benchmarker, and hence it is assumed
that the specific benchmarker is a "dyna" bench.

Multiple output directories are used to store the results.


Post processing
===============

During the post processing stage, various statistics are computed and
saved:

  $ rocfft-perf post DOCDIR OUTPUT [OUTPUT ...]

The first directory is the 'document directory'.  When comparing
multiple runs, comparative statistics are saved here in `.sdat` files.

For each `.dat` file in the output directories, summary statistics are
saved in `.mdat` files.


Plotting
========

Based on the results from post processing, generate either an html or
pdf report:

  $ rocfft-perf html DOCDIR OUTPUT [OUTPUT ...]
  $ rocfft-perf pdf DOCDIR OUTPUT [OUTPUT ...]

"""

import argparse
import logging
import statistics
import sys
import os
import tempfile
import re
import collections

from pathlib import Path

import multiprocessing

top = Path(__file__).resolve().parent
sys.path.append(str(top))

import perflib

console = logging.StreamHandler()

import types

#
# Helpers
#


def update(attr, dst, src):
    """Set attribute `attr` on dst if it is not None on `src`."""
    value = getattr(src, attr, None)
    if value is not None:
        setattr(dst, attr, value)


#
# Commands
#


def command_test(arguments):
    """Test for regressions."""

    # FIXME: rename and replace above comment

    verbose = arguments.verbose
    significance = arguments.significance
    multitest = arguments.multitest

    print("statistical test:", arguments.method)
    print("significance:", significance)
    print("multi-hypothesis framework:", multitest)

    outdirs = [Path(x) for x in arguments.runs]
    all_runs = perflib.utils.read_runs(outdirs, verbose)

    if len(all_runs) != 2:
        print(
            "Error: one must provide exactly two runs for statistical comparison"
        )
        sys.exit(1)

    runs = perflib.utils.by_dat(all_runs)
    refdir, testdir = outdirs

    label = ["reference", "test"]
    if arguments.label != None:
        for idx in range(len(arguments.label)):
            label[idx] = arguments.label[idx]

    print("reference data:", outdirs[0], label[0])
    print("comparison data:", outdirs[1], label[1])

    print()

    ncompare = perflib.utils.find_ncompare(arguments.runs)

    slower, faster, new_sig = perflib.utils.find_slower_faster(
        outdirs, arguments.method, multitest, significance, ncompare, verbose)

    if verbose:
        print("faster:")
        for case in faster:
            print(case)
        print("slower:")
        for case in slower:
            print(case)

    nh1 = len(faster) + len(slower)
    nh0 = ncompare - nh1

    print("ncompare:", ncompare)
    print("nh0:", nh0)
    print("nh1:", nh1)
    print("faster (" + label[1] + " wins): " + str(len(faster)))
    print("slower (" + label[0] + " wins): " + str(len(slower)))

    import numpy as np
    geomean = 1.0
    if nh1 > 0:
        sig_speedups = []
        for case in slower:
            sig_speedups.append(case[1] / case[2])
        for case in faster:
            sig_speedups.append(case[1] / case[2])
        import scipy
        geomean = scipy.stats.mstats.gmean(sig_speedups)
    print("geomean (significant cases):", geomean)

    globalgeomean = perflib.utils.find_geomean(outdirs, verbose)
    print("geomean:", globalgeomean)

    return len(slower) > 0


def generate_mdat(dat, measure, confidence):
    import numpy
    vals = [['token', 'median_sample', 'median_low', 'median_high']]
    for token, sample in dat.get_samples():
        if measure == "median":
            median = statistics.median(sample.times)
        elif measure == "mean":
            median = numpy.mean(sample.times)
        low, high = perflib.analysis.confidence_interval(sample.times,
                                                         measure=measure,
                                                         confidence=confidence)
        vals.append([sample.label, median, low, high])
    path = dat.path.with_suffix('.mdat')
    perflib.utils.write_tsv(path, vals, meta=dat.meta, overwrite=True)


def generate_pts_dat(dat):
    """
    For PTS system, extract data from raw dat and mdat.
    """
    import pandas
    mdat = dat.path.with_suffix('.mdat')
    mdat_df = pandas.read_csv(mdat, delimiter='\t', comment='#')

    # The parsing rule subjects to changes in the future
    ss = dat.tag
    input_params = []
    # placeness
    input_params.append(ss[ss.rfind('_') + 1:])
    ss = ss[:ss.rfind('_')]
    # transform type
    input_params.append(ss[ss.rfind('_', 0, ss.rfind('_') - 1) + 1:])
    ss = ss[:ss.rfind('_', 0, ss.rfind('_') - 1)]
    # precision
    input_params.append(ss[ss.rfind('_') + 1:])
    # suite
    input_params.append(ss[:ss.rfind('_')])

    input_params.reverse()

    dimensions = set()

    rows = []
    for row_idx, sample in enumerate(dat.get_samples()):
        new_row = []

        token = sample[0]

        # sanity check
        if not token or token.count('_') < 3:
            print(f"Skipping malformed token on row {row_idx}: '{token}'")
            continue

        try:
            transform_type, placeness, length, batch, precision, bricks, gpus, ranks = perflib.utils.parse_token(
                token)
        except Exception as e:
            print(
                f"Error parsing token on row {row_idx}: '{token}'\nException: {e}"
            )
            continue

        new_row.extend(input_params)
        new_row.insert(1, token)
        dimensions.add(len(length))
        new_row.append(len(length))
        new_row.extend(length)
        if len(batch) == 1:
            new_row.append(batch[0])
        else:
            print("multi-batch data format; exiting abnormally")
            sys.exit(1)

        # Multi-process and multi-gpu performance
        ngpus_per_node = max(1, len(gpus))
        nranks = max(1, len(ranks))

        nnodes = max(1,
                     (nranks + 1) / ngpus_per_node)  # assumes 1 gpu per rank

        new_row.append(nranks)
        new_row.append(ngpus_per_node)
        new_row.append(nnodes)

        # Input layout
        input_split_type = perflib.utils.get_decomposition_type(
            bricks['ifield'], label="ifield")
        input_grid = perflib.utils.get_proc_grid(bricks['ifield'])
        new_row.append(input_split_type)
        new_row.extend(input_grid)

        # Output layout
        output_split_type = perflib.utils.get_decomposition_type(
            bricks['ofield'], label="ofield")
        output_grid = perflib.utils.get_proc_grid(bricks['ofield'])
        new_row.append(output_split_type)
        new_row.extend(output_grid)

        # Statistics
        new_row.extend(
            mdat_df.loc[row_idx,
                        ['median_sample', 'median_low', 'median_high']].
            to_numpy().tolist())
        times = sample[1].times
        new_row.append(len(times))
        new_row.extend(times)
        rows.append(new_row)

    if len(set(dimensions)) > 1:
        print("mixed dimensions in the set; exiting abnormally")
        sys.exit(1)

    if len(set(dimensions)) == 0:
        print("PTS data set empty")
        return

    dimension = list(dimensions)[0]

    header = [
        'suite', 'token', 'precision', 'transform type', 'placeness',
        'dimension', 'xlength'
    ]
    if dimension == 2:
        header.append('ylength')
    elif dimension == 3:
        header.extend(['ylength', 'zlength'])

    header.append('nbatch')

    # Multi-process headers
    header.extend([
        'nprocs', 'ngpus_per_node', 'nnodes', 'input_split_type', 'ingrid_x',
        'ingrid_y', 'ingrid_z', 'output_split_type', 'outgrid_x', 'outgrid_y',
        'outgrid_z'
    ])

    header.extend(
        ['median_sample', 'median_low', 'median_high', 'nsample', 'samples'])

    content = [header]
    content.extend(rows)

    perflib.utils.write_pts_dat(dat.path.with_suffix('.ptsdat'),
                                content,
                                meta=dat.meta)


def command_post(arguments):
    """Post process results in directories listed in `outdirs`.

    Median confidence intervals for each run are written in 'mdat'
    files.

    Speedups and pvals are written in 'sdat' files.

    """

    outdirs = arguments.runs
    docdir = arguments.output
    verbose = arguments.verbose

    import itertools

    if verbose:
        print("docdir:", docdir)
        print("outdirs:", outdirs)

    outdirs = [Path(x) for x in outdirs]

    all_runs = perflib.utils.read_runs(outdirs, verbose)

    # median confidence intervals
    for run in all_runs:
        with multiprocessing.Pool(None) as p:
            p.starmap(
                generate_mdat,
                itertools.product(run.dats.values(), [arguments.measure],
                                  [arguments.confidence]))
            p.map(generate_pts_dat, run.dats.values())

    # speedup and pvals
    if len(outdirs) > 1:
        docdir = Path(docdir)
        docdir.mkdir(parents=True, exist_ok=True)

        import scipy.stats, numpy

        runs = perflib.utils.by_dat(all_runs)

        group_outdirs = []
        if arguments.ngroup == None:
            group_outdirs.append(outdirs)
        else:
            totalgroups = (len(outdirs) + arguments.ngroup -
                           1) // arguments.ngroup
            for gidx in range(totalgroups):
                group_outdirs.append(
                    outdirs[gidx * arguments.ngroup:(gidx + 1) *
                            arguments.ngroup])
        #print(group_runs)
        #sys.exit(1)

        for gidx, group in enumerate(group_outdirs):

            refdir, *otherdirs = group

            for dat_name, dat_runs in runs.items():
                refdat = dat_runs[refdir]
                for otherdat in [
                        dat_runs[otherdir] for otherdir in otherdirs
                        if otherdir in dat_runs
                ]:
                    speedups = [[
                        'token', 'speedup', 'speedup_low', 'speedup_high',
                        'speedup_pval'
                    ]]
                    for token, sample in refdat.get_samples():
                        if token not in otherdat.samples:
                            continue
                        sample = refdat.samples[token]
                        Avals = refdat.samples[token].times
                        Bvals = otherdat.samples[token].times
                        if arguments.measure == "median":
                            speedup = statistics.median(
                                Avals) / statistics.median(Bvals)
                        elif arguments.measure == "mean":
                            speedup = numpy.mean(Avals) / numpy.mean(Bvals)
                        low, high = perflib.analysis.ratio_confidence_interval(
                            Avals, Bvals)
                        pval = -1
                        if arguments.method == 'moods':
                            _, pval, _, _ = scipy.stats.median_test(
                                Avals, Bvals)
                        elif arguments.method == 'ttest':
                            _, pval = scipy.stats.ttest_ind(Avals, Bvals)
                        elif arguments.method == 'mwu':
                            _, pval = scipy.stats.mannwhitneyu(Avals, Bvals)
                        else:
                            print("unsupported statistical method")
                            sys.exit(1)

                        speedups.append(
                            [sample.token, speedup, low, high, pval])

                    path = docdir / ('group_' + str(gidx) + '-' + dat_name +
                                     '.sdat')
                    perflib.utils.write_tsv(path,
                                            speedups,
                                            meta=refdat.meta,
                                            overwrite=True)


def command_generate(arguments):
    """Generate PDF/HTML/DOCX from run results."""

    import perflib.pdf
    import perflib.html

    Figure = {
        'pdf': perflib.pdf.PDFFigure,
        'html': perflib.html.HTMLFigure,
        'docx': perflib.pdf.PDFFigure,
    }[arguments.command]

    docdir = Path(arguments.output)
    docdir.mkdir(parents=True, exist_ok=True)

    outdirs = [Path(outdir) for outdir in arguments.runs]
    label = arguments.label
    if label is None:
        label = [outdir.stem for outdir in outdirs]
    reference = perflib.utils.read_run(outdirs[0])

    import pandas
    import perflib.utils
    ncompare = perflib.utils.find_ncompare(arguments.runs)
    print("ncompare:", ncompare)

    slower, faster, new_sig = perflib.utils.find_slower_faster(
        outdirs, arguments.method, arguments.multitest, arguments.significance,
        ncompare, arguments.verbose)

    figures = []
    for datname in perflib.utils.list_runs(outdirs[0]):
        tag = datname.stem
        title = reference.dats[datname.stem].meta.get('title', tag)
        caption = reference.dats[datname.stem].meta.get('caption',
                                                        title).replace(
                                                            '_', ' ')
        figtype = reference.dats[datname.stem].meta.get('figtype', 'linegraph')
        primary, secondary = perflib.utils.get_post_processed(
            tag, docdir, outdirs, arguments.ngroup)
        figure = Figure(tag, title, caption, docdir, label, primary, secondary,
                        figtype, reference.dats[datname.stem].meta)
        figures.append(figure)

    for figure in figures:
        figure.make(new_sig)

    if arguments.command == 'pdf':
        if False:
            for figure in figures:
                figure.runasy()
        else:
            pool = multiprocessing.Pool(None)
            for figure in figures:
                pool.map_async(Figure.runasy, [figure])
            pool.close()
            pool.join()

    if arguments.command == 'pdf':
        perflib.pdf.make_tex(figures, docdir, outdirs, label, new_sig,
                             ncompare)
    if arguments.command == 'html':
        title = f"Performance report: {perflib.utils.cjoin(outdirs)}"
        perflib.html.make_html(figures, title, docdir, outdirs,
                               arguments.significance)
    if arguments.command == 'docx':
        import perflib.docx
        perflib.docx.make_docx(figures, docdir, outdirs,
                               arguments.significance)


def command_run(arguments):
    """Run dyna-bench or bench."""

    # build generator
    generator = None
    if arguments.suite is not None:
        generator = perflib.generators.SuiteProblemGenerator(arguments.suite)
    else:
        generator = perflib.generators.RadixProblemGenerator()
        for attr in [
                'radix', 'xmin', 'xmax', 'ymin', 'ymax', 'zmin', 'zmax',
                'verbose', 'timeout', 'hipskip'
        ]:
            update(attr, generator, arguments)

    for attr in ['nbatch', 'mp_size', 'mp_exec', 'ingrid', 'outgrid', 'ngpus']:
        update(attr, generator, arguments)

    # build filter
    filtered = perflib.generators.FilteredProblemGenerator()
    if arguments.direction is not None:
        filtered.direction = [arguments.direction]
    if arguments.inplace:
        filtered.inplace = [True]
    if arguments.outplace:
        filtered.inplace = [False]
    if arguments.real:
        filtered.real = [True]
    if arguments.complex:
        filtered.real = [False]
    if arguments.precision:
        filtered.precision = arguments.precision
    if arguments.dimension:
        filtered.dimension = arguments.dimension
    if arguments.mp_size:
        filtered.mp_size = arguments.mp_size
    if arguments.mp_exec:
        filtered.mp_exec = arguments.mp_exec
    if arguments.ingrid:
        filtered.ingrid = arguments.ingrid
    if arguments.outgrid:
        filtered.outgrid = arguments.outgrid
    if arguments.ngpus:
        filtered.ngpus = arguments.ngpus

    if arguments.list:
        for test in filtered(generator).generate_problems():
            print(test)
        return

    # build timer
    if arguments.bench is None:
        print("No benchmarker set... use -w /path/to/benchmarker.")
        return
    dyna = 'dyna' in arguments.bench
    if dyna:
        if not arguments.lib:
            print(
                "Need to set dynamically loaded library when using dyna-bench."
            )
            return
    if not arguments.out:
        nout = len(arguments.lib) if dyna else 1
        arguments.out = ['out' + str(i) for i in range(nout)]

    timer = perflib.timer.GroupedTimer()
    for attr in [
            'device', 'bench', 'accutest', 'lib', 'out', 'device', 'ntrial',
            'verbose', 'timeout', 'sequence', 'mp_size', 'mp_exec', 'ingrid',
            'outgrid', 'ngpus'
    ]:
        update(attr, timer, arguments)

    specs = perflib.specs.get_machine_specs(timer.device, arguments.specs_type)
    for out in timer.out:
        specs_file = Path(out) / 'specs.txt'
        specs_file.parent.mkdir(parents=True, exist_ok=True)
        specs_file.write_text(str(specs))

    failed_tokens = timer.run_cases(filtered(generator))

    print()

    if failed_tokens:
        logging.info("failed tokens: " + "\n".join(failed_tokens))
        print("failed tokens:\n" + "\n".join(failed_tokens))


def command_autoperf(arguments):
    """Compare performance of two builds automagically."""

    workdir = arguments.workdir
    reference_commit = arguments.reference_commit
    reference_repository = arguments.reference_repository
    reference_label = arguments.reference_label
    commit = arguments.commit
    repository = arguments.repository
    label = arguments.label
    suite = arguments.suite
    format = arguments.format
    static = arguments.static
    timeout = arguments.timeout

    # Use the short version of the hashes (default length: 7)
    if commit != None:
        commit = commit[0:6]
    if reference_commit != None:
        reference_commit = reference_commit[0:6]

    from perflib.build import build_rocfft

    if reference_repository is None:
        reference_repository = repository

    if reference_label is None:
        reference_label = reference_commit

    if label is None:
        label = commit

    top = Path(workdir).resolve()
    build1 = top / f'build-{reference_commit}'
    build2 = top / f'build-{commit}'
    output = top / f'doc-{commit}'

    # build rocFFTs
    top.mkdir(parents=True, exist_ok=True)
    os.chdir(str(top))

    lib1 = build1 / 'lib' / 'librocfft.so'
    lib1.parent.mkdir(parents=True, exist_ok=True)
    if not lib1.exists():
        build_rocfft(reference_commit, dest=build1, repo=reference_repository)

    lib2 = build2 / 'lib' / 'librocfft.so'
    lib2.parent.mkdir(parents=True, exist_ok=True)
    if not lib2.exists():
        build_rocfft(commit, dest=build2, repo=repository)

    # run cases
    if static:
        # use more trials for static bench
        timer1 = perflib.timer.GroupedTimer()
        timer1.bench = build1 / 'rocfft-bench'
        timer1.lib = None
        timer1.out = [build1]
        timer1.ntrial = 20
        timer1.timeout = timeout

        timer2 = perflib.timer.GroupedTimer()
        timer2.bench = build2 / 'rocfft-bench'
        timer2.lib = None
        timer2.out = [build2]
        timer2.ntrial = 20
        timer2.timeout = timeout
        timers = [timer1, timer2]
    else:
        timer = perflib.timer.GroupedTimer()
        timer.bench = build1 / 'dyna-rocfft-bench'
        timer.lib = [lib1, lib2]
        timer.out = [build1, build2]
        timer.timeout = timeout
        timers = [timer]

    specs = perflib.specs.get_machine_specs(timers[0].device,
                                            arguments.specs_type)
    for t in timers:
        for out in t.out:
            specs_file = Path(out) / 'specs.txt'
            specs_file.write_text(str(specs))

    generator = perflib.generators.SuiteProblemGenerator(suite)
    for t in timers:
        t.run_cases(generator)

    # post-process results
    arguments.runs = [build1, build2]
    arguments.output = output
    arguments.label = [reference_label, label]
    command_post(arguments)

    # generate report
    for report_type in format:
        command_generate(arguments)


def command_bweff(arguments):
    """Collect bandwidth efficiency information."""

    # build generator from suite
    generator = perflib.generators.SuiteProblemGenerator(arguments.suite)

    Path(arguments.out).mkdir(parents=True, exist_ok=True)

    all_problems = collections.defaultdict(list)
    for problem in generator.generate_problems():
        all_problems[problem.tag].append(problem)

    # create temporary file
    fp = tempfile.NamedTemporaryFile()

    # set environment variables
    os.environ['ROCFFT_LAYER'] = '4'
    os.environ['ROCFFT_LOG_PROFILE_PATH'] = fp.name

    data = []
    for i, (tag, problems) in enumerate(all_problems.items()):
        print(
            f'\n{tag} (group {i} of {len(all_problems)}): {len(problems)} problems'
        )

        bench = Path(arguments.bench)
        if not bench.is_file():
            raise RuntimeError(
                f"Unable to find benchmarker: {arguments.bench}")

        effdat_paths = [Path(arguments.out) / (tag + '.effdat')]
        generator = perflib.generators.VerbatimGenerator(problems)

        for prob in generator.generate_problems():

            # determine appropriate batch size
            if prob.precision == "half":
                elem_size_bytes = 4
            elif prob.precision == "single":
                elem_size_bytes = 8
            elif prob.precision == "double":
                elem_size_bytes = 16

            for length in prob.length:
                elem_size_bytes *= length

            nbatch = (arguments.target_size << 30) // elem_size_bytes

            # run bench
            token = perflib.bench.run(arguments.bench,
                                      prob.length,
                                      direction=prob.direction,
                                      real=prob.real,
                                      inplace=prob.inplace,
                                      precision=prob.precision,
                                      nbatch=nbatch,
                                      ntrial=arguments.ntrial,
                                      mp_size=prob.mp_size,
                                      mp_exec=prob.mp_exec,
                                      ingrid=prob.ingrid,
                                      outgrid=prob.outgrid,
                                      ngpus=prob.ngpus)[0]

            fp.seek(0)

            # parse profile log
            profile_log = []
            for line in fp:

                line = line.decode('UTF-8').strip('\n')

                perf_info = {}
                items = re.split(r',(?![^\[]*[\]])', line)

                for i in range(1, len(items), 2):
                    perf_info.update({items[i]: items[i + 1]})

                profile_log.append(perf_info)

            fp.truncate(0)

            # collect data in tab-separated .effdat files
            for path in effdat_paths:
                out = Path(path)
                logging.info("output: " + str(out))
                meta = {'title': prob.tag}
                meta.update(prob.meta)
                for row in profile_log:
                    records = [
                        token,  # testcase token
                        row['scheme'],  # scheme
                        row['duration_ms'],  # kernel duration in milliseconds
                        row['bw_efficiency_pct'],  # estimated efficiency
                        row['kernel_index']  # index number of this kernel in the execution plan
                    ]
                    data.append(records)
                    perflib.utils.write_tsv(out, [records], meta=meta)

    # close temporary file
    fp.close()

    # unset environment variables
    if 'ROCFFT_LAYER' in os.environ:
        del os.environ['ROCFFT_LAYER']
        del os.environ['ROCFFT_LOG_PROFILE_PATH']

    # determine median duration and efficiency by token and index
    medians = collections.defaultdict(list)

    for entry in data:
        token = entry[0]
        scheme = entry[1]
        duration = float(entry[2])
        efficiency = float(entry[3])
        index = int(entry[4])

        medians[(token, index, scheme)].append((duration, efficiency))

    # collect median data in tab-separated .effdat files
    out = Path(arguments.out) / ("median_values.effdat")
    logging.info("output: " + str(out))
    meta = {'title': "median values"}
    for key in medians:
        if arguments.mesaure == "median":
            records = [
                key[0],  # token
                key[1],  # index
                key[2],  # scheme
                statistics.median(medians[key][0]),  # duration_ms
                statistics.median(medians[key][1])  # bw_efficiency_pct
            ]
        elif arguments.mesaure == "mean":
            records = [
                key[0],  # token
                key[1],  # index
                key[2],  # scheme
                numpy.mean(medians[key][0]),  # duration_ms
                numpy.mean(medians[key][1])  # bw_efficiency_pct
            ]
        perflib.utils.write_tsv(out, [records], meta=meta)


#
# Main
#


def main():

    parser = argparse.ArgumentParser(
        prog='rocfft-perf',
        epilog="For a detailed usage overview, run: %(prog)s overview")
    parser.add_argument('-v', '--verbose', action='store_true', default=False)
    parser.add_argument('--logfile',
                        type=str,
                        help="log file name",
                        default="rocfft-perf.log")

    subparsers = parser.add_subparsers(dest='command')

    subparsers.add_parser('overview', help='print a general usage overview')
    specs_parser = subparsers.add_parser('specs', help='print machine specs')

    run_parser = subparsers.add_parser('run', help='run!')
    post_parser = subparsers.add_parser('post', help='post processing')
    pdf_parser = subparsers.add_parser('pdf', help='generate pdf plots')
    html_parser = subparsers.add_parser('html', help='generate html plots')
    docx_parser = subparsers.add_parser('docx', help='generate docx plots')
    test_parser = subparsers.add_parser('test', help='test for regressions')
    autoperf_parser = subparsers.add_parser(
        'autoperf',
        help='clone, build, run, post, and plot two rocFFT commits')

    specs_parser.add_argument(dest='specs_type',
                              type=str,
                              default='default',
                              nargs='?',
                              choices=['default', 'host', 'device'],
                              help="type of specs")

    for p in [post_parser, pdf_parser, html_parser, docx_parser]:
        p.add_argument('output', type=str)
    for p in [post_parser, pdf_parser, html_parser, docx_parser, test_parser]:
        p.add_argument('runs', type=str, nargs='+')

    for p in [post_parser, autoperf_parser]:
        p.add_argument('--confidence',
                       type=str,
                       choices=["bootstrap", "stdev"],
                       help="method for generating confidence interval",
                       default="bootstrap")
    for p in [post_parser, pdf_parser, html_parser]:
        p.add_argument('--ngroup',
                       type=int,
                       help="size of comparison subgroup")

    for p in [
            post_parser, pdf_parser, test_parser, autoperf_parser, html_parser
    ]:
        p.add_argument('--method',
                       type=str,
                       choices=["moods", "ttest", "mwu"],
                       help="statistical method",
                       default="moods")
    for p in [
            post_parser, pdf_parser, html_parser, docx_parser, test_parser,
            autoperf_parser
    ]:
        p.add_argument('--measure',
                       type=str,
                       choices=["mean", "median"],
                       help="measure of central tendency: median or mean",
                       default="median")
    for p in [
            pdf_parser, html_parser, docx_parser, test_parser, autoperf_parser
    ]:
        p.add_argument('--significance',
                       type=float,
                       help='moods significance threshold',
                       default=0.05)
        p.add_argument('--multitest',
                       type=str,
                       choices=["none", "bonferroni", "bh"],
                       help='Multi-hypothesis correction method',
                       default="bonferroni")
    for p in [pdf_parser, html_parser, docx_parser, test_parser]:
        p.add_argument('-l',
                       '--label',
                       type=str,
                       help='label (appendable)',
                       action='append')

    run_parser.add_argument('-g', '--device', type=int, help='device number')
    run_parser.add_argument(
        '--hipskip',
        action=argparse.BooleanOptionalAction,
        help='Do not count hip runtime error codes as failures')
    run_parser.add_argument('-l',
                            '--list',
                            help='list runs (but do not run them)',
                            action='store_true',
                            default=False)
    run_parser.add_argument('-o',
                            '--out',
                            type=str,
                            help='output (appendable)',
                            action='append')
    run_parser.add_argument('-S',
                            '--suite',
                            type=str,
                            help='test suite name (appendable)',
                            action='append')
    run_parser.add_argument('-w',
                            '--bench',
                            type=str,
                            help='test executable path')
    run_parser.add_argument('-i',
                            '--lib',
                            type=str,
                            help='test library path (appendable)',
                            action='append')
    run_parser.add_argument('-r', '--radix', type=int, help='radix')
    run_parser.add_argument('-x',
                            '--xmin',
                            type=int,
                            help='minimum problem size in x direction')
    run_parser.add_argument('-X',
                            '--xmax',
                            type=int,
                            help='maximum problem size in x direction')
    run_parser.add_argument('-y',
                            '--ymin',
                            type=int,
                            help='minimum problem size in y direction')
    run_parser.add_argument('-Y',
                            '--ymax',
                            type=int,
                            help='maximum problem size in y direction')
    run_parser.add_argument('-z',
                            '--zmin',
                            type=int,
                            help='minimum problem size in z direction')
    run_parser.add_argument('-Z',
                            '--zmax',
                            type=int,
                            help='maximum problem size in z direction')
    run_parser.add_argument('-D',
                            '--direction',
                            type=int,
                            help='direction of transform')
    run_parser.add_argument('-I',
                            '--inplace',
                            help='make transform in-place',
                            action='store_true',
                            default=False)
    run_parser.add_argument('-O',
                            '--outplace',
                            help='make transform out-of-place',
                            action='store_true',
                            default=False)
    run_parser.add_argument('-R',
                            '--real',
                            help='make transform real/complex',
                            action='store_true',
                            default=False)
    run_parser.add_argument('-C',
                            '--complex',
                            help='make transform complex/complex',
                            action='store_true',
                            default=False)
    run_parser.add_argument('-d',
                            '--dimension',
                            type=int,
                            help='dimension of transform',
                            action='append')
    run_parser.add_argument('-b',
                            '--nbatch',
                            type=int,
                            help='number of batches')
    run_parser.add_argument('-N',
                            '--ntrial',
                            type=int,
                            help='number of trials',
                            default=20)
    run_parser.add_argument('-mp_size',
                            '--mp_size',
                            type=int,
                            help='number of MPI processors',
                            default=1)

    run_parser.add_argument('-mp_exec',
                            '--mp_exec',
                            type=str,
                            help="path to mpi executable",
                            default="/usr/bin/mpirun")

    run_parser.add_argument('-ngpus',
                            '--ngpus',
                            type=int,
                            help='number of GPUs',
                            default=1)
    run_parser.add_argument(
        '-T',
        '--timeout',
        type=int,
        help='test timeout in seconds (0 disables timeout)',
        default=600)
    run_parser.add_argument('--sequence',
                            type=int,
                            help='dyna-bench test sequence',
                            default=0)

    run_parser.add_argument('--ingrid',
                            type=int,
                            required=False,
                            nargs='+',
                            help='input processors grid')

    run_parser.add_argument('--outgrid',
                            type=int,
                            required=False,
                            nargs='+',
                            help='output processors grid')

    run_parser.add_argument('-f',
                            '--precision',
                            type=str,
                            help='precision',
                            action='append')
    run_parser.add_argument('-t',
                            '--accutest',
                            type=str,
                            help='accuracy test executable path')
    run_parser.add_argument('--specs_type',
                            type=str,
                            default='default',
                            nargs='?',
                            choices=['default', 'host', 'device'],
                            help="type of specs")

    autoperf_parser.add_argument('--workdir',
                                 type=str,
                                 help='Working directory',
                                 default='.')
    autoperf_parser.add_argument('--reference_commit',
                                 type=str,
                                 help='Reference commit',
                                 required=True)
    autoperf_parser.add_argument(
        '--reference_repository',
        type=str,
        help='Reference repository (if different from repository)')
    autoperf_parser.add_argument(
        '--reference_label',
        type=str,
        help='Reference label (if different from reference commit)')
    autoperf_parser.add_argument('--commit',
                                 type=str,
                                 help='Commit to test',
                                 required=True)
    autoperf_parser.add_argument('--repository',
                                 type=str,
                                 help='Repository to test',
                                 required=True)
    autoperf_parser.add_argument(
        '--label', type=str, help='Test label (if different from test commit)')
    autoperf_parser.add_argument('--suite',
                                 type=str,
                                 help='Test suite name (appendable)',
                                 action='append',
                                 required=True)
    autoperf_parser.add_argument('--format',
                                 type=str,
                                 help='Output format (appendable)',
                                 action='append',
                                 default=['html'])
    autoperf_parser.add_argument('--static',
                                 help='Use static bench instead of dyna',
                                 action='store_true',
                                 default=False)
    autoperf_parser.add_argument(
        '-T',
        '--timeout',
        type=int,
        help='test timeout in seconds (0 disables timeout)',
        default=600)
    autoperf_parser.add_argument('--specs_type',
                                 type=str,
                                 default='default',
                                 nargs='?',
                                 choices=['default', 'host', 'device'],
                                 help="type of specs")

    bweff_parser = subparsers.add_parser(
        'bweff', help='bandwidth efficiency collection')
    # suite of tests to run
    bweff_parser.add_argument('-S',
                              '--suite',
                              type=str,
                              help='test suite name (appendable)',
                              action='append',
                              required=True)
    # path to bench executable
    bweff_parser.add_argument('-w',
                              '--bench',
                              type=str,
                              help='test executable path',
                              required=True)
    # output directory for results
    bweff_parser.add_argument('-o',
                              '--out',
                              type=str,
                              help='output',
                              default='out')
    # number of trials to run per test case
    bweff_parser.add_argument('-N',
                              '--ntrial',
                              type=int,
                              help='number of trials',
                              default=10)

    bweff_parser.add_argument('-mp_size',
                              '--mp_size',
                              type=int,
                              help='number of MPI processors',
                              default=1)

    bweff_parser.add_argument('-mp_exec',
                              '--mp_exec',
                              type=str,
                              help="path to mpi executable",
                              default="/usr/bin/mpirun")

    bweff_parser.add_argument('-ngpus',
                              '--ngpus',
                              type=int,
                              help='number of GPUs',
                              default=1)

    # target transform size
    bweff_parser.add_argument('--target_size',
                              type=int,
                              help='target transform size in GiB',
                              default=5)

    arguments = parser.parse_args()

    logging.basicConfig(filename=arguments.logfile,
                        format='%(asctime)s %(levelname)s: %(message)s',
                        level=logging.DEBUG)

    console.setLevel(logging.WARNING)
    console.setFormatter(logging.Formatter('%(levelname)-8s: %(message)s'))
    logging.getLogger('').addHandler(console)

    if arguments.verbose:
        console.setLevel(logging.INFO)

    if arguments.command == 'specs':
        # Todo: find unified way to specify device id
        print(perflib.specs.get_machine_specs(0, arguments.specs_type))

    if arguments.command == 'overview':
        print(globals()['__doc__'])

    if arguments.command == 'run':
        command_run(arguments)

    if arguments.command == 'post':
        command_post(arguments)

    if arguments.command == 'test':
        sys.exit(command_test(arguments))

    if arguments.command == 'pdf':
        command_generate(arguments)

    if arguments.command == 'html':
        command_generate(arguments)

    if arguments.command == 'docx':
        command_generate(arguments)

    if arguments.command == 'autoperf':
        command_autoperf(arguments)

    if arguments.command == 'bweff':
        command_bweff(arguments)

    sys.exit(0)


if __name__ == '__main__':
    main()