TheRock/rocm-libraries/projects/rocfft/scripts/perf/perflib/pdf.py

# Copyright (C) 2021 - 2022 Advanced Micro Devices, Inc. All rights reserved.
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
"""Utilities to generate PDF plots (via Asymptote)."""

import logging
import subprocess

import pandas

from dataclasses import dataclass
from pathlib import Path
from perflib.utils import sjoin, cjoin
from typing import List
import tempfile
import numpy as np
import os
import scipy.stats

import perflib.utils

top = Path(__file__).resolve().parent.parent


@dataclass
class BaseFigure:
    tag: str
    title: str
    caption: str
    docdir: Path
    labels: List[str]
    primary: List[Path]
    secondary: List[Path]
    figtype: str
    metadata: dict[str, str]


class PDFFigure(BaseFigure):

    def asycmd(self):
        asycmd = ['asy', '-f', 'pdf']

        ndata = 0
        for filename in self.primary:
            df = pandas.read_csv(filename, sep="\t", comment='#')
            ndata = max(ndata, len(df.index))

        if ndata > 1 and self.figtype == "linegraph":
            asycmd.append(str(top / "datagraphs.asy"))
            scaling = self.metadata.get('scaling')
        elif ndata == 1 or self.figtype == "bargraph":
            asycmd.append(str(top / "bargraph.asy"))

        ivariable = self.metadata.get('ivariable')
        if ivariable == None:
            scaling = self.metadata.get('scaling')
            if scaling != None:
                ivariable = 'ndev'
                asycmd.extend(['-u', f'scaling="{scaling}"'])
        if ivariable != None:
            asycmd.extend(['-u', f'ivariable="{ivariable}"'])

        primary = [x.resolve() for x in self.primary]
        asycmd.extend(['-u', f'filenames="{cjoin(primary)}"'])

        if self.labels is not None:
            asycmd.extend(['-u', f'legendlist="{cjoin(self.labels)}"'])

        if self.secondary is not None:
            secondary = [x for x in self.secondary]
            asycmd.extend(['-u', f'secondary_filenames="{cjoin(secondary)}"'])

        asycmd.extend(['-o', self.filename])

        return [str(x) for x in asycmd]

    def runasy(self):
        asycommand = self.asycmd()
        logging.info('ASY: ' + sjoin(asycommand))

        fout = tempfile.TemporaryFile(mode="w+")
        ferr = tempfile.TemporaryFile(mode="w+")

        proc = subprocess.Popen(asycommand, cwd=top, stdout=fout, stderr=ferr)

        try:
            proc.wait(timeout=20)
        except subprocess.TimeoutExpired:
            logging.info("Asy command killed: " + sjoin(asycommand))
            proc.kill()

            fout.seek(0)
            ferr.seek(0)
            cout = fout.read()
            cerr = ferr.read()

            print(cout)
            print(cerr)

        if proc.returncode != 0:
            logging.warn('ASY command failed: ' + sjoin(asycommand))

            fout.seek(0)
            ferr.seek(0)
            cout = fout.read()
            cerr = ferr.read()

            print(cout)
            print(cerr)

    def make(self, significance):
        self.filename = (Path(self.docdir) / (self.tag + '.pdf')).resolve()


gflopstext = '''\
GFLOP/s are computed based on the Cooley--Tukey operation count \
for a radix-2 transform, and half that for in the case of \
real-complex transforms.  The rocFFT operation count may differ from \
this value: GFLOP/s is provided for the sake of comparison only.'''

efficiencytext = '''\
Efficiency is computed for an idealised FFT which requires exactly \
one read and one write to global memory.  In practice, this \
isn't possible for most problem sizes, as the data does \
not fit into cache, and one must use global memory to store \
intermediary results.  As FFTs are bandwidth-limited on modern hardware, \
the efficiency is measured against the theoretical maximum bandwidth \
for the device.'''


def make_tex(figs,
             docdir,
             outdirs,
             label,
             significance,
             ncompare,
             secondtype=None):
    """Generate PDF containing performance figures."""

    docdir = Path(docdir)

    header = '''\
\\documentclass[12pt]{article}
\\usepackage[margin=1in]{geometry}
\\usepackage{graphicx}
\\usepackage{url}
\\usepackage{hyperref}
\\usepackage{float}
\\usepackage{longtable}
\\begin{document}
\\hypersetup{
  pdfborder={0,0,0},
  linkcolor=blue,
  citecolor=blue,
  urlcolor=blue
}
'''
    globalgeomean = perflib.utils.find_geomean(outdirs, False)
    print("geomean:", globalgeomean)

    tex = header

    tex += "\n\\section{Introduction}\n"

    # tex += "Each data point represents the median of " + str(nsample) + " values, with error bars showing the 95\\% confidence interval for the median.  All transforms are " + precision + "-precision.\n\n"

    if secondtype == "gflops":
        tex += gflopstext + "\n\n"

    tex += "\\vspace{1cm}\n"

    tex += "\n\\section{Device Specification}\n"
    for idx in range(len(outdirs)):
        tex += "\n\\subsection{" + str(label[idx]) + "}\n"
        path = Path(outdirs[idx]) / "specs.txt"
        if path.is_file():
            specs = path.read_text()

            for line in specs.split("\n"):
                if line.startswith("Host info"):
                    tex += "\\noindent " + line
                    tex += "\\begin{itemize}\n"
                elif line.startswith("Device info"):
                    tex += "\\end{itemize}\n"
                    tex += line
                    tex += "\\begin{itemize}\n"
                else:
                    if line.strip() != "":
                        tex += "\\item \\verb|" + line + "|\n"
            tex += "\\end{itemize}\n"
            tex += "\n"

    tex += "\\clearpage\n"
    tex += "\\section{Figures}\n"
    tex += "\\listoffigures\n"
    tex += "\\clearpage\n"

    # Data frames for significant speedups and slowdowns
    df_all_good = pandas.DataFrame()
    df_all_bad = pandas.DataFrame()

    # We need a list of speedups to compute the geometric mean via
    # sicpy.stats; the naive calculation suffers from issues with
    # finite precision.
    speedups = []

    figtex = ""

    for idx, fig in enumerate(figs):
        figtex += '''
\\centering
\\begin{figure}[H]
   \\includegraphics[width=\\textwidth]{'''
        figtex += str(fig.filename.name)
        figtex += '''}
   \\caption{''' + fig.caption + '''}
\\end{figure}
'''
        for p in fig.secondary:
            df = pandas.read_csv(p, sep="\t", comment='#')

            for row in df.itertuples(index=False):
                speedups.append(row.speedup)

            # Significant results:
            df_sig = df.loc[df['speedup_pval'] <= significance]

            # Significant results that are good or bad:
            df_good = df_sig.loc[df_sig['speedup'] > 1]
            df_bad = df_sig.loc[df_sig['speedup'] < 1]

            if not df_good.empty:

                df_all_good = pandas.concat([df_all_good, df_good])

                figtex += "\\begin{longtable}{l|l|l|}\n"
                figtex += "transform & speedup \% & significance\\\\ \n"
                figtex += "\\hline\n"
                for row in df_good.itertuples(index=False):
                    #figtex += str(row.token).replace("_", "\\_")
                    #figtex += "token"
                    transform_type, placeness, length, batch, precision = perflib.utils.parse_token(
                        row.token)
                    figtex += "$" + "\\times{}".join(str(x)
                                                     for x in length) + "$"

                    if np.prod(batch) > 1:
                        figtex += " by $" + "\\times{}".join(
                            str(x) for x in batch) + "$"

                    speedup = '{0:.3f}'.format((row.speedup - 1) * 100)
                    pval = '{0:.3f}'.format(row.speedup_pval)
                    figtex += " & " + str(speedup) + " & " + str(pval) + "\\\\"
                figtex += "\\hline\n"
                figtex += "\\caption{Improvements for " + fig.caption + "}\n"
                figtex += "\\end{longtable}\n"

            if not df_bad.empty:

                df_all_bad = pandas.concat([df_all_bad, df_bad])

                figtex += "\\begin{longtable}{l|l|l|}\n"
                figtex += "transform & slowdown \% & significance\\\\ \n"
                figtex += "\\hline\n"
                for row in df_bad.itertuples(index=False):
                    #figtex += str(row.token).replace("_", "\\_")
                    #figtex += "token"
                    transform_type, placeness, length, batch, precision = perflib.utils.parse_token(
                        row.token)
                    figtex += "$" + "\\times{}".join(str(x)
                                                     for x in length) + "$"

                    if np.prod(batch) > 1:
                        figtex += " by $" + "\\times{}".join(
                            str(x) for x in batch) + "$"

                    speedup = '{0:.3f}'.format((1 - 1 / row.speedup) * 100)

                    pval = '{0:.3f}'.format(row.speedup_pval)
                    figtex += " & " + str(speedup) + " & " + str(pval) + "\\\\"
                figtex += "\\hline\n"
                figtex += "\\caption{Regressions for " + fig.caption + "}\n"
                figtex += "\\end{longtable}\n"

        figtex += "\\clearpage\n"

    nspeedup = len(df_all_good.index)
    nslowdown = len(df_all_bad.index)

    if len(outdirs) > 1:
        print(
            "nspeedup  (" + label[1] + " is faster): " +
            " " * max(len(label[0]) - len(label[1]), 0), nspeedup)
        print(
            "nslowdown (" + label[0] + " is faster): " +
            " " * max(len(label[1]) - len(label[0]), 0), nslowdown)
        tex += "geometric mean overall cases: " + str(globalgeomean) + "\n"

        if ncompare > 0:
            geometric_mean = 1.0
            if len(speedups) > 1:
                geometric_mean = scipy.stats.mstats.gmean(speedups)
            print("geometric mean:", geometric_mean)
            tex += "\\begin{table}[H]\n"
            tex += "\\centering\n"
            tex += "\\begin{tabular}{l|p{5cm}|p{5cm}|l|}\n"
            tex += "ncompare & nspeedup " + "(" + label[
                1] + " faster)" + " & nslowdown (" + label[
                    0] + " faster)" + " & gmean\\\\ \n"
            tex += "\\hline\n"
            tex += str(ncompare) + "&" + str(nspeedup) + "&" + str(
                nslowdown) + "&" + '{0:.3f}'.format(geometric_mean) + "\\\\\n"
            tex += "100\\%" + "&" + '{0:.3f}'.format(
                100 * nspeedup / ncompare) + "\\% " + "&" + '{0:.3f}'.format(
                    100 * nslowdown / ncompare) + "\\% " + "&\\\\\n"
            tex += "\\hline\n"
            tex += "\\end{tabular}\n"
            tex += "\\caption{Overall Performance Changes}\n"
            tex += "\\end{table}\n"

        if nspeedup + nslowdown > 0:
            vals = []
            for row in df_all_good.itertuples(index=False):
                vals.append(100 * (row.speedup - 1))
            for row in df_all_bad.itertuples(index=False):
                vals.append(100 * (1 - (1 / row.speedup)))

            histdatname = os.path.join(docdir, "histogram.dat")

            with open(histdatname, 'w') as f:
                f.write("\t".join(str(x) for x in vals))
                f.write("\n")

            fout = tempfile.TemporaryFile(mode="w+")
            ferr = tempfile.TemporaryFile(mode="w+")

            asycmd = ["asy", "-f", "pdf", "histogram.asy"]
            asycmd.extend(['-u', 'filename="' + histdatname + '"'])
            asycmd.extend(['-o', os.path.join(docdir, "histogram.pdf")])

            asyproc = subprocess.Popen(asycmd,
                                       cwd=top,
                                       stdout=fout,
                                       stderr=ferr)
            try:
                asyproc.wait(timeout=20)
            except subprocess.TimeoutExpired:
                logging.info("asy command killed: " + sjoin(asycmd))
                asyproc.kill()

            if asyproc.returncode != 0:
                logging.warn('ASY command failed: ' + sjoin(asycmd))

                fout.seek(0)
                ferr.seek(0)
                cout = fout.read()
                cerr = ferr.read()

                print(cout)
                print(cerr)

            tex += '''\\centering
    \\begin{figure}[H]
    \\includegraphics[width=\\textwidth]{'''
            tex += "histogram.pdf"
            tex += '''}
    \\caption{''' + "Histogram of performance changes" + '''}\n\\end{figure}'''

        tex += "\\clearpage\n"

    tex += figtex

    if nspeedup > 0:
        tex += "\\clearpage\n"
        tex += "tokens for improved performance\n"
        tex += "\\begin{tiny}"
        tex += "\\begin{verbatim}"
        for row in df_all_good.itertuples(index=False):
            #print(row.token)

            tex += str(row.token) + "\n"
            #tex += "\\small\\texttt{" + str(row.token).replace("_", "\\_") + "}\n"
            #tex += str(row.token).replace("_", "\\_") + "\n"
        tex += "\\end{verbatim}"
        tex += "\\end{tiny}"

    if nslowdown > 0:
        print("There were", nslowdown, "regressions.  The tokens are:")
        tex += "\\clearpage\n"
        tex += "tokens for regressed performance\n"
        tex += "\\begin{tiny}"
        tex += "\\begin{verbatim}"
        for row in df_all_bad.itertuples(index=False):
            print(row.token, row.speedup)
            tex += str(row.token) + "\n"
            #tex += "\\small\\texttt{" + str(row.token).replace("_", "\\_") + "}\n"
        tex += "\\end{verbatim}"
        tex += "\\end{tiny}"

    tex += "\n\\end{document}\n"

    fname = docdir / 'figs.tex'
    fname.write_text(tex)

    log = docdir / 'tex.log'
    with log.open('w') as f:
        #latexcmd = ['latexmk','-pdf', fname.name]
        latexcmd = ['latexmk', '-interaction=batchmode', '-pdf', fname.name]
        texproc = subprocess.Popen(latexcmd,
                                   cwd=fname.parent,
                                   stdout=f,
                                   stderr=f)
        try:
            texproc.wait(timeout=60)
        except subprocess.TimeoutExpired:
            logging.info("tex command killed: " + sjoin(latexcmd))
            texproc.kill()

        if texproc.returncode != 0:
            print("****tex fail****")