#!/usr/bin/python3

import sys, getopt
import numpy as np
from math import *
import subprocess
import os
import re # regexp package
import shutil
import tempfile
import csv
import random

import perflib
import timing

usage = '''A timing script to generate perf data and plot for major fft 1D/2D/3D cases

Usage:
\talltime.py
\t\t-b          Specify dload executable(optional)
\t\t-i          Specify test libraries for dload or test executables
\t\t              for regular test(appendable)
\t\t-o          Specify output directories for raw data
\t\t              appendable; defaults to "dir0", "dir1", etc.
\t\t-l          Specify labels for runs
\t\t              appendable; defaults to "dir0", "dir1", etc.
\t\t-w          output directory for graphs and final document
\t\t-S          plot speedup (default: 1, disabled: 0)
\t\t-t          data type: time (default) or gflops or roofline
\t\t-y          secondary axis type: none, gflops, efficiency, p
\t\t-s          short run
\t\t-T          do not perform FFTs; just generate document
\t\t-f          document format: pdf (default) or docx
\t\t-g          generate graphs via Asymptote: 0(default) or 1
\t\t-d          device number (default: 0)
\t\t-N          Number of samples (default: 10)
\t\t-D          dims to test. default: 1,2,3
\t\t-R          runtype: report benchmark or efficiency
\t\t-F          filename to read problem sizes from
\t\t-p          precision: single or double(default)
\t\t-B          specify bandwidth for efficiency computation
\t\t-v          verbose output
'''

def nextpow(val, radix):
    x = 1
    while(x <= val):
        x *= radix
    return x

# A class for generating data for figures.
class rundata:
    def __init__(self, label,
                 dimension, minsize, maxsize, nbatch, radix, ratio, ffttype,
                 direction, inplace, precision):
        self.dimension = dimension
        self.minsize = minsize
        self.maxsize = maxsize
        self.nbatch = nbatch
        self.radix = radix
        self.ratio = ratio
        self.ffttype = ffttype
        self.precision = precision
        self.inplace = inplace
        self.direction = direction
        self.label = label
        
    def outfilename(self, odir):
        outfile = ""
        outfile += "radix" + str(self.radix)
        outfile += "_dim" + str(self.dimension)
        outfile += "_" + self.precision
        outfile += "_n" + str(self.nbatch)
        if self.direction == 1:
            outfile += "_inv"
        if self.dimension > 1:
            outfile += "_ratio" + "_" + str(self.ratio[0])
        if self.dimension > 2:
            outfile += "_" + str(self.ratio[1])
        outfile += "_" + self.ffttype
        if self.inplace:
            outfile += "_inplace"
        else:
            outfile += "_outofplace"
        outfile += ".dat"
        outfile = os.path.join(odir, outfile)
        return outfile

    def runcmd(self, nsample, inlist, outdirlist, dloadexe, problem_file):
        timer = timing.Timer()

        if dloadexe == None:
            # When not using dload, we just have one input and output dir.
            timer.prog = os.path.abspath(inlist[0])
            timer.out = [ self.outfilename(outdirlist[0]) ]
        else:
            timer.prog = dloadexe
            timer.lib = inlist
            timer.out = [ self.outfilename(x) for x in outdirlist ]

        timer.ntrial = nsample
        timer.nbatch = self.nbatch
        timer.xmin = self.minsize
        timer.xmax = self.maxsize
        if self.dimension > 1:
            timer.ymin = self.minsize * self.ratio[0]
            timer.ymax = self.maxsize * self.ratio[0]
        if self.dimension > 2:
            timer.zmin = self.minsize * self.ratio[1]
            timer.zmax = self.maxsize * self.ratio[1]

        timer.radix = self.radix
        timer.inplace = self.inplace
        timer.direction = self.direction
        timer.dimension = self.dimension
        timer.precision = self.precision
        timer.real = self.ffttype == "r2c"
        timer.problem_file = problem_file

        return timer

    def executerun(self, nsample, inlist, outdirlist, dloadexe, problem_file):
        if dloadexe != None:
            self.runcmd(nsample, inlist, outdirlist, dloadexe, problem_file).run_cases()
        else:
            for idx in range(min(len(inlist), len(outdirlist))):
                print(idx, ":", inlist[idx], "->", outdirlist[idx])
                self.runcmd(nsample, [inlist[idx]], [outdirlist[idx]], None, problem_file).run_cases()

# Compute the alpha-confidence interval for the given values using
# boot-strap resampling.
def confidence_interval(vals, alpha=0.95, nboot=2000):
    medians = []
    for iboot in range(nboot):
        resample = []
        for i in range(len(vals)):
            resample.append(vals[random.randrange(len(vals))])
        medians.append(np.median(resample))
    medians = sorted(medians)
    low = medians[int(np.floor(nboot * 0.5*(1.0 - alpha)))]
    high = medians[int(np.ceil(nboot * (1.0 - 0.5*(1.0 - alpha))))]
    return low, high

# Compute the alpha-confidence interval for the ratio of the given
# sets of values using boot-strap resampling.
def ratio_confidence_interval(Avals, Bvals, alpha=0.95, nboot=2000):
    ratios = []
    for i in range(nboot):
        ratios.append( Avals[random.randrange(len(Avals))] / Bvals[random.randrange(len(Bvals))])
    ratios = sorted(ratios)
    low = ratios[int(np.floor(len(ratios) * 0.5*(1.0 - alpha)))]
    high = ratios[int(np.ceil(len(ratios) * (1.0 - 0.5*(1.0 - alpha))))]
    return low, high

# Figure class, which contains runs and provides commands to generate figures.
class figure:
    def __init__(self, name, caption):
        self.name = name
        self.runs = []
        self.caption = caption

    def inputfiles(self, outdirlist):
        import os
        files = []
        for run in self.runs:
            for outdir in outdirlist:
                files.append(run.outfilename(outdir))
        print(files)
        return files

    def labels(self, labellist):
        labels = []
        for run in self.runs:
            for label in labellist:
                labels.append(label + run.label)
        return labels

    def filename(self, outdir):
        outfigure = self.name
        outfigure += ".pdf"
        return os.path.join(outdir, outfigure)

    def asycmd(self, docdir, primary_fnames, secondary_fnames, labellist):
        asycmd = ["asy"]

        asycmd.append("-f")
        asycmd.append("pdf")

        asycmd.append(os.path.join(sys.path[0], "datagraphs.asy"))
       
        asycmd.append("-u")
        labels = self.labels(labellist)
        asycmd.append('filenames="' + ",".join(primary_fnames) + '"')

        if labellist != None:
            asycmd.append("-u")
            labels = self.labels(labellist)
            asycmd.append('legendlist="' + ",".join(labels) + '"')
        
        if secondary_fnames != None:
            asycmd.append("-u")
            asycmd.append('secondary_filenames="' + ",".join(secondary_fnames) + '"')
            
        asycmd.append("-o")
        asycmd.append(self.filename(docdir) )

        return asycmd

    # Remove anything that starts with #, which we use for a comment character.
    def decomment(self, csvfile):
        for row in csvfile:
            raw = row.split('#')[0].strip()
            if raw: yield raw

    # Collect the data from individual runs into one data object.
    def collect_data(self, outdirs):
        inputfiles = self.inputfiles(outdirs)
        dataout = []
        for datafile in inputfiles:
            idata = []
            with open(datafile) as csvfile:
                csvfile.seek(0)
                reader = csv.reader(self.decomment(csvfile), delimiter='\t')
                for row in reader:
                    dim = int(row[0])
                    lengths = []
                    for idim in range(dim):
                        lengths.append(int(row[1 + idim]))
                    nsample = int(row[2 + dim])
                    vals = []
                    offset = 3 + dim
                    for i in range(nsample):
                        vals.append(float(row[offset + i]))
                    idata.append([lengths,  vals])
                dataout.append(idata)
        return dataout

    # Compute the median and confidence interval.
    def process_data(self, data):
        dataout = []
        for idata in data:
            idataout = []
            for row in idata:
                low, high =  confidence_interval(row[1])
                idataout.append([row[0], np.median(row[1]), low, high])
            dataout.append(idataout)
        return dataout

    # Compare data from different runs
    def compare_data(self, data, ncompare, comparison="speedup"):
        dataout = []
        for i in range(len(data) // ncompare):
            for icompare in range(1, ncompare):
                idataout = []
                ibase = i * ncompare
                icomp = ibase + icompare
                for ibcase in range(len(data[ibase])):
                    lengths = data[ibase][ibcase][0]
                    imatch = -1
                    for iccase in range(len(data[icomp])):
                        if lengths == data[icomp][iccase][0]:
                            imatch = iccase
                            break
                    if imatch != -1:
                        Avals = data[ibase][ibcase][1]
                        Bvals = data[icomp][imatch][1]
                        if comparison == "speedup":
                            speedup = np.median(Avals) / np.median(Bvals)
                            low, high = ratio_confidence_interval(Avals, Bvals)
                            idataout.append([lengths, speedup, low, high])
                        elif comparison == "pval":
                            import scipy.stats
                            stat, pm, med, tbl = scipy.stats.median_test(Avals, Bvals, ties="ignore")
                            idataout.append([lengths, pm])
                dataout.append(idataout)
        return dataout

    # In the case where we want a linear horizontal axis, we choose a
    # single value for the indetifier.  One file per curve.
    def write_linear_output_bounds(self, data, docdir, suffix = ".dat"):
        filenames = []
        for idx, idata  in enumerate(data):
            fname = os.path.join(docdir, self.name + suffix + str(idx))
            filenames.append(fname)
            with open(fname, 'w', newline='') as ofile:
                for row in idata:
                    ofile.write(str(row[0][0]) + "\t" + str(row[1]) + "\t" + str(row[2]) + "\t" + str(row[3]) + "\n")
        return filenames

    # In the case where we want a linear horizontal axis, we choose a
    # single value for the indetifier.  One file per curve.
    def write_linear_output(self, data, docdir, suffix = ".dat"):
        filenames = []
        for idx, idata  in enumerate(data):
            fname = os.path.join(docdir, self.name + suffix + str(idx))
            filenames.append(fname)
            with open(fname, 'w', newline='') as ofile:
                for row in idata:
                    ofile.write(str(row[0][0]) + "\t" + str(row[1]) + "\n")
        return filenames
                         
    # Read the raw data, process, and create a figure.
    def makefig(self, docdir, outdirs, labellist, docformat, datatype, ncompare, secondtype, bandwidth, verbose):
        fout = tempfile.TemporaryFile(mode="w+")
        ferr = tempfile.TemporaryFile(mode="w+")
        
        data = self.collect_data(outdirs)

        dataout = self.process_data(data)

        # Write primary-axis data:
        primary_fnames = self.write_linear_output_bounds(dataout, docdir)

        print(primary_fnames)

        secondary_fnames = None
        if ncompare > 1:
            speedupdata = self.compare_data(data, ncompare, "speedup")
            # Write secondary-axis data:
            secondary_fnames = self.write_linear_output_bounds(speedupdata, docdir, suffix=".sec")

            pvaldata = self.compare_data(data, ncompare, "pval")
            print("pvals")
            print(pvaldata)

            # TODO: roofline
            # TODO: efficiency
            # TODO: glfop/s (whatever that means).
            print(secondary_fnames)

        asycommand = self.asycmd(docdir, primary_fnames, secondary_fnames, labellist)
        print(asycommand)
        asyproc = subprocess.Popen(asycommand,
                                   stdout=fout, stderr=ferr, env=os.environ.copy(),
                                   cwd = sys.path[0])
        asyproc.wait()
        asyrc = asyproc.returncode
        if asyrc != 0:
            print("****asy fail****")
        if verbose or (asyrc != 0):
            fout.seek(0)
            cout = fout.read()
            print(cout)
            ferr.seek(0)
            cerr = ferr.read()
            print(cerr)
        return asyrc

# Function for generating figures for benchmark output
def benchfigs(rundims, shortrun, precision):
    figs = []
    # FFT directions
    forwards = -1
    backwards = 1

    if 1 in rundims:
        dimension = 1

        nbatch = 1

        min1d = 256 if shortrun else 1024
        max1d = 4000 if shortrun else 536870912

        for inplace in [True, False]:
            fig = figure("1d_c2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "1D complex transforms " + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension, nextpow(min1d, radix), max1d, nbatch,
                                         radix, [], "c2c", forwards, inplace, precision) )
            figs.append(fig)

        for inplace in [True, False]:
            fig = figure("1d_r2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "1D real-to-complex transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension, nextpow(min1d, radix), max1d, nbatch,
                                         radix, [], "r2c", forwards, inplace, precision) )
            figs.append(fig)


        for inplace in [True, False]:
            fig = figure("1d_c2r" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "1D complex-to-real transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix) ,
                                         dimension, nextpow(min1d, radix), max1d, nbatch,
                                         radix, [], "r2c", backwards, inplace, precision) )
            figs.append(fig)


    if 2 in rundims:
        dimension = 2

        nbatch = 1
        min2d = 64 if shortrun else 128
        max2d = 8192 if shortrun else 32768

        for inplace in [True, False]:
            fig = figure("2d_c2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "2D complex transforms " + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min2d, radix), max2d, nbatch, radix, [1],
                                         "c2c",
                                         forwards, inplace, precision) )
            figs.append(fig)

        for inplace in [True, False]:
            fig = figure("2d_r2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "2D real-to-complex transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min2d, radix), max2d, nbatch, radix, [1],
                                         "r2c",
                                         forwards, inplace, precision) )
            figs.append(fig)

        for inplace in [True, False]:
            fig = figure("2d_c2r" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "2D complex-to-real transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min2d, radix), max2d, nbatch, radix, [1],
                                         "r2c",
                                         backwards, inplace, precision) )
            figs.append(fig)

    if 3 in rundims:
        dimension = 3
        min3d = 16
        max3d = 128 if shortrun else 1024
        nbatch = 1

        for inplace in [True, False]:
            fig = figure("3d_c2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "3D complex transforms " + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3, 5]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min3d, radix), max3d, nbatch, radix, [1,1],
                                         "c2c",
                                         forwards, inplace, precision) )
            figs.append(fig)

        for inplace in [True, False]:
            fig = figure("3d_r2c" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "3D real-to-complex transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min3d, radix), max3d, nbatch, radix, [1,1],
                                         "r2c",
                                         forwards, inplace, precision) )
            figs.append(fig)

        for inplace in [True, False]:
            fig = figure("3d_c2r" + ("inplace" if inplace else "outofplace") + "_" + precision,
                         "3D complex-to-real transforms " \
                         + ("in-place" if inplace else "out-of-place"))
            for radix in [2, 3]:
                fig.runs.append( rundata("radix " + str(radix), dimension,
                                         nextpow(min3d, radix), max3d, nbatch, radix, [1,1],
                                         "r2c",
                                         backwards, inplace, precision) )
            figs.append(fig)

    return figs

def efficiencyfigs(rundims, shortrun, precision):
    figs = []

    # FFT directions
    forwards = -1
    backwards = 1

    inplace = True
    dimension = 1

    radix = 2

    min1d = 1024
    max1d = 1048576 if shortrun else 268435456 #pow(2,28) gives a floating type :(
    nbatch = 1
    while max1d > min1d:
        fig = figure("1d_c2c_batch" + str(nbatch) + "_radix" + str(radix) + "_" + precision,
                     "1D complex transforms " + ("in-place" if inplace else "out-of-place") + " radix " + str(radix) + " batch " + str(nbatch) )

        fig.runs.append( rundata("radix " + str(radix),
                                 dimension, nextpow(min1d, radix), max1d, nbatch,
                                 radix, [], "c2c", forwards, inplace, precision) )
        figs.append(fig)
        nbatch *= 2
        max1d //= 2
        min1d //= 2
        min1d = max(min1d, 2^5)
    return figs

# Function for generating figures for a performance report
def reportfigs(rundims, shortrun, precision):
    figs = []

    # FFT directions
    forwards = -1
    backwards = 1

    inplace = True

    if 1 in rundims:
        dimension = 1

        for min1d, max1d, nbatch in [[1024,536870912,1], [8,32768,100000]]:

            for radix in [2, 3, 5, 7]:
                fig = figure("1d_c2c" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "1D complex transforms with radix " + str(radix)\
                             + " and batch size " + str(nbatch) + "." )

                fig.runs.append( rundata("radix " + str(radix),
                                         dimension, nextpow(min1d, radix),
                                         max1d, nbatch,
                                         radix, [], "c2c", forwards,
                                         inplace, precision) )
                figs.append(fig)

            for radix in [2, 3, 5, 7]:
                fig = figure("1d_r2c"\
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "1D real-to-complex transforms with radix "\
                             + str(radix) \
                             + " and batch size " + str(nbatch) + ".")
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension, nextpow(min1d, radix),
                                         max1d, nbatch,
                                         radix, [], "r2c", forwards,
                                         inplace, precision) )
                figs.append(fig)

            for radix in [2, 3, 5, 7]:
                fig = figure("1d_c2r" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "1D complex-to-real transforms with radix " \
                             + str(radix) \
                             + " and batch size " + str(nbatch) + "." )
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension, nextpow(min1d, radix),
                                         max1d, nbatch,
                                         radix, [], "r2c", backwards,
                                         inplace, precision) )
                figs.append(fig)

    if 2 in rundims:
        dimension = 2

        for min2d, max2d, nbatch in [[128,32768,1], [64,8192,100]]:

            for radix in [2, 3, 5]:
                fig = figure("2d_c2c" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "2D complex transforms with radix " + str(radix)\
                             + " and batch size " + str(nbatch) + ".")
                fig.runs.append( rundata( "radix "+ str(radix),
                                         dimension,
                                         nextpow(min2d, radix), max2d,
                                         nbatch,
                                         radix, [1], "c2c",
                                         forwards, inplace, precision) )
                figs.append(fig)

            for radix in [2, 3, 5]:
                fig = figure("2d_r2c" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "2D real-to-complex transforms with radix "\
                             + str(radix) \
                             + " and batch size " + str(nbatch) + ".")

                fig.runs.append( rundata( "radix " + str(radix),
                                         dimension,
                                         nextpow(min2d, radix), max2d,
                                         nbatch,
                                         radix, [1], "r2c",
                                         forwards, inplace, precision) )
                figs.append(fig)

            for radix in [2, 3, 5]:
                fig = figure("2d_c2r" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "2D complex-to-real transforms with radix "\
                             + str(radix) +\
                             " and batch size " + str(nbatch) + ".")
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension,
                                         nextpow(min2d, radix), max2d,
                                         nbatch,
                                         radix, [1], "r2c",
                                         backwards, inplace, precision) )
                figs.append(fig)


            for radix in [2]:
                fig = figure("2d_c2c_r2" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "2D complex transforms "\
                             + "with aspect ratio N:2N with radix "\
                             + str(radix) + " and batch size " + str(nbatch) \
                             + ".")
                fig.runs.append( rundata( "radix 2",
                                         dimension, min2d, max2d, nbatch, 2,
                                         [2], "c2c",
                                         forwards, inplace, precision) )
                figs.append(fig)

            for radix in [2]:
                fig = figure("2d_r2c_r2" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "2D real-to-complex transforms with radix "\
                             + str(radix) \
                             + " and batch size " + str(nbatch) + ".")
                fig.runs.append( rundata("radix 2",
                                         dimension, min2d, max2d, nbatch, 2,
                                         [2], "r2c",
                                         forwards, inplace, precision) )
                figs.append(fig)

    if 3 in rundims:
        dimension = 3

        for min3d, max3d, nbatch in [[16,128,1],[4,64,100]]:

            for radix in [2, 3, 5]:
                fig = figure("3d_c2c" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "3D complex transforms with radix "\
                             + str(radix) \
                             + " and batch size " + str(nbatch) + ".")
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension,
                                         nextpow(min3d, radix), max3d,
                                         nbatch,
                                         radix, [1,1], "c2c",
                                         forwards, inplace, precision) )
                figs.append(fig)

            for radix in [2, 3]:
                fig = figure("3d_r2c" \
                             + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                             "3D real-to-complex transforms with radix "\
                             + str(radix)\
                             + " and batch size " + str(nbatch) + ".")

                fig.runs.append( rundata("radix " + str(radix),
                                         dimension,
                                         nextpow(min3d, radix), max3d,
                                         nbatch,
                                         radix, [1,1], "r2c",
                                         forwards, inplace, precision) )
                figs.append(fig)


            fig = figure("3d_c2r" \
                         + "_radix" + str(radix) \
                         + "_batch" + str(nbatch) + "_" + precision,
                         "3D complex-to-real transforms with radix "\
                         + str(radix)
                         + " and batch size " + str(nbatch) + ".")
            for radix in [2]:
                fig.runs.append( rundata("radix " + str(radix),
                                         dimension,
                                         nextpow(min3d, radix), max3d,
                                         nbatch,
                                         radix, [1,1], "r2c",
                                         backwards, inplace, precision) )
            figs.append(fig)

            fig = figure("3d_c2c_aspect" \
                         + "_radix" + str(radix) \
                         + "_batch" + str(nbatch) + "_" + precision,
                         "3D complex transforms "\
                         + "with aspect ratio N:N:16N with radix "\
                         + str(radix)\
                         + " and batch size " + str(nbatch) + ".")
            fig.runs.append( rundata("radix 2",
                                     dimension, min3d, max3d, nbatch, 2,
                                     [1,16], "c2c",
                                     forwards, inplace, precision) )
            figs.append(fig)

            fig = figure("3d_r2c_aspect" \
                         + "_radix" + str(radix) \
                             + "_batch" + str(nbatch) + "_" + precision,
                         "3D real-to-complex transforms " \
                         + "with aspect ratio N:N:16N with radix " \
                         + str(radix)\
                         + " and batch size " + str(nbatch) + ".")
            fig.runs.append( rundata("radix 2",
                                     dimension, min3d, max3d, nbatch, 2,
                                     [1,16], "r2c",
                                     forwards, inplace, precision) )
            figs.append(fig)

    return figs


def main(argv):
    dloadexe = None

    inlist = []
    outdirlist = []
    labellist = []

    docdir = "doc"

    dryrun = False
    nbatch = 1
    speedup = True
    datatype = "time"
    shortrun = False
    docformat = "pdf"
    devicenum = 0
    doAsy = True
    nsample = 10
    rundims = [1,2,3]
    runtype = "benchmark"
    secondtype = "none"
    precision = "double"
    problem_file = None
    verbose = False
    bandwidth = None
    
    try:
        opts, args = getopt.getopt(argv,"hb:D:f:Tt:i:o:l:S:sg:d:N:R:w:y:F:p:B:v")
    except getopt.GetoptError:
        print("error in parsing arguments.")
        print(usage)
        sys.exit(2)
    for opt, arg in opts:
        if opt in ("-h"):
            print(usage)
            exit(0)
        elif opt in ("-b"):
            dloadexe = os.path.abspath(arg)
        elif opt in ("-i"):
            inlist.append(arg)
        elif opt in ("-o"):
            outdirlist.append(arg)
        elif opt in ("-l"):
            labellist.append(arg)
        elif opt in ("-w"):
            docdir = arg
        elif opt in ("-T"):
            dryrun = True
        elif opt in ("-s"):
            shortrun = True
        elif opt in ("-g"):
            if int(arg) == 0:
                doAsy = False
            if int(arg) == 1:
                doAsy = True
        elif opt in ("-d"):
            devicenum = int(arg)
        elif opt in ("-D"):
            rundims = []
            for val in arg.split(','):
                rundims.append(int(val))
        elif opt in ("-N"):
            nsample = int(arg)
        elif opt in ("-S"):
            if int(arg) == 0:
                speedup = False
            if int(arg) == 1:
                speedup = True
        elif opt in ("-t"):
            if arg not in ["time", "gflops", "roofline"]:
                print("data type must be time or gflops or roofline")
                print(usage)
                sys.exit(1)
            datatype = arg
        elif opt in ("-y"):
            if arg not in ["none", "gflops", "efficiency"]:
                print("data type must be gflops or none")
                print(usage)
                sys.exit(1)
            secondtype = arg
        elif opt in ("-R"):
            if arg not in ["report", "benchmark", "efficiency"]:
                print("data type must be gflops or none")
                print(usage)
                sys.exit(1)
            runtype = arg
            if runtype == "efficiency":
                datatype = "roofline"
        elif opt in ("-f"):
            goodvals = ["pdf", "docx"]
            if arg not in goodvals:
                print("error: format must in " + " ".join(goodvals))
                print(usage)
                sys.exit(1)
            docformat = arg
        elif opt in ("-p"):
            if arg not in ["single", "double"]:
                print("precision type must be single or double")
                print(usage)
                sys.exit(1)
            precision = arg
        elif opt in ("-F"):
            problem_file = arg
        elif opt in ("-B"):
            bandwidth = float(arg)
        elif opt in ("-v"):
            verbose = True

    print("rundims:")
    print(rundims)

    if not dryrun:
        if dloadexe != None:
            if not os.path.isfile(dloadexe):
                print("unable to find " + dloadexe)
                sys.exit(1)

        for i in inlist:
            if not os.path.isfile(i):
                print("unable to find " + i)
                print("please specify with -i")
                sys.exit(1)

    print("inputs:", inlist)

    if len(inlist) > len(labellist):
        for i in range(len(labellist), len(inlist)):
            labellist.append("dir" + str(i))
    print("run labels:", labellist)

    for idx in range(len(inlist)):
        inlist[idx] = os.path.abspath(inlist[idx])

    if len(inlist) > len(outdirlist):
        for i in range(len(outdirlist), len(inlist)):
            outdirlist.append(os.path.abspath("dir" + str(i)))

    for idx in range(len(outdirlist)):
        outdirlist[idx] = os.path.abspath(outdirlist[idx])
    print("data output directories:", outdirlist)


    if shortrun:
        print("short run")
    print("output format: " + docformat)
    print("device number: " + str(devicenum))
    print("precision: " + precision)

    docdir = os.path.abspath(docdir)

    print("document output in", docdir)
    if not os.path.exists(docdir):
        os.makedirs(docdir)

    for outdir in outdirlist:
        if not os.path.exists(outdir):
            os.makedirs(outdir)


    if not dryrun:
        machine_specs = perflib.get_machine_specs(devicenum)

        for outdir in outdirlist:
            with open(os.path.join(outdir, "specs.txt"), "w+") as f:
                f.write(str(machine_specs))

            with open(os.path.join(outdir, "gpuid.txt"), "w") as f:
                f.write(machine_specs.gpuid)

    figs = []

    if runtype == "benchmark":
        figs = benchfigs(rundims, shortrun, precision)
    if runtype == "report":
        figs = reportfigs(rundims, shortrun, precision)
    if runtype == "efficiency":
        figs = efficiencyfigs(rundims, shortrun, precision)

    for idx, fig in enumerate(figs):
        for idx2, fig2 in enumerate(figs):
            if idx != idx2 and fig.name == fig2.name:
                print("figures have the same name!")
                print(fig.name)
                print(fig2.name)
                sys.exit(1)

    for fig in figs:
        print(fig.name)
        # Run the tests and put output in the outdirs:
        for run in fig.runs:
            if not dryrun:
                run.executerun(nsample, inlist, outdirlist, dloadexe, problem_file)
            # HACK: problem file should have all the problem sizes
            # that need running, so just one execution should produce
            # all the data we need
            if problem_file:
                break


                
        # Compile the data in the outdirs into figures in docdir:
        ncompare = len(labellist) if dryrun else len(inlist)
            
        print(fig.labels(labellist))
        if doAsy:
            fig.makefig(docdir, outdirlist, labellist, docformat, datatype, ncompare, secondtype, bandwidth, verbose)

    if doAsy:
        # Make the document in docdir:
        #
        # HACK: problem file implies html report
        if problem_file:
            from html_report import graph_dirs
            graph_dirs(outdirlist, problem_file, docdir)
        else:
            # otherwise, make other doc types using asymptote figs
            if docformat == "pdf":
                maketex(figs, docdir, outdirlist, labellist, nsample, secondtype, precision)
            if docformat == "docx":
                makedocx(figs, docdir, nsample, secondtype, precision)
                
        print("Finished!  Output in " + docdir)

def binaryisok(dirname, progname):
    prog = os.path.join(dirname, progname)
    return os.path.isfile(prog)

gflopstext = '''\
GFLOP/s are computed based on the Cooley--Tukey operation count \
for a radix-2 transform, and half that for in the case of \
real-complex transforms.  The rocFFT operation count may differ from \
this value: GFLOP/s is provided for the sake of comparison only.'''


efficiencytext = '''\
Efficiency is computed for an idealised FFT which requires exactly \
one read and one write to global memory.  In practice, this \
isn't possible for most problem sizes, as the data does \
not fit into cache, and one must use global memory to store \
intermediary results.  As FFTs are bandwidth-limited on modern hardware, \
the efficiency is measured against the theoretical maximum bandwidth \
for the device.'''


# Function for generating a tex document in PDF format.
def maketex(figs, docdir, outdirlist, labellist, nsample, secondtype, precision):

    header = '''\
\\documentclass[12pt]{article}
\\usepackage[margin=1in]{geometry}
\\usepackage{graphicx}
\\usepackage{url}
\\begin{document}
'''
    texstring = header

    texstring += "\n\\section{Introduction}\n"

    texstring += "Each data point represents the median of " + str(nsample) + " values, with error bars showing the 95\\% confidence interval for the median.  All transforms are " + precision + "-precision.\n\n"

    if secondtype == "gflops":
        texstring += gflopstext + "\n\n"


    texstring += "\\vspace{1cm}\n"

    # texstring += "\\begin{tabular}{ll}"
    # texstring += labelA +" &\\url{"+ dirA+"} \\\\\n"
    # if not dirB == None:
    #     texstring += labelB +" &\\url{"+ dirB+"} \\\\\n"
    # texstring += "\\end{tabular}\n\n"

    # texstring += "\\vspace{1cm}\n"

    texstring += "\n\\section{Device Specification}\n"
    for idx in range(len(outdirlist)):
        texstring += "\n\\subsection{" + labellist[idx]  + "}\n"
        specfilename = os.path.join(outdirlist[idx], "specs.txt")
        if os.path.isfile(specfilename):
            specs = ""
            with open(specfilename, "r") as f:
                specs = f.read()

            for line in specs.split("\n"):
                if line.startswith("Host info"):
                    texstring += "\\noindent " + line
                    texstring += "\\begin{itemize}\n"
                elif line.startswith("Device info"):
                    texstring += "\\end{itemize}\n"
                    texstring += line
                    texstring += "\\begin{itemize}\n"
                else:
                    if line.strip() != "":
                        texstring += "\\item \\verb|" + line + "|\n"
            texstring += "\\end{itemize}\n"
            texstring += "\n"

    texstring += "\\clearpage\n"

    texstring += "\n\\section{Figures}\n"

    for idx, fig in enumerate(figs):
        print(fig.filename(docdir))
        print(fig.caption)
        texstring += '''
\\centering
\\begin{figure}[htbp]
   \\includegraphics[width=\\textwidth]{'''
        texstring += fig.filename("")
        texstring += '''}
   \\caption{''' + fig.caption + '''}
\\end{figure}
'''
        if (idx % 2) == 0:
            texstring += "\\clearpage\n"

    texstring += "\n\\end{document}\n"

    fname = os.path.join(docdir, 'figs.tex')

    with open(fname, 'w') as outfile:
        outfile.write(texstring)

    fout = open(os.path.join(docdir, "texcmd.log"), 'w+')
    ferr = open(os.path.join(docdir, "texcmd.err"), 'w+')

    latexcmd = ["latexmk", "-pdf", 'figs.tex']
    print(" ".join(latexcmd))
    texproc =  subprocess.Popen(latexcmd, cwd=docdir, stdout=fout, stderr=ferr,
                                env=os.environ.copy())
    texproc.wait()
    fout.close()
    ferr.close()
    texrc = texproc.returncode
    if texrc != 0:
        print("****tex fail****")

# Confert a PDF to an EMF using pdf2svg and inkscape.
def pdf2emf(pdfname):
    svgname = pdfname.replace(".pdf",".svg")
    cmd_pdf2svg = ["pdf2svg", pdfname, svgname]
    proc = subprocess.Popen(cmd_pdf2svg, env=os.environ.copy())
    proc.wait()
    if proc.returncode != 0:
        print("pdf2svg failed!")
        sys.exit(1)

    emfname = pdfname.replace(".pdf",".emf")
    cmd_svg2emf = ["inkscape", svgname, "-M", emfname]
    proc = subprocess.Popen(cmd_svg2emf, env=os.environ.copy())
    proc.wait()
    if proc.returncode != 0:
        print("svg2emf failed!")
        sys.exit(1)

    return emfname

# Function for generating a docx using emf files and the docx package.
def makedocx(figs, outdir, nsample, secondtype, precision):
    import docx

    document = docx.Document()

    document.add_heading('rocFFT benchmarks', 0)

    document.add_paragraph("Each data point represents the median of " + str(nsample) + " values, with error bars showing the 95% confidence interval for the median.  Transforms are " + precision + "-precision, forward, and in-place.")

    if secondtype == "gflops":
        document.add_paragraph(gflopstext)
    if secondtype == "efficiency":
        document.add_paragraph(efficiencytext)

        
    specfilename = os.path.join(outdir, "specs.txt")
    if os.path.isfile(specfilename):
        with open(specfilename, "r") as f:
            specs = f.read()
        for line in specs.split("\n"):
            document.add_paragraph(line)

    for fig in figs:
        print(fig.filename(outdir, "docx"))
        print(fig.caption)
        emfname = pdf2emf(fig.filename(outdir, "docx"))
        document.add_picture(emfname, width=docx.shared.Inches(6))
        document.add_paragraph(fig.caption)

    document.save(os.path.join(outdir,'figs.docx'))

if __name__ == "__main__":
    main(sys.argv[1:])


    
