#!/usr/bin/python
#
# Copyright (C) 2014 Canonical, Ltd.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
#
# Author: Colin Ian King <colin.king@canonical.com>
#
import os
import sys
import stat
import json
import numpy as np
import re
import gzip
import string
import errno
import operator

default_colors = [ '#ff0000', '#00ff00', '#0000ff', '#ff00ff', '#00ffff', '#ffff00' ]
#
# Default graph sizes in pixels
#
default_width = 1000
default_height = 350
default_max_plots_in_scatter_plot = 1500

job_config_cache = {}
job_info_cache = {}

if len(sys.argv) < 2:
	sys.exit('Usage: %s results-path' % sys.argv[0])

if not os.path.exists(sys.argv[1]):
	sys.exit('Path %s not found' % sys.argv[1])

#
#  generate a html file containing links to graphs
#
def generate_html(html_subdirs, html, root_path, sub_path, html_file, title, hover_hint):
	page = '''
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
  <script type="text/javascript" src="https://www.google.com/jsapi"></script>
  <meta content="text/html; charset=ISO-8859-1"
 http-equiv="content-type">
  <link rel="stylesheet" href="http://kernel.ubuntu.com/media/kernel-style.css" type="text/css" media="screen" />
  <style>
    div.index-bottom-section {
    border-radius: 0px;
    box-shadow: 0 1px 3px rgba(0,0,0,0.3);
    background: #f7f5f6;
    padding: 20px;
    margin-top: -16px;
    font-size: 11px;
    line-height: 16px;
    color: #333;
    }
    a {
      color: #333;
    }
    a:hover {
      text-decoration: underline;
    }
    a:active a:link, a:visited {
      text-decoration: none;
    }
  </style>
  <title>''' + title + '''</title>
</head>
  <body class="dash-body">
    <div class="dash-center-wrap">
      <div class="dash-center">
        <div class="dash-center-content">
          <div id="dash-header">
            <div id="dash-timestamp">
              <a href="http://ubuntu.com" title="Home" rel="home"><img src="media/ubuntu-logo.png" alt="Home" /></a>
            </div>
            <h1>Ubuntu Kernel Team</h1>
          </div>
          <h2><center>''' + title + '''</center></h2>
          <center>
            <table width=100% cellspacing="2">'''
	colors = [ "#ffffff", "#e0f0ff" ]
	i = 0
	for (t, f, h) in sorted(html_subdirs, key=lambda d: d[1]):
		common = os.path.commonprefix([sub_path, f])
		relname = f[len(common)+1:]
		page += '\n              <tr><td><center><a href="' + relname + '" title="' + h + '">' + t + '</a></center></td></tr>'
	page += html

	for f in sorted(os.listdir(sub_path)):
		(filename, extension) = os.path.splitext(f)
		if extension in [ ".png" ]:
			page += '\n              <tr><td bgcolor="' + colors[i % 2] + '"><center><img src=' + f + '></center></td></tr>'
			i += 1
	page += '''
            </table>
          </center>
        </div>
      </div>
    </div>
  </body>
</html>'''
	mkdirp(sub_path)
	filename = os.path.join(sub_path, "index.html")
	f = open(filename, 'w')
	f.write(page)
	f.close()
	return (title, filename, hover_hint)

#
#  fio data log mappings
#
#
samples_info = [
	#  key		human readable title			regex on log name	generated filename path
	('Bandwidth',	'Fio I/O Bandwidth (MB/s)',		'job.*_bw\.log*', 	'daily-fio-stats-bandwidth',
	 'Charts derived from flexible I/O tester (fio) bandwidth measurements during a test run'),
	('IOops',	'Fio I/O operations (ops/s)', 		'job.*_iops\.log*', 	'daily-fio-stats-io-ops',
	 'Charts derived from flexible I/O tester (fio) I/O operation measurements during a test run'),
	('IOLatency',	'Fio I/O Latency (us)', 		'job.*_lat\.log*', 	'daily-fio-stats-io-latency',
	 'Charts derived from flexible I/O tester (fio) I/O latency  measurements during a test run'),
	('IOCLatentcy',	'Fio I/O Completion Latency (us)',	'job.*_clat\.log*', 	'daily-fio-stats-clatency',
	 'Charts derived from flexible I/O tester (fio) I/O completion latency measurements during a test run') ]


fio_map = {
	'Context Switches'	: 'Scheduler context switches',
	'Disk Read I/Os'	: 'Disk read I/O operations',
	'Disk Read Merges'	: 'Read I/O requests merged',
	'Disk Read Ticks'	: 'Number of system ticks that the disk reads kept the disk busy',
	'Disk utilisation'	: 'Percentage of disk utilisation',
	'Disk Write I/Os'	: 'Disk write I/O operations measured',
	'Disk Write Merges'	: 'Write I/O requests merged',
	'Disk Write Ticks'	: 'Number of system ticks that the disk writes kept the disk busy',
	'Latency Depth'		: '',
	'Latency Percentile'	: '',
	'Read Bandwidth'	: 'Mean read bandwidth',
	'Read Bandwidth Aggregate': 'Aggreate percentage of total read bandwidth',
	'Read Bandwidth Max'	: 'Maximum read bandwidth',
	'Read Bandwidth Mean'	: 'Mean read bandwidth',
	'Read Bandwidth Min'	: 'Minumum read bandwidth',
	'Read Bandwidth StdDev'	: 'Read bandwidth standard deviation',
	'Read Mean CLatency'	: 'Mean read completion latency (usecs)',
	'Read Mean Latency'	: 'Mean read submission latency (usecs)',
	'Read Ops'		: 'Read I/O operations per second',
	'Read Run time'		: 'Read run time',
	'System CPU'		: 'CPU usage in kernel (System)',
	'User CPU'		: 'CPU usage in userspace',
	'Write Bandwidth'	: 'Mean write bandwidth',
	'Write Bandwidth Aggregate': 'Aggreate percentage of total write bandwidth',
	'Write Bandwidth Max'	: 'Maximum write bandwidth',
	'Write Bandwidth Mean'	: 'Mean write bandwidth',
	'Write Bandwidth Min'	: 'Minimum write bandwidth',
	'Write Bandwidth StdDev': 'Write bandwidth standard deviation',
	'Write Mean CLatency'	: 'Mean write completion latency (usecs)',
	'Write Mean Latency'	: 'Mean write submission latency (usecs)',
	'Write Ops'		: 'Write I/O operations per second',
	'Write Run time'	: 'Write run time'
	}
#
#
#  Create all dirs in a given path
#
def mkdirp(path):
	try:
		os.makedirs(path)
	except OSError as exc:
		if exc.errno == errno.EEXIST and os.path.isdir(path):
			pass
		else: raise

def kernel_canonical(kernel):
	# 3.11.0-26-generic
	# 3.17.0-031700rc1-generic
	pattern = re.compile(r"""(\d*).(\d*).(\d*)-(\S*)-(\S*)""", re.VERBOSE)
	m = pattern.match(kernel)
        if m:
		return "%4.4d.%4.4d.%4.4d-%s-%s" \
			% (int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4), m.group(5))
	else:
		pattern = re.compile(r"""(\d*).(\d*).(\d*)-(\S*)""", re.VERBOSE)
		m = pattern.match(kernel)
		if m:
			return "%4.4d.%4.4d.%4.4d-%s-unknown" \
				% (int(m.group(1)), int(m.group(2)), int(m.group(3)), m.group(4))
		else:
			pattern = re.compile(r"""(\d*).(\d*).(\d*)""", re.VERBOSE)
			m = pattern.match(kernel)
			if m:
				return "%4.4d.%4.4d.%4.4d-unknown-unknown" \
					% (int(m.group(1)), int(m.group(2)), int(m.group(3)))
			else:
				return ""

#
#  Load sysinfo
#
def sysinfo_load(file):
	sysinfo = {}
	# 3.11.0-26-generic
	# 3.17.0-031700rc1-generic

	try:
		f = gzip.open(file + '.gz')
	except:
		try:
			f = open(file, 'r')
		except:
			print "Can't open " + file
			return sysinfo

	got_kernel_release = False

	for line in f.read().splitlines():
		if ':' in line:
			[key, val] = line.split(':',1)
			#
			# Temp hack for 2.6.32, need to fix in
			# fio.sh stats gathering
			#
			val = val.lstrip()
			if key in ["Processor", "Hardware"] and val == "unknown":
				val = "x86_64"
			sysinfo[key] = val
			#
			# Convert kernel release into something that
			# can be easily sorted into order
			#
			if key == 'Kernel-release':
				k = kernel_canonical(val)
				if k != "":
					got_kernel_release = True
					sysinfo['Kernel-release-canonical'] = k
	f.close()

	if not got_kernel_release:
		print "Did not find kernel release info for " + file
		sys.exit("Failed")
	return sysinfo

#
#  Load perf data
#
def perf_report_load(file):
	pattern1 = re.compile(r"""\s+(\d*\.\d+|\d+)%\s+(\d*\.\d+|\d+)%\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)""", re.VERBOSE)
	# 0.00%               sh  [kernel.kallsyms]   [k] native_write_msr_safe
	pattern2 = re.compile(r"""\s+(\d*\.\d+|\d+)%\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)""", re.VERBOSE)

	if os.path.isfile(file):
		f = open(file, 'r')
	elif os.path.isfile(file + '.gz'):
		f = gzip.open(file + '.gz')
	else:
		return []

	perf_report = []
	for line in f.read().splitlines():
		m = pattern1.match(line)
		if m:
			percent = float(m.group(2))
			address = m.group(6)
			perf_report.append((address, percent))
		else:
			m = pattern2.match(line)
			if m:
				percent = float(m.group(1))
				address = m.group(5)
				perf_report.append((address, percent))

	return perf_report

#
#  Get and cache job configuration data
#
def job_config_get(job):
	if job in job_config_cache:
		return job_config_cache[job]

	config = []
	keys = [ 'Label', 'Units', 'Scale', 'JsonPath' ]
	try:
		f = open(os.path.join('jobs', job + '.conf'), 'r')
	except:
		return []
	for line in f.read().splitlines():
		if line[:1] == '#':
			continue
		if not "," in line:
			continue
		info = {}
		fields = line.split(',', 4)
		if ',' in line:
			for i in range(0, len(fields)):
				info[keys[i]] = fields[i].lstrip().rstrip()

		config.append(info)
	f.close()
	job_config_cache[job] = config
	return config

#
#  Get and cache job info
#
def job_info_get(job):
	if job in job_info_cache:
		return job_info_cache[job]
	try:
		f = open(os.path.join('jobs', job + '.info'), 'r')
	except:
		return ""

	info = f.read()
	f.close()
	job_info_cache[job] = info

	return info

#
#  Variants of keys?
#
def keys_variant_collection(part_stats):
	skipkeys = [ 'Perfreport', 'Fiostats', 'Datapath', 'Samples', \
		     'Time-run','Time', 'Kernel-version', 'Full Date', \
		     'Kernel-version-canonical']
	keys = set([])
	for stats in part_stats:
		for sysinfo1 in stats:
			for sysinfo2 in stats:
				for key in sysinfo2:
					try:
						if not key in skipkeys and sysinfo1[key] != sysinfo2[key]:
							keys.add(key)
					except:
						pass
	return list(keys)

#
#  Remove trailing comma and spaces from label
#
def clean_label(label):
	str = label
	while str.endswith(' ') or str.endswith(','):
		str = str[:-2]
	return str

#
#  Take an array of words and return words as a comma separated string
#
def array_to_str(array):
	return ', '.join(array)

#
#  keys in sysinfo to a label
#
def sysinfo_keys_to_label(sysinfo, keys):
	#
	#  ignore_keys are keys special keys we want to hide from the label
	#
	ignore_keys = ['Kernel-release-canonical']

	return ', '.join([ sysinfo[key] for key in keys if key in sysinfo and key not in ignore_keys ])

#
#  Convert a string into a filename containing letters and digits
#
def str_to_filename(str):
	filename = ''
	for c in str.lower():
		if c in string.ascii_letters or c in string.digits:
			filename += c
		else:
			filename += '-'

	return filename

#
#  horizontal bar chart
#
def bar_horiz(html_subdirs, root_path, keys, config, collection):
	title = config['Label']
	units = config['Units']
	if title in fio_map:
		hint = fio_map[title]
	else:
		hint = ""
	print "Bar charting " + title
	#
	#  Filename based on title..
	#
	filename = str_to_filename(title)
	newpath = os.path.join(root_path, filename)
	mkdirp(newpath)
	#
	#  Sort collection on xlabel
	#
	collection = sorted(collection, key=lambda d: d[3])

	max_label_len = 0
	fs = set([])
	kernels = set([])
	schedulers = set([])
	jobs = set([])
	#labels = {}
	for (subtitle, xlabels, yvalues, xlabel, sysinfos) in collection:
		#
		#  Collect up unique filesystems;
		#  this allows us to determine how to color the bars
		#
		fs = fs.union({ sysinfo['Filesystem'] for sysinfo in sysinfos})
		kernels = kernels.union({ sysinfo['Kernel-release'] for sysinfo in sysinfos})
		schedulers = schedulers.union({ sysinfo['IO Scheduler'] for sysinfo in sysinfos})
		jobs = jobs.union({ sysinfo['Job'] for sysinfo in sysinfos})
		#
		# get label and find longest one
		#
		max_label_len = max(max_label_len, max([ len( \
			sysinfo['Kernel-release'] + ", " + \
		        sysinfo['IO Scheduler'] + ", " + \
			sysinfo['Filesystem']) for sysinfo in sysinfos ]))
	fs = sorted(fs)

	html = "<h3>Comparing " + hint + " by test, kernel, I/O scheduler and file system as measured by the flexible I/O tester (fio)</h3>"

	values = {}
	for (subtitle, xlabels, yvalues, xlabel, sysinfos) in collection:
		indexes = range(len(sysinfos))
		for i in indexes:
			sysinfo = sysinfos[i]
			key = sysinfo['Kernel-release'] + sysinfo['Filesystem'] + sysinfo['IO Scheduler'] + sysinfo['Job']
			values[key] = yvalues[i]


	html += '''
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
        var data = google.visualization.arrayToDataTable([
	'''
	html += "['Configuration', "
	for job in jobs:
		html += "'" + job + "', "
	html += " { role: 'annotation' }],\n"

	bar_height = 0
	for kernel in kernels:
		for f in fs:
			for scheduler in schedulers:
				html += "['" + kernel + " " + f + " " + scheduler + "', "
				for job in jobs:
					key = kernel + f + scheduler + job
					try:
						v = values[key]
					except:	
						v = 0.0
					html += str(v) + ","
				html += "''],\n"
				bar_height += 25

	html += "]);\n"
	max_height = bar_height + 100

	html += '''
        var options = {
          title: 'Stacked results for all jobs: ''' + title + " (" + units + ") " + subtitle + ''' ',
          vAxis: {title: 'Configuration' },
          hAxis: {title: 'Total of ''' + title + " (" + units + ") for all tests" + ''' ', minValue: 0 },
	  chartArea: {left: ' ''' + str(10 * max_label_len) + ''' ', height: ' ''' + str(bar_height) + ''' '},
	  bar: {groupWidth: '90%'},
	  isStacked: true,
	  legend: { position: 'bottom'},
	  height: ''' + str(max_height) + '''
        };

        var chart = new google.visualization.BarChart(document.getElementById(''' + "'chart_div_0'" + '''));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_0" style="width: ''' + str(default_width) + '''px"></div>'''

	chart = 1
	for (subtitle, xlabels, yvalues, xlabel, sysinfos) in collection:
		#
		# sort by kernel and file system order
		#
		indexes = sorted(range(len(sysinfos)), key=lambda x : sysinfos[x]['Kernel-release-canonical'] + sysinfos[x]['Filesystem'])
		html += '''
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
        var data = google.visualization.arrayToDataTable([
	'''
		html += "['Configuration', '" + title + " (" + units + ")'," + \
			" { role: 'style' }],\n"

		bar_height = 25 * len(indexes)
		max_height = bar_height + 100
		for i in indexes:
			color = default_colors[fs.index(sysinfos[i]['Filesystem'])]
			label = sysinfos[i]['Kernel-release'] + ", " + \
				sysinfos[i]['IO Scheduler'] + ", " + \
				sysinfos[i]['Filesystem']
			html += "['" + label + "'," + str(yvalues[i]) + "," + \
			"'fill-color: " + color + "; " + \
			"fill-opacity: 0.9'],\n"

		html += "]);\n"
		html += '''

        var options = {
          title: ' ''' + xlabel + ": " + title + " (" + units + ") " + subtitle + ''' ',
          vAxis: {title: 'Configuration' },
          hAxis: {title: ' ''' + title + " (" + units + ")" + ''' ', minValue: 0 },
	  chartArea: {left: ' ''' + str(10 * max_label_len) + ''' ', height: ' ''' + str(bar_height) + ''' '},
	  bar: {groupWidth: '90%'},
	  legend: {position: 'none'},
	  height: ''' + str(max_height) + '''
        };

        var chart = new google.visualization.BarChart(document.getElementById(''' + "'chart_div_" + str(chart) + "'" + '''));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_''' + str(chart) + '''" style="width: ''' + str(default_width) + '''px"></div>'''
		chart += 1

	html_subdirs.append(generate_html([], html, root_path, newpath, filename, title, hint))

#
#  trend line graph plot
#
def plot_horiz_trend(html_subdirs, path, keys, config, collection):
	title = config['Label']
	units = config['Units']
	if title in fio_map:
		hint = fio_map[title]
	else:
		hint = title
	#
	# Determine all the xlabels in the collection
	# and collect up unique filesystems and ioscheds
	#
	filesystems = set([])
	ioscheds = set([])
	all_xlabels = set([])
	xlabel_map = { }
	for (subtitle, xlabels, yvalues, xlabel, sysinfos) in collection:
		for sysinfo in sysinfos:
			human_xlabel = sysinfo['Kernel-release']
			sort_xlabel = sysinfo['Kernel-release-canonical']
			xlabel_map[sort_xlabel] = human_xlabel

			all_xlabels.add(sort_xlabel)
			filesystems.add(sysinfo['Filesystem'])
			ioscheds.add(sysinfo['IO Scheduler'])

	all_xlabels = sorted(all_xlabels)
	ioscheds = sorted(ioscheds)
	filesystems = sorted(filesystems)

	new_path = os.path.join(path, str_to_filename(title))
	mkdirp(new_path)

	chart = 0
	html = ''

	#
	#  Generate a graph for each I/O scheduler found..
	#
	for io in ioscheds:
		html += '''
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
        var data = google.visualization.arrayToDataTable([
	'''
		html += "['Run #' "
		for fs in filesystems:
			html += ", '" + fs + "'"
		html += "],\n"

		for x in all_xlabels:
			#
			#  We map munged sorted names to human readable text
			#
			xl = xlabel_map[x]
			html += "[ '" + xl + "' "
			values = {}
			#
			#  Scan across file systems and look for the matching
			#  value that we need to plot
			#
			for fs in filesystems:
				value = " null "
				for (subtitle, xlabels, yvalues, xlabel, sysinfos) in collection:
					for j in range(0, len(sysinfos)):
						sysinfo = sysinfos[j]
						if xl == sysinfo['Kernel-release'] and \
						   sysinfo['Filesystem'] == fs and \
						   sysinfo['IO Scheduler'] == io:
							value = str(yvalues[j])
							break
				html += "," + value
			html += "],\n"

		html += "]);\n"
		html += '''
        var options = {
          title: ' ''' + io + ": " + title + " (" + units + ") " + subtitle + ''' ',
          hAxis: {slantedTextAngle: 90, maxAlternation: 1, title:'Run #' },
          height: 600,
          chartArea: { top:50, height:300 },
          vAxis: {title: ' ''' + title + " (" + units + ")" + ''' ', minValue: 0  },
	  interpolateNulls: true,
	  pointShape: 'diamond',
	  pointSize: 4
        };

        var chart = new google.visualization.AreaChart(document.getElementById(''' + "'chart_div_" + io + "'" + '''));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_''' + io + '''" style="width: ''' + str(default_width) + '''px; height: ''' + str(600) + '''px;"></div>'''

	html_subdirs.append(generate_html([], html, new_path, new_path, title, title, hint))

#
#  trend line graph plot for perf stats
#
def plot_perf_horiz_trend(html_subdirs, job, path, sysinfos):
	job_info = job_info_get(job)
	html = "<center>" + job_info + "<center><br>"
	#
	# Determine all the xlabels in the collection
	# and collect up unique filesystems and ioscheds
	#
	filesystems = set([])
	ioscheds = set([])
	all_xlabels = set([])
	xlabel_map = {}
	for sysinfo in sysinfos:
		all_xlabels.add(sysinfo['Kernel-release-canonical'])
		xlabel_map[sysinfo['Kernel-release-canonical']] = sysinfo['Kernel-release']

		filesystems.add(sysinfo['Filesystem'])
		ioscheds.add(sysinfo['IO Scheduler'])

	all_xlabels = sorted(all_xlabels)
	ioscheds = sorted(ioscheds)
	filesystems = sorted(filesystems)

	new_path = os.path.join(path, str_to_filename(job))
	mkdirp(new_path)

	#
	# Collect symbols and get ranking order
	#
	percents = {}
	for sysinfo in sysinfos:
		data = sysinfo['Perfreport']
		for (address, percent) in data:
			if address[0:1] != "0":
				try:
					percents[address] += percent
				except:
					percents[address] = percent

	if len(percents) == 0:
		html_subdirs.append(generate_html([], "<tr><td><center>No perf data for this test</center></td></tr>", path, new_path, job, job, job_info))
		return
		
	indexes = sorted(range(len(sysinfos)), key=lambda x : sysinfos[x]['Kernel-release-canonical'])

	#
	# Now produce a char for each symbol(!)
	#
	for (symbol, val) in sorted(percents.items(), key = lambda x: x[1])[::-1][:100]:
		html += '''
		<script type="text/javascript">
		google.load("visualization", "1", {packages:["corechart"]});
		google.setOnLoadCallback(drawChart);
		function drawChart() {
		var data = google.visualization.arrayToDataTable([
		'''
		#
		# Define all headings (we may not use all the columns)
		#
		row = [ "'Run #'" ]
		for io in ioscheds:
			for fs in filesystems:
				row.append("'" + io + " " + fs + "'")
		rows = [ row ]

		#
		#  This is ugly and inefficient, it needs optimising, but
		#  for now, it will do
		#
		cols = set()
		for xl in all_xlabels:
			i = 0
			row = [ "'" + xlabel_map[xl] + "'" ]
			cols.add(i)
			allnull = True
			for io in ioscheds:
				for fs in filesystems:
					i += 1
					val = "null"
					for sysinfo in sysinfos:
						if sysinfo['Kernel-release-canonical'] == xl and \
						   sysinfo['Filesystem'] == fs and \
						   sysinfo['IO Scheduler'] == io:
							for (address, percent) in sysinfo['Perfreport']:
								if address == symbol:
									val = str(percent)
									allnull = False
									cols.add(i)
									break
					row.append(val)
			if not allnull:
				rows.append(row)

		cols = list(cols)
		#
		#  Got to ensure we don't emit columns that are all
		#  null, so only push out a column if numcols[i] is defined
		#
		txt = ""
		for row in rows:
			html += "[" + ','.join([ row[i] for i in cols]) + "],\n"
		html += "]);\n"

		html += '''
        var options = {
          title: ' ''' + job + " (" + symbol + ") " + ''' ',
          hAxis: {title: 'Run #' },
          vAxis: {title: ' total utilisation % ', minValue: 0  },
	  interpolateNulls: true,
	  pointShape: 'diamond',
          isNull: true,
	  pointSize: 4
        };

        var chart = new google.visualization.AreaChart(document.getElementById(''' + "'chart_div_" + symbol + "'" + '''));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_''' + symbol + '''" style="width: ''' + str(default_width) + '''px; height: ''' + str(default_height) + '''px;"></div>'''

	html_subdirs.append(generate_html([], html, new_path, new_path, job, job, job_info))

#
#  Calculate heatmap colors
#
def heatmap_colors(heatmap):
	max_val = 0
	for datarow in heatmap:
		max_val = max(max_val, max(datarow))

	#
	# Create a range of colours 0..max_val
	#
	max_val += 1
	red = float(255)
	green = float(0)
	step = 2 * (float(255) / max_val)
	colors = []

	while green < 255:
		green += step
		green = min(green, 255)
		color = (65536 * int(red)) + (256 * int(green))
		colors.append(format(color, '06x'))

	while red > 0:
		red -= step
		red = max(red, 0)
		color = (65536 * int(red)) + (256 * int(green))
		colors.append(format(color, '06x'))

	return colors[::-1]

#
#  Calculate moving average of data with a specificed window size
#
def moving_average(values, window):
	weights = np.repeat(1.0, window)/window
	if len(weights) == 0:
		return []
	else:
		return np.convolve(values, weights, 'valid')


#
#  plot samples and heatmaps from raw fio data
#
def plot_samples_raw_fio_data(job, root_path, key, keys, title, sysinfos):
	job_info = job_info_get(job)
	html = "<center>" + job_info + "<center><br>"
	chart = 0

	filesystems = sorted({ sysinfo['Filesystem'] for sysinfo in sysinfos })
	ioscheds = sorted({ sysinfo['IO Scheduler'] for sysinfo in sysinfos })

	#
	# Part #1, barchart of means
	#
	mean_y = []
	vticks = []

	max_label_len = max([ len( \
				sysinfo['Kernel-release'] + ", " + \
		        	sysinfo['IO Scheduler'] + ", " + \
				sysinfo['Filesystem']) for sysinfo in sysinfos ])
	indexes = sorted(range(len(sysinfos)), key=lambda x : sysinfos[x]['Filesystem'] + sysinfos[x]['IO Scheduler'] + sysinfos[x]['Kernel-release-canonical'])

	html += '''
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
        var data = google.visualization.arrayToDataTable([
	'''

	html += "['Filesytem', '" + title + "'," + \
		" { role: 'style' }],\n"

	for i in indexes:
		samples = sysinfos[i]['Samples'][key]
		total_y = 0
		total_n = 0
		for (x, y) in samples:
			total_y += sum(y)
			total_n += len(y)
			if total_n != 0:
				mean_y = float(total_y) / total_n
			else:
				mean_y = 0

		color = default_colors[filesystems.index(sysinfos[i]['Filesystem'])]
		label = sysinfos[i]['Kernel-release'] + ", " + \
			sysinfos[i]['IO Scheduler'] + ", " + \
			sysinfos[i]['Filesystem']

		html += "['" + label + "', " + str(mean_y) + "," + \
			"'fill-color: " + color + "; " + \
			"fill-opacity: 0.9'],\n"


	bar_height = 25 * len(indexes)
	max_height = bar_height + 100
	html += "]);\n"
	html += '''
        var options = {
          title: 'Mean ''' + title + ''' ',
          vAxis: {title: 'Configuration' },
	  hAxis: {title: 'Mean ''' + title + ''' ', minValue: 0 },
	  chartArea: {left: ' ''' + str(10 * max_label_len) + ''' ', height: ''' + str(bar_height) + '''},
	  height: ''' + str(max_height) + ''',
	  legend: {position: 'none' }
        };

        var chart = new google.visualization.BarChart(document.getElementById('chart_div_barchart'));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_barchart" style="width: ''' + str(default_width) + '''px; height: ''' + str(max_height) + '''px;"></div>'''

	html += "<br><center><h3>Scatter plots of " + title + " and equivalent data as a heatmap</h3></center>"
	html += "<center>(horizontal axis 'Sample #' represents the Nth data sample measured during the run time of the test)</center>"
	#
	# Now do scatter plot and heat maps
	#
	for i in indexes:
		sysinfo = sysinfos[i]
		subtitle = ""

		samples = sysinfo['Samples'][key]
		max_x = 0
		max_y = 0
		mean_y = 0
		all_x = []
		all_y = []
		#
		#  Data is bundled in 1 or more sets of results
		#  For scatter plot and heatmap we can combine
		#  it all into one big hunk of data
		#
		for (x, y) in samples:
			all_x += x
			all_y += y

		len_all_x = len(all_x)
		len_all_y = len(all_y)

		if len_all_x == 0 and len_all_y == 0:
			html += "<center>" + subtitle + sysinfo_keys_to_label(sysinfo, keys) + ": "
			html += "No fio samples gathered</center>"
			continue
			
		if len_all_y > 0:
			max_y = max(all_y)
			avg_y = moving_average(all_y, len(all_y) / 20)
			mean_y = float(sum(all_y)) / len_all_y
		if len_all_x > 0:
			max_x = max(all_x)

		rows=32
		cols=64
		tick_interval = 5
		#
		# We add in the corner co-ordinates to force the histrogram2d
		# cover the entire space, otherwise it will crop it
		#
		heatmap, yedges, xedges = np.histogram2d(all_y + [ 0, max_y, 0, max_y ], all_x + [ 0, 0, max_x, max_x ], bins=(rows,cols))

		rows_delta = float(max_y) / float(tick_interval - 1)
		cols_delta = float(max_x) / float(tick_interval)
		heatmap_vticks = []
		for k in range(0, tick_interval + 1):
			heatmap_vticks.append("{v:" + str(k * float(rows)/tick_interval) + ", f:'" + str(k * float(max_y)/tick_interval) + "'}")
		heatmap_hticks = []
		for k in range(0, tick_interval + 1):
			heatmap_hticks.append("{v:" + str(k * float(cols)/tick_interval) + ", f:'" + str(k * float(max_x)/tick_interval) + "'}")
		vticks = []
		for k in range(0, tick_interval + 1):
			vticks.append("{v:" + str(k * float(max_y)/tick_interval) + ", f:'" + str(k * float(max_y)/tick_interval) + "'}")
		hticks = []
		for k in range(0, tick_interval + 1):
			hticks.append("{v:" + str(k * float(max_x)/tick_interval) + ", f:'" + str(k * float(max_x)/tick_interval) + "'}")

		skip = float(len_all_y) / float(default_max_plots_in_scatter_plot)
		if skip > 1:
			all_x = [ all_x[int(r)] for r in np.arange(0, len(all_x) - 1, skip)]
			all_y = [ all_y[int(r)] for r in np.arange(0, len(all_y) - 1, skip)]
			avg_y = [ avg_y[int(r)] for r in np.arange(0, len(avg_y) - 1, skip)]

		len_avg_g = len(avg_y)
		html += '''
    <script type="text/javascript" src="http://www.google.com/jsapi"></script>
    <script type="text/javascript">
      google.load("visualization", "1", {});
      google.load("prototype", "1.6");
    </script>
    <script type="text/javascript">
      google.load("visualization", "1", {packages:["corechart"]});
      google.setOnLoadCallback(drawChart);
      function drawChart() {
        var data = google.visualization.arrayToDataTable(['''
		#
		#  Rolling average may be empty of we don't have enough samples..
		#
		if len_avg_g == 0:
			html += "['Sample #','Value', 'Mean'],\n"
		else:
			html += "['Sample #','Value', 'Rolling Average', 'Mean'],\n"
		n_avg_y = len(avg_y)
		offset = (len(all_y) - n_avg_y) / 2
		for k in range(0,len(all_y)):
			if len_avg_g == 0:
				html += "[%d,%d,%.2f],\n" % (all_x[k], all_y[k], mean_y)
			else:
				if k >= offset and (k - offset) < n_avg_y:
					avg = "%.2f" % avg_y[k - offset]
				else:
					avg = "null"
				html += "[%d,%d,%s,%.2f],\n" % (all_x[k], all_y[k], avg, mean_y)
		html += '''
	]);

        var options = {
          title: ''' + "'" + subtitle + sysinfo_keys_to_label(sysinfo, keys) + "'" + ''',
          legend: { position: 'right' },
          pointSize: 0.1,
	  width: 500,
	  height: 300,
	  hAxis: { ticks:[''' + ','.join(hticks) + '''], title:'Sample #' },
	  vAxis: { ticks:[''' + ','.join(vticks) + '''], title:''' "'" + title + "'" +''' },
        };

        var chart = new google.visualization.ScatterChart(document.getElementById(''' + \
		"'chart_div_scatter_" + str(chart) + "'" + '''));

        chart.draw(data, options);
      }
    </script>
    <script type="text/javascript">
      google.load("visualization", "1", {packages:["corechart"]});
      google.setOnLoadCallback(drawChart);
      function drawChart() {
        var data = google.visualization.arrayToDataTable([
['Sample #','Value',{ role: 'style'}],
'''
		colors = heatmap_colors(heatmap)
		row=0
		for datarow in heatmap:
			col = 0
			for d in datarow:
				html += "[" + str(col + 0.5)  + "," + str(row + 0.5) + ",'color:#" + colors[int(d)] + "'],"
				col += 1
			row += 1

		html += '''
	]);

        var options = {
          title: ''' + "'" + subtitle + sysinfo_keys_to_label(sysinfo, keys) + "'" + ''',
	  pointShape: 'square',
	  pointSize: 6,
          legend: 'none',
	  hAxis: { ticks:[''' + ','.join(heatmap_hticks) + '''], title:'Sample #' },
	  vAxis: { ticks:[''' + ','.join(heatmap_vticks) + '''], title:''' "'" + title + "'" +''' },
	  width: 500,
	  height: 300,
	  tooltip: { trigger: 'none' }
        };

        var chart = new google.visualization.ScatterChart(document.getElementById(''' + \
		"'chart_div_heatmap_" + str(chart) + "'" + '''));
        chart.draw(data, options);
      }
    </script>
    <table>
    <tr>
      <td>
        <div id="chart_div_scatter_''' + str(chart) + '''"></div>
      </td>
      <td>
        <div id="chart_div_heatmap_''' + str(chart) + '''"></div>
      </td>
    <tr>
    </table>
'''
		chart += 1

	return html

#
#  Traverse a path to find json values
#
def get_json_val(json_data, fullpath):
	path = fullpath.split('.')
	j = json_data
	try:
		for p in path:
			if p.isdigit():
				j = j[int(p)]
			else:
				j = j[p]
		return j
 	except:
		return 0


#
# Do statistic data gathering
#
def graph_stats(html_subdirs, root_path, graph_func, stats_collection, collection_keys):
	#
	# Determine which jobs were run
	# and hence all the unique configs
	#
	unique_configs = []
	keys = keys_variant_collection(stats_collection)

	for stats in stats_collection:
		for sysinfo in stats:
			job = sysinfo['Job']
			config = job_config_get(job)
			for c in config:
				if not c in unique_configs:
					unique_configs.append(c)

	#
	#  The following are config values that we may be
	#  interested in labelling
	#
	sysinfokeys = [ 'Blocksize', 'Device', 'Hardware',
		       	'Job', 'Kernel-release', 'Date', 'Filesize', 'Processor', 'Filesystem', 'IO Scheduler' ]
	#
	#   gather the data
	#
	for config in unique_configs:
		sysinfovals = {}
		collection = []
		#
		#  find all the different sysinfo fields that change and don't
		#  change across the tests
		#
		for stats in stats_collection:
			for sysinfo in stats:
				if config in job_config_get(sysinfo['Job']):
					for c in sysinfokeys:
						try:
							if not sysinfo[c] in sysinfovals[c]:
								sysinfovals[c].append(sysinfo[c])
						except:
							sysinfovals[c] = [sysinfo[c]]
		sysinfo_fields_vary = []
		sysinfo_fields_static = []
		for s in sysinfovals:
			if len(sysinfovals[s]) > 1:
				if not s in collection_keys:
					sysinfo_fields_vary.append(s)
			else:
				sysinfo_fields_static.append(s)

		#
		# And sort them in sysinfokey order
		#
		sysinfo_fields_vary = sorted(sysinfo_fields_vary, key = lambda x:sysinfokeys.index(x))
		sysinfo_fields_static = sorted(sysinfo_fields_static, key = lambda x:sysinfokeys.index(x))

		label = config['Label']
		units = config['Units']
		scale = float(config['Scale'])
		jsonpath = config['JsonPath']
		subtitle = [ sysinfo[f] for f in sysinfo_fields_static ]

		for stats in stats_collection:
			yvalues = []
			xlabels = []
			sysinfos = []
			for sysinfo in stats:
				if config in job_config_get(sysinfo['Job']):
					val = get_json_val(sysinfo['Fiostats'], jsonpath)
					yvalues.append(val / scale)
					xlabel = ""
					for f in sysinfo_fields_vary:
						xlabel += sysinfo[f] + ", "

					xlabels.append(clean_label(xlabel))
					sysinfos.append(sysinfo)

					for c in sysinfokeys:
						try:
							if not sysinfo[c] in sysinfovals[c]:
								sysinfovals[c].append(sysinfo[c])
						except:
							sysinfovals[c] = [sysinfo[c]]
				xlabel = []
				for f in collection_keys:
					xlabel.append(sysinfo[f])
			#
			# Add them to the collection
			#
			if len(xlabels) > 0:
				collection.append((array_to_str(subtitle), xlabels, yvalues, array_to_str(xlabel), sysinfos))

		graph_func(html_subdirs, root_path, keys, config, collection)

#
#  Load in fio-stats.json and sysinfo.log for the given tests
#
def collect_stats(path):
	stats = []
	for datapath, _dirs, files in os.walk(path):
		head, tail = os.path.split(datapath)
		if tail in [ 'stats', 'perf']:
			sysinfo = sysinfo_load(os.path.join(datapath, 'sysinfo.log'))
			job_config_get(sysinfo['Job'])
			job_info_get(sysinfo['Job'])

			sysinfo['Samples'] = None	# Loaded on demand later
			sysinfo['Datapath'] = datapath
			sysinfo['Perfreport'] = perf_report_load(os.path.join(datapath, 'perf.report'))

			try:
				f = open(os.path.join(datapath, 'fio-stats.json'), 'r')
				sysinfo['Fiostats'] = json.load(f)
				f.close()
				stats.append(sysinfo)
			except:
				print "Failed to read fio-stats.json for " + datapath

	return stats

#
#  Given a set of criteria, find all the gathered stats that match
#
def find_matching_stats(stats, criteria):
	matches = []
	for sysinfo in stats:
		keep = True
		for (key, values) in criteria:
			if values != []:
				found = False
				for value in values:
					if key in sysinfo and sysinfo[key] == value:
						found = True
						break
				keep = keep and found
		if keep:
			matches.append(sysinfo)

	return matches

def find_unique_key_value_stats(stats, criteria, key):
	values = []
        matches = find_matching_stats(stats, criteria)
	for sysinfo in matches:
		if not sysinfo[key] in values:
			values.append(sysinfo[key])

	#print "Found %d unique stats" % len(values)
	return values

#
#  Load samples given a path and a regex pattern for the
#  raw data name.  Loads fio raw samples.
#
def samples_load_by_pattern(path, pattern):
	regex = re.compile(pattern)
	values = []
	for l in sorted(os.listdir(path)):
		for m in [regex.search(l)]:
			if m:
				filename = os.path.join(path, l)
				if filename[-3:] == '.gz':
					f = gzip.open(filename)
				else:
					f = open(filename, 'r')

				lines = f.readlines()
				f.close()
				x = [ int(col.split(',')[0]) for col in lines]
				y = [ int(col.split(',')[1]) for col in lines]
				values.append((x,y))
	return values

#
#  Load samples from file from a given path
#
def samples_load(path):
	print "Loading data: " + path
	samples = {}
	for (key, title, pattern, name, hint) in samples_info:
		samples[key] = samples_load_by_pattern(path, pattern)

	return samples

def flamegraph_perf_data(html_subdirs, root_path, path, stats, keys, title):
	#
	# Find all unique jobs
	#
	jobs = sorted({ sysinfo['Job'] for sysinfo in stats})
	#
	# Process data from each unique job:
	#
	html = ""
	for job in jobs:
		print "Flamegraphing " + title + " (" + job + ")"
		ordered_stats = []
		titles = []
		for sysinfo in stats:
			if sysinfo['Job'] == job:
				ordered_stats.append(sysinfo)
				titles.append(sysinfo_keys_to_label(sysinfo, keys))
		#
		# Sort based on the chart titles
		#
		indexes = np.argsort(titles)
		titles = [ titles[i] for i in indexes ]
		ordered_stats = [ ordered_stats[i] for i in indexes ]

		new_path = os.path.join(path, job)
		mkdirp(new_path)

		html = "\n"
		if len(ordered_stats):
			i = 0
			for sysinfo in ordered_stats:
				flamegraphname = "flamegraph-" + str(i) + ".svg"
				flamegraph = os.path.join(new_path, flamegraphname)
				perffile = os.path.join(sysinfo['Datapath'], 'perf.script')
				if os.path.isfile(perffile + '.gz'):
					cmd = "zcat"
					perffile += ".gz"
				elif os.path.isfile(perffile):
					cmd = "cat"
				else:
					html += "              <tr><td><center>No perf data for " + titles[i] + "</center></td></tr>\n"
					i = i + 1
					continue

				os.system(cmd + " " + perffile + " | ../tools/FlameGraph/stackcollapse-perf.pl | ../tools/FlameGraph/flamegraph.pl --minwidth=5 --width=" + str(default_width - 15) + " --title=\"" + titles[i] + "\" > " + flamegraph)

				html += "<tr><td><center><iframe src=\"" + flamegraphname
				html += "\" width=" + str(default_width)
				html += " height=500 frameborder=0 seamless" + "></center></iframe>"
				html += "</tr></td>"
				i = i + 1

			html_subdirs.append(generate_html([], html, root_path, new_path, job, job, ""))
		else:
			print "No relevant perf data can be found"

	return html


def barchart_perf_report(html_subdirs, root_path, path, stats, keys, title, symbols, threshold):
	#
	# Find all unique jobs
	#
	jobs = sorted({ sysinfo['Job'] for sysinfo in stats})
	filesystems = sorted({ sysinfo['Filesystem'] for sysinfo in stats})
	ioscheds = sorted({ sysinfo['IO Scheduler'] for sysinfo in stats})

	#
	# Process data from each unique job:
	#
	for job in jobs:
		job_info = job_info_get(job)
		#
		#  Collect up unique perf kernel syms
		#
		percents = {}
		for sysinfo in stats:
			if sysinfo['Job'] == job:
				data = sysinfo['Perfreport']
				for (address, percent) in data:
					if symbols == [] or address in symbols:
						try:
							percents[address] += percent
						except:
							percents[address] = percent

		new_path = os.path.join(path, job)
		mkdirp(new_path)

		if len(percents) == 0:
			html_subdirs.append(generate_html([], "<tr><td><center>No perf data for this test</center></td></tr>", root_path, new_path, job, job, ""))
			continue
			
		html = "<center>" + job_info + "</center><br>"
		print "Bar chart of " + title + " (" + job + ")"
		ordered_stats = []
		titles = []

		pc = {}
		chart = 0
		for sysinfo in stats:
			for fs in filesystems:
				for io in ioscheds:
					tag = fs + ' ' + io
					if sysinfo['Job'] == job and \
					   sysinfo['Filesystem'] == fs and \
					   sysinfo['IO Scheduler'] == io:
						data = sysinfo['Perfreport']
						for (address, percent) in data:
							if symbols == [] or address in symbols:
								try:
									pcs = pc[address]
								except:
									pc[address] = {}
								try:
									pc[address][tag] += percent
								except:
									pc[address][tag] = percent


		html += '''
	<tr><td><center>
	<script type="text/javascript">
	google.load("visualization", "1", {packages:["corechart"]});
	google.setOnLoadCallback(drawChart);
	function drawChart() {
        var data = google.visualization.arrayToDataTable([
	'''
		txt = "'Symbol/Address'"
		for fs in filesystems:
			for io in ioscheds:
				txt += ",'" + fs + " " + io + "'"

		html += "[" + txt + "],\n"

		for (key,val) in sorted(percents.items(), key = lambda x: x[1])[::-1][:threshold]:
			txt = "'" + key + "'"
			for fs in filesystems:
				for io in ioscheds:
					tag = fs + ' ' + io
					txt += ", "
					try:
						txt += str(pc[key][tag])
					except:
						txt += "null"
			html += "[" + txt + "],\n"
		html += "]);\n"
		html += '''
        var options = {
          title: 'Perf Report: % utilisation',
	  isStacked: true,
	  bar: { groupWidth: "95%"},
	  chartArea: {left:'15%',top:20,width:'70%',height:'95%'},
	  fontSize: 14,
        };

        var chart = new google.visualization.BarChart(document.getElementById(''' + "'chart_div_" + str(chart) + "'" + '''));
        chart.draw(data, options);
      }
	</script>
	<div id="chart_div_''' + str(chart) + '''" style="width: ''' + str(default_width) + '''px; height: ''' + str(min(len(percents), threshold) * 16) + '''px"></div>
	</td></tr></center>'''
		html_subdirs.append(generate_html([], html, root_path, new_path, job, job, job_info))

	return ""


def graph_samples(html_subdirs, root_path, path, key, stats, keys, title):
	#
	# Find all unique jobs
	#
	jobs = sorted({ sysinfo['Job'] for sysinfo in stats})
	#
	# Process data from each unique job:
	#
	for job in jobs:
		job_info = job_info_get(job)
		print "Bar charting " + title + " (" + job + ")"
		ordered_stats = []
		titles = []
		for sysinfo in stats:
			if sysinfo['Job'] == job:
				if sysinfo['Samples'] == None:
					sysinfo['Samples'] = samples_load(sysinfo['Datapath'])
				samples = sysinfo['Samples']
				ordered_stats.append(sysinfo)
				titles.append(sysinfo_keys_to_label(sysinfo, keys))
		#
		# Sort based on the chart titles
		#
		indexes = np.argsort(titles)
		titles = [ titles[i] for i in indexes ]
		ordered_stats = [ ordered_stats[i] for i in indexes ]

		new_path = os.path.join(path, job)
		mkdirp(new_path)

		html = "<center>" + job_info + "</center><br>"

		if len(ordered_stats):
			html = plot_samples_raw_fio_data(job, new_path, key, keys, title, ordered_stats)
			html_subdirs.append(generate_html([], html, root_path, new_path, job, job, job_info))
		else:
			print "No relevant data can be found"

	return html

#
#  break stats into collections based on the group_by_keys
#
def compare_stats_ordered_traverse(stats, group_by_keys, collection_keys, traversed_stats):
	if (len(group_by_keys) <= 1):
		sortstats = sorted(stats, key = lambda d: d[group_by_keys[0]])
		traversed_stats.append(sortstats)
		return
	#
	#  For all key values, partition into "collections"
	#  where the key value is identical and sub-divide
	#  or graph on next key value
	#
	k = group_by_keys[0]
	suborder_by_keys = group_by_keys[1:len(group_by_keys)]

	if not k in collection_keys:
		compare_stats_ordered_traverse(stats, suborder_by_keys, collection_keys, traversed_stats)
	else:
		values = set([])
		for sysinfo in stats:
			values.add(sysinfo[k])

		for v in values:
			substats = [ sysinfo for sysinfo in stats if sysinfo[k] == v ]
			compare_stats_ordered_traverse(substats, suborder_by_keys, collection_keys, traversed_stats)

#
#  group up data by group_by_keys and then graph it using graph_func
#
def compare_stats_ordered(html_subdirs, path, graph_func, stats, group_by_keys, collection_keys):
	traversed_stats = []
	compare_stats_ordered_traverse(stats, group_by_keys, collection_keys, traversed_stats)
	graph_stats(html_subdirs, path, graph_func, traversed_stats, collection_keys)


#
#  Bar chart of fs + tests on a given day
#
def daily_tests_barchart(html_dirs, root_path, date):
	info = "Comparison of flexible I/O tester (fio) metrics such as Disk Read/Writes, CPU utilisation, etc., that are relevant to each test."
	
	tests = find_matching_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Kernel-release', []),
			('Filesystem', []),
			('IO Scheduler', [])
		])
	#
	# Compare filesystems
	#
	newpath = os.path.join(root_path, 'daily-fio-barchart-by-filesystem')
	mkdirp(newpath)
	html_subdirs = []
	compare_stats_ordered(html_subdirs, newpath, bar_horiz, tests,
		['IO Scheduler', 'Job', 'Filesystem'], ['Job', 'IO Scheduler'])

	html = "<center>" + info + "</center><br>"
	html_dirs.append(generate_html(html_subdirs, html, root_path, newpath, 'daily-tests-barchart', 'Fio metrics' + ', '.join(date), info))

#
#  Bar chart of fs + tests by kernel
#
def kernel_tests_barchart(html_dirs, root_path):
	info = "Comparison of flexible I/O tester (fio) metrics such as Disk Read/Writes, CPU utilisation, etc., that are relevant to each test."

	kernels = find_unique_key_value_stats(stats, [], 'Kernel-release')
	canonical_kernels = {}
	for kernel in kernels:
		canonical_kernels[kernel_canonical(kernel)] = kernel

	new_path = os.path.join(root_path, 'kernel-tests-barchart')
	mkdirp(new_path)

	html_subdirs = []
	for canonical_kernel in sorted(canonical_kernels):
		kernel = canonical_kernels[canonical_kernel]
		html = "<center>" + kernel + "</center><br>"
		html_kernelsubdirs = []
		print "Generating charts for kernel " + kernel
		kernel_path = os.path.join(new_path, canonical_kernel)
		mkdirp(kernel_path)

		tests = find_matching_stats(stats,
			[
				('Kernel-release', [kernel])
			])

		#
		# Compare filesystems
		#
		compare_stats_ordered(html_kernelsubdirs, kernel_path, bar_horiz, tests,
			['IO Scheduler', 'Job', 'Filesystem'], ['Job', 'IO Scheduler'])
	
		html_subdirs.append(generate_html(html_kernelsubdirs, html, new_path, kernel_path, 'daily-tests-barchart', 'Fio metrics for ' + kernel , info))

	html = ""
	html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, 'kernel-tests-barchart', 'Fio metrics by kernel', "Fio metrics by kernel"))

#
#  Samples of fs on a given day
#
def daily_tests_samples(html_dirs, root_path, date):
	tests = find_matching_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', []),
			('Kernel-release', [])
		])
	keys = keys_variant_collection([tests])
	#
	#  Process raw samples from fio logs
	#
	for (key, title, pattern, name, hint) in samples_info:
		new_path = os.path.join(root_path, name)
		mkdirp(new_path)
		html_subdirs = []
		html = "<center><b>Note: these charts may take a while to render on a web browser<b></center><br>"
		graph_samples(html_subdirs, root_path, new_path, key, tests, keys, title)
		if len(date) > 0:
			html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, name, title + ": " + ', '.join(date), ""))
		else:
			html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, name, title, hint))

#
#  Samples of fs for all kernels
#
def kernel_tests_samples(html_dirs, root_path):
	kernels = find_unique_key_value_stats(stats, [], 'Kernel-release')
	canonical_kernels = {}
	for kernel in kernels:
		canonical_kernels[kernel_canonical(kernel)] = kernel

	new_path = os.path.join(root_path, 'kernel-tests-samples')
	mkdirp(new_path)

	html_subdirs = []
	for canonical_kernel in sorted(canonical_kernels):
		kernel = canonical_kernels[canonical_kernel]
		html = "<center>" + kernel + "</center><br>"
		print "Generating charts for kernel " + kernel
		kernel_path = os.path.join(new_path, canonical_kernel)
		mkdirp(kernel_path)

		tests = find_matching_stats(stats,
			[
				('Kernel-release', [kernel])
			])
		keys = keys_variant_collection([tests])
		html_kernelsubdirs = []
		#
		#  Process raw samples from fio logs
		#
		for (key, title, pattern, name, hint) in samples_info:
			name_path = os.path.join(kernel_path, name)
			mkdirp(name_path)
			graph_subdirs = []
			html = "<center><b>Note: these charts may take a while to render on a web browser<b></center><br>"
			graph_samples(graph_subdirs, root_path, name_path, key, tests, keys, title)
			html_kernelsubdirs.append(generate_html(graph_subdirs, html, root_path, name_path, name, title, hint))

		html_subdirs.append(generate_html(html_kernelsubdirs, "", root_path, kernel_path, 'kernel-tests-samples', 'Fio samples by kernel ' + kernel, 'Fio samples by kernel ' + kernel))
	html_dirs.append(generate_html(html_subdirs, "", root_path, new_path, 'kernel-tests-samples', 'Fio samples by kernel', "Fio samples by kernel"))

#
#  Flamegraphs of tests on given day
#
def daily_tests_flamegraph(html_dirs, root_path, date):
	tests = find_matching_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', []),
			('Kernel-release', [])
		])
	keys = keys_variant_collection([tests])
	new_path = os.path.join(root_path, 'flamegraph')
	mkdirp(new_path)
	html_subdirs = []
	flamegraph_perf_data(html_subdirs, root_path, new_path, tests, keys, 'flamegraph')
	html_dirs.append(generate_html(html_subdirs, "", root_path, new_path, 'flamegraph', "Perf flamegraph (by test) " + ', '.join(date), ""))

#
#  Perf reports of tests on given day, all symbols
#
def daily_tests_perf_report(html_dirs, root_path, date):
	info = "Utilisaton of CPU by each kernel function for each test (for all kernel symbols)"
	html = "<center>" + info + "</center><br>"

	tests = find_matching_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', []),
			('Kernel-release', [])
		])
	keys = keys_variant_collection([tests])
	new_path = os.path.join(root_path, 'perfreport')
	mkdirp(new_path)
	html_subdirs = []
	barchart_perf_report(html_subdirs, root_path, new_path, tests, keys, 'perfreport', [], 100000)
	html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, 'perfreport', "Perf Report (all symbols) (by test) " + ', '.join(date), info))

#
#  Perf reports of tests on given day, reduced symbols
#
def daily_tests_perf_report_reduced(html_dirs, root_path, date):
	info = "Utilisaton of CPU by each kernel function for each test (top 100 utilisation hits)"
	html = "<center>" + info + "</center><br>"

	tests = find_matching_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', []),
			('Kernel-release', [])
		])

	f = open('symbols-perf-histogram.txt', 'r')
	symbols = f.read().splitlines()
	f.close()

	keys = keys_variant_collection([tests])
	new_path = os.path.join(root_path, 'perfreport-reduced')
	mkdirp(new_path)
	html_subdirs = []
	barchart_perf_report(html_subdirs, root_path, new_path, tests, keys, 'perfreport-reduced', symbols, 100)
	html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, 'perfreport-reduced', "Perf Report (reduced symbols, top 100) (by test) " + ', '.join(date), info))


#
#  Plot trends by kernel
#
def kernel_trends(html_dirs, root_path):
	info = "Chart the flexible I/O tester (fio) metrics for each test and compare these over different kernels"
	html = "<center>" + info + "</center><br>"

	new_path = os.path.join(root_path, 'kernel-trends')
	mkdirp(new_path)

	kernel_release = []
	file_systems = []
	date = []

	jobs = find_unique_key_value_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', file_systems),
			('Kernel-release', kernel_release)
		], 'Job')

	html_subdirs = []
	for job in jobs:
		job_info = job_info_get(job)
		html = "<center>" + job_info + "</center><br>"
		html_jobsubdirs = []
		print "Generating trends for job " + job
		job_path = os.path.join(new_path, job)
		mkdirp(job_path)
		tests = find_matching_stats(stats,
			[
				('Date', date),
				('Job', [job]),
				('Filesystem', file_systems),
				('Kernel-release', kernel_release )
			])
		compare_stats_ordered(html_jobsubdirs, job_path, plot_horiz_trend, tests,
			[ 'Job', 'Filesystem', 'IO Scheduler', 'Kernel-release'], ['Filesystem', 'IO Scheduler', 'Job'])
		html_subdirs.append(generate_html(html_jobsubdirs, html, new_path, job_path, 'kernel-trends', job + ": " + 'trends by kernel', job_info))

	html = "<center>" + info + "</center><br>"
	html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, 'kernel-trends', 'Fio trends', info))


#
#  Plot perf trends by kernels
#
def kernel_perf_trends(html_dirs, root_path):
	info = "Chart the utilisation for each of the top used 100 kernel symbols and compare these over different kernels"
	html = "<center>" + info + "</center><br>"

	new_path = os.path.join(root_path, 'overtime-perf-trends')
	mkdirp(new_path)

	kernel_release = []
	file_systems = []
	date = []

	jobs = find_unique_key_value_stats(stats,
		[
			('Date', date),
			('Job', []),
			('Filesystem', file_systems),
			('Kernel-release', kernel_release)
		], 'Job')

	html_subdirs = []
	for job in jobs:
		html_jobsubdirs = []
		print "Generating trends for job " + job
		job_path = os.path.join(new_path, job)
		mkdirp(job_path)
		tests = find_matching_stats(stats,
			[
				('Date', date),
				('Job', [job]),
				('Filesystem', file_systems),
				('Kernel-release', kernel_release )
			])
		plot_perf_horiz_trend(html_subdirs, job, new_path, tests)

	html_dirs.append(generate_html(html_subdirs, html, root_path, new_path, 'overtime-trends', 'Perf trends by kernel (top 100 kernel symbols)', info))
#
#  Gather up stats
#
dates = []
html_dirs = []
stats = []
tmpdir = "/tmp/test"
print "Results in " + tmpdir

print "Reading metadata.."
for i in range(1, len(sys.argv)):
	stats += collect_stats(sys.argv[i])

mkdirp(tmpdir)

#
# Daily tests run on one "daily" run of data
#
#daily_tests_perf_report(html_dirs, tmpdir, dates)
#daily_tests_perf_report_reduced(html_dirs, tmpdir, dates)
#daily_tests_flamegraph(html_dirs, tmpdir, dates)
#daily_tests_barchart(html_dirs, tmpdir, dates)
#daily_tests_samples(html_dirs, tmpdir, dates)

#
# The following graph trends on all data sets
#
kernel_trends(html_dirs, tmpdir)
#kernel_perf_trends(html_dirs, tmpdir)
kernel_tests_barchart(html_dirs, tmpdir)
kernel_tests_samples(html_dirs, tmpdir)

generate_html(html_dirs, "", tmpdir, tmpdir, 'results', 'All results', "")
