#!/usr/bin/env python3

import json, os, os.path, argparse, glob, re, ast

parser = argparse.ArgumentParser(description='Generate JSON configuration file for SystemDataScope')
parser.add_argument('root_dir', type=str,
                    help='Root directory with collectd RRD databases, for example /tmp/collectd/Jolla')

args = parser.parse_args()

Root = args.root_dir
os.chdir(Root)


######################################################################################
#
# These maps are used to specify several parameters of the graphs
# 

Units = { "DEFAULT": "",
          "CPU": "%",
          "Battery/voltage": "V",
          "Battery/charge": "%",
          "Battery/capacity": "%",
          "Battery/current": "A",
          "Battery/energy_wh": "Wh",
          "Battery/power": "W",
          "Battery/temperature": "C",
          "Battery/duration-low": "h",
          "Battery/duration-full": "h",
          "Context/context_switch": "1/s",
          "Entropy/entropy": "bits",
          "Memory": "bytes",
          "Swap": "bytes",
          "Swap/inout": "bytes/s",
          "Network/octets": "bytes/s",
          "NetworkTotal/octets": "B",
          "Network": "1/s",
          "NetworkTotal": "",
          "Processes/cputime": "%",
          "Processes/disk_ops": "1/s",
          "Processes/disk_octets": "bytes/s",
          "Processes/vm": "bytes",
          "Processes/data": "bytes",
          "Processes/code": "bytes",
          "Processes/rss": "bytes",
          "Processes/pagefaults": "1/s",
          "Processes/stacksize": "bytes",
          "Disk/io_time": "%",
          "Disk/ops": "1/s",
          "Disk/octets": "bytes/s",
          #"Disk/time": "1/s",
          "Storage": "bytes",
          "Uptime/uptime": "days",
          "CPUstate/cpusleep": "%",
          "Radio/Cellular": "%",
          "Radio/Internet": "%",
          "Suspend": "1/s",
          "CPUfreq/values": "Hz",
}

Formats = { "DEFAULT": "%5.2lf",
            "Battery/charge": "%3.1lf",
            "Battery/capacity": "%3.1lf",
            "Battery/temperature": "%3.1lf",
            "Battery/energy_wh": "%3.2lf",
            "Battery/power": "%3.2lf",
            "Battery/voltage": "%1.3lf%S",
            "Battery/current": "%1.0lf%S",
            "Context/context_switch": "%1.0lf%S",
            "Entropy/entropy": "%1.0lf%S",
            "Memory/Overview": "%4.0lf%S",
            "Memory": "%4.0lf%S",
            "Swap": "%5.2lf%S",
            "Swap/inout": "%6.0lf%S",
            "Load": "%5.2lf",
            "Network/octets": "%6.0lf%S",
            "Network/packets": "%6.0lf%S",
            "Network": "%6.2lf%S",
            "Processes/code": "%5.1lf%S",
            "Processes/data": "%5.1lf%S",
            "Processes/rss": "%5.1lf%S",
            "Processes/stacksize": "%5.1lf%S",
            "Processes/vm": "%5.1lf%S",
            "Processes/cputime": "%5.3lf",
            "Processes/disk_ops": "%4.0lf%S",
            "Processes/disk_octets": "%4.0lf%S",
            "Processes": "%5.1lf",
            "Processes/Overview": "%5.1lf",
            "Disk": "%4.0lf%S",
            "Disk/merged": "%5.2lf",
            "Disk/io_time": "%5.1lf",
            "Disk/ops": "%5.2lf",
            "CPUstate/cpusleep": "%0.0lf",
            "CPU/Overview": "%6.0lf",
            "CPU": "%0.0lf",
            "Storage": "%6.1lf%S",
            "Radio/Cellular": "%4.0lf",
            "Radio/Internet": "%4.0lf",
            "Radio/Bluetooth": "%4.2lf",
            "Radio/active": "%4.2lf",
            "Suspend": "%4.2lf",
            "CPUfreq/values": "%4.0lf%S",
            "CPUidle/values": "%4.0lf%S",
}

Heads = { "DEFAULT": None,
          "CPU/Overview": 6,
          "Storage": 7,
          "Disk": 5,
          "Memory/Overview": 5,
          "Network": 7,
          "Load/load": 5,
          "Swap": 6,
          "Swap/inout": 7,
          "Processes/Overview": 5,
          "Processes/cputime": 5,
          "Processes/count": 5,
          "Processes/pagefaults": 5,
          "Processes/disk_octets": 5,
          "Processes/disk_ops": 5,
          "Radio/Cellular": 4,
          "Radio/Internet": 4,
          "Radio/Bluetooth": 4,
          "Radio/active": 4,
          "Suspend/Overview": 4,
          "CPUfreq/values": 5,
          "CPUidle/values": 5,
}

Human = { "DEFAULT": None,
          "Battery/power": "Battery power consumption",
          "Battery/energy_wh": "Battery energy",
          "Battery/duration-full": "Battery: time to full",
          "Battery/duration-low": "Battery: time to low",
          "Disk/io_time": "I/O time",
          "Disk/merged": "Merged operations",
          "Disk/octets": "Traffic",
          "Disk/ops": "Operations",
          "Disk/time": "Average time per operation",
          "Network/octets": "Traffic",
          "Processes/cputime": "CPU time",
          "Processes/stacksize": "Stack size",
          "Processes/vm": "Virtual memory",
          "Processes/data": "Data size",
          "Processes/code": "Code size",
          "Processes/rss": "Resident segment size",
          "Processes/pagefaults": "Page faults",
          "Processes/disk_ops": "Disk operations",
          "Processes/disk_octets": "Disk traffic",
          "Processes/count": "# processes",
          "Radio/Cellular": "Cellular signal strength",
          "Radio/Internet": "Internet signal strength",
          "Radio/Bluetooth": "Bluetooth activity",
          "Radio/active": "Active radio(s)",
          "CPUidle/values": "CPU idle state",
}

######################################################################################
## Helper classes

######################################################################################
# Colors

class ColorSingle:
    def __init__(self, color, color_opa):
        self.color = color
        self.color_opa = color_opa

    def set_number_of_lines(self, n):
        # noop
        self.n = n

    def get_color(self, i, opacity = 1.0):
        if opacity > 0.9: return self.color
        return self.color_opa


class Colors:
    # internally stored as RGBA integers
    # Colorschemes from http://colorbrewer2.org/ , qualitative
    def __init__(self, last_transparent = False):
        self.colors = None
        self.n = None
        self.last_transparent = last_transparent

        self.colorschemes = {}
        for cbrewer in [ "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)','rgb(255,127,0)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)','rgb(255,127,0)','rgb(255,255,51)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)','rgb(255,127,0)','rgb(255,255,51)','rgb(166,86,40)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)','rgb(255,127,0)','rgb(255,255,51)','rgb(166,86,40)','rgb(247,129,191)'",
                         "'rgb(228,26,28)','rgb(55,126,184)','rgb(77,175,74)','rgb(152,78,163)','rgb(255,127,0)','rgb(255,255,51)','rgb(166,86,40)','rgb(247,129,191)','rgb(153,153,153)'"
        ]:
            cbrewer = "[ " +  cbrewer.replace("'rgb(", "[" ).replace(")'", "]" ) + " ]"
            colors = ast.literal_eval( cbrewer )
            for i in colors: i.append(255)
            self.colorschemes[ len(colors) ] = colors

    def set_number_of_lines(self, n):
        self.n = n
        if self.last_transparent: k = n-1
        else: k = n

        if k in self.colorschemes:
            self.colors = self.colorschemes[k]
        elif k < min(self.colorschemes.keys()):
            self.colors = self.colorschemes[ min(self.colorschemes.keys()) ]
        else:
            self.colors = self.colorschemes[ max(self.colorschemes.keys()) ]

    def makestr(self, c, opacity):
        s = "#"
        for i in c[:-1]:
            s += ("%02X" % int(round(i)) )
        s += ("%02X" % int(round(c[-1]*opacity)) )
        return s

    def get_color(self, i, opacity=1.0):
        if i == 0: return self.makestr(self.colors[0], opacity)
        if self.last_transparent:
            if i >= self.n-1: return self.makestr([0,0,0,0], opacity)
            elif i == self.n-2: return self.makestr(self.colors[-1], opacity)
        else:
            if i >= self.n-1: return self.makestr(self.colors[-1], opacity)

        if self.last_transparent: nc = self.n-2
        else: nc = self.n-1

        dline = float(i) / float(nc)
        color0 = int( (len(self.colors) - 1) * dline )
        factor = (len(self.colors) - 1) * dline - color0

        c = []
        for i in range(len(self.colors[color0])):            
            c.append( (1-factor)*self.colors[color0][i] + factor*self.colors[color0+1][i] )
        return self.makestr(c, opacity)

### Useful color objects

cs = Colors( )
csTr = Colors( True )
csSingle = ColorSingle( "$COLOR_LINE_SINGLE$", "$COLOR_LINE_SINGLE_SUB$")

############################################################################################

# Helper function to print table heading
def makeheads(l):
    s = ''
    fmt = '%' + str(l) + 's'
    heads = ["Avr", "Min", "Max", "Last"]
    for i, t in enumerate(heads):
        s += 'COMMENT:"' + (fmt % t)
        if i == len(heads)-1: s += '\\r" '
        else: s += '" '
    return s

# Class for making multiline(-area) plots or stacks

class StackOrLines:
    def __init__(self, colors, Type, name, Title = None,
                 Unit = None, isStack = False, minmax = False,
                 SkipLegend = False, HeadSize = None,
                 t = "LINE",
                 extra_command = "" ):
        self.lines = []
        self.gt = t
        self.colors = colors
        self.isStack = isStack
        self.minmax = minmax
        self.fullname = Type + "/" + name
        self.SkipLegend = SkipLegend

        if Unit is None: self.u = getunit(self.fullname)
        else: self.u = Unit

        if HeadSize is None: self.HeadSize = gethead(self.fullname)
        else: self.HeadSize = HeadSize

        command_def = '-t " '
        if Title is not None: command_def += Title
        elif self.fullname in Human: command_def += Human[ self.fullname ]
        else: command_def += Type + " " + name

        if len(self.u) > 0: command_def += ", " + self.u
        command_def += '"  '  + " " + defColors + " " + extra_command + " "

        self.command_def = command_def

    # def add(self, name, width, options, extra="", makeLine=False, var_name = "value"):
    #     self.lines.append( { "name": name,
    #                          "width": width,
    #                          "options": options,
    #                          "extra": extra,
    #                          "makeLine": makeLine,
    #                          "var_name": var_name,
    #     } )

    def add(self, name, g, width="$LINE_WIDTH_PRIMARY$", makeLine=False, var_name = "value", labelName = None, Format = None, scale_data = None):
        if Format is None: Format = getf( self.fullname + "/" + name )
        if labelName is None: labelName = name
        self.lines.append( { "name": name,
                             "g": g,
                             "width": width,
                             "makeLine": makeLine,
                             "var_name": var_name,
                             "format": Format,
                             "scale_data": scale_data,
                             "labelName": labelName,
        } )

    def compose(self):
        command_def = self.command_def
        files = []

        # make all defs
        for i in self.lines:
            if i["scale_data"] is None: scale = False
            else: scale = True

            if i["g"] not in files:
                files.append( i["g"] )

            vd = i["name"]
            if scale: vd += "_data"

            command_def += "DEF:" + vd + "=" + i["g"] + ":" + i["var_name"] + ":AVERAGE "
            if self.minmax or not self.SkipLegend:
                command_def += "DEF:" + vd + "_min=" + i["g"] + ":" + i["var_name"] + ":MIN "
                command_def += "DEF:" + vd + "_max=" + i["g"] + ":" + i["var_name"] + ":MAX "

            if scale:
                command_def += "CDEF:" + i["name"] + "=" + vd + "," + i["scale_data"] + " "
                if self.minmax or not self.SkipLegend:
                    command_def += "CDEF:" + i["name"] + "_min=" + vd + "_min," + i["scale_data"] + " "
                    command_def += "CDEF:" + i["name"] + "_max=" + vd + "_max," + i["scale_data"] + " "

            if self.minmax or not self.SkipLegend:    
                command_def += "CDEF:" + i["name"] + "_max_min_delta=" + i["name"] + "_max," + i["name"] + "_min,- "

        if not self.SkipLegend and self.HeadSize is not None:
            command_def += makeheads( self.HeadSize ) + " "

        s = command_def

        self.colors.set_number_of_lines(len(self.lines))
        if self.minmax:
            for idx, i in enumerate(self.lines):
                s += "LINE:" + i["name"] + "_min AREA:" + i["name"] + "_max_min_delta" + self.colors.get_color(idx, 0.5) + "::STACK "

        for idx, i in enumerate(self.lines):
            color = self.colors.get_color(idx) 
            s += self.gt
            if self.gt == "LINE": s += i["width"]
            s += ":" + i["name"] + color + ":"
            if not self.SkipLegend:
                if len(self.lines) > 1 or self.HeadSize is not None:
                    s += '"' + i["labelName"] + '\\l"'
                else:
                    s += '" \\l"'

            if self.isStack and idx > 0 and not i["makeLine"]: s += ":STACK"
            s += " " #+ i["extra"] + " "

            if not self.SkipLegend:
                f = i["format"]
                name = i["name"]
                if self.HeadSize is None:
                    s += "COMMENT:\\u GPRINT:"+name+":AVERAGE:\"Avr " + f + "\" GPRINT:"+name+"_min:MIN:\"Min " + f + "\" GPRINT:"+name+"_max:MAX:\"Max " + f + "\" GPRINT:"+name+":LAST:\"Last " + f + "\\r\" "
                else:
                    s += "COMMENT:\\u GPRINT:"+name+":AVERAGE:\"" + f + "\" GPRINT:"+name+"_min:MIN:\"" + f + "\" GPRINT:"+name+"_max:MAX:\"" + f + "\" GPRINT:"+name+":LAST:\"" + f + "\\r\" "

        gt = { "command": s,
               "files": files }
        plot = { "type": self.fullname }

        return gt, plot, self.fullname

######################################################################################
# Helper function for a single value plot
def maketypesplot(Type, name, g, Title = None, Format = None, Unit = None, minmax=True, SkipLegend = False,
                  var_name = "value", extra_command = "", scale_data = None):

    s = StackOrLines( csSingle, Type, name, Title=Title, Unit=Unit, SkipLegend=SkipLegend, minmax=minmax, extra_command=extra_command)
    s.add( name, g, var_name=var_name, Format=Format, scale_data=scale_data )

    return s.compose()


# used when more control is needed
class StackOrLinesOld:
    def __init__(self, col, isStack = False, minmax = False, t = "LINE"):
        self.lines = []
        self.gt = t
        self.colors = col
        self.isStack = isStack
        self.minmax = minmax

    def add(self, name, width, options, extra="", makeLine=False):
        self.lines.append( { "name": name,
                             "width": width,
                             "options": options,
                             "extra": extra,
                             "makeLine": makeLine } )

    def str(self):
        s = ""
        self.colors.set_number_of_lines(len(self.lines))
        if self.minmax:
            for idx, i in enumerate(self.lines):
                s += "LINE:" + i["name"] + "_min AREA:" + i["name"] + "_max_min_delta" + self.colors.get_color(idx, 0.5) + "::STACK "

        for idx, i in enumerate(self.lines):
            color = self.colors.get_color(idx) 
            s += self.gt
            if self.gt == "LINE": s += i["width"]
            s += ":" + i["name"] + color + ":" + i["options"]
            if self.isStack and idx > 0 and not i["makeLine"]: s += ":STACK"
            s += " " + i["extra"] + " "

        return s    


######################################################################################

def sorter(glob_arg, re_search = None, front = [], end = [], useNumericInt = False):
    a = []
    for g in glob.glob( glob_arg ):
        if re_search is not None: m = re.search( re_search, g ).group(1)
        else: m = g

        if useNumericInt: m = int(m)

        added = False
        for k in [ [0,front], [10000,end] ] :
            if m in k[1]:
                a.append( [ k[0] + k[1].index( m ), g ] )
                added = True
        if not added:
            if not useNumericInt: a.append( [ 5000, g ] )
            else: a.append( [ 5000 + m, g ] )

    a.sort()
    r = []
    for i in a: r.append(i[1])
    return r


def getit(name, D):
    if name in D: return D[name]

    sname = name.split("/")
    if len(sname) > 1:
        s = ''
        for i in sname[:-1]: s += i + "/"        
        return getit( s[:-1], D )

    return D["DEFAULT"]

def getunit(name): return getit(name, Units)
def getf(name): return getit(name, Formats)
def gethead(name): return getit(name, Heads)

def gethuman(name):
    n = getit(name, Human)
    if n is None: return name
    return n

def settype(Plot, name):
    if "type" not in Plot or Plot["type"] is None:
        Plot["type"] = name

######################################################################################
# Start definition of types                

Config = {}

Config["variables"] = {
    "COLOR_BACKGROUND": "#00000000",
    "COLOR_CANVAS": "#00000000",
    "COLOR_FONT": "#000000FF",
    "COLOR_AXIS": "#000000FF",
    "COLOR_ARROW": "#000000FF",
    "COLOR_LINE_SINGLE": "#0000FFFF",
    "COLOR_LINE_SINGLE_SUB": "#0000FF80",

    "LINE_WIDTH_PRIMARY": "3",
    "LINE_WIDTH_SECONDARY": "1",
    }

Config["page"] = {

    "title": "Overview",
    "plots": [
    ]
}

Config["cover"] = [
    ]

Config["types"] = {}

defColors = "--color BACK$COLOR_BACKGROUND$ --color SHADEA$COLOR_BACKGROUND$ --color SHADEB$COLOR_BACKGROUND$ --color CANVAS$COLOR_CANVAS$  "
defColors += "--color FONT$COLOR_FONT$ --color AXIS$COLOR_AXIS$ --color ARROW$COLOR_ARROW$ "


######################################################################################
#
# DEFINITION OF GRAPHS
#
######################################################################################

######################################################################################
# CPU
if os.path.exists( "cpu" ) or os.path.exists( "cpusleep" ):

    Plots = { "type": None, "subplots": { "title": "CPU details", "plots": [ ] } }

    if os.path.exists( "cpu" ) :
        SubPlots = { "subplots": { "title": "CPU usage", "plots": [ { "type": "CPU/Overview" } ] } }

        # CPU overview

        s = StackOrLines( csTr, "CPU", "Overview", isStack = True, t = "AREA", extra_command = "--upper-limit 100 --lower-limit 0 --rigid" )
        cpustates = sorter( "cpu/*.rrd", "^cpu.*/.*-(.*).rrd",
                            ["interrupt", "softirq", "steal", "wait", "system"], 
                            ["user", "nice", "idle"] )

        for g in cpustates:
            name = re.search( "^cpu.*/.*-(.*).rrd", g ).group(1)
            s.add( name, g )

        gt, plot, fullname = s.compose()
        Config["types"][fullname] = gt
        SubPlots["type"] = fullname
        settype(Plots, fullname)

        # CPU overview : cover
        s = StackOrLines( csTr, "CPU", "Overview", Title="CPU", isStack = True, t = "AREA", extra_command = "--upper-limit 100 --lower-limit 0 --rigid", SkipLegend = True )

        for g in cpustates:
            name = re.search( "^cpu.*/.*-(.*).rrd", g ).group(1)
            s.add( name, g )

        gt, plot, fullname = s.compose()

        Config["types"]["CPU/Overview/cover"] = gt
        Config["cover"].append( "CPU/Overview/cover" )

        # Make CPU subplots
        cpustates.reverse()
        for g in cpustates:
            name = re.search( "^cpu.*/.*-(.*).rrd", g ).group(1)
            gt, plot, fullname = maketypesplot( "CPU", name, g, extra_command = "--upper-limit 100 --lower-limit 0 --rigid")

            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        Plots["subplots"]["plots"].append( SubPlots )


    if os.path.exists( "cpusleep" ):

        ###################################################
        # CPU sleep needs some math
        g = "cpusleep/total_time_in_ms.rrd"
        name = "cpusleep"
        gt, plot, fullname = maketypesplot( "CPUstate", name, g, Title="CPU sleep",
                                            extra_command='--upper-limit 100 --lower-limit 0 --rigid',
                                            scale_data = "10,/" )

        Config["types"]["CPUstate/" + name] = gt
        settype(Plots, fullname)
        CpuSleepPlots = { "type": fullname,
                          "subplots": { "title": "CPU sleep details", "plots": [ {"type": fullname} ] } }

        gt, plot, fullname = maketypesplot( "CPUstate", name, g, Title="CPU sleep",
                                            extra_command='--upper-limit 100 --lower-limit 0 --rigid',
                                            scale_data = "10,/", SkipLegend = True )

        Config["types"]["CPUstate/" + name + "/cover"] = gt
        Config["cover"].append( "CPUstate/" + name + "/cover" )

        # CPU sleep subplots
        if os.path.exists("suspend"):

            # success/failed graph
            s = StackOrLines( cs, "Suspend", "Overview", isStack = True, t = "AREA", extra_command = "--lower-limit 0" )
            for name in ["fail", "success"]:
                g = "suspend/attempts-" + name + ".rrd"
                s.add( name, g )

            gt, plot, fullname = s.compose()
            Config["types"][fullname] = gt

            SuspendPlots = { "type": fullname,
                             "subplots": { "title": "CPU suspend details", "plots": [ {"type": fullname} ] } }

            for g in sorter("suspend/attempts-*.rrd",
                            "^suspend/attempts-(.*).rrd",
                            ["success", "fail"] ):
                name = re.search( "^suspend/attempts-(.*).rrd", g ).group(1)
                gt, plot, fullname = maketypesplot( "Suspend", name, g, 
                                                    extra_command='--lower-limit 0' )

                Config["types"][fullname] = gt
                SuspendPlots["subplots"]["plots"].append( plot )

            CpuSleepPlots["subplots"]["plots"].append( SuspendPlots )

            ##############################################
            # graph showing the duration of a single sleep

            command_def = '-t "Duration of a single suspend, s" ' + defColors #+ makeheads(7)
            command_line = ""
            command_extra = ""
            files = ["cpusleep/total_time_in_ms.rrd", "suspend/attempts-success.rrd"]
            s = StackOrLinesOld( csSingle, minmax=False )
            command_def += "DEF:cpusleep_data=cpusleep/total_time_in_ms.rrd:value:AVERAGE "
            command_def += "DEF:suspend_data=suspend/attempts-success.rrd:value:AVERAGE "
            command_def += "CDEF:sleep_per_suspend_ms=cpusleep_data,suspend_data,/ "
            command_def += "CDEF:sleep_sus=sleep_per_suspend_ms,1000,/ "
            s.add( "sleep_sus", "$LINE_WIDTH_PRIMARY$", '"\\l"' ,
                   "COMMENT:\\u GPRINT:sleep_sus:AVERAGE:\"Avr %4.2lf%s"
                   "\" GPRINT:sleep_sus:LAST:\"Last %4.2lf%s\\r\" " + " ")

            command_line = s.str() + command_extra

            fullname = "CPUstate/Sleep_per_suspend"
            gt = { "command": command_def + command_line,
                   "files": files }
            plot = { "type": fullname }

            Config["types"][fullname] = gt
            CpuSleepPlots["subplots"]["plots"].append( plot )
            # single sleep duration: done
            ##############################################

        Plots["subplots"]["plots"].append( CpuSleepPlots )

    # CPU frequency
    if len(glob.glob("cpufreq*")) > 0:
        SubPlots = { "subplots": { "title": "CPU frequency details", "plots": [ ] } }

        # distributions 
        if len(glob.glob("cpufreq*/total_time_in_ms*.rrd")) > 0:

            for D in sorter( "cpufreq*", front=["cpufreq"]):
                if len(glob.glob(D + "/total_time_in_ms*.rrd")) > 0:

                    if D == "cpufreq": ht = ""
                    else: ht = ": CPU " + re.search("^cpufreq-(.*)", D).group(1)

                    # CPU frequency distribution overview
                    s = StackOrLines( cs, "CPUfreq", "Overview " + ht,
                                      Title = "Used frequencies" + ht,
                                      Unit="%", HeadSize = 5,
                                      t = "AREA", isStack = True, extra_command = "--lower-limit 0" )

                    srted = sorter( D + "/total_time_in_ms-*.rrd", "^" + D + "/total_time_in_ms-(.*).rrd", useNumericInt = True )
                    srted.reverse()
                    for g in srted:
                        freq = int(re.search( "^" + D + "/total_time_in_ms-(.*).rrd", g ).group(1))/1000
                        name = str( int(round(freq)) ) + "MHz"
                        s.add( name, g, scale_data="0.1,*", Format="%4.0lf%s" )

                    gt, plot, fullname = s.compose()
                    Config["types"][fullname] = gt
                    settype(SubPlots, fullname)
                    SubPlots["subplots"]["plots"].append( plot )

        # single shot measurements
        if len(glob.glob("cpufreq/cpufreq-*.rrd")) > 1:
            s = StackOrLines( cs, "CPUfreq", "values", minmax=True, extra_command="--lower-limit 0"  )
            for g in sorter( "cpufreq/cpufreq-*.rrd", "^cpufreq/cpufreq-(.*).rrd", useNumericInt = True ):
                name = "cpu_" + re.search( "^cpufreq/cpufreq-(.*).rrd", g ).group(1)
                s.add( name, g )

            gt, plot, fullname = s.compose()
            Config["types"][fullname] = gt
            settype(SubPlots, fullname)
            SubPlots["subplots"]["plots"].append( plot )


        # add frequency plots
        if len(SubPlots["subplots"]["plots"]) > 1:
            Plots["subplots"]["plots"].append( SubPlots )
        elif "type" in SubPlots:
            Plots["subplots"]["plots"].append( { "type": SubPlots["type"] } )

    ###############################################################################
    # CPU idle
    if len(glob.glob("cpuidle*")) > 0:
        SubPlots = { "subplots": { "title": "CPU idle states details", "plots": [ ] } }

        # # make local dictionary
        # idleName2Desc = {}
        # for d in glob.glob("/sys/devices/system/cpu/cpu*/cpuidle/state*"):
        #     f = open(d + "/name", "r")
        #     name = f.readline()
        #     name = name.strip()
        #     for c in [" ", "-", "/"]: name = name.replace(c, "_")
        #     f.close()

        #     f = open(d + "/desc", "r")
        #     desc = f.readline()
        #     desc = desc.strip()
        #     f.close()

        #     idleName2Desc[name] = desc

        for D in sorter( "cpuidle*", front=["cpuidle"]):
            if len(glob.glob(D + "/total_time_in_ms-*.rrd")) > 0:

                if D == "cpuidle": ht = ""
                else: ht = ": CPU " + re.search("^cpuidle-(.*)", D).group(1)

                # CPU idle distribution overview
                s = StackOrLines( cs, "CPUidle", "Overview " + ht,
                                  Title = "Used idle states" + ht,
                                  Unit="%", HeadSize = 5,
                                  t = "AREA", isStack = True, extra_command = "--lower-limit 0" )

                srted = sorter( D + "/total_time_in_ms-*.rrd", "^" + D + "/total_time_in_ms-(.*).rrd" )
                for g in srted:
                    name = re.search( "^" + D + "/total_time_in_ms-(.*).rrd", g ).group(1)
                    s.add( name , g, scale_data="10,*", Format="%4.0lf%s" ) #, labelName = name + "[" + idleName2Desc[name] + "]" )

                gt, plot, fullname = s.compose()
                Config["types"][fullname] = gt
                settype(SubPlots, fullname)
                SubPlots["subplots"]["plots"].append( plot )

        # add idle plots
        if len(SubPlots["subplots"]["plots"]) > 1:
            Plots["subplots"]["plots"].append( SubPlots )
        elif "type" in SubPlots:
            Plots["subplots"]["plots"].append( { "type": SubPlots["type"] } )


    # Add all CPU plots
    Config["page"]["plots"].append( Plots )


######################################################################################
# Battery 

if os.path.exists( "battery-0" ):
    BatteryPlots = { "subplots": { "title": "Battery details", "plots": [ ] } }

    Factors = { "DEFAULT": None,
                "duration-low": "3600,/",
                "duration-full": "3600,/",
                }

    VarNames = { "DEFAULT": "value",
                "duration-low": "seconds",
                "duration-full": "seconds",
                }


    Extra = { "DEFAULT": "",
              "capacity": "--upper-limit 100 --lower-limit 0 --rigid",
              "charge": "--upper-limit 100 --lower-limit 0 --rigid",
              "duration-low": "--lower-limit 0",
              "duration-full": "--lower-limit 0",
              "current": "HRULE:0$COLOR_LINE_SINGLE_SUB$",
              "power": "HRULE:0$COLOR_LINE_SINGLE_SUB$",
              }

    for g in sorter( "battery-0/*.rrd" ):
        name = re.search( "^battery-0/(.*).rrd", g ).group(1)
        gt, plot, fullname = maketypesplot( "Battery", name, g,
                                            Title = getit("Battery/"+name, Human),
                                            var_name = getit(name, VarNames),
                                            extra_command = getit(name, Extra),
                                            scale_data=getit(name, Factors) )

        Config["types"][fullname] = gt
        BatteryPlots["subplots"]["plots"].append( plot )

    # Add all Battery plots
    BatteryPlots["type"] = "Battery/charge"
    Config["page"]["plots"].append( BatteryPlots )

    # Cover
    gt, plot, fullname = maketypesplot( "Battery", "charge",
                                        "battery-0/charge.rrd",
                                        Title = "Battery",
                                        extra_command = getit("capacity", Extra),
                                        SkipLegend = True )
    Config["types"]["Battery/charge/cover"] = gt
    Config["cover"].append( "Battery/charge/cover" )


######################################################################################
# CPU states 


######################################################################################
# Storage & RAM

if len( glob.glob( "df-*" ) ) > 0 or os.path.exists( "memory" ) or os.path.exists( "swap" ):
    Plots = { "type": None, "subplots": { "title": "Memory & Storage", "plots": [ ] } }

    ######################################################################################
    # Memory
    if os.path.exists( "memory" ):
        SubPlots = { "subplots": { "title": "Memory details", "plots": [ ] } }

        # Memory overview
        s = StackOrLines( cs, "Memory", "Overview", isStack = True, t = "AREA", extra_command="--lower-limit 0"  )

        memory = sorter( "memory/*.rrd", "^memory/.*-(.*).rrd",
                         ["slab_unrecl", "slab_recl", "used", "buffered", "system"],
                         ["cached", "free"] )

        for g in memory:
            name = re.search( "^memory/.*-(.*).rrd", g ).group(1)
            s.add( name, g )

        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        settype(SubPlots, fullname)
        SubPlots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

        # Memory overview: cover
        s = StackOrLines( cs, "Memory", "Overview", Title = "Memory, B", Unit = "", SkipLegend = True, isStack = True, t = "AREA", extra_command="--lower-limit 0"  )

        for g in memory:
            name = re.search( "^memory/.*-(.*).rrd", g ).group(1)
            s.add( name, g )

        gt, plot, fullname = s.compose()

        Config["types"]["Memory/overview/cover"] = gt
        Config["cover"].append( "Memory/overview/cover" )

        # Swap is added as Memory subplot
        if os.path.exists( "swap" ):
            SubSubPlots = { "subplots": { "title": "Swap details", "plots": [ ] } }

            # Swap overview
            s = StackOrLines( cs, "Swap", "Overview", isStack = True, t = "AREA", extra_command="--lower-limit 0"  )

            swap = sorter( "swap/swap-*.rrd", "^swap/.*-(.*).rrd",
                           ["used"],
                           ["free"] )

            for g in swap:
                name = re.search( "^swap/swap-(.*).rrd", g ).group(1)
                s.add( name, g )

            gt, plot, fullname = s.compose()

            Config["types"][fullname] = gt
            settype(SubSubPlots, fullname)
            SubSubPlots["subplots"]["plots"].append( plot )

            # in and out
            if os.path.exists("swap/swap_io-in.rrd" ) and os.path.exists("swap/swap_io-out.rrd" ):
                s = StackOrLines( cs, "Swap", "inout", Title="In/Out",
                                  minmax=True, extra_command='--lower-limit 0' )
                for var_name in ["in", "out"]:
                    name = var_name
                    g = "swap/swap_io-%s.rrd" % var_name
                    s.add( name, g )

                gt, plot, fullname = s.compose()        

                Config["types"][fullname] = gt
                SubSubPlots["subplots"]["plots"].append( plot )

            # detailed plots
            swap.reverse()
            for g in swap:
                name = re.search( "^swap/swap-(.*).rrd", g ).group(1)
                gt, plot, fullname = maketypesplot( "Swap", name, g )

                Config["types"][fullname] = gt
                SubSubPlots["subplots"]["plots"].append( plot )

        SubPlots["subplots"]["plots"].append( SubSubPlots )

        # detailed plots
        memory.reverse()
        for g in memory:
            name = re.search( "^memory/.*-(.*).rrd", g ).group(1)
            gt, plot, fullname = maketypesplot( "Memory", name, g )

            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        Plots["subplots"]["plots"].append( SubPlots )

    ######################################################################################
    # Storage
    if len( glob.glob( "df-*" ) ) > 0:
        SubPlots = { "subplots": { "title": "Storage details", "plots": [ ] } }

        for gd in sorter( "df-*", "^df-(.*)", ["root"] ):
            part_name = re.search( "^df-(.*)", gd ).group(1)

            # special treatment of tmp filesystem
            # remove it if you don't need it
            if part_name == "tmp":
                skip_free = True
            else:
                skip_free = False

            s = StackOrLines( cs, "Storage", part_name, isStack = True, t = "AREA",
                              extra_command = '--lower-limit 0' )

            for g in sorter( gd + "/*.rrd", "^df-.*/df_complex-(.*).rrd", front = ["used"], end=["free"] ):
                name = re.search( "^df-.*/df_complex-(.*).rrd", g ).group(1)
                if skip_free and name == "free": continue
                s.add( name, g )

            gt, plot, fullname = s.compose()

            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )
            settype(SubPlots, fullname)
            settype(Plots, fullname)


        # disk access rates
        DiskStats = { "time": [ "read", "write" ],
                      "octets": [ "read", "write" ],
                      "ops": [ "read", "write" ],
                      "merged": [ "read", "write" ],
                      "io_time": [ "io_time" ],
        }

        DiskScale = { "DEFAULT": None,
                      "io_time": "10,/", # from ms/s to %: 1ms / 1000ms * 100%
        }

        for gd in sorter( "disk-*" ):
            dname = re.search( "^disk-(.*)", gd ).group(1)
            if re.search( "(.*)p.*", dname ) is not None:
                continue # skip partitions

            for g in sorter( gd + "/disk_*.rrd" ):
                name = re.search( "^disk-.*/disk_(.*).rrd", g ).group(1)
                if name == "time":
                    continue # skip this stat, not sure what it means [units]

                if name in DiskStats.keys():
                    col = cs
                    h = gethead( "Disk/" + name )
                    if len( DiskStats[ name ] ) == 1:
                        col = csSingle
                        h = None

                    s = StackOrLines( col, "Disk/" + dname, name, minmax=True,
                                      Title = dname + ': ' + gethuman("Disk/"+name),
                                      Unit = getunit("Disk/" + name),
                                      HeadSize = h,
                                      extra_command = "--lower-limit 0")

                    for li, load_type in  enumerate(DiskStats[ name ]):
                        s.add( load_type, g, Format = getf("Disk/" + name), var_name = load_type, scale_data=getit( load_type, DiskScale ) )

                    gt, plot, fullname = s.compose()

                else:
                    gt, plot, fullname = maketypesplot( "Disk/" + dname, name, g, 
                                                        Title = dname + ': ' + getit("Disk/"+name, Human),
                                                        Format = getf("Disk/" + name),
                                                        Unit = getunit("Disk/" + name) )

                Config["types"][fullname] = gt
                SubPlots["subplots"]["plots"].append( plot )

        Plots["subplots"]["plots"].append( SubPlots )

    # detailed plots for selected processes
    ProcStats = { "rss": [ ],
                  "vm": [ ],
                  "data": [ ],
                  "code": [ ],
                  "stacksize": [ ],
    }

    ProcScale = { "DEFAULT": None,
    }
    for gp in sorter("processes-*"):
        procname = re.search("^processes-(.*)", gp).group(1)
        myplots = { "subplots": {"plots": [], "title": "Process " + procname } }

        #for g in sorter("processes-" + procname + "/*rrd"):
        for g in [
                "processes-" + procname + "/ps_rss.rrd",
                "processes-" + procname + "/ps_pagefaults.rrd",
                "processes-" + procname + "/ps_vm.rrd",
                "processes-" + procname + "/ps_data.rrd",
                "processes-" + procname + "/ps_stacksize.rrd",
                "processes-" + procname + "/ps_code.rrd",
            ]:
            name = re.search( "^processes-" + procname + "/ps_(.*).rrd", g )
            if (name is None):
                continue
            else:
                name = name.group(1)

            title_start = ""
            if name=="rss": title_start += procname + ': '

            if name in ProcStats.keys():
                f = getf("Processes/" + name)
                u = getunit("Processes/" + name)

                s = StackOrLines( cs, "Processes/"+procname, name,
                                  Title = title_start + gethuman("Processes/"+name),
                                  HeadSize = gethead("Processes/"+name),
                                  extra_command = '--lower-limit 0',
                                  Unit = u,
                                  minmax=True )

                for li, load_type in  enumerate(ProcStats[ name ]):
                    s.add( load_type, g, var_name = load_type, Format=f, scale_data=getit(name,ProcScale)  )

                gt, plot, fullname = s.compose()

            else:
                gt, plot, fullname = maketypesplot( "Processes/" + procname, name, g, 
                                                    Title = title_start + getit("Processes/"+name, Human),
                                                    Format = getf("Processes/" + name),
                                                    Unit = getunit("Processes/" + name),
                                                    scale_data=getit(name,ProcScale) )

            Config["types"][fullname] = gt
            myplots["subplots"]["plots"].append( plot )
        myplots["type"] = "Processes/" + procname + "/rss"
        Plots["subplots"]["plots"].append( myplots )

    Config["page"]["plots"].append( Plots )


######################################################################################
# Network interfaces

class Interface2Human:

    def __init__(self):
        # collect all interface names
        interfaces = []
        for g in glob.glob("interface-*"):
            intname = re.search( "^interface-(.*)", g ).group(1)
            intname = intname.replace("-", "_")
            interfaces.append( intname )

        # how many cellular & wlan ifaces we have?
        ncell = 0
        nwlan = 0
        for i in interfaces:
            if i.find("rmnet") == 0: ncell += 1
            elif i.find("wlan") == 0: nwlan += 1

        # build dictionary
        self.translate = {}
        for i in interfaces:
            if i == "lo": t = "local"
            elif i.find("rmnet") == 0:
                t = "cellular"
                if ncell > 1:
                    if i.find("rmnet_") == 0: t += i[ len("rmnet_"): ]
                    else: t += i[ len("rmnet"): ]

            elif i.find("wlan") == 0:
                if nwlan > 1: t = i
                else: t = "wlan"

            else: t = i

            self.translate[i] = t

    def str(self, i):
        return self.translate[i]


if len( glob.glob("interface-*") ) > 0:

    Plots = { "subplots": { "title": "Network details", "plots": [ ] } }
    int2hum = Interface2Human()

    # overall traffic

    command_def = '-t "Network traffic, bytes" ' + defColors #+ makeheads(7)
    command_line = ""
    command_extra = ""
    files = []
    s = StackOrLinesOld( cs, minmax=False, isStack=True, t="AREA" )
    for li, DT in  enumerate([ ["tx", s, -1], ["rx", s, +1] ]):
        data_type = DT[0]            
        humandt = { "tx": "outgoing", "rx": "incoming" }[data_type]
        first = True

        for g in sorter("interface-*/if_octets.rrd", "^interface-(.*)/if_octets.rrd", [], ["lo"]):
            intname = re.search( "^interface-(.*)/if_octets.rrd", g ).group(1)
            intname = intname.replace("-", "_")

            if intname == "lo": # skip lo
                continue

            files.append(g)
            name = re.search( "^interface-.*/if_(.*).rrd", g ).group(1)
            f = "%5.0lf%s"

            command_def += "DEF:" + intname + data_type + "_data=" + g + ":" + data_type + ":AVERAGE "
            command_def += "CDEF:" + intname + data_type + "=" + intname + data_type + "_data," + str(DT[2]) + ",* "
            command_def += "VDEF:" + intname + data_type + "_total=" + intname + data_type + "_data,TOTAL "

            makeLine = first

            DT[1].add( intname + data_type, "$LINE_WIDTH_PRIMARY$", '"' + int2hum.str(intname) + " " + humandt + '\\l"',
                       "COMMENT:\\u GPRINT:"+ intname + data_type+"_data:AVERAGE:\"Avr " + f +
                       "\" GPRINT:"+ intname + data_type+"_data:LAST:\"Last " + f + "\\r\" " + " ",
                       makeLine=makeLine )

            command_extra += 'GPRINT:' + intname + data_type + '_total:"' + \
                             int2hum.str(intname) + " " + humandt + ' Total %8.0lf%s' + getunit("NetworkTotal/" + name) + '\\r" ' \
                             + "HRULE:0$COLOR_LINE_SINGLE_SUB$ "


            first = False

    command_line = s.str() + command_extra

    fullname = "Network/Overall_traffic"
    gt = { "command": command_def + command_line,
           "files": files }
    plot = { "type": fullname }

    Config["types"][fullname] = gt
    Plots["subplots"]["plots"].append( plot )
    settype(Plots, fullname)

    # interface one by one
    for CNet in sorter("interface-*/if_octets.rrd", "^interface-(.*)/if_octets.rrd", [], ["lo"]):
        intname = re.search( "^interface-(.*)/if_octets.rrd", CNet ).group(1)
        intname = intname.replace("-", "_")

        SubPlots = { "subplots": { "title": "Network details: " + int2hum.str(intname), "plots": [ ] } }

        for g in glob.glob("interface-" + intname + "/if_*.rrd"):
            name = re.search( "^interface-.*/if_(.*).rrd", g ).group(1)
            fullname = "Network/" + intname + "/" + name
            f = getf("Network/" + name)
            u = getunit("Network/" + name)

            if name == "octets": hname = "traffic"
            else: hname = name

            command_def = '-t "' + int2hum.str(intname) + ' ' + hname + ", " + u + '" --lower-limit 0 ' + defColors + makeheads(7)
            command_line = ""
            command_extra = ""
            files = [g]
            s = StackOrLinesOld( cs, minmax=True )
            for li, data_type in  enumerate([ "tx", "rx" ]):
                frm = f + u
                humandt = { "tx": "Outgoing", "rx": "Incoming" }[data_type]

                command_def += "DEF:" + data_type + "=" + g + ":" + data_type + ":AVERAGE "
                command_def += "DEF:" + data_type + "_min=" + g + ":" + data_type + ":MIN "
                command_def += "DEF:" + data_type + "_max=" + g + ":" + data_type + ":MAX "
                command_def += "CDEF:" + data_type + "_max_min_delta=" + data_type + "_max," + data_type + "_min,- "
                command_def += "VDEF:" + data_type + "_total=" + data_type + ",TOTAL "

                s.add( data_type, "$LINE_WIDTH_PRIMARY$", '"' + humandt + '\\l"',
                       "COMMENT:\\u GPRINT:"+data_type+":AVERAGE:\"" + f + "\" GPRINT:"+data_type+"_min:MIN:\"" + f +
                       "\" GPRINT:"+data_type+"_max:MAX:\"" + f + "\" GPRINT:"+data_type+":LAST:\"" + f + "\\r\" " + " " 
                )

                command_extra += 'GPRINT:' + data_type + '_total:"' + humandt + ' Total %8.0lf%s' + getunit("NetworkTotal/" + name) + '\\r" ' 

            command_line = s.str() + command_extra

            gt = { "command": command_def + command_line,
                   "files": files }
            plot = { "type": fullname }

            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        SubPlots["type"] = "Network/" + intname + "/octets"
        Plots["subplots"]["plots"].append( SubPlots )
        settype(Plots, "Network/" + intname + "/octets")

    Config["page"]["plots"].append( Plots )


######################################################################################
if os.path.exists("ofono") or \
   os.path.exists("connman") or \
   len( glob.glob("radio-*") ) > 1 or \
   os.path.exists("statefs_cellular") or \
   os.path.exists("statefs_internet"):

    Plots = { "subplots": { "title": "Radio details", "plots": [ ] } }

    cpath = None
    if os.path.exists("ofono"): cpath = "ofono"
    elif os.path.exists("statefs_cellular"): cpath = "statefs_cellular"

    if cpath is not None:
        s = StackOrLines( cs, "Radio", "Cellular",
                          minmax = True,
                          extra_command = "--upper-limit 100 --lower-limit 0 --rigid" )
        for g in sorter( cpath + "/signal_strength-*.rrd" ):
            name = re.search( "^" + cpath + "/signal_strength-(.*).rrd", g ).group(1)
            s.add( name, g )
        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        Plots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

    cpath = None
    if os.path.exists("connman"): cpath = "connman"
    elif os.path.exists("statefs_internet"): cpath = "statefs_internet"

    if cpath is not None:

        s = StackOrLines( cs, "Radio", "Internet",
                          minmax = True,
                          extra_command = "--upper-limit 100 --lower-limit 0 --rigid" )
        for g in sorter( cpath + "/signal_strength-*.rrd" ):
            name = re.search( "^" + cpath + "/signal_strength-(.*).rrd", g ).group(1)
            s.add( name, g )
        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        Plots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

    if len( glob.glob("radio-*") ) > 1:

        s = StackOrLines( cs, "Radio", "active",
                          minmax = False,
                          isStack = True,                          
                          t = "AREA" )

        for dtype in sorter( "radio-*" ):
            names = sorter( dtype + "/" + "active-*.rrd" )
            simple = ( len(names) == 1 )
            for g in names:
                name = re.search( "^radio-(.*)", dtype ).group(1)
                if not simple:
                    name += "_" +  re.search( "^" + dtype + "/active-(.*).rrd", g).group(1)

                s.add( name, g )
        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        Plots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

    if os.path.exists( "statefs_bluetooth" ):

        s = StackOrLines( cs, "Radio", "Bluetooth",
                          minmax = False,
                          isStack = True,
                          t = "AREA" )
        for g in sorter( "statefs_bluetooth/*.rrd",
                         "^statefs_bluetooth/bluetooth_(.*).rrd",
                         front=["enabled"] ):
            name = re.search( "^statefs_bluetooth/bluetooth_(.*).rrd", g ).group(1)
            s.add( name, g )
        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        Plots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

    Config["page"]["plots"].append( Plots )


######################################################################################
# Processes and Load
if os.path.exists( "processes" ) or os.path.exists( "load" ):
    Plots = { "type": None, "subplots": { "title": "Processes & Load", "plots": [ ] } }

    if os.path.exists( "processes" ):

        SubPlots = { "type": None, "subplots": { "title": "Processes details", "plots": [ ] } }

        # Processes overview
        s = StackOrLines( cs, "Processes", "Overview", isStack = True, t = "AREA", extra_command = '--lower-limit 0' )
        processes = sorter( "processes/ps_state-*.rrd", "^processes/ps_state-(.*).rrd",
                            ["zombies", "paging", "blocked", "running"], ["stopped", "sleeping"] )
        for g in processes:
            name = re.search( "^processes/ps_state-(.*).rrd", g ).group(1)
            s.add( name, g )

        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        settype(SubPlots, fullname)
        SubPlots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)

        # detailed plots
        processes.reverse()
        for g in processes:
            name = re.search( "^processes/.*-(.*).rrd", g ).group(1)
            gt, plot, fullname = maketypesplot( "Processes", name, g )

            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        Plots["subplots"]["plots"].append( SubPlots )

        # fork rate goes directly to Plot
        g = "processes/fork_rate.rrd"
        name = "fork_rate"
        gt, plot, fullname = maketypesplot( "Processes", name, g, "Fork rate, 1/s" )
        Config["types"][fullname] = gt
        Plots["subplots"]["plots"].append( plot )

    ######################################################################################
    # Load
    if os.path.exists( "load" ):
        SubPlots = { "subplots": { "title": "Load details", "plots": [ ] } }

        s = StackOrLines( cs, "Load", "load", Title="Load",
                          minmax=True, extra_command='--lower-limit 0' )

        for li, load_type in  enumerate([ "shortterm", "midterm", "longterm" ]):
            Type = "Load"
            name = load_type
            g = "load/load.rrd"

            s.add( name, g, var_name=name )

        gt, plot, fullname = s.compose()

        Config["types"][fullname] = gt
        settype(SubPlots, fullname)
        SubPlots["subplots"]["plots"].append( plot )
        settype(Plots, fullname)
        #### Other Load related graphs

        g = "contextswitch/contextswitch.rrd"
        if os.path.exists(g):
            name = "context_switch"
            gt, plot, fullname = maketypesplot( "Context", name, g, "Context switch", extra_command='--lower-limit 0' )
            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        g = "entropy/entropy.rrd"
        if os.path.exists(g):
            name = "entropy"
            gt, plot, fullname = maketypesplot( "Entropy", name, g, "Entropy", extra_command='--lower-limit 0' )
            Config["types"][fullname] = gt
            SubPlots["subplots"]["plots"].append( plot )

        Plots["subplots"]["plots"].append( SubPlots )


    # detailed plots for selected processes
    ProcStats = { "cputime": [ "user", "syst" ],
                  "pagefaults": [ "minflt", "majflt" ],
                  "count": [ "processes", "threads" ],
                  "disk_octets": [ "read", "write" ],
                  "disk_ops": [ "read", "write" ],

    }

    ProcScale = { "DEFAULT": None,
                  "cputime": "10000,/", # from micros/s to %: 1us / 1000000us * 100%
    }
    for gp in sorter("processes-*"):
        procname = re.search("^processes-(.*)", gp).group(1)
        myplots = { "subplots": {"plots": [], "title": "Process " + procname } }

        for g in sorter("processes-" + procname + "/*rrd"):
            name = re.search( "^processes-" + procname + "/ps_(.*).rrd", g )
            if name is None:
                continue
            else:
                name = name.group(1)

            title_start = ""
            if name=="cputime": title_start += procname + ': '

            if name in ProcStats.keys():
                f = getf("Processes/" + name)
                u = getunit("Processes/" + name)

                s = StackOrLines( cs, "Processes/"+procname, name,
                                  Title = title_start + gethuman("Processes/"+name),
                                  HeadSize = gethead("Processes/"+name),
                                  extra_command = '--lower-limit 0 --upper-limit 1',
                                  Unit = u,
                                  minmax=True )

                for li, load_type in  enumerate(ProcStats[ name ]):
                    s.add( load_type, g, var_name = load_type, Format=f, scale_data=getit(name,ProcScale)  )

                gt, plot, fullname = s.compose()

            else:
                gt, plot, fullname = maketypesplot( "Processes/" + procname, name, g, 
                                                    Title = title_start + getit("Processes/"+name, Human),
                                                    Format = getf("Processes/" + name),
                                                    Unit = getunit("Processes/" + name),
                                                    scale_data=getit(name,ProcScale) )

            Config["types"][fullname] = gt
            myplots["subplots"]["plots"].append( plot )
        myplots["type"] = "Processes/" + procname + "/cputime"
        Plots["subplots"]["plots"].append( myplots )

    Config["page"]["plots"].append( Plots )


###################################################
# Misc

Plots = { }


###################################################
# uptime 
g = "uptime/uptime.rrd"
if os.path.exists(g):    
    name = "uptime"
    gt, plot, fullname = maketypesplot( "Uptime", "uptime", g, Title="Uptime", scale_data="86400,/" )

    Config["types"][fullname] = gt
    Plots["type"] = fullname

    # Add all Misc plots: depends on uptime
    Config["page"]["plots"].append( Plots )



# Print resulting JSON configuration
print(json.dumps(Config, indent=3))
