# CubETL
# Copyright (c) 2013-2019
# See AUTHORS and LICENSE files for more information.

# This file is part of the CubETL library.

# This file is meant to be included from other CubETL configuration files,
# it provides data types, schema and other objects.


from cubetl.olap import Dimension, Key, Attribute, HierarchyDimension, Hierarchy, Fact, Measure, DimensionAttribute
from cubetl.text import RegExp
from cubetl import table


'''
CubETL HTTP types library.

This file provides OLAP dimensions for typical HTTP-related datasets,
eg dimensions for "HTTP Method", "HTTP Response Code", "User agent",
"MIME type" and others.
'''


def cubetl_config(ctx):

    ctx.include('${ ctx.library_path }/datetime.py')
    ctx.include('${ ctx.library_path }/geo.py')
    ctx.include('${ ctx.library_path }/net.py')

    ctx.add('cubetl.http.request.client_address',
            Dimension(name='client_address', label='Address'))

    ctx.add('cubetl.http.request.client_domain',
            Dimension(name='client_domain', label='Domain'))

    ctx.add('cubetl.http.request.username',
            Dimension(name='username', label='Username'))

    ctx.add('cubetl.http.request.method',
            Dimension(name='http_method', label='HTTP Method'))

    ctx.add('cubetl.http.request.path', HierarchyDimension(
        name='request_path',
        label='Path',
        hierarchies=[Hierarchy(name='path14', label='Path', levels=["path1", "path2", "path3", "path4"])],
        attributes=[DimensionAttribute(dimension=Dimension(name='path1', label='Path 1')),
                    DimensionAttribute(dimension=Dimension(name='path2', label='Path 2')),
                    DimensionAttribute(dimension=Dimension(name='path3', label='Path 3')),
                    DimensionAttribute(dimension=Dimension(name='path4', label='Path 4'))]))

    ctx.add('cubetl.http.protocol',
            Dimension(name='protocol', label='Protocol'))


    ctx.add('cubetl.http.response.status.code',
            Dimension(name='status_code', label='Status', attributes=[
                Attribute(name='status_code', type='Integer', label='Status Code'),
                Attribute(name='status_description', type='String', label='Status Description')]))

    ctx.add('cubetl.http.response.status.type',
            Dimension(name='status_type', label='Status Type', attributes=[
                Attribute(name='status_type_label', type='String', label='Status Type'),
                Attribute(name='status_type_code', type='String', label='Status Type Code')]))

    ctx.add('cubetl.http.response.status', HierarchyDimension(
        name='response_status',
        label='Status',
        hierarchies=[Hierarchy(name='http_status', label='Status', levels=['status_type', 'status_code'])],
        attributes=[DimensionAttribute(ctx.get('cubetl.http.response.status.type')),
                    DimensionAttribute(ctx.get('cubetl.http.response.status.code'))]))

    ctx.add('cubetl.http.request.referer_path',
            Dimension(name='referer_path', label='Referer Path'))

    '''

    !!python/object:cubetl.olap.HierarchyDimension
    id: cubetl.http.referer
    name: referer
    label: Referer
    hierarchies:
    - name: referer
      label: Referer
      levels: referer_domain, referer_path
    levels:
    - !ref cubetl.http.referer.domain
    - !ref cubetl.http.referer.path
    '''

    ctx.add('cubetl.http.user_agent', HierarchyDimension(
        name='user_agent',
        label='User Agent',
        attributes=[DimensionAttribute(Dimension(name='user_agent_family', label='User Agent')),
                    DimensionAttribute(Dimension(name='user_agent_version', label='Version'))
                    ]))

    ctx.add('cubetl.os.operating_system', HierarchyDimension(
        name='operating_system',
        label='Operating System',
        attributes=[DimensionAttribute(Dimension(name='operating_system_family', label='OS')),
                    DimensionAttribute(Dimension(name='operating_system_version', label='Version'))
                    ]))

    ctx.add('cubetl.http.mimetype', HierarchyDimension(
        name='mimetype',
        label='MIME Type',
        attributes=[DimensionAttribute(Dimension(name='mimetype_type', label='Type')),
                    DimensionAttribute(Dimension(name='mimetype_subtype', label='Subtype'))
                    ]))

    ctx.add('cubetl.os.device',
            Dimension(name='device', label='Device'))

    ctx.add('cubetl.http.request.is_bot',
            Dimension(name='is_bot', label='Is Bot', attributes=[
                Attribute(name='is_bot', type='Boolean', label='Is Bot')]))

    ctx.add('cubetl.http.request.is_mobile',
            Dimension(name='is_mobile', label='Is Mobile', attributes=[
                Attribute(name='is_mobile', type='Boolean', label='Is Mobile')]))

    ctx.add('cubetl.http.request.is_pc',
            Dimension(name='is_pc', label='Is PC', attributes=[
                Attribute(name='is_pc', type='Boolean', label='Is PC')]))

    ctx.add('cubetl.http.request.is_tablet',
            Dimension(name='is_tablet', label='Is Tablet', attributes=[
                Attribute(name='is_tablet', type='Boolean', label='Is Tablet')]))

    ctx.add('cubetl.http.request.file_extension',
            Dimension(name='file_extension', label='File Extension'))

    ctx.add('cubetl.http.request.referer_origin',
            Dimension(name='referer_origin', label='Referer Origin'))

    ctx.add('cubetl.http.response.is_download',
            Dimension(name='is_download', label='Is Download', attributes=[
                Attribute(name='is_download', type='Boolean', label='Is Download')]))

    ctx.add('cubetl.http.request', Fact(
        name='http_request',
        label='HTTP Request',
        #natural_key=
        #notes='',
        attributes=[
            DimensionAttribute(ctx.get('cubetl.datetime.date'), name='request_date', label="Request Date"),
            #ctx.get('cubetl.datetime.date'),
            DimensionAttribute(ctx.get('cubetl.http.protocol')),
            DimensionAttribute(ctx.get('cubetl.http.request.client_address')),
            DimensionAttribute(ctx.get('cubetl.http.request.username')),
            DimensionAttribute(ctx.get('cubetl.http.request.method')),
            DimensionAttribute(ctx.get('cubetl.http.request.path')),
            DimensionAttribute(ctx.get('cubetl.http.request.file_extension')),
            DimensionAttribute(ctx.get('cubetl.http.request.referer_origin')),
            DimensionAttribute(ctx.get('cubetl.http.request.is_bot')),
            DimensionAttribute(ctx.get('cubetl.http.request.is_pc')),
            DimensionAttribute(ctx.get('cubetl.http.request.is_tablet')),
            DimensionAttribute(ctx.get('cubetl.http.request.is_mobile')),
            DimensionAttribute(ctx.get('cubetl.geo.contcountry')),
            DimensionAttribute(ctx.get('cubetl.http.response.status')),
            DimensionAttribute(ctx.get('cubetl.http.mimetype')),
            DimensionAttribute(ctx.get('cubetl.http.response.is_download')),
            DimensionAttribute(ctx.get('cubetl.net.domain'), name="referer_domain", label="Referer Domain"),
            DimensionAttribute(ctx.get('cubetl.http.request.referer_path')),
            #ctx.get('cubetl.http.referer'),  # TODO: try nested hierarchydimensions in "referer" hierarchydimension
            DimensionAttribute(ctx.get('cubetl.http.user_agent')),
            DimensionAttribute(ctx.get('cubetl.os.operating_system')),
            DimensionAttribute(ctx.get('cubetl.os.device')),

            Attribute(name='user_agent_string', type='String', label='User Agent String'),
            Attribute(name='verb', type='String', label='Verb'),
            #Attribute(name='referer', type='String', label='Referer')

            Measure(name='served_bytes', type='Integer', label="Served Bytes"),
        ]))

    '''

    - !ref
    #- !ref cubetl.http.request.client_domain
    #- !ref cubetl.http.request.rlogname
    #- !ref cubetl.http.request.username
    '''


    ctx.add('cubetl.http.parse.apache_combined',
            RegExp(regexp=r'([\d\.]+) (-) (-) \[(.*?)\] "(.*?)" (\d+) (\S+) "(.*?)" "(.*?)"',
                   errors=RegExp.ERRORS_WARN,
                   names='address, rlogname, username, date_string, verb, status_code, served_bytes, referer, user_agent_string'))


    ctx.add('cubetl.http.status.table', table.CSVMemoryTable(
        data='''
            status_code,status_description,status_type
            100,Continue,Informational
            101,Switching Protocols,Informational
            200,OK,Successful
            201,Created,Successful
            202,Accepted,Successful
            203,Non-Authoritative Information,Successful
            204,No Content,Successful
            205,Reset Content,Successful
            206,Partial Content,Successful
            300,Multiple Choices,Redirection
            301,Moved Permanently,Redirection
            302,Found,Redirection
            303,See Other,Redirection
            304,Not Modified,Redirection
            305,Use Proxy,Redirection
            307,Temporary Redirect,Redirection
            400,Bad Request,Client Error
            401,Unauthorized,Client Error
            402,Payment Required,Client Error
            403,Forbidden,Client Error
            404,Not Found,Client Error
            405,Method Not Allowed,Client Error
            406,Not Acceptable,Client Error
            407,Proxy Authentication Required,Client Error
            408,Request Timeout,Client Error
            409,Conflict,Client Error
            410,Gone,Client Error
            411,Length Required,Client Error
            412,Precondition Failed,Client Error
            413,Request Entity Too Large,Client Error
            414,Request-URI Too Long,Client Error
            415,Unsupported Media Type,Client Error
            416,Requested Range Not Satisfiable,Client Error
            417,Expectation Failed,Client Error
            500,Internal Server Error,Server Error
            501,Not Implemented,Server Error
            502,Bad Gateway,Server Error
            503,Service Unavailable,Server Error
            504,Gateway Timeout,Server Error
            505,HTTP Version Not Supported,Server Error
        '''))





