#!/usr/bin/env python3
# Import from SDV canonical representation to Neo4j.
#
# It might make more sense to write this in Javascript, but it's much quicker
# for me to write Python right now. Also, the current implementation takes a
# simplistic approach that would not scale to huge amounts of data. There are
# hopefully more efficient ways to batch insert data into Neo4j. (One way is
# to generate .csv files and then use neo4j-import; this doc:
# <http://neo4j.com/docs/stable/cypherdoc-loading-data.html> is also relevant).

# Requires 'py2neo' Python bindings for Neo4j.
# Requires 'pyld' JSON-LD processor.
# Requires 'pyyaml' YAML library.
#
# If you want to empty your Neo4j database, you can use this Cypher command:
#
#   MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;


import py2neo
import pyld.jsonld
import yaml

import argparse
import datetime
import os
import warnings

DEFAULT_ENDPOINT =  'http://neo4j:insecure@localhost:7474/db/data'

def argument_parser():
    parser = argparse.ArgumentParser(
        description="Import s-d-v data into Neo4j graph database")
    parser.add_argument('files', metavar='FILE', nargs='+',
                        help="JSON-LD data to import (YAML allowed)")
    parser.add_argument('--endpoint', '-e', metavar='URL',
                        default=DEFAULT_ENDPOINT,
                        help="Neo4J endpoint (default: %s)" % DEFAULT_ENDPOINT)
    parser.add_argument('--wipe', '-w', action='store_true',
                        help="Wipe the neo4j database before import (default: no)")
    return parser


def compact_uri(uri, context):
    # A rather awful way of getting the compact URI for a full URI.
    # The Python JSON-LD module really doesn't make this very easy right now.
    result = pyld.jsonld.compact({uri: 0}, ctx=context)
    del result['@context']
    return list(result.keys())[0]


def find_node_by_uri(graph, uri, uri_property='uri'):
    # This is probably not very efficient. An alternative way of doing this
    # is to store URI as a label, BUT... the browser looks a lot less usable
    # that way, because the 'Labels' list contains millions of entries

    # This can't be done, which seems like a massive flaw in Py2neo.
    # Maybe neo4j-rest-client is better?
    #query_result = graph.find(
    #    '*', property_key=uri_property, property_value=uri, limit=2)

    query_result = graph.cypher.execute(
        'MATCH (var {`%s`: "%s"}) RETURN var LIMIT 2' % (uri_property, uri))

    nodes = list(query_result)
    if len(nodes) > 1:
        warnings.warn("Multiple nodes exist already for %s" % uri)
    if len(nodes) > 0:
        return nodes[0][0]
    else:
        return None


def find_or_create_node_by_uri(graph, uri, uri_property='uri'):
    node = find_node_by_uri(graph, uri, uri_property=uri_property)
    if node is None:
        node = py2neo.Node()
        node.properties[uri_property] = uri
        graph.create(node)
    return node


def data_type(values):
    '''Returns 'property' or 'relationship' a given set of values.'''
    kind = None
    for value_dict in values:
        if '@value' in value_dict:
            if kind is None:
                kind = 'property'
            elif kind == 'relationship':
                raise ValueError("Data mixes up @value and @id")
        elif '@id' in value_dict:
            if kind is None:
                kind = 'relationship'
            elif kind == 'property':
                raise ValueError("Data mixes up @value and @id")
        else:
            raise ValueError("Data specifies neither @value nor @id")
    return kind


def createRelationships(node, relationships, context, graph, wipeDatabase):
    for key, value in relationships.items():
        # We always use the short version of a key name to name
        # relationships. This means that the data requires the context
        # to make sense. But it is totally unreadable to have
        # properties named <http://example.com/property-name>, and we
        # avoid that this way.

        key = compact_uri(key, context)

        if not isinstance(value, list):
            value = [value]
        # Value must be a list now.
        if len(value) == 0:
            value = [None]
        for target in value:
            if isinstance(target, dict):
                if '@id' not in target:
                    warnings.warn("Invalid value for %s %s:\n\t%s\nMaybe you "
                                "forgot to use @id to refer to another "
                                "resource?" % (resource['@id'], key,
                                                target))
                    continue
                target_uri = target['@id']
            else:
                target_uri = target

            target = find_or_create_node_by_uri(graph, target_uri)
            if wipeDatabase:
                existing_rels = []
            else:
                existing_rels = graph.match(
                    start_node=node,
                    rel_type=key,
                    end_node=target,
                    limit=2)
            print("Existing relations for %s->%s: %d"%(node, target, len(list(existing_rels))))
            if len(list(existing_rels)) == 0:
                relationship = py2neo.Relationship(node, key, target)
                relationship.properties['name'] = compact_uri(key, context)
                graph.create_unique(relationship)

def main():
    args = argument_parser().parse_args()

    graph = py2neo.Graph(args.endpoint)

    # Wipe everything
    if args.wipe:
        print ("Wiping database...")
        query_result = graph.cypher.execute('MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE n,r;')

    # The 'Resource' label is applied to all nodes in our graph. Neo4j indexes
    # and constraints must be attached to a label. Every node we create has a
    # URI, so to create a uniqueness constraint and an index for the 'uri'
    # and 'compact_uri' properties, we need to explicitly declare every node
    # as a Resource, too.
    all_labels = set('Resource')

    with open(os.path.join(os.path.dirname(__file__), '../data/context.jsonld.yaml')) as f:
        default_context = yaml.load(f)

    for filename in args.files:
        with open(filename) as f:
            data = yaml.load(f)

        # Avoid fetching a file that we have available locally. This is a bit
        # hacky though.
        default_context_url = 'http://localhost:8000/context.jsonld'
        if default_context_url in data['@context']:
            index = data['@context'].index(default_context_url)
            data['@context'][index] = default_context
        context = data['@context']

        data = pyld.jsonld.expand(data)
        data = pyld.jsonld.flatten(data)

        nodes = []
        resourceNo = 0
        startTime = datetime.datetime.now()
        for resource in data:
            resourceNo+=1
            currentTime = datetime.datetime.now()
            delta = currentTime - startTime
            delta.total_seconds()
            rate = (float(delta.total_seconds()) / resourceNo)
            print ("Processing resource %d of %d (%d seconds elapsed; average %.1fs/resource; estimated finish in %ds)"% (resourceNo,len(data),delta.total_seconds(),rate,len(data)*rate-delta.total_seconds()))
            node = find_or_create_node_by_uri(graph, resource['@id'])

            # The 'compact_uri' property is just for convenience when exploring
            # the graph directly within Neo4j.
            node.properties['compact_uri'] = compact_uri(
                resource['@id'], context)

            # Separate out labels, properties and values from the JSON-LD data.
            labels = []
            properties = {}
            relationships = {}
            for key, values in resource.items():
                if key == '@id':
                    continue

                # Empty arrays cause problems, so ignore any key with that as a value.
                if values == []:
                    continue

                # JSON-LD seems to turn everything into lists, other than @id,
                # so we assume value is always a list here.
                assert isinstance(values, list)

                if key == '@type':
                    labels.extend(values)
                elif not key.startswith('@'):
                    if data_type(values) == 'relationship':
                        relationships[key] = [d['@id'] for d in values]
                    else:
                        if key == 'https://schema.org/name':
                            key = 'name'
                        if len(values) == 1:
                            node.properties[key] = values[0]['@value']
                        else:
                            node.properties[key] = [d['@value'] for d in values]

            # Labels are displayed in the Neo4j browser. Showing full URIs
            # there is totally unwieldy, and as long as you have the JSON-LD
            # context to hand there's no ambiguity about the compact URIs, so
            # we just use those.
            node.labels.add('Resource')
            for label in labels:
                label_id = compact_uri(label, context)
                node.labels.add(label_id)
                all_labels.add(label_id)

            createRelationships(node, relationships, context, graph, args.wipe)

            node.push()

    # The uniqueness constraint created here also creates an index on
    # Resource:uri and Resource.compact_uri. This will only get used if the
    # Resource label is specified explicitly. If you match this, for example:
    #
    #   (node:Package {compact_uri: 'id:foo'})
    #
    # ...then Neo4j won't use an index. We would need to create indices for
    # everything in `all_labels` for this to work. I don't yet know what the
    # tradeoffs are.
    existing_constraints = graph.schema.get_uniqueness_constraints('Resource')
    for property_key in ['uri', 'compact_uri']:
        if property_key not in existing_constraints:
            graph.schema.create_uniqueness_constraint('Resource', property_key)


main()
