'''
indexer for suche Search Engine
copyright (c) 2014 Suche
'''
from indexer.models import SucheURL,Link,Word,Result
from indexer.htmlparser import HTMLParser
from crawler.models import *

class Indexer:

    def set_raw(self,raw):
        '''
        sets the raw data row to which the indexer is to operate.
        '''
        self.raw = raw

    def operate(self):
        '''
        operate on the data. This will extract links, etc
        First of all, we have to revert back the changes of previous website data for eg reduce the number 
        of links, reduce word counts, etc. Then, we have to apply the changes due to new data. Finally, move 
        the new data to old data and set oeprated to true
        '''

        parser = HTMLParser(self.raw.new_data, self.raw.url)

        parser.parse()

        #TODO: parser will detect error in the HTML document. If the HTML
        # is ill formed or less than certain length, ignore it.

        #now extract information from the parse and update database
        urls = []

        # we filter out the valid URLs from the list generated by the html parser
        # for example, we want to save only the URLs from certain domains
        # for testing purpose

        filteredurls = []
        for url,text in parser.get_links():
            if len(url) < 200:
                if SucheURL.isvalid(url):
                    filteredurls.append((url,text))

        for url,text in filteredurls:
            urls.append(text+"-"+url)
            newurl,created = SucheURL.objects.get_or_create(url = url)
            if created:
                #create the crawl record for the newly generated url
                newdata = CrawlData(url = newurl)
                newdata.save()

        #now go on to create the links
        thisurl = SucheURL.objects.get(url = self.raw.url)

        # actually, we retrieve the old links, and compare them with the new links
        # to leave the older links and delete or add new links only as needed
        # but for now, we empty the table and add the links again
        Link.objects.filter(fromurl = thisurl).delete()

        for url,text in filteredurls:
            desturl = SucheURL.objects.get(url = url)

            if desturl != thisurl:
                link = Link(fromurl = thisurl, tourl = desturl, text = text)
                link.save()

        #if any new words are found in the document, add them to the words table
        for word in parser.get_word_dict().keys():
            wordrow, created = Word.objects.get_or_create(word = word)

        # now get the rank of the current URL for the words and place it in index if it has higher rank than
        # the current URLs or if the current URL list is incomplete
        for word in parser.get_word_dict().keys():
            wordobj = Word.objects.get(word = word)
            wordobj.wordcount += 1

            #delete any previous result
            Result.objects.filter(word = wordobj, url = thisurl).delete()

            #create a result for that word
            result = Result(word = wordobj, url = thisurl)

            # get the word rank for this page
            result.wordrank = parser.word_rank(word)

            # count the number of times this word appear in the title of the website
            result.titlewordcount = parser.title_word_count(word)

            #count the number of times this word has been linked with text containing this word
            result.linkswordcount = Link.objects.filter(tourl = thisurl, text__icontains = word).count()

            # we ignore the user rank for now

            #calculate the url point as
            # urlpoint = wordrank + pagerank of website + max(6 , No of words in title) + max(6, 2 * no of words in URL) + max( 20, linkswordcount / 10)
            result.urlpoint = result.wordrank + min(6, result.titlewordcount) + min( 6, 2 * thisurl.url.count(word)) + min( 20, result.linkswordcount / 10)

            # if the url doesnot contain any title, reduce the URL point as it can be ajax script or comment section
            if len(parser.get_title()) < 5:
                result.urlpoint -= 6

            #if there are not enough results for the word or if the current
            # URL is better than the previous URLs, save the result
            if Result.objects.filter(word = wordobj).count() < 100:
                # if less result, simple add this result
                result.save()
            else:
                # if the current URL is better than the worst result, delete the worst result and add this result
                worstresult = Result.objects.filter(word = wordobj).order_by('urlpoint')[0]
                if result.urlpoint > worstresult.urlpoint:
                    worstresult.delete()
                    result.save()

        # set the data as operated
        try:
            self.raw.operated = True
            self.raw.old_data = self.raw.new_data
            self.raw.new_data = ''
            self.raw.save()
        except:
            pass #do nothing in case of error
        self.raw.url.title = parser.get_title()
        self.raw.url.body = parser.get_content()

        self.raw.url.save()
        return  parser.get_info()

