"""

This file retrieves stock information for all symbols generated by
the gen_symbols file. It will do an http request to gather information
then store it into the mongo database

Meant to be executed once a day (at market close)

"""
import sqlite3
import urllib2
import re
import time

from Stock import Stock
from threading import Thread
from Queue import Queue

import db_operations

NUMB_WORKERS = 50
FILES = ["Finance.csv","Technology.csv","Favourites.csv"]
PREFIX = "data/"
BASE_URL = "http://finance.yahoo.com/q?s="

#create the database
db_operations.create_db()

stock_queue = Queue()

# worker function for thread. Calls get_url
# then calls process_response
def fetch_stock():
    while True:
        obj = q.get()
        symbol = obj[0]
        name = obj[1]
        sect = obj[2]

        content = urllib2.urlopen(BASE_URL + symbol).read()
        stock = Stock(content, symbol, name)

        if stock:
            stock_queue.put((stock, sect))

        q.task_done()


def add_stock():

    db = sqlite3.connect('db/stock_finder')
    count = 0

    while True:

        while stock_queue.qsize() > 0:
            obj = stock_queue.get()
            db_operations.insert_row(obj[0], obj[1], db)
            count = 0

        if count == 10:
            break;
        else:
            time.sleep(1)
            print "Queue empty sleeping ..."
            count += 1;

    db.close()

# Initialize thread pool
q = Queue(NUMB_WORKERS)
for i in range(NUMB_WORKERS):
    t = Thread(target=fetch_stock)
    t.daemon = True
    t.start()


w = Thread(target=add_stock)
w.start();

# Read every file
for file_name in FILES:

    f = open(PREFIX + file_name, 'r')
    sector = re.match(r"(\w*)", file_name).group(1)

    # Add sector record
    db_operations.insert_sectors(sector)

    # retrieve stock info for each stock
    for line in f:

        tokens = line.split(";")

        if len(tokens) == 2:
            q.put((tokens[0], tokens[1], sector))


q.join()

