# GitHub Source: https://github.com/hardikvasa/google-images-download/blob/master/google-images-download.py
'''
    Selin's Notes:
    This script initially would take in a list of keywords, creates a folder for each keyword searched,
    and populates that folder with the first 100 images which came from the search results.
    My modifications focused on taking data directly from the products.csv file containing products
    from the Instacart database and limiting it to downloading the first 100 images (only one of each)
    this time in order to make sure that a large amount of computer memory wasn't taken up with these
    high-resolution photos. Photos can be viewed by switching the database to view the small, 100-item
    database.

'''

# coding: utf-8

# In[ ]:

#Searching and Downloading Google Images/Image Links

#Import Libraries

#coding: UTF-8

import time       #Importing the time library to check the time of code execution
import sys    #Importing the System Library
import os
import urllib2


import csv
search_keyword = []
file = open('products.csv', "r")
reader = csv.reader(file)
for line in reader:
    product_name = line[1]
    if len(search_keyword) < 100 and 'product_name' not in product_name:
        search_keyword.append(product_name)

#print all_products


#This list is used to search keywords. You can edit this list to search for google images of your choice. You can simply add and remove elements of the list.
#search_keyword = ['Chocolate Sandwich Cookies', 'All-Seasons Salt', 'Robust Golden Unsweetened Oolong Tea']

#This list is used to further add suffix to your search term. Each element of the list will help you download 100 images. First element is blank which denotes that no suffix is added to the search keyword of the above list. You can edit the list by adding/deleting elements from it.So if the first element of the search_keyword is 'Australia' and the second element of keywords is 'high resolution', then it will search for 'Australia High Resolution'
keywords = [' high resolution']

#Downloading entire Web Document (Raw Page Content)
def download_page(url):
    version = (3,0)
    cur_version = sys.version_info
    if cur_version >= version:     #If the Current Version of Python is 3.0 or above
        import urllib.request    #urllib library for Extracting web pages
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36"
            req = urllib.request.Request(url, headers = headers)
            resp = urllib.request.urlopen(req)
            respData = str(resp.read())
            return respData
        except Exception as e:
            print(str(e))
    else:                        #If the Current Version of Python is 2.x
        import urllib2
        try:
            headers = {}
            headers['User-Agent'] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"
            req = urllib2.Request(url, headers = headers)
            response = urllib2.urlopen(req)
            page = response.read()
            return page
        except:
            return"Page Not found"


#Finding 'Next Image' from the given raw page
def _images_get_next_item(s):
    start_line = s.find('rg_di')
    if start_line == -1:    #If no links are found then give an error!
        end_quote = 0
        link = "no_links"
        return link, end_quote
    else:
        start_line = s.find('"class="rg_meta"')
        start_content = s.find('"ou"',start_line+1)
        end_content = s.find(',"ow"',start_content+1)
        content_raw = str(s[start_content+6:end_content-1])
        return content_raw, end_content


#Getting all links with the help of '_images_get_next_image'
def _images_get_all_items(page):
    item, end_content = _images_get_next_item(page)
    if item == "no_links":
        return None
    else:
        time.sleep(0.1)        #Timer could be used to slow down the request for image downloads
    return item

def generateURLs(search_keyword):
    urls = []
    for i in xrange(len(search_keyword)):
        search_keywords = search_keyword[i]
        search = search_keywords.replace(' ','%20')

        pure_keyword = keywords[j].replace(' ','%20')
        url = 'https://www.google.com/search?q=' + search + pure_keyword + '&espv=2&biw=1366&bih=667&site=webhp&source=lnms&tbm=isch&sa=X&ei=XosDVaCXD8TasATItgE&ved=0CAcQ_AUoAg'
        urls.append(url)
    return urls



############## Main Program ############
t0 = time.time()   #start the timer

#Download Image Links
i = 0
j = 0

 #make a search keyword  directory
try:
    os.makedirs("grocery_items")
except OSError, e:
    if e.errno != 17:
        raise   
    # time.sleep might help here

urls = generateURLs(search_keyword)
#print urls

k = 0
while i < len(urls):
    items = []
    iteration = "Item no.: " + str(i+1) + " -->" + " Item name = " + str(search_keyword[i])
    print (iteration)
    print ("Evaluating...")
    url = urls[i]

    raw_html =  (download_page(url))
    time.sleep(0.1)
    item = (_images_get_all_items(raw_html))

    t1 = time.time()    #stop the timer
    total_time = t1-t0   #Calculating the total time required to crawl, find and download all the links of 60,000 images
    print("Total time taken: "+str(total_time)+" Seconds")
    print ("Starting Download...")

    ## To save imges to the same directory
    # IN this saving process we are just skipping the URL if there is any error

    errorCount = 0
    from urllib2 import Request,urlopen
    from urllib2 import URLError, HTTPError

    try:
        req = Request(item, headers={"User-Agent": "Mozilla/5.0 (X11; Linux i686) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.27 Safari/537.17"})
        response = urlopen(req,None,15)
        search_keyword_str = search_keyword[i].replace(" ", "_")
        output_file = open("grocery_items"+"/"+search_keyword_str+".jpg",'wb')
        
        data = response.read()
        output_file.write(data)
        response.close();

        print("completed ====> "+str(k+1))

        k = k + 1;

    except IOError:   #If there is any IOError

        errorCount += 1
        print("IOError on image " + str(k+1))
        k = k + 1;

    except HTTPError as e:  #If there is any HTTPError

        errorCount += 1
        print("HTTPError" + str(k))
        k = k + 1;
    except URLError as e:

        errorCount += 1
        print("URLError " + str(k))
        k = k + 1;

    i = i + 1

print("\n")
print("Everything downloaded!")
print("\n"+str(errorCount)+" ----> total Errors")

#----End of the main program ----#


# In[ ]: