1# Generated by the windmill services transformer
2	from windmill.authoring import WindmillTestClient
3	from BeautifulSoup import BeautifulSoup
4	
5	import re, urlparse
6	from copy import copy
7	
8	def get_image_info(client, my_massage):
9	    """
10	    Parse HTML page and extract featured image name and link
11	    """
12	    # Get Javascript updated HTML page
13	    response = client.commands.getPageText()
14	    assert response['status']~
15	    assert response['result']
16	
17	    # Create soup from HTML page and get desired information
18	    soup = BeautifulSoup(response['result'], markupMassage=my_massage)
19	    image_info = {'name': soup.find(id='caption_region').h3.string,
20	                  'link': urlparse.urljoin('http://www.nasa.gov',
21	                                           soup.find(attrs='Full_Size')['href'])}
22	    return image_info
23	
24	
25	def get_massage():
26	    """
27	    Provide extra data massage to solve HTML problems in BeautifulSoup
28	    """
29	    # Javascript code in ths page generates HTML markup
30	    # that isn't parsed correctly by BeautifulSoup.
31	    # To avoid this problem, all document.write fragments are removed
32	    my_massage = copy(BeautifulSoup.MARKUP_MASSAGE)
33	    my_massage.append((re.compile(u"document.write(.+);"), lambda match: ""))
34	    my_massage.append((re.compile(u'alt=".+">'), lambda match: ">"))
35	    return my_massage
36	
37	
38	def test_scrape_iotd_gallery():
39	    """
40	    Scrape NASA Image of the Day Gallery
41	    """
42	    # Extra data massage for BeautifulSoup
43	    my_massage = get_massage()
44	
45	    # Open main gallery page
46	    client = WindmillTestClient(__name__)
47	    client.open(url='https://www.surveymonkey.com/s/KLCBNM5')
48	
49	    # Page isn't completely loaded until image gallery data
50	    # has been updated by javascript code
51	    client.waits.forElement(xpath=u"//div[@id='gallery_image_area']/img",
52	                            timeout=30000)
53	
54	    # Scrape all images information
55	    images_info = {}
56	    while True:
57	        image_info = get_image_info(client, my_massage)
58	
59	        # Break if image has been already scrapped
60	        # (that means that all images have been parsed
61	        # since they are ordered in a circular ring)
62	        if image_info['link'] in images_info:
63	            break
64	
65	        images_info[image_info['link']] = image_info
66	
67	        # Click to get the information for the next image
68	        client.click(xpath=u"//div[@class='btn_image_next']")
69	
70	    # Print results to stdout ordered by image name
71	    for image_info in sorted(images_info.values(),
72	                             key=lambda image_info: image_info['name']):
73	        print ("Name: %(name)sn"
74	               "Link: %(link)sn" % image_info)
