old_contents: import clamd
  import requests
  import itertools
  import lxml.html
  from urlparse import urlparse
  from eventlet.green import urllib2
  from eventlet.timeout import Timeout
  
  MAX_FILE_SIZE = 104857600 #100MB
  MAX_DOWNLOAD_TIME = 60
  
  clamd_socket = clamd.ClamdUnixSocket()
  
  #download with size and time limits
  def download_link(url):
  	r = requests.get(url, prefetch=False, timeout=10.0)
  	if r.headers['content-length'] == None or int(r.headers['content-length']) < MAX_FILE_SIZE:
  		response = bytearray()
  		with Timeout(MAX_DOWNLOAD_TIME, False):
  			response = urllib2.urlopen(url).read()
  		if (len(r)):
  			return response
  	else:
  		return 0
  
  def crawl_html(url):
      """
      Naive html crawler.
  
      To be truly effective we need to determine what files might actually effect
      the rendering of the page and what files a user might be tricked into
      downloading.
  
      * JavaScript in <script> tags
      * Executable files
  
      Things we might not be interested in:
      * images
      * links to other domains (example.com and example.net differ, but www.example.com and example.com do not) that are pure HTML
      * CSS
  
      Also not we need to be careful with how much we download!
  
      We don't want to sit and download an 8GB DVD or even an unlimited byte radio
      stream!
  
      We also want to avoid downloading the same page twice (once to get the links
      then again to scan the page itself).
      """
      response = requests.get(url, timeout=10.0)
      html = lxml.html.fromstring(
          response.content,
          base_url=response.url
      )
      html.make_links_absolute(resolve_base_href=True)
      links = {link for element, attribute, link, pos in html.iterlinks()}
      domains = {urlparse(resp.url).netloc for resp in itertools.chain(response.history, (response,))}
  	#print domains
      return map(clamd_scan, links)
  
  
  def clamd_scan(url):
  	content = download_link(url)
  	if(content == 0):
  		print 'File is too large or cannot be downloaded'
  	else:
  		return clamd_socket.scan_stream(content)
  
  if __name__ == "__main__":
      print clamd_scan("https://secure.eicar.org/eicar.com")
      print crawl_html("https://secure.eicar.org/")
  
new_contents: import clamd
  import requests
  import itertools
  import lxml.html
  from urlparse import urlparse
  
  clamd_socket = clamd.ClamdUnixSocket()
  
  
  def crawl_html(url):
      """
      Naive html crawler.
  
      To be truly effective we need to determine what files might actually effect
      the rendering of the page and what files a user might be tricked into
      downloading.
  
      * JavaScript in <script> tags
      * Executable files
  
      Things we might not be interested in:
      * images
      * links to other domains (example.com and example.net differ, but www.example.com and example.com do not) that are pure HTML
      * CSS
  
      Also not we need to be careful with how much we download!
  
      We don't want to sit and download an 8GB DVD or even an unlimited byte radio
      stream!
  
      We also want to avoid downloading the same page twice (once to get the links
      then again to scan the page itself).
      """
      response = requests.get(url, timeout=10.0)
      html = lxml.html.fromstring(
          response.content,
          base_url=response.url
      )
      html.make_links_absolute(resolve_base_href=True)
      links = {link for element, attribute, link, pos in html.iterlinks()}
      domains = {urlparse(resp.url).netloc for resp in itertools.chain(response.history, (response,))}
      return map(clamd_scan, links)
  
  
  def clamd_scan(url):
      return clamd_socket.scan_stream(requests.get(url, timeout=10.0).content)
  
  if __name__ == "__main__":
      print clamd_scan("https://secure.eicar.org/eicar.com")
      print crawl_html("https://secure.eicar.org/")
  
subject: Revert "clamav now downloads links with size and time limits."