old_contents: import requests
  from urlparse import urlparse, parse_qs
  import lxml.html
  import itertools
  
  from celery import task
  
  from six.moves import map
  from django.conf import settings
  
  
  def dogpile_link_handle(url):
      # parse the click handler from dogpile to get the real URL
      return parse_qs(urlparse(url).query)["du"][0]
  
  
  @task
  def dogpile(keyword):
  
      links = (
          dogpile_multilink(keyword, i) for i in range(
              1, 100 if not settings.DEBUG else 20, 10
          )
      )
  
      return list(itertools.chain.from_iterable(links))
  
  
  @task
  def tchain(iterable):
      return list(itertools.chain.from_iterable(iterable))
  
  
  @task
  def dogpile_multilink(keyword, qsi):
      response = requests.get(
          "http://www.dogpile.co.uk/search/web",
          params={"q": keyword, "qsi": qsi}
      )
  
      html = lxml.html.fromstring(
          response.content,
          base_url=response.url
      )
      html.make_links_absolute(resolve_base_href=True)
  
      return [
          dogpile_link_handle(anchor.get("href")) for anchor in html.cssselect(
              ".webResult .resultDisplayUrl"
          )
      ]
  
  
  search_engines = (dogpile,)
  
  if __name__ == "__main__":
      print dogpile("foo")
  
new_contents: import requests
  from urlparse import urlparse, parse_qs
  import lxml.html
  import itertools
  
  from celery import task, chord, group, chain
  
  from six.moves import map
  from django.conf import settings
  
  
  def dogpile_link_handle(url):
      # parse the click handler from dogpile to get the real URL
      return parse_qs(urlparse(url).query)["du"][0]
  
  
  @task
  def dogpile_multilink(keyword, qsi):
      response = requests.get(
          "http://www.dogpile.co.uk/search/web",
          params={"q": keyword, "qsi": qsi}
      )
  
      html = lxml.html.fromstring(
          response.content,
          base_url=response.url
      )
      html.make_links_absolute(resolve_base_href=True)
  
      return [
          dogpile_link_handle(anchor.get("href")) for anchor in html.cssselect(
              ".webResult .resultDisplayUrl"
          )
      ]
  
  
  @task
  def tchain(iterable):
      return list(itertools.chain.from_iterable(iterable))
  
  
  @task
  def dogpile(keyword):
      callbacks = dogpile.request.callbacks
      dogpile.request.callbacks = []
  
      header = [dogpile_multilink.s(keyword, i) for i in range(
          1, 100 if not settings.DEBUG else 20, 10)
      ]
  
      if callbacks:
          chord(
              header,
              (tchain.s() | group(callbacks))
          )()
  
  search_engines = (dogpile,)
  
subject: use callback re-routing to parralelize dogpile