old_contents: import urllib2
  from jug import Task
  from collections import defaultdict
  from time import sleep
  import re
  from string import lower
  from os.path import exists
  
  
  def getdata(title):
      sleep(8)
      if exists('text-data'):
          return file('text-data/' + title).read()
  
      title = urllib2.quote(title)
      url = 'http://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=txt&titles=' + title
      text = urllib2.urlopen(url).read()
      text = re.sub(ur'(?x) \[ [^]] *? \]\]', '', text)
      text = re.sub('(?x) {{ [^}]*? }}', '', text)
      text = text.strip()
      return text
  
  def isstopword(titlewords, w):
      if not re.match('^\w+$', w): return True
      if w in titlewords: return True
      return False
  
  def countwords(title, document):
      '''
      Takes a file name and returns a wordcount.
      '''
      sleep(4)
      titlewords = map(lower, title.split())
      counts = defaultdict(int)
      for w in document.split():
          w = lower(w)
          if not isstopword(titlewords, w):
              counts[w] += 1
      return dict(counts)
  
  def addcounts(counts):
      '''
      Takes intermediate word counts and puts them together
      '''
      sleep(24)
      allcounts = defaultdict(int)
      for c in counts:
          for k,v in c.iteritems():
              allcounts[k] += v
      return dict(allcounts)
  
  def divergence(global_counts, nr_documents, counts):
      '''
      Takes the global word counts as well as a single count vector
      and returns a set of words *in this document* that are document
      specific.
      '''
      sleep(8)
      specific = []
      for w,n in counts.iteritems():
          if n > global_counts[w]//100:
              specific.append(w)
      specific.sort(key=counts.get)
      return list(reversed(specific))
  
  counts = []
  for mp in file('MPs.txt'):
      mp = mp.strip()
      document = Task(getdata, mp)
      counts.append(Task(countwords, mp, document))
  avgs = Task(addcounts, counts)
  results = []
  for c in counts:
      results.append(Task(divergence,avgs, len(counts), c))
  
new_contents: import urllib2
  from jug import Task
  from collections import defaultdict
  from time import sleep
  import re
  from string import lower
  from os.path import exists
  
  
  def getdata(title):
      sleep(8)
      if exists('text-data'):
          return file('text-data/' + title).read()
  
      title = urllib2.quote(title)
      url = 'http://en.wikipedia.org/w/api.php?action=query&prop=revisions&rvprop=content&format=txt&titles=' + title
      text = urllib2.urlopen(url).read()
      text = re.sub(ur'(?x) \[ [^]] *? \]\]', '', text)
      text = re.sub('(?x) {{ [^}]*? }}', '', text)
      text = text.strip()
      return text
  
  def isstopword(titlewords, w):
      if not re.match('^\w+$', w): return True
      if w in titlewords: return True
      return False
  
  def countwords(title, document):
      '''
      Takes a file name and returns a wordcount.
      '''
      sleep(4)
      titlewords = map(lower, title.split())
      counts = defaultdict(int)
      for w in document.split():
          w = lower(w)
          if not isstopword(titlewords, w):
              counts[w] += 1
      return dict(counts)
  
  def addcounts(counts):
      '''
      Takes intermediate word counts and puts them together
      '''
      sleep(24)
      allcounts = defaultdict(int)
      for c in counts:
          for k,v in c.iteritems():
              allcounts[k] += v
      return dict(allcounts)
  
  def divergence(global_counts, nr_documents, counts):
      '''
      Takes the global word counts as well as a single count vector
      and returns a set of words *in this document* that are document
      specific.
      '''
      sleep(8)
      specific = []
      for w,n in counts.iteritems():
          if n > global_counts[w]//100:
              specific.append(w)
      specific.sort(key=counts.get)
      specific.reverse()
      return specific
  
  counts = []
  for mp in file('MPs.txt'):
      mp = mp.strip()
      document = Task(getdata, mp)
      counts.append(Task(countwords, mp, document))
  avgs = Task(addcounts, counts)
  results = []
  for c in counts:
      results.append(Task(divergence,avgs, len(counts), c))
  
subject: Simplify code.