old_contents: import logging
  
  import requests
  import httplib
  
  from bijgeschaafd import diff_match_patch
  
  logger = logging.getLogger('parsers')
  
  
  def hash_djb2(s):
      hash = 5381
      for x in s:
          hash = ((hash << 5) + hash) + ord(x)
      return hash & 0xFFFFFFFF
  
  
  CHARSET_LIST = """EUC-JP GB2312 EUC-KR Big5 SHIFT_JIS windows-1252
  IBM855
  IBM866
  ISO-8859-2
  ISO-8859-5
  ISO-8859-7
  KOI8-R
  MacCyrillic
  TIS-620
  windows-1250
  windows-1251
  windows-1253
  windows-1255""".split()
  
  
  def is_boring(old, new):
      oldu = canonicalize(old.decode('utf8'))
      newu = canonicalize(new.decode('utf8'))
  
      def extra_canonical(s):
          """Ignore changes in whitespace"""
          return s.split()
  
      if extra_canonical(oldu) == extra_canonical(newu):
          return True
  
      for charset in CHARSET_LIST:
          try:
              if oldu.encode(charset) == new:
                  logger.debug('Boring!')
                  return True
          except UnicodeEncodeError:
              pass
      return False
  
  
  def get_diff_info(old, new):
      dmp = diff_match_patch.diff_match_patch()
      dmp.Diff_Timeout = 3  # seconds; default of 1 is too little
      diff = dmp.diff_main(old, new)
      dmp.diff_cleanupSemantic(diff)
      chars_added = sum(len(text) for (sign, text) in diff if sign == 1)
      chars_removed = sum(len(text) for (sign, text) in diff if sign == -1)
      return dict(chars_added=chars_added, chars_removed=chars_removed)
  
  
  def http_get(url):
      response = requests.get(url, stream=True)
  
      addr, port = None, None
  # Does not work for SSL connections
  #    if isinstance(response.raw._fp, httplib.HTTPResponse):
  #        addr, port = response.raw._fp.fp._sock.getpeername()
  
      request_info = {
          'addr': addr,
          'port': port,
      }
  
      return response.content, request_info
  
  
  def strip_whitespace(text):
      lines = text.split('\n')
      return '\n'.join(x.strip().rstrip(u'\xa0') for x in lines).strip() + '\n'
  
  
  def canonicalize(text):
      return strip_whitespace(text)
  
  
new_contents: import logging
  
  import requests
  import httplib
  
  from bijgeschaafd import diff_match_patch
  
  logger = logging.getLogger('parsers')
  
  
  def hash_djb2(s):
      hash = 5381
      for x in s:
          hash = ((hash << 5) + hash) + ord(x)
      return hash & 0xFFFFFFFF
  
  
  CHARSET_LIST = """EUC-JP GB2312 EUC-KR Big5 SHIFT_JIS windows-1252
  IBM855
  IBM866
  ISO-8859-2
  ISO-8859-5
  ISO-8859-7
  KOI8-R
  MacCyrillic
  TIS-620
  windows-1250
  windows-1251
  windows-1253
  windows-1255""".split()
  
  
  def is_boring(old, new):
      oldu = canonicalize(old.decode('utf8'))
      newu = canonicalize(new.decode('utf8'))
  
      def extra_canonical(s):
          """Ignore changes in whitespace"""
          return s.split()
  
      if extra_canonical(oldu) == extra_canonical(newu):
          return True
  
      for charset in CHARSET_LIST:
          try:
              if oldu.encode(charset) == new:
                  logger.debug('Boring!')
                  return True
          except UnicodeEncodeError:
              pass
      return False
  
  
  def get_diff_info(old, new):
      dmp = diff_match_patch.diff_match_patch()
      dmp.Diff_Timeout = 3  # seconds; default of 1 is too little
      diff = dmp.diff_main(old, new)
      dmp.diff_cleanupSemantic(diff)
      chars_added = sum(len(text) for (sign, text) in diff if sign == 1)
      chars_removed = sum(len(text) for (sign, text) in diff if sign == -1)
      return dict(chars_added=chars_added, chars_removed=chars_removed)
  
  
  def http_get(url):
      response = requests.get(url, stream=True, timeout=120)
  
      addr, port = None, None
  # Does not work for SSL connections
  #    if isinstance(response.raw._fp, httplib.HTTPResponse):
  #        addr, port = response.raw._fp.fp._sock.getpeername()
  
      request_info = {
          'addr': addr,
          'port': port,
      }
  
      return response.content, request_info
  
  
  def strip_whitespace(text):
      lines = text.split('\n')
      return '\n'.join(x.strip().rstrip(u'\xa0') for x in lines).strip() + '\n'
  
  
  def canonicalize(text):
      return strip_whitespace(text)
  
  
subject: Add a timeout of 2 minutes. The requests library, by default, waits forever for a request to respond. This caused scraper processes to block indefinitely, and eventually make the server unresponsive because of the large amount of memory usage.