old_contents: """
  Miscelanius utilities which are helpful sometime.
  """
  import logging
  from urlparse import urlsplit
  from hashlib import sha1
  
  def unique_file(path):
      """
      Drop non-unique lines in the file.
      Return number of unique lines.
      """
  
      lines = set()
      count = 0
      with open(path) as inf:
          for line in inf:
              lines.add(line)
              count += 1
      logging.debug('Read %d lines from %s, unique: %d' % (count, path,
                                                           len(lines)))
      with open(path, 'w') as out:
          out.write(''.join(lines))
      return len(lines)
  
  
  def unique_host(path):
      """
      Filter out urls with duplicated hostnames.
      """
  
      hosts = set()
      lines = []
      count = 0
      with open(path) as inf:
          for line in inf:
              host = urlsplit(line).netloc
              if not host in hosts:
                  lines.append(line)
                  hosts.add(host)
              count += 1
      logging.debug('Read %d lines from %s, unique hosts: %d' % (count, path,
                                                                 len(lines)))
      with open(path, 'w') as out:
          out.write(''.join(lines))
      return len(lines)
  
  def hash_path(url, ext=None):
      _hash = sha1(url).hexdigest()
      a, b, tail = _hash[:2], _hash[2:4], _hash[4:]
      if ext is not None:
          tail = '%s.%s' % (tail, ext)
      rel_path = '%s/%s/%s' % (a, b, tail)
      return rel_path
  
new_contents: """
  Miscelanius utilities which are helpful sometime.
  """
  import logging
  from urlparse import urlsplit
  from hashlib import sha1
  
  def unique_file(path):
      """
      Drop non-unique lines in the file.
      Return number of unique lines.
      """
  
      lines = set()
      count = 0
      with open(path) as inf:
          for line in inf:
              lines.add(line)
              count += 1
      logging.debug('Read %d lines from %s, unique: %d' % (count, path,
                                                           len(lines)))
      with open(path, 'w') as out:
          out.write(''.join(lines))
      return len(lines)
  
  
  def unique_host(path):
      """
      Filter out urls with duplicated hostnames.
      """
  
      hosts = set()
      lines = []
      count = 0
      with open(path) as inf:
          for line in inf:
              host = urlsplit(line).netloc
              if not host in hosts:
                  lines.append(line)
                  hosts.add(host)
              count += 1
      logging.debug('Read %d lines from %s, unique hosts: %d' % (count, path,
                                                                 len(lines)))
      with open(path, 'w') as out:
          out.write(''.join(lines))
      return len(lines)
  
  
  def hash_path(url, ext='jpg'):
      _hash = sha1(url).hexdigest()
      a, b, tail = _hash[:2], _hash[2:4], _hash[4:]
      if ext is not None:
          tail = '%s.%s' % (tail, ext)
      rel_path = '%s/%s/%s' % (a, b, tail)
      return rel_path
  
  
  def hashed_path(url, ext='jpg'):
      return hash_path(url, ext=ext)
  
subject: Add alias of hash_path function -> hashed_path