old_contents: from collections import namedtuple
  
  Token = namedtuple('Token', ['text'])
  
  
  def tokenize(text):
      for word in text.split():
          word = word.strip(',.;!?-()')
          if word:
              yield Token(word)
  
new_contents: from collections import namedtuple
  import re
  
  Token = namedtuple('Token', ['text'])
  word_pattern = re.compile(r'\S+')
  
  
  def tokenize(text):
      offset = 0
      while True:
          match = word_pattern.search(text, offset)
          if match is None:
              break
          word = match.group()
          offset = match.end()
          word = word.strip(',.;!?-()')
          if word:
              yield Token(word)
  
subject: reimplement tokenizer using regular expression