old_contents: from typing import Any, Dict, List, Text
  
  import regex, re
  
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.nlu.training_data import Message
  
  
  class WhitespaceTokenizer(Tokenizer):
  
      defaults = {
          # Flag to check whether to split intents
          "intent_tokenization_flag": False,
          # Symbol on which intent should be split
          "intent_split_symbol": "_",
          # Text will be tokenized with case sensitive as default
          "case_sensitive": True,
      }
  
      def __init__(self, component_config: Dict[Text, Any] = None) -> None:
          """Construct a new tokenizer using the WhitespaceTokenizer framework."""
  
          super().__init__(component_config)
  
          self.case_sensitive = self.component_config["case_sensitive"]
  
      def tokenize(self, message: Message, attribute: Text) -> List[Token]:
          text = message.get(attribute)
  
          if not self.case_sensitive:
              text = text.lower()
  
          # we need to use regex instead of re, because of
          # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters
  
          # remove 'not a word character' if
          words = regex.sub(
              # there is a space or an end of a string after it
              r"[^\w#@&]+(?=\s|$)|"
              # there is a space or beginning of a string before it
              # not followed by a number
              r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
              # not in between numbers and not . or @ or & or - or #
              # e.g. 10'000.00 or blabla@gmail.com
              # and not url characters
              r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
              " ",
              text,
          ).split()
          # if we removed everything like smiles `:)`, use the whole text as 1 token
          if not words:
              words = [text]
  
          # convert to ascii and check if the resultant string is empty
          words = [w for w in words if w.encode("ascii", "ignore").strip()]
  
          return self._convert_words_to_tokens(words, text)
  
new_contents: from typing import Any, Dict, List, Text
  
  import regex
  
  from rasa.nlu.tokenizers.tokenizer import Token, Tokenizer
  from rasa.nlu.training_data import Message
  
  
  class WhitespaceTokenizer(Tokenizer):
  
      defaults = {
          # Flag to check whether to split intents
          "intent_tokenization_flag": False,
          # Symbol on which intent should be split
          "intent_split_symbol": "_",
          # Text will be tokenized with case sensitive as default
          "case_sensitive": True,
      }
  
      def __init__(self, component_config: Dict[Text, Any] = None) -> None:
          """Construct a new tokenizer using the WhitespaceTokenizer framework."""
  
          super().__init__(component_config)
  
          self.case_sensitive = self.component_config["case_sensitive"]
  
      def tokenize(self, message: Message, attribute: Text) -> List[Token]:
          text = message.get(attribute)
  
          if not self.case_sensitive:
              text = text.lower()
  
          # we need to use regex instead of re, because of
          # https://stackoverflow.com/questions/12746458/python-unicode-regular-expression-matching-failing-with-some-unicode-characters
  
          # remove 'not a word character' if
          words = regex.sub(
              # there is a space or an end of a string after it
              r"[^\w#@&]+(?=\s|$)|"
              # there is a space or beginning of a string before it
              # not followed by a number
              r"(\s|^)[^\w#@&]+(?=[^0-9\s])|"
              # not in between numbers and not . or @ or & or - or #
              # e.g. 10'000.00 or blabla@gmail.com
              # and not url characters
              r"(?<=[^0-9\s])[^\w._~:/?#\[\]()@!$&*+,;=-]+(?=[^0-9\s])",
              " ",
              text,
          ).split()
          # if we removed everything like smiles `:)`, use the whole text as 1 token
          if not words:
              words = [text]
  
          # convert to ascii and check if the resultant string is empty
          words = [w for w in words if w.encode("ascii", "ignore").strip()]
  
          return self._convert_words_to_tokens(words, text)
  
subject: remove unused import