old_contents: #!/usr/bin/env python
  # -*- coding: UTF-8 -*-
  # Copyright 2016 Eddie Antonio Santos <easantos@ualberta.ca>
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #   http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from pprint import pprint
  
  from tqdm import tqdm
  from blessings import Terminal
  
  from corpus import Corpus
  from stringify_token import stringify_token
  from vocabulary import START_TOKEN, END_TOKEN
  
  t = Terminal()
  
  
  def summarize(vocab):
      """
      >>> summarize({'var', '{', '/regexp/'})
      The size of vocabulary is 5
      """
      # Add /*start*/ and /*end*/ tokens to the count.
      size = len(vocab) + 2
  
      if size < 128:
          size = t.green(str(size))
      elif 128 <= size < 256:
          size = t.yellow(str(size))
      else:
          size = t.red(str(size))
  
      print("The size of vocabulary is", size)
      total_vocab = [START_TOKEN] + sorted(list(vocab)) + [END_TOKEN]
  
      with open('autogenerated_vocabulary.py', 'wt', encoding='utf-8') as vocab_file:
          vocab_file.write('VOCAB = ')
          pprint(total_vocab, stream=vocab_file)
  
  
  if __name__ == '__main__':
      import sys
      _, filename = sys.argv
      corpus = Corpus.connect_to(filename)
  
      vocab = set()
  
      for file_tokens in tqdm(corpus, total=len(corpus)):
          for token in file_tokens:
              vocab.add(stringify_token(token))
  
      summarize(vocab)
  
new_contents: #!/usr/bin/env python
  # -*- coding: UTF-8 -*-
  # Copyright 2016 Eddie Antonio Santos <easantos@ualberta.ca>
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #   http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  
  from pprint import pprint
  
  from tqdm import tqdm
  from blessings import Terminal
  
  from corpus import Corpus
  from stringify_token import stringify_token
  from vocabulary import START_TOKEN, END_TOKEN
  
  t = Terminal()
  
  
  def summarize(vocab, write_to_file=True):
      """
      >>> summarize({'var', '{', '/regexp/'}, write_to_file=False)
      The size of vocabulary is 5
      """
      # Add /*start*/ and /*end*/ tokens to the count.
      size = len(vocab) + 2
  
      if size < 128:
          size = t.green(str(size))
      elif 128 <= size < 256:
          size = t.yellow(str(size))
      else:
          size = t.red(str(size))
  
      print("The size of vocabulary is", size)
      total_vocab = [START_TOKEN] + sorted(list(vocab)) + [END_TOKEN]
  
      if not write_to_file:
          return
  
      with open('autogenerated_vocabulary.py', 'wt', encoding='utf-8') as vocab_file:
          vocab_file.write('VOCAB = ')
          pprint(total_vocab, stream=vocab_file)
  
  
  if __name__ == '__main__':
      import sys
      _, filename = sys.argv
      corpus = Corpus.connect_to(filename)
  
      vocab = set()
  
      for file_tokens in tqdm(corpus, total=len(corpus)):
          for token in file_tokens:
              vocab.add(stringify_token(token))
  
      summarize(vocab)
  
subject: Stop writing all the time...