old_contents: """Script that converts a field in a tabular data file to cptm input files
  
  Used for the CAP vragenuurtje data.
  
  Uses frog to pos-tag and lemmatize the data.
  
  Usage: python tabular2cpt_input.py <csv of excel file> <full text field name>
  <dir out>
  """
  
  import pandas as pd
  import logging
  import sys
  import argparse
  import re
  
  from pynlpl.clients.frogclient import FrogClient
  
  from cptm.utils.inputgeneration import Perspective, remove_trailing_digits
  from cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types
  
  logger = logging.getLogger(__name__)
  logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)
  
  parser = argparse.ArgumentParser()
  parser.add_argument('in_file', help='excel or csv file containing text data')
  parser.add_argument('text_field', help='name of the text field')
  parser.add_argument('out_dir', help='directory where output should be stored')
  args = parser.parse_args()
  
  port = 8020
  
  try:
      frogclient = FrogClient('localhost', port)
  except:
      logger.error('Cannot connect to the Frog server. '
                   'Is it running at port {}?'.format(port))
      logger.info('Start the Frog server with "docker run ''-p 127.0.0.1:{}:{} '
                  '-t -i proycon/lamachine frog -S {}"'.format(port, port, port))
      sys.exit(1)
  
  regex = re.compile(r'\(.*\)')
  
  if args.in_file.endswith('.xls') or args.in_file.endswith('.xlsx'):
      input_data = pd.read_excel(args.in_file)
  else:
      input_data = pd.read_csv(args.in_file)
  
  for i, text in enumerate(input_data[args.text_field]):
      p = Perspective('', pos_topic_words(), pos_opinion_words())
      if i % 25 == 0:
          logger.info('Processing text {} of {}'.format(i + 1,
                      len(input_data[args.text_field])))
      if pd.notnull(text):
          for data in frogclient.process(text):
              word, lemma, morph, ext_pos = data[:4]
              if ext_pos:  # ext_pos can be None
                  pos = regex.sub('', ext_pos)
                  if pos in word_types():
                      p.add(pos, remove_trailing_digits(lemma))
          file_name = '{}.txt'.format(i)
          p.write2file(args.out_dir, file_name)
  
new_contents: """Script that converts a field in a tabular data file to cptm input files
  
  Used for the CAP vragenuurtje data.
  
  Uses frog to pos-tag and lemmatize the data.
  
  Usage: python tabular2cpt_input.py <csv of excel file> <full text field name>
  <dir out>
  """
  
  import pandas as pd
  import logging
  import sys
  import argparse
  import re
  
  from pynlpl.clients.frogclient import FrogClient
  
  from cptm.utils.inputgeneration import Perspective, remove_trailing_digits
  from cptm.utils.dutchdata import pos_topic_words, pos_opinion_words, word_types
  
  logger = logging.getLogger(__name__)
  logging.basicConfig(format='%(levelname)s : %(message)s', level=logging.DEBUG)
  
  parser = argparse.ArgumentParser()
  parser.add_argument('in_file', help='excel or csv file containing text data')
  parser.add_argument('text_field', help='name of the text field')
  parser.add_argument('out_dir', help='directory where output should be stored')
  args = parser.parse_args()
  
  port = 8020
  
  try:
      frogclient = FrogClient('localhost', port)
  except:
      logger.error('Cannot connect to the Frog server. '
                   'Is it running at port {}?'.format(port))
      logger.info('Start the Frog server with "docker run -p 127.0.0.1:{}:{} '
                  '-t -i proycon/lamachine frog -S {}"'.format(port, port, port))
      sys.exit(1)
  
  regex = re.compile(r'\(.*\)')
  
  if args.in_file.endswith('.xls') or args.in_file.endswith('.xlsx'):
      input_data = pd.read_excel(args.in_file)
  else:
      input_data = pd.read_csv(args.in_file)
  
  for i, text in enumerate(input_data[args.text_field]):
      p = Perspective('', pos_topic_words(), pos_opinion_words())
      if i % 25 == 0:
          logger.info('Processing text {} of {}'.format(i + 1,
                      len(input_data[args.text_field])))
      if pd.notnull(text):
          for data in frogclient.process(text):
              word, lemma, morph, ext_pos = data[:4]
              if ext_pos:  # ext_pos can be None
                  pos = regex.sub('', ext_pos)
                  if pos in word_types():
                      p.add(pos, remove_trailing_digits(lemma))
          try:
              file_name = '{}.txt'.format(input_data['id'][i])
          except:
              file_name = '{}.txt'.format(i)
          p.write2file(args.out_dir, file_name)
  
subject: Update script to use id field for file names