#!/usr/bin/python3
#
# xpandlatex - eXpand LaTeX macros etc
# maneesh 2016/04/19
#
# Process a LaTeX source file to:
#
#   - expand locally defined macros
#   - replace \refs with corresponding targets
#   - replace \cite* with citation targets 
#   - import files base on:
#	\include{file}
#	\tableofcontents
#	\listoffigures
#	\listoftables
#       \bibliography
#   - interpret special commands beginning with '%XP'


# based on:
# xplatex (perl)
# maneesh 2007/09/25.
#
# which was based on:
# tme - a TeX macro expander
# @(#) tme 0.5.6 28/Oct/1996 jonathan



# This file contains a python class to do the actual processing; and
# simple arg parsing (at the end) to make a standalone program.



import re
import sys
import argparse


class XpandLaTeX:

    # ###############################################################################
    # Error handling.

    def die(self, msg):
        sys.exit('%s:%d: %s'%(self.inputFileName, self.inputLineNumber, msg))

    warnFile = sys.stderr;
    def warn(self, msg):
        print('%s:%d: %s'%(self.inputFileName, self.inputLineNumber, msg), file=self.warnFile)


    # ###############################################################################
    # Debug output handling.

    debugLevel = 0
    debugFile = sys.stderr
    
    def debug(self, msg=None, level=1):
        if self.debugLevel >= level:
            print('%s:%d: %s'%(self.inputFileName, self.inputLineNumber, msg), file=self.debugFile)

 
    # ###############################################################################
    # File handling

    inputFile = None
    inputFileName = 'undef'
    inputLineNumber = 0

    def open_input(self, filename):
        self.inputFileName = filename
        self.inputLineNumber = 0
        try:
            self.inputFile = open(self.inputFileName, 'r')
            return True
        except IOError:
            return False

    def close_input(self):
        self.inputFile.close()

    outputFile = sys.stdout
    outputFileName = '-'

    def open_ouput(self, filename, flags='w'):
        self.outputFileName = filename;
        self.outputFile = open(self.outputFileName, flags)



    # ###############################################################################
    # Token handling

    commentMode = 'discard'  # how to handle comments 'discard'|'copy'
    tokenBuffer = []         # pushed-back "input" tokens
    charBuffer = ""          # untokenized input characters


    # put_token(tok) -- send token to output stream

    def put_token(self, tok):
        self.debug("put token <" + tok + ">", level=10)
        print(tok, end='', file=self.outputFile)
        

    # put_tokens(tok) -- send a list of tokens to output stream

    def put_tokens(self, toks):
        for tok in toks:
            self.put_token(tok);


    # push_tokens(tokens) - re-buffer tokens to be read again by later get_token calls.  
    # 
    # tokens is reversed, so the first call to get_token() (which
    # reads the end of the stack) will return the first item in the list.

    def push_tokens(self, tokens):
        self.debug('push_tokens <' + ':'.join(reversed(tokens)) + '>', level=5)
        self.tokenBuffer.extend(reversed(tokens))
        


    # get_token([mode], [comment], [space]) - get a token with some filtering
    #
    # get next token, either from push_token buffer or from file.
    #   mode='parsing' 	       => delay comment tokens and discard whitespace
    #   mode='all' 	       => return everything
    #   comment_mode='discard' => discard comment tokens in stream forever
    #   comment_mode='copy'    => return comment token if next in stream
    #   comment_mode='delay'   => return non-comment, but push any comments onto tokenBuffer
    #   space_mode='discard'   => discard whitespace tokens in stream forever
    #   space_mode='copy'      => return whitespace token if next in stream
    
    def get_token(self, mode=None, comment_mode=None, space_mode='copy', read='any'):

        if mode == 'parsing': 
            space_mode = 'discard'
            comment_mode = 'delay'

        if mode == 'all':
            space_mode = 'copy'
            comment_mode = 'copy'

        if comment_mode is None: 
            comment = self.commentMode;

        delay_tokens = []
        tok=self.get_bare_token(read=read)
        while tok is not None:
            if len(tok) == 0: tok=self.get_bare_token(read=read);  continue

            if re.fullmatch(r'\s+', tok): # whitespace
                if space_mode=='copy' : break

                # else discard, get new token and check again
                tok=self.get_bare_token(read=read); 
                continue

            if re.match('%XP', tok):      # %XP special token - return (FIXME: delay when parsing?)
                break
                
            if tok[0] == '%':             # comment 
                if comment_mode=='copy' : break

                # else delay or discard, get new token and check again
                if comment_mode=='delay': delay_tokens.append(tok)
                tok=self.get_bare_token(read=read)                
                continue

            # else non-comment, non-space
            break

        if len(delay_tokens): self.push_tokens(delay_tokens)

        self.debug('got token <%s>' % tok, level=10);
        return tok


    # tok=get_bare_token() - get a token from buffer, or new input stream (unfiltered)

    # A simple approximation to TeX's actual tokenizing algorithm (TeXbook
    # pp. 46-47; from tme.pl), with the local addition of %XP tokens.
    #
    # The untokenized input is compared to the following regular expressions,
    # taking the first match found as the next token: 
    # 	\\[a-zA-Z]+ 	% eg \foo
    # 	\\[^a-zA-Z] 	% eg \$ or \, 
    # 	#[1-9#] 	% eg #1 or ##
    #   %XP[a-zA-z:]*   % XpandLaTeX special tokens
    #	%.*\n		% TeX comments
    #	(.|\n)		% anything else			
    #
    # %XP lines are (obviously) comments to tex, but interpreted by XPLATEX


    # compile the token regular expression globally
    tokenizer = re.compile('\\\\[a-zA-Z]+|\\\\[^a-zA-Z]|#[1-9#]|%XP[a-zA-Z:]*|%.*\n|\s+|.|\n');

    def get_bare_token(self, read="any"):
    
        # return pushed-back token if any are available
        if read in ['any', 'buffer']:
            if len(self.tokenBuffer) > 0:
                self.debug('got buffered token <' + self.tokenBuffer[-1] + '>', level=10)
                return self.tokenBuffer.pop();

        if read == 'buffer':
            return None;

        # refill charBuffer if needed
        if len(self.charBuffer) == 0:
            self.charBuffer = self.inputFile.readline()
            self.inputLineNumber+=1
            if len(self.charBuffer) == 0:
                return None

        # grab the next token in charBuffer

        tok_match = self.tokenizer.match(self.charBuffer);
        if tok_match:
            tok = tok_match.group() # get the matched token
        else:
            self.die("can't find token in: " + charBuffer)

        # remove the matched token from the buffer
        self.charBuffer = self.charBuffer[tok_match.end():] 
            
        return tok



    # stat=gobble_tokens(token_list) - discard tokens as long as they match token_list

    # stat=True if all tokens match, False otherwise
    # all tokens that do match will be discarded, even if False

    def gobble_tokens(self, token_list, mode="parsing"):
        for token in token_list:
            tok = self.get_token(mode)
            if not tok == token:
                self.push_tokens([tok]) # push back the mismatched token
                return False
        return True



    # token_list = get_tokens_to_match(end_list) 
    #  - read tokens up to (and including) the first appearance of end_list at same tex group level

    # token_list (may be empty) does not include end_list
    # end_list tokens are discarded

    def get_tokens_to_match(self, end_list, mode=''):
        end_count = len(end_list)
        token_list = []

        # we don't want any matches that are nested inside {}-blocks,
        # so we use 'nesting' to count how many levels deep we are
        # while parsing; but if end_list has braces then we need to
        # account for these.
        nesting = 0;
        end_nesting = end_list.count('{') - end_list.count('}')

        token = self.get_token(mode=mode)
    
        while token is not None:
            token_list.append(token)

            if token == '{':
                nesting+=1
            elif  token == '}':
                nesting-=1

            if nesting == end_nesting: # only check for end_list when at correct {}-block level
                if len(token_list) >= end_count and token_list[-end_count:] == end_list:
                    # tail matches end_list => chop off match, return preceding tokens
                    del token_list[-end_count:]
                    return token_list

            token=self.get_token(mode=mode)      # end while

        # trouble...
        self.die('EOF encountered while looking for <' +
            ':'.join(end_list) + '>');





    # ###############################################################################
    # Macro handling

    macroParamCount = {}
    macroParamDelim = {}
    macroDefinition = {}

    # # A macro definition is taken to have one of the (equivalent) forms
    # # (TeXbook p.203-204)
    # #	\def _ <name token> _ <parameter token list> { <expansion token list> }
    # #	\gdef _ <name token> _ <parameter token list> { <expansion token list> }
    # #	\newcommand _ { <name token> } _ { <expansion token list> }
    # #	\newcommand _ { <name token> } _ [N] _ { <expansion token list> }
    # #	\renewcommand _ { <name token> } _ { <expansion token list> }
    # #	\renewcommand _ { <name token> } _ [N] _ { <expansion token list> }
    # # where in each case _ denotes optional whitespace (which will be ignored).
    # #
    # # Regardless of which form of definition is used, a macro is stored as
    # # the following data structures:
    # #	$macro_N_pars{$macro_name} = number of parameters
    # #	$macro_par_delim{$macro_name} = { <parameter delimiter hash> }
    # #	$macro_defn{$macro_name} = [ <expansion token list> ]
    # # where { <parameter delimiter hash> } is a hash:
    # # - keys are '#0', '#1', '#2', ...
    # # - values are references to (possibly empty) token lists of the
    # #   "parameter delimiter" tokens following the keys, with the '#0'
    # #   case being the (possibly empty) list of nonblank tokens (blank
    # #   tokens being ignored) immediately following the macro name and
    # #   preceding the #1 token (if any) or the { token.
    # #
    # # For example, the macro definition
    # #	\def\Jac[#1]{{\bf J} \Bigl[ #1 \Bigr]}
    # # would be stored as the data structures
    # #	$macro_N_pars{'\Jac'} = 1
    # #	$macro_par_delim{'\Jac'} = {
    # #				   '#0' => [ '[' ],
    # #				   '#1' => [ ']' ],
    # #				   }
    # #	$macro_defn{'\Jac'} = [
    # #			      '{', '\bf', ' ', 'J', '}', ' ',
    # #			      '\Bigl', '[', ' ', '#1', ' ', '\Bigr', ']'
    # #			      ]



    # stat = is_defined_macro(token) - is macro token in list of definitions

    def is_defined_macro(self, token):
        if token in self.macroDefinition:
            self.debug('defined macro ' + token, level=4);
            return 1
        else:
            self.debug('undefined macro ' + token, level=4);
            return 0
        


    # define_macro_def() - store macro definition from input stream using \def format

    # the \def keywords should already have been read, so the first
    # token will be the macro name

    def define_macro_def(self):
        # get macro name
        macro_name = self.get_token('parsing')

        self.debug(r'defining macro <%s> using \(g)def ...' % macro_name);

        # get macro parameters

        macro_npar = 0;
        macro_delim = {};       # will build up <parameter delimiter hash>


        # loop over parameters #0, #1, #2, ...
        par = '#0'              # pseudo-parameter for tokens preceding #1
        while True:
            delim = []                     # delimiter tokens for this par

            tok = self.get_token(comment_mode='discard')
            while tok is not None:
                if tok == '{' or re.fullmatch('#[0-9]', tok): break
                delim.append(tok)
                tok = self.get_token(comment_mode='discard')
            else:                           # reached end of file
                self.die('EOF when reading arglist for %s' % macro_name)

            macro_delim[par] = delim

            if tok == '{': break        # done with param list

            macro_npar += 1             # otherwise found another param
            par = tok

        # get macro definition
        macro_defn = self.get_tokens_to_match(['}']);

        self.debug('\t' + str(macro_delim))
        self.debug('\t' + str(macro_defn))

        self.macroParamCount[macro_name] = macro_npar
        self.macroParamDelim[macro_name] = macro_delim
        self.macroDefinition[macro_name] = macro_defn




    # define_macro_newcommand() - store macro definition from input stream using \newcommand format

    # the \(re)newcommand keyword should already have been read, so the first
    # token will be the macro name (possibly in {})

    def define_macro_newcommand(self):
        die_msg = r"\(re)newcommand syntax error: ";

        # get macro name

        macro_name = self.get_token('parsing')
        if macro_name == '{':        		# name in {}
            macro_name = self.get_token('parsing');
            if not self.get_token('parsing') == '}': 	# {} must close after single token 
                self.die(die_msg);

        self.debug(r'defining macro <%s> via \(re)newcommand ...' % macro_name);


        # get number of parameters

        macro_npar = 0;				# default value
        tok = self.get_token('parsing')
        if tok == '[':
            npar_str = self.get_token('parsing')
            if not re.fullmatch('[1-9]', npar_str): # [] contains non-int
                self.die(die_msg + macro_name + ': bad arg  <' + npar_str + '>'); 
            
            macro_npar = int(npar_str)		# convert to integer (we hope) ...

            if not self.get_token('parsing') == ']':      # [] must close after one token
                self.die(die_msg + 'bad arg format');
        else:
            self.push_tokens([tok])


        # build macro parameter hash

        macro_delim = {};		# will build up <parameter delimiter hash>

        for i in range (macro_npar + 1):
            par = '#' + str(i);
            macro_delim[par] = [];	# empty delimiter token list


        # look for optional arg

        if macro_npar > 0:
            tok = self.get_token('parsing')
            if tok == '[':
                default = self.get_tokens_to_match([']'])
                macro_delim['#1'] = ['[]', default]
            else:
                self.push_tokens([tok])


        # get macro definition

        tok = self.get_token('parsing')
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        macro_defn = self.get_tokens_to_match(['}']);

        self.debug('\t' + str(macro_delim))
        self.debug('\t' + str(macro_defn))

        self.macroParamCount[macro_name] = macro_npar
        self.macroParamDelim[macro_name] = macro_delim
        self.macroDefinition[macro_name] = macro_defn




    # arg_dict = get_macro_args(macro_name) - read macro arguments from input stream

    # arguments are returned in a new dict object, with '#?' keys

    def get_macro_args(self, macro, nparam=None, delims=None):
        args = {}

        self.debug("getting args for %s ..." % macro)
 
        if nparam is None:
            nparam = self.macroParamCount[macro]

        if delims is None:
            delims = self.macroParamDelim[macro]

        # special-case the pseudo-parameter #0
        param = '#0';
        delim = delims[param]

        if len(delim) > 0:
            if not self.gobble_tokens(delim):
                self.die('usage of %s does not match definition' % macro)

        # get arguments for "normal" parameters
        for i in range(1, nparam + 1):
            param = '#' + str(i)
            delim = delims[param]

            # get the argument following the rules in the TeXbook (pp. 203-204)
            # (or LaTeX optional param)
            if len(delim) == 0:                    # undelimited parameter
                token = self.get_token('parsing');
                if token == '{':
                    arg = self.get_tokens_to_match(['}'])
                else:
                    arg = [token]
                self.debug('\t %s (undelimited) = <%s>' % (param, str(arg)) )

            elif delim[0] == '[]': 		    # LaTeX optional parameter
                token = self.get_token('parsing');
                if token == '[':
                    arg = self.get_tokens_to_match([']'])
                else:
                    self.push_tokens([token]) 	    # no []'s => push back token
                    arg = delim[1]                  # default value
                self.debug('\t %s (optional) = <%s>' % (param, str(arg)) )
                
            else:                                   # delimited parameter
                arg = self.get_tokens_to_match(delim);

                # if arg is a tex block strip off enclosing {}
                if arg[0] == '{' and arg[-1] == '}':
                    del arg[-1:0]

                self.debug('\t %s (delimited) = <%s>' % (param, str(arg)))

            args[param] = arg;
        #end for

        self.debug('\t <%s>' % str(args))
        return args




    # expand_macro(macro) - expand a macro, push the expansion back onto the input stream

    # assume the macro name has already been read, but args are still
    # on input stream

    def expand_macro(self, macro, args=None, defn=None, extra_args=None):

        self.debug('expanding macro ' + macro + ":")

        if args is None: args = self.get_macro_args(macro)
        if defn is None: defn = self.macroDefinition[macro]

        if extra_args is not None: args.update(extra_args)
        
        self.debug ('\t' + str(args))

        expansion = []
        for token in defn:
            if re.match('#[1-9#]', token):
                expansion.extend(args[token])
            else:
                expansion.append(token)
            
        self.debug ('\t' + str(expansion))
        self.push_tokens(expansion);



    # ###############################################################################
    # Label/ref handling

    labelDefinition = {}
    labelPagenumber = {}

    def define_label(self, prefix=""):
        die_msg = r"\newlabel syntax error: ";

        # get label name
        tok = self.get_token('parsing')
        if not tok == '{':
            self.die(die_msg + 'unexpected token <' + tok + '>')

        label_name = prefix
        tok = self.get_token()
        while not tok ==  '}':
            label_name += tok
            tok = self.get_token()

        self.debug(r'defining label <%s> via \newlabel ...' % label_name);


        # parse label definition: {{item}{page}}

        tok = self.get_token('parsing') # '{'{item}{page}}
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        tok = self.get_token('parsing') # '{'item}{page}}
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')
        self.labelDefinition[label_name] = self.get_tokens_to_match(['}']);

        tok = self.get_token('parsing') # '{'page}}
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')
        self.labelPagenumber[label_name] = self.get_tokens_to_match(['}']);

        tok = self.get_token('parsing') # '}'
        if not tok == '}': self.die(die_msg + 'unexpected token <' + tok + '>')

        self.debug('\t' + str(self.labelDefinition[label_name]) 
                   +" " + str(self.labelPagenumber[label_name]));




    # expand_ref(prefix="") - read ref name and push corresponding label tokens onto input stack
    #
    # assume '\ref' has already been read; prefix is prepended to
    # label name (for example, to handle \subref).
    
    def expand_ref(self, prefix=""):
        die_msg = r"\ref syntax error: ";

        # get label name
        tok = self.get_token('parsing')
        if not tok == '{':
            self.die(die_msg + 'unexpected token <' + tok + '>')  

        label_name = prefix
        tok = self.get_token()
        while not tok ==  '}':
            label_name += tok
            tok = self.get_token()

        if label_name in self.labelDefinition:
            defn = self.labelDefinition[label_name]
            self.debug('expanding ref <%s> to <%s>' % (label_name, str(defn)));
            self.push_tokens(defn)
        else:
            self.warn('unknown ref <%s>' % label_name);
  




    # ###############################################################################
    # Citation handling

    citeDefinition = {}

    citeOpenType = {'cite': '[', 
                    'citep': '(', 
                    'citet': '', 
                    'citealt': '', 
                    'citeauthor': ''}
    citeSepType = {'cite': ', ',
                   'citep': '; ',
                   'citet': ' and ',
                   'citealt': '; ', 
                   'citeauthor': '; '}
    citeCloseType = {'cite': ']',
                     'citep': ')',
                     'citet': '',
                     'citealt': '',
                     'citeauthor': ''}
    citeMacroType = {'cite': r'\XPcite',
                     'citep': r'\XPcitep',
                     'citet': r'\XPcitet', 
                     'citealt': r'\XPcitealt', 
                     'citeauthor': r'\XPciteauthor'}

    def define_bibcite(self):
        die_msg = r"\bibcite syntax error: ";

        # get label name

        tok = self.get_token('parsing')
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        label_name = ""
        tok = self.get_token()
        while not tok ==  '}':
            label_name += tok
            tok = self.get_token()

        self.debug(r'defining cite label <%s> via \bibcite ...' % label_name);

        # get citation definition

        tok = self.get_token('parsing')
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        self.citeDefinition[label_name] = self.get_tokens_to_match(['}']);

        self.debug('\t' + str(self.citeDefinition[label_name]) );



    def expand_cite(self, type="cite"):
        die_msg = r"\%s syntax error: " % type;

        xpnd_tokens = []
        pre_tokens = []
        post_tokens = []

        # check for pre- and post- options
        tok = self.get_token('parsing')
        if tok == '[':
            post_tokens = self.get_tokens_to_match([']'])
            tok = self.get_token('parsing')

        if tok == '[':
            pre_tokens = post_tokens[:]
            post_tokens = self.get_tokens_to_match([']'])
            tok = self.get_token('parsing')

        if len(pre_tokens) > 0:
            pre_tokens.append(" ")
        if len(post_tokens) > 0:
            post_tokens.insert(0, " ")

        xpnd_tokens.append(self.citeOpenType[type]);
        xpnd_tokens.extend(pre_tokens);

        if not tok == '{':
            self.die(die_msg + 'unexpected token <' + tok + '>')  

        # Get the citation names
        while not tok == '}':
            name = ""
        
            tok = self.get_token('parsing');
            while tok not in [',', '}']:
                name += tok;
                tok = self.get_token('parsing');

            if name in self.citeDefinition:
                defn = self.citeDefinition[name];
                self.debug('expanding cite <%s> to <%s>' % (name, str(defn)))
                xpnd_tokens.append(self.citeMacroType[type])
                xpnd_tokens.extend(defn)
            else:
                self.warn('unknown cite name <%s>' % name);

            if tok == ',':
                xpnd_tokens.append(self.citeSepType[type])
            
        xpnd_tokens.extend(post_tokens)
        xpnd_tokens.append(self.citeCloseType[type])

        self.debug(r'\%s expanded: %s' % (type, ''.join(xpnd_tokens)) )
        self.push_tokens(xpnd_tokens)




    # ###############################################################################
    # included file handling

    # update_definitions(xpander) - incorporate macro/label/cite definitions from peer

    def update_definitions(self, other):

        self.debug('merging definitions from ' + other.inputFileName);
        if len(other.macroDefinition):
            self.debug('\t %d macros' % len(other.macroDefinition) )
            self.macroParamCount.update(other.macroParamCount)
            self.macroParamDelim.update(other.macroParamDelim)
            self.macroDefinition.update(other.macroDefinition)
        if len(other.labelDefinition):
            self.debug('\t %d labels' % len(other.labelDefinition))
            self.labelDefinition.update(other.labelDefinition)
            self.labelPagenumber.update(other.labelPagenumber)
        if len(other.citeDefinition):
            self.debug('\t %d cites' % len(other.citeDefinition))
            self.citeDefinition.update(other.citeDefinition)
            


    # read_aux() - read definitions from .aux file

    def read_aux(self):
        aux_file = re.sub(r'\.tex$', '.aux', self.inputFileName)

        aux_xpander = XpandLaTeX(copy=self)
        aux_xpander.process_file(aux_file, type="aux")
        self.update_definitions(aux_xpander)



    # read_external_aux() - read definitions from another .aux
    #
    # triggered by reading \externaldocument (from xr.sty); filename
    # still on stream

    def read_external_aux(self):

        # check for prefix (optional arg)
        tok = self.get_token('parsing')
        if tok == '[':
            xr_prefix = ''.join(self.get_tokens_to_match([']']))
        else:
            self.push_tokens([tok])

        tok = self.get_token('parsing')
        if not tok == '{': self.die(r'\externaldocument: unexpected token <' + tok + '>')
    
        xr_file = ''.join(self.get_tokens_to_match(['}'])) + '.aux'

        self.debug('reading \externaldocument[%s]{%s} ...'%(xr_prefix, xr_file))

        xr_xpander = XpandLaTeX(type="aux")
        xr_xpander.readLabelPrefix=xr_prefix
        xr_xpander.process_file(xr_file)
        
        self.update_definitions(xr_xpander)



    # merge_input() - read a \input file
    #
    # triggered by reading \input: filename still on stream

    def merge_input(self):
        
        # get the filename
        tok = self.get_token('parsing')
        if not tok == '{': self.die(r'\input: unexpected token <' + tok + '>')
        file = ''.join(self.get_tokens_to_match(['}'])) + ".tex"

        file_xpander = XpandLaTeX(copy=self)
        file_xpander.process_file(file)
        self.update_definitions(file_xpander)

    
    # merge_tocs(type) - read a toc/lof/lot file
    #
    # Pushes the section header (as defined by article.cls);   

    # default title macros for sections
    contentsTitleMacro = {'toc': '\contentsname', 'lof': '\listfigurename', 'lot': '\listtablename'}
    contentsOpenMacro  = {'toc': '\XPtocbegin', 'lof': '\XPlofbegin', 'lot': '\XPlotbegin'}
    contentsCloseMacro = {'toc': '\XPtocend', 'lof': '\XPlofend', 'lot': '\XPlotend'}
    
    def merge_tocs(self, type="toc"):

        # delay processing any remaining pushed tokens
        saveTokenBuffer = self.tokenBuffer
        self.tokenBuffer = []

        # push and expand the section header
        self.push_tokens(['\section', '*', '{', self.contentsTitleMacro[type], '}'])
        self.process_tokens(read='buffer');

        # push and expand the XP opening macro IF DEFINED (eg to create an environment)
        if self.contentsOpenMacro[type] in self.macroDefinition:
            self.push_tokens([self.contentsOpenMacro[type]])
            self.process_tokens(read='buffer');
    
        # read the file
        con_file = re.sub(r'\.tex$', '.'+type, self.inputFileName)
        con_xpander = XpandLaTeX(copy=self)
        con_xpander.process_file(con_file)
        # self.update_definitions(con_xpander)  # no definitions or labels in contents files?


        # push and expand the XP closing macro IF DEFINED (eg to end an environment)
        if self.contentsCloseMacro[type] in self.macroDefinition:
            self.push_tokens([self.contentsCloseMacro[type]])
            self.process_tokens(read='buffer');

        # restore pushed tokens
        self.tokenBuffer = saveTokenBuffer




    # merge_bbl() - read bibliography
    #
    # differs from contents files in:
    #  1. need to gobble .bib file names
    #  2. no \section or \begin\end wrapper (.bbl includes thebibliography env already)

    def merge_bbl(self):
    
        # gobble the (unwanted) bibfile names
        tok = self.get_token('parsing')
        if not tok == '{': self.die(r'reading \bibliography: unexpected token <' + tok + '>')
        self.get_tokens_to_match(['}'])

        bbl_file = re.sub(r'\.tex$', '.bbl', self.inputFileName)
        bbl_xpander = XpandLaTeX(copy=self)
        bbl_xpander.process_file(bbl_file)





    # ###############################################################################
    # float and other environment handling

    envBodyAction = {}
    envUseCounter = {}

    def is_defined_environment(self, token):
        return token in self.envDefinition

    def define_environment(self, type="new"):
        die_msg = r"\%senvironment syntax error: "%type;

        # get environment name
        tok = self.get_token('parsing');
        if not tok == '{': self.die(die_msg + "missing name?")
        name_toks = self.get_tokens_to_match(['}']);
        name = ''.join(name_toks)

        self.debug(r'defining env <%s> via \%senvironment ...' % (name, type));

        # use the latex names for begin and end macros
        begin_macro = '\\' + name
        end_macro = '\\end' + name

        
        #
        # build begin macro
        #

        # get number of parameters

        macro_npar = 0;				# default value
        tok = self.get_token('parsing')
        if tok == '[':
            npar_str = self.get_token('parsing')
            if not re.fullmatch('[1-9]', npar_str): # [] contains non-int
                self.die(die_msg + macro_name + ': bad arg  <' + npar_str + '>'); 
            
            macro_npar = int(npar_str)		# convert to integer (we hope) ...

            if not self.get_token('parsing') == ']':      # [] must close after one token
                self.die(die_msg + 'bad arg format');
        else:
            self.push_tokens([tok])


        # build parameter hash

        macro_delim = {};		# will build up <parameter delimiter hash>

        for i in range (macro_npar + 1):
            par = '#' + str(i);
            macro_delim[par] = [];	# empty delimiter token list


        # get begin macro definition

        tok = self.get_token('parsing')
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        macro_defn = self.get_tokens_to_match(['}']);

        self.debug('\tbegin args:' + str(macro_delim))
        self.debug('\tbegin defn:' + str(macro_defn))

        self.macroParamCount[begin_macro] = macro_npar
        self.macroParamDelim[begin_macro] = macro_delim
        self.macroDefinition[begin_macro] = macro_defn

        

        #
        # build end macro
        #

        # no params
        macro_npar = 0;				# default value
        macro_delim = {'#0': [] };		# will build up <parameter delimiter hash>

        # get macro definition

        tok = self.get_token('parsing')
        if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')

        macro_defn = self.get_tokens_to_match(['}']);

        self.debug('\tend args:' + str(macro_delim))
        self.debug('\tend defn:' + str(macro_defn))

        self.macroParamCount[end_macro] = macro_npar
        self.macroParamDelim[end_macro] = macro_delim
        self.macroDefinition[end_macro] = macro_defn

        
        #
        # get body action
        #

        if type == 'XP':
            tok = self.get_token('parsing')
            if not tok == '{': self.die(die_msg + 'unexpected token <' + tok + '>')
            self.envBodyAction[name] = self.get_tokens_to_match(['}']);
        else:
            self.envBodyAction[name] = ['%XPcopy'] # default to copy

        self.debug('\tbody defn:' + str(self.envBodyAction[name]))
        self.envUseCounter[name] = 0

    

    def process_environment(self):

        # called after \begin is read (if expandMacros)
        
        # read the name
        tok = self.get_token('parsing')
        if not tok == '{': self.die('\begin: unexpected token <' + nx + '>')
        name_toks = self.get_tokens_to_match(['}'])
        name = ''.join(name_toks)
        
        if not name in self.envUseCounter:
            # not a known environment -- return to sender
            self.debug('unknown environment: ' + name)
            return '\\begin{' + name + '}'


        self.envUseCounter[name] += 1 
        self.debug('processing environment: %s (%d)' % (name, self.envUseCounter[name]))


        begin_macro = '\\' + name
        end_macro = '\\end' + name

        end_toks = ['\end', '{']
        end_toks.extend(name_toks)
        end_toks.append('}')

        body = self.get_tokens_to_match(end_toks, mode='all')

        # push tokens in reverse order!
        self.expand_macro(end_macro)

        for tok in self.envBodyAction[name]:
        
            if tok == r'%XPcopy':
                self.push_tokens(body)

            if tok == r'%XPverbatim':
                if self.copyText: self.put_tokens(body)

            if tok == r'%XPdiscard':
                pass

            if tok == r'%XPwritefile':
                filename = '%s_%d.tex' % (name, self.envUseCounter[name])
                file = open(filename, 'w')
                for tok in body:
                    print(tok, end='', file=file)
                file.close()

        self.expand_macro(begin_macro, extra_args={'##':[str(self.envUseCounter[name])]})
    # ###############################################################################

    readMacros = True
    expandMacros = True
    
    readLabels = True
    readLabelPrefix = ''
    expandRefs = True
    
    readExternalLabels = True

    readCites = True
    expandCites = True
    
    copyText = True
    copyComments = False

    readAuxFile = True                  # do read aux files
    mergeInputs = False              # don't read input files 
    mergeTOCs = True                 # do read .toc etc.
    mergeBBL = True             # do read bbl 

    def process_defaults(self, type="tex"):
        if type == "tex":
            pass

        elif type == "macro":
            self.readMacros = True
            self.copyText = False
            self.copyComments = False

            self.readAuxFile = False
            self.mergeInputs = False
            self.mergeTOCs = False
            self.mergeBBL = False

        elif type == "aux":
            self.expandMacros = False

            self.readLabels = True
            self.readLabelPrefix = ''
            self.expandRefs = False

            self.readExternalLabels = False

            self.readCites = True
            self.expandCites = False

            self.copyText = False
            self.copyComments = False

            self.readAuxFile = False
            self.mergeInputs = False
            self.mergeTOCs = False
            self.mergeBBL = False

        elif type == "bbl":
            pass


    def process_tokens(self, read='any'):

        tok=self.get_token(read=read)
        while tok is not None:
    
            # expand macros
            if self.expandMacros and len(tok) and tok[0] == '\\' and self.is_defined_macro(tok):
                self.expand_macro(tok)

                # process environments
            elif self.expandMacros and (tok == r'\begin'):
                tok = self.process_environment()
                if tok is not None and self.copyText: 
                    self.put_token(tok)

                # read macros
            elif self.readMacros and (tok in ['\\newcommand', '\\renewcommand']):
                self.define_macro_newcommand()
            elif self.readMacros and (tok in ['\\def', '\\gdef']):
                self.define_macro_def()

                # read environments
            elif self.readMacros and (tok in ['\\newenvironment', '\\renewenvironment']):
                self.define_environment(type="new")

            elif self.readMacros and (tok == r'\XPenvironment'):
                self.define_environment(type="XP")

                # expand references
            elif self.expandRefs and tok == r'\ref':
                self.expand_ref()
            elif self.expandRefs and tok == r'\subref':
                self.expand_ref('sub@')
        
                # read labels
            elif self.readLabels and  tok == r'\newlabel':
                self.define_label(self.readLabelPrefix)

                # read external label file
            elif self.readExternalLabels and tok == r'\externaldocument':
                self.read_external_aux()

                # expand citations
            elif self.expandCites and re.match(r'\\cite.*', tok):
                cite_type = re.match(r'\\(cite[a-zA-Z]*)', tok).group(1)
                self.expand_cite(cite_type)
                
                # read citations
            elif self.readCites and tok == r'\bibcite':
                self.define_bibcite()
                
                # merge \input files
            elif self.mergeInputs and tok == r'\input':
                self.merge_input()

                # merge bibliography
            elif self.mergeBBL and tok == r'\bibliography':
                self.merge_bbl()

                # merge tableofcontents
            elif self.mergeTOCs and tok == r'\tableofcontents':
                self.merge_tocs('toc')

                # merge listoffigures
            elif self.mergeTOCs and tok == r'\listoffigures':
                self.merge_tocs('lof')

                # merge listoftables
            elif self.mergeTOCs and tok == r'\listoftables':
                self.merge_tocs('lot')

                # %XP tokens
            elif (len(tok) >= 3) and (tok[0:3] == '%XP'):
                if tok == '%XP':        # just '%XP': ignore this token but process rest of line
                    pass
                elif tok == '%XPCUT': 	# gobble tokens to %XPTUC
                    self.debug('processing %XPCUT segment')
                    self.get_tokens_to_match(['%XPTUC'], mode='all')
                elif tok == '%XPVERB':  # copy tokens verbatim to %XPBREV
                    self.debug('processing %XPVERB segment')
                    toks = self.get_tokens_to_match(['%XPBREV'], mode='all')
                    if self.copyText: self.put_tokens(toks)
                # comments
            elif (tok[0] == '%'):
                if self.copyComments: self.put_token(tok)
                
                # anything else
            elif self.copyText:
                self.put_token(tok)

            tok=self.get_token(read=read)




    def process_file(self, filename= "", type="tex"):
        
        if type == "macro":
            self.readMacros = True
            self.copyText = False
            self.copyComments = False
            self.readAuxFile = False

        if type == "aux":
            self.readLabels = True
            self.readCites = True
            self.copyText = False
            self.copyComments = False
            self.readAuxFile = False

        if len(filename): 
            if not self.open_input(filename):
                self.warn('cannot read ' + filename);
                return

        self.debug('Processing file: ' + self.inputFileName)
        self.debug('\t readMacros = ' + str(self.readMacros), 2)
        self.debug('\t expandMacros = ' + str(self.expandMacros), 2)
        self.debug('\t readAuxFile = ' + str(self.readAuxFile), 2)
        self.debug('\t copyText = ' + str(self.copyText), 2)
        

        if self.inputFileName and re.match('.*\\.tex$', self.inputFileName) and self.readAuxFile:
            self.read_aux()
        self.process_tokens(read='any')
        self.close_input()


    def __init__(self, type="tex", copy=None, input_name=""):
        self.process_defaults(type)
        if len(input_name):
            self.open_input(input_name)
        if copy is not None:
            self.readMacros         	= copy.readMacros
            self.expandMacros       	= copy.expandMacros
            self.readLabels         	= copy.readLabels         
            self.readLabelPrefix    	= copy.readLabelPrefix    
            self.expandRefs         	= copy.expandRefs         
            self.readExternalLabels 	= copy.readExternalLabels 
            self.readCites          	= copy.readCites          
            self.expandCites        	= copy.expandCites        
            self.copyText           	= copy.copyText           
            self.copyComments       	= copy.copyComments       
            self.readAuxFile        	= copy.readAuxFile        
            self.mergeInputs	    	= copy.mergeInputs     
            self.mergeTOCs 	    	= copy.mergeTOCs
            self.mergeBBL   	= copy.mergeBBL   
                                    	
            self.outputFileName     	= copy.outputFileName
            self.outputFile         	= copy.outputFile
                                    	
            self.debugLevel         	= copy.debugLevel
            self.debugFile          	= copy.debugFile

            self.update_definitions(copy)

###############################################################################
## MAIN:

argparser = argparse.ArgumentParser(description='expand LaTeX elements', 
                                    formatter_class=argparse.ArgumentDefaultsHelpFormatter)
argparser.add_argument('files', metavar='<file>', nargs='+',
                  help='file to process');


## build up a standard on/off argument format
def onoffkw(default, help):
    kwargs = {'choices':['on', 'off'], 'const':'on', 'nargs':'?', 'metavar':'on|off'}
    kwargs['default'] = default;
    kwargs['help'] = help;
    return kwargs


argparser.add_argument('-x', '--expand-macros', 
                       **onoffkw('on', 'expand macros'))

argparser.add_argument('-r', '--expand-refs',   
                       **onoffkw('off','expand \\refs'))

argparser.add_argument('-b', '--expand-cites',  
                       **onoffkw('off','expand bibliography \\cite, \\citep, \\citet ...'))


argparser.add_argument('-m', '--read-macros', 
                       **onoffkw('off', 'read macro definitions that appear in main file'))


argparser.add_argument('-X', '--expand-all', 
                       **onoffkw(None, 'control all expansion (overrides individual flags'))


argparser.add_argument('-I', '--merge-inputs', 
                       **onoffkw('off', 'merge \\input files'))

argparser.add_argument('-T', '--merge-tocs', 
                       **onoffkw('off', 'merge <name>.{toc,lof,lot}'))

argparser.add_argument('-B', '--merge-bibliography',
                       **onoffkw('off','merge compiled bibliography from <name>.bbl'))

argparser.add_argument('-M', '--merge-all', 
                       **onoffkw(None, 'control all merges (overrides individual flags)'))




argparser.add_argument('-f', '--macro-file', action='append', default=[], metavar="<file>",
                  help='read macros from file')

argparser.add_argument('-D', dest='debug', type=int, default=0, metavar="#",
                  help='debug level')


args = argparser.parse_args()


xp = XpandLaTeX()
xp.debugLevel = args.debug

if args.expand_all is not None:
    xp.expandMacros = (args.expand_all == 'on')
    xp.expandRefs = (args.expand_all == 'on')
    xp.expandCites = (args.expand_all == 'on')
else:
    xp.expandMacros = (args.expand_macros == 'on')
    xp.expandRefs = (args.expand_refs == 'on')
    xp.expandCites = (args.expand_cites == 'on')


xp.readMacros = (args.read_macros == 'on')
xp.readLabels = xp.expandRefs   # read labels iff expanding refs
xp.readExternalLabels = xp.expandRefs
xp.readCites = xp.expandCites   # read \bibcite iff expanding cites

if args.merge_all is not None:
    xp.mergeInputs = (args.merge_all == 'on')
    xp.mergeTOCs = (args.merge_all == 'on')
    xp.mergeBBL = (args.merge_all == 'on')
else:
    xp.mergeInputs = (args.merge_inputs == 'on')
    xp.mergeTOCs = (args.merge_tocs == 'on')
    xp.mergeBBL = (args.merge_bibliography == 'on')


if args.debug:
    print ('expandMacros = ', xp.expandMacros);
    print ('expandRefs = ', xp.expandRefs);
    print ('expandCites = ', xp.expandCites);
    print ('readMacros = ', xp.readMacros);
    print ('mergeInputs = ', xp.mergeInputs);
    print ('mergeTOCs = ', xp.mergeTOCs);
    print ('mergeBBL = ', xp.mergeBBL);



for file in args.macro_file:
    macro_xp = XpandLaTeX(copy=xp)
    macro_xp.process_file(file, type="macro")
    xp.update_definitions(macro_xp)
    
for file in args.files:
    if re.fullmatch(r'.*\.aux', file):
        xp.process_file(file, type="aux")
    elif re.fullmatch(r'.*\.cfg', file):
        xp.process_file(file, type="macro")
    elif re.fullmatch(r'.*\.tex', file):
        xp.process_file(file, type="tex")


             
    
################################################################################################
#
# These notes are copied from the original tme implementation, in case helpful.

#
# # tme has only a limited understanding of TeX.  It handles macro
# # definitions and uses such as
# #
# #	% cat macros.tex
# #	% macro definitions
# # 	\def\h{{\sf h}}
# # 	\def\H{{\sf H}}
# # 	\def\Jac[#1]{{\bf J} \Bigl[ #1 \Bigr]}
# #	\newcommand{\papertitle}[1]{{\it #1\/}}
# #	\renewcommand{\thesistitle}[1]{{\it #1\/}}
# #
# #	% cat input.tex
# #	% text to expand
# # 	$\Jac[\H(\h)]$
# #	\bibitem{Choptuik-91}%%%
# #	M.~W.~Choptuik,
# #	% \papertitle{Consistency of Finite-Difference Solutions
# #	%	      of Einstein's Equations},
# #	Phys.~Rev.~D {\bf 44}(10), 3124--3135 (1991).
# #
# # Notice that:
# # - Nested macros are ok, i.e. macros may reference other macros.
# # - Both TeX's native \def and LaTex's \newcommand/\renewcommand are
# #   supported.
# # - For \def, "delimited" macro parameters (eg for \Jac above) are
# #   supported, as per pp. 202-204 of the TeXbook.
# # - Macro arguments can span multiple lines of text.  (This is hard
# #   to do if you try to fake tme with editor scripts.)
# #
# # For the example just given, the output of tme is
# #
# #	% tme macros.tex <input.tex
# #	% text to expand
# #	${\bf J} \Bigl[ {\sf H}({\sf h}) \Bigr]$
# #	\bibitem{Choptuik-91}%%%
# #	M.~W.~Choptuik,
# #	% \papertitle{Consistency of Finite-Difference Solutions
# #	%	      of Einstein's Equations},
# #	Phys.~Rev.~D {\bf 44}(10), 3124--3135 (1991).
# #
# # or if macros should be expanded in comments (see below),
# #
# #	% tme -c macros.tex <input.tex
# #	% text to expand
# #	${\bf J} \Bigl[ {\sf H}({\sf h}) \Bigr]$
# #	\bibitem{Choptuik-91}%%%
# #	M.~W.~Choptuik,
# #	% {\it Consistency of Finite-Difference Solutions
# #	%	      of Einstein's Equations\/},
# #	Phys.~Rev.~D {\bf 44}(10), 3124--3135 (1991).
# #
# # Among (many) other things, tme does _not_ know about \catcode, any
# # of TeX's variant macro-defining commands \edef, or \xdef, or the \def
# # modifiers \long or \outer.  tme recognizes \gdef, but treats it as
# # identical to \def.  More precisely, tme treats _all_ macros as global.
# # tme makes no attempt to handle conditional macros.  tme allows only
# # letter sequences or single non-letters as macro names.
# #
# # tme discards TeX comments (and anything else except macro definitions)
# # in macro definition files.  When reading text from standard input to
# # be macro-expanded, tme by default transparently copies TeX comments
# # to standard output, but if the  -c  option is specified, tme instead
# # expands macros in comments here as well as elsewhere in the input text.
# #
# # tme doesn't (yet) know about LaTex's \newenvironment/\renewenvironment.
# # I might add this feature to tme in the future.
# #
# # Right now tme doesn't interpret \input in any way.  The "texexpand"
# # program can be used to expand \input files if desired.  I might add
# # this feature to tme in the future.
# #
# # tme's output may contain very long lines.  Right now tme doesn't do
# # anything special about this, but I might add an output-line-splitting
# # option sometime in the future.  (Or I might write a separate program
# # to do it.)
# #
# # Comments, suggestions, improvements, bug fixes, etc, are always welcome.
# #
# 
# #
# # Bugs (user-level):
# # - tme is painfully slow.  On my computer it takes $\sim 5$ times as
# #   long to process a file as LaTeX does to _typeset_ it!  I wrote it
# #   without any particular concern for efficiency: there are lots of
# #   perl array and hash-table copies floating around, and a number of
# #   functions use append-to-scratch-array quadratic algorithms.  But
# #   it's still vastly faster than hacking the macro expansion in a
# #   text editor...
# # - Error checking?  If you have to ask, tme doesn't have enough for you!
# #   tme is easily confused by TeX syntax errors.  Don't try running your
# #   text through tme until TeX is reasonably happy with it (no fatal
# #   errors at least).
# # - tme's handling of blanks in the input is only an approximation to
# #   TeX's.  In particular, there may be problems with macro definitions
# #   where blanks separate \def, the macro name, the macro parameters,
# #   and/or the macro expansion.  Basically, tme ignores blanks following
# #   the \def and the macro name token, but not elsewhere.  The simple case
# #   (with no macro arguments)
# #	\def \foo {blah blah blah}
# #   should thus be ok: tme ignores the blanks, which matches what TeX
# #   seems to do, though not what a naive reading of the TeXbook would
# #   suggest.  (I think TeX ignores the blanks when tokenizing the input;
# #   the TeXbook's description of input tokenizing on pp. 46-47 doesn't
# #   discuss the handling of macro definitions.)
# #   The cases
# #	\def \foo#1 {blah #1 blah}
# #	\def \foo #1 {blah #1 blah}
# #   are accepted, and I believe tme handles them correctly, but note that
# #   "correctly" here involves quite different semantics from the previous
# #   cases: here the blank following the #1 is _not_ ignored, rather it
# #   changes the semantics of how the macro argument is delimited, as
# #   discussed in the TeXbook (pp. 203-204).  For macros defined with
# #   \newcommand or \renewcommand, tme ignores the blanks in any of
# #	\newcommand \foo {blah blah}
# #	\renewcommand \foo {blah blah}
# #	\newcommand \foo [1] {blah #1 blah}
# #	\renewcommand \foo [1] {blah #1 blah}
# # - tme doesn't do anything special for macro definitions appearing in
# #   text to be expanded.  This means that if a macro \foo has been defined,
# #   the text-to-be-expanded
# #	\def\foo{new definition}
# #   won't work properly.  I could fix this if someone cares about it...
# # - Please note the disclaimers of warranty in the (GNU public license)
# #   copyright statement below.  tme works well enough to process my own
# #   papers, but if it mangles your life's masterpiece and costs you the
# #   Nobel prize, Big Contract, Thesis, and/or Job, don't sue me.
# # - Seriously, before submitting tme output for publication I suggest
# #   you check that it's semantically identical to the tme input.  The
# #   easy way to do this is to just TeX/LaTeX both files and compare
# #   the dvi files with 'cmp -l'.  (You could also dvips and compare
# #   the postscript files.)  Except for the dvi timestamps, the files
# #   should be byte-for-byte identical.  If they're not, please report
# #   this to me as a tme bug.
# #
# # Bugs (implementation):
# # - I did _try_ to use the Getopt library routines to parse the argument
# #   list, but they don't seem to be able to handle switches which _don't_
# #   allow a space between the switch and the (optional) argument.  Sigh...
# # - There's a fair bit of duplicated code between  define_macro_via_def()
# #   and  define_macro_via_newcommand() .
# #
# 
# ###############################################################################
# 
# #
# # Copyright (C) 1996, Jonathan Thornburg <thornbur@theory.physics.ubc.ca>
# #
# # This program is free software; you can redistribute it and/or modify
# # it under the terms of the GNU General Public License as published by
# # the Free Software Foundation; either version 2 of the License, or
# # (at your option) any later version.
# #
# # This program is distributed in the hope that it will be useful,
# # but WITHOUT ANY WARRANTY; without even the implied warranty of
# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# # GNU General Public License for more details.
# #
# # You should have received a copy of the GNU General Public License
# # along with this program; see the file COPYING.  If not, write to
# # the Free Software Foundation, 59 Temple Place - Suite 330, Boston,
# # MA 02111-1307, USA.
# #
# 
# ###############################################################################
# 
# #
# # <<<user documentation>>>
# # <<<copyright>>>
# # <<<this table of contents>>>
# # <<<overall design notes>>>
# # <<<design notes for data types>>>
# # main - driver
# #
# # <<<global data for the macro engine>>>
# # is_macro - is a given token defined as a macro?
# # define_macro_via_def - define a macro via \def
# # define_macro_via_newcommand - define a macro via \newcommand or \renewcommand
# # get_macro_args - get a macro's arguments
# # expand_macro - expand a macro
# #
# # match_tokens - match input stream against token list
# # get_tokens_to_delimiter - get tokens until (and including) delimiter
# # get_token_group - get a {}-balanced token group
# # skipBlankTokens - skip blank (actully any whitespace) tokens
# #
# # subarray_eq - are two subarrays element-by-element eq?
# #
# # <<<design notes on comment handling>>
# # <<<global data for the input routines>>>
# # tme_die - die() with input file/line message
# # set_comment_handling - set the comment-handling policy for the input routines
# # open_input_file - open an input file (also updates input-comment handling)
# # close_input_file - close an input file
# # push_tokens - push back a token or tokens into the input stream
# # getToken - get the next (non-comment) token from the input stream
# # get_bare_token - get the next bare-token from the input stream
# #
# # put_token - output a token
# #
# 
# ###############################################################################
# 
# #
# # <<<overall design notes>>>
# #
# 
# #
# # TeX uses a 2-stage processing scheme, where input characters are
# # first tokenized in TeX's "mouth", then macros are expanded on the
# # token stream in TeX's "stomach".  tme follows a similar design,
# # since otherwise we'd have prefix problems, eg macros \e and \end
# # would be confused.
# #
# 
# #
# # The tme macro engine operates by reading the input token stream,
# # and copying it to the output, except that when a previously defined
# # macro is seen, the macro is expanded and the expansion is "pushed back"
# # FIFO-fashion into the input stream, since it might itself contain
# # further macros.
# #
# # Some examples may clarify this:
# #
# #
# # First, consider a simple case:
# #
# # macro defn:	\def\foo{123}
# # input:		\a\foo\b
# #
# # unread input		read tokens	output tokens	action
# # ------------		-----------	-------------	------
# # \a\foo\b
# # \foo\b			\a			read token
# # 					\a		copy token to output
# # \b			\foo				read token
# # 123\b							push macro expansion
# # 							   back into input
# # 			123\b				read tokens
# # 					123\b		copy tokens to output
# #
# #
# # Next, consider a nested macro:
# #
# # macro defns:	\def\h{{\sf h}}
# # 		\def\H{{\sf H}}
# # 		\def\Jac[#1]{{\sf J} \Bigl[ #1 \Bigr]}
# # input:		$\Jac[\H(\h)]$
# #
# # unread input		read tokens	output tokens	action
# # ------------		-----------	-------------	------
# # $\Jac[\H(\h)]$
# # \Jac[\H(\h)]$		$				read token
# # 					$		copy token to output
# # [\H(\h)]$		\Jac				read token
# # \H(\h)]$		[				read macro delimiter
# # 							   token list
# # ]$			\H(\h)				read macro arg tokens
# # $			]				read macro delim token
# # {\sf J} \Bigl[ \H(\h) \Bigr]$				push macro expansion
# # 							   back into input
# # \H(\h) \Bigr]$		{\sf J} \Bigl[ 			read tokens
# # 					{\sf J} \Bigl[	copy tokens to output
# # (\h) \Bigr]$		\H				read token
# # {\sf H}(\h) \Bigr]$					push macro expansion
# # 							   back into input
# # \h) \Bigr]$		{\sf H}(			read tokens
# # 					{\sf H}(	copy tokens to output
# # ) \Bigr]$		\h				read token
# # {\sf h}) \Bigr]$					push macro expansion
# # 							   back into input
# # 			{\sf h}) \Bigr]$		read tokens
# # 					{\sf h}) \Bigr]$ copy tokens to output
# #
# # Notice how pushed-back input is pushed back so as to be read *before*
# # the unread input.
# #
# #
# # Finally, consider a simple macro, invoked with blanks (denoted here by _)
# # separating the parameters:
# #
# # macro defn:	\def\foo#1{#1#1}
# # input:		\a\foo_\x\b
# # unread input		read tokens	output tokens	action
# # ------------		-----------	-------------	------
# # \a\foo_\x\b
# # \foo_\x\b		\a				read token
# # \foo_\x\b				\a		copy token to output
# # _\x\b			\foo				read token
# # \x\b			_				skip over blank token
# # \b			\x				read macro arg token
# # \x\x\b						push macro expansion
# # 							   back into input
# # 			\x\x\b				read tokens
# # 					\x\x\b		copy tokens to output
# #
# # Notice how the blank between \foo and \x was skipped over (discarded)
# # in parsing the macro arguments.
# #
# # Actually, this last example doesn't quite work the way we've just
# # described it.  We implement "skipping over blank tokens" by reading
# # until a non-blank token is found, then pushing that token back into
# # the input, so the example really works like this:
# #
# # macro defn:	\def\foo#1{#1#1}
# # input:		\a\foo_\x\b
# # unread input		read tokens	output tokens	action
# # ------------		-----------	-------------	------
# # \a\foo_\x\b
# # \foo_\x\b		\a				read token
# # \foo_\x\b				\a		copy token to output
# # _\x\b			\foo				read token
# # \x\b			_				read token,
# # 							   see that it's blank,
# # 							   discard it
# # \b			\x				read token
# # \x\b							push token
# # 							   back into input
# # \b			\x				read macro arg token
# # \x\x\b						push macro expansion
# # 							   back into input
# # 			\x\x\b				read tokens
# # 					\x\x\b		copy tokens to output
# #
# 
# ###############################################################################
# 
# #
# # <<<design notes for data types>>>
# #
# 
# #
# # tme uses the following main data types:
# # - A "bare-token" is represented as a (scalar) nonempty perl character
# #   string, the empty string representing EOF.  We don't bother with
# #   TeX's separate character and category codes; we compare tokens as
# #   character strings only.  A bare-token may be a TeX comment.
# # - A "token" (with no qualifiers) is a bare-token, except that comments
# #   may have been screened out.  See the input-routines comments for
# #   details on how comments are handled.
# # - A "token list" is represented as a (possibly empty) perl array of
# #   tokens.
# # - A "token group" is a {}-balanced token list.
# #

# ###############################################################################
# ###############################################################################
# ###############################################################################
# 
# #
# # <<<design notes on comment handling>>
# #
# 
# #
# # The handling of comments is a tricky point in tme:
# # - When we're reading macro-definition files, we want to ignore comments.
# #   In particular, TeX ignores comments in it's "mouth", i.e. when tokenizing
# #   the input, so comments never make it into macro definitions.  We want
# #   to follow this same behavior.
# # - When we're reading text from standard input to be macro-expanded, there
# #   are two cases:
# #   - By default, we want to transparently copy comments to standard output.
# #     output.
# #   - If the -c command line flag is specified, we want to treat comments
# #     just like any other input text, i.e. we want to process their bodies
# #     through our macro engine.
# #
# # We implement this as follows:
# # - When we're reading macro-definition files,  get_bare_token()  returns
# #   comment tokens containing the entire input-stream text text from '%'
# #   through "\n" inclusive;  self.get_token()  then discards these tokens.
# # - When we're reading TeX in which to expand macros:
# #   - By default,  get_bare_token()  continues to return comment tokens
# #     containing the entire input-stream text from '%' through "\n"
# #     inclusive;  self.get_token()  then transparently copies these to the
# #     output stream.
# #   - If the -c command line flag is specified,  get_bare_token()  and
# #      self.get_token()  give no special treatment to '%', it's just treated
# #     as a normal single-character token and passed up into the macro
# #     engine.
# #
# 

















