old_contents: 
  from itertools import groupby
  
  from nltk.translate.gleu_score import sentence_gleu as gleu
  from nltk.tokenize.toktok import ToktokTokenizer
  
  _tokenize = ToktokTokenizer().tokenize
  
  def select_first(p):
      """
      Return (hypothesis, reference) translation pairs using the first
      realization result per item.
      """
      pairs = []
      rows = p.join('item', 'g-result')
      for i_id, group in groupby(rows, key=lambda row: row['g-result:i-id']):
          row = next(group)
          pairs.append((row['g-result:surface'], row['item:i-translation']))
      return pairs
  
  
  def select_oracle(p):
      """
      Return (hypothesis, reference) translation pairs using the
      realization result per item with the highest GLEU score.
      """
      pairs = []
      rows = p.join('item', 'g-result')
      for i_id, group in groupby(rows, key=lambda row: row['g-result:i-id']):
          hrs = ((r['g-result:surface'], r['item:i-translation']) for r in group)
          ranked = [(gleu(_tokenize(r), _tokenize(h)), h, r) for h, r in hrs]
          _, hyp, ref = sorted(ranked, key=lambda r: r[0])[-1]
          pairs.append((hyp, ref))
      return pairs
  
new_contents: 
  from itertools import groupby
  
  from nltk.translate import bleu_score
  from nltk.tokenize.toktok import ToktokTokenizer
  
  _tokenize = ToktokTokenizer().tokenize
  _smoother = bleu_score.SmoothingFunction().method3
  bleu = bleu_score.sentence_bleu
  
  def select_first(p):
      """
      Return (hypothesis, reference) translation pairs using the first
      realization result per item.
      """
      pairs = []
      rows = p.join('item', 'g-result')
      for i_id, group in groupby(rows, key=lambda row: row['g-result:i-id']):
          row = next(group)
          pairs.append((row['g-result:surface'], row['item:i-translation']))
      return pairs
  
  
  def select_oracle(p):
      """
      Return (hypothesis, reference) translation pairs using the
      realization result per item with the highest GLEU score.
      """
      pairs = []
      rows = p.join('item', 'g-result')
      for i_id, group in groupby(rows, key=lambda row: row['g-result:i-id']):
          scored = []
          for res in group:
              ref = res['item:i-translation']
              hyp = res['g-result:surface']
              scored.append(
                  (bleu([_tokenize(ref)], _tokenize(hyp),
                        smoothing_function=_smoother), hyp, ref)
              )
          _, hyp, ref = sorted(scored, key=lambda r: r[0])[-1]
          pairs.append((hyp, ref))
      return pairs
  
subject: Use NIST-BLEU instead of GLEU for oracle.