old_contents: 
  from itertools import product
  from collections import defaultdict
  
  
  from scipy.spatial.distance import euclidean
  import pulp
  
  
  def tokens_to_fracdict(tokens):
      cntdict = defaultdict(lambda : 0)
      for token in tokens:
          cntdict[token] += 1
      totalcnt = sum(cntdict.values())
      return {token: float(cnt)/totalcnt for token, cnt in cntdict.items()}
  
  
  # use PuLP
  def word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=None):
      all_tokens = list(set(first_sent_tokens+second_sent_tokens))
      wordvecs = {token: wvmodel[token] for token in all_tokens}
  
      first_sent_buckets = tokens_to_fracdict(first_sent_tokens)
      second_sent_buckets = tokens_to_fracdict(second_sent_tokens)
  
      T = pulp.LpVariable.dicts('T_matrix', list(product(all_tokens, all_tokens)), lowBound=0)
  
      prob = pulp.LpProblem('WMD', sense=pulp.LpMinimize)
      prob += pulp.lpSum([T[token1, token2]*euclidean(wordvecs[token1], wordvecs[token2])
                          for token1, token2 in product(all_tokens, all_tokens)])
      for token2 in second_sent_buckets:
          prob += pulp.lpSum([T[token1, token2] for token1 in first_sent_buckets])==second_sent_buckets[token2]
      for token1 in first_sent_buckets:
          prob += pulp.lpSum([T[token1, token2] for token2 in second_sent_buckets])==first_sent_buckets[token1]
  
      if lpFile!=None:
          prob.writeLP(lpFile)
  
      prob.solve()
  
      return prob
  
  
  def word_mover_distance(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=None):
      prob = word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, lpFile=lpFile)
      return pulp.value(prob.objective)
new_contents: 
  from itertools import product
  from collections import defaultdict
  
  
  from scipy.spatial.distance import euclidean
  import pulp
  
  
  def tokens_to_fracdict(tokens):
      cntdict = defaultdict(lambda : 0)
      for token in tokens:
          cntdict[token] += 1
      totalcnt = sum(cntdict.values())
      return {token: float(cnt)/totalcnt for token, cnt in cntdict.items()}
  
  
  # use PuLP
  def word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel, distancefunc=euclidean, lpFile=None):
      all_tokens = list(set(first_sent_tokens+second_sent_tokens))
      wordvecs = {token: wvmodel[token] for token in all_tokens}
  
      first_sent_buckets = tokens_to_fracdict(first_sent_tokens)
      second_sent_buckets = tokens_to_fracdict(second_sent_tokens)
  
      T = pulp.LpVariable.dicts('T_matrix', list(product(all_tokens, all_tokens)), lowBound=0)
  
      prob = pulp.LpProblem('WMD', sense=pulp.LpMinimize)
      prob += pulp.lpSum([T[token1, token2]*distancefunc(wordvecs[token1], wordvecs[token2])
                          for token1, token2 in product(all_tokens, all_tokens)])
      for token2 in second_sent_buckets:
          prob += pulp.lpSum([T[token1, token2] for token1 in first_sent_buckets])==second_sent_buckets[token2]
      for token1 in first_sent_buckets:
          prob += pulp.lpSum([T[token1, token2] for token2 in second_sent_buckets])==first_sent_buckets[token1]
  
      if lpFile!=None:
          prob.writeLP(lpFile)
  
      prob.solve()
  
      return prob
  
  
  def word_mover_distance(first_sent_tokens, second_sent_tokens, wvmodel, distancefunc=euclidean, lpFile=None):
      prob = word_mover_distance_probspec(first_sent_tokens, second_sent_tokens, wvmodel,
                                          distancefunc=distancefunc, lpFile=lpFile)
      return pulp.value(prob.objective)
subject: allow flexible distance metric