new_contents: #! /usr/bin/env python
  
  """Processes the Gigaword Corpus (https://catalog.ldc.upenn.edu/LDC2011T07).
  Generates source and target files where sources correspond to the first N
  sentences of each article, and targets corresponds to article headlines.
  
  Usage:
  
  bin/data/gigaword_summary < GIGAWORD \
    -f gigaword_data
    -o output/
  """
  
  from __future__ import absolute_import
  from __future__ import division
  from __future__ import print_function
  from __future__ import unicode_literals
  
  import sys
  import os
  import argparse
  
  from tensorflow.python.platform import gfile
  from bs4 import BeautifulSoup
  
  PARSER = argparse.ArgumentParser(
      description="Processes the Gigaword corpus.")
  PARSER.add_argument(
      "-f", "--file", type=str, required=True,
      help="path to the Gigaword SGML file")
  PARSER.add_argument(
      "-o", "--output_dir", type=str, required=True,
      help="path to the output directory")
  PARSER.add_argument(
      "-n", "--num_sentences", type=str, required=False, default=2,
      help="Use the first N sentences as source text")
  ARGS = PARSER.parse_args()
  
  def gigaword_iter(path, n_sentences=2):
    """Creates an iterator that yields (source, target) tuples.
    """
    soup = BeautifulSoup(open(path))
    for doc in soup.find_all("doc"):
      # Skip docs without headline
      if doc.headline is None:
        continue
      # Find first N sentences
      sentences = doc.find_all("p")[:n_sentences]
      if not sentences:
        continue
      sentences = [_.text.strip().replace("\n", "") for _ in sentences]
      # Returns sentencs and headline
      yield " ".join(sentences), doc.headline.text.strip()
  
  def main():
    """The entrypoint for the script"""
  
    gfile.MakeDirs(ARGS.output_dir)
  
    sources_filename = os.path.join(ARGS.output_dir, "sources.txt")
    targets_filename = os.path.join(ARGS.output_dir, "targets.txt")
    sources_file = gfile.GFile(sources_filename, "w")
    targets_file = gfile.GFile(targets_filename, "w")
  
    records = gigaword_iter(ARGS.file, ARGS.num_sentences)
    for i, (source, target) in enumerate(records):
      sources_file.write(source + "\n")
      targets_file.write(target + "\n")
      if i % 1000 == 0:
        sys.stderr.write(".")
      if i % 100000 == 0:
        sys.stderr.write("\n")
  
    sources_file.close()
    targets_file.close()
  
  if __name__ == "__main__":
    main()
  
subject: Add Gigaword summarization preprocessing script