old_contents: # Load the on-time parquet file
  on_time_dataframe = spark.read.parquet('data/on_time_performance.parquet')
  
  # The first step is easily expressed as SQL: get all unique tail numbers for each airline
  on_time_dataframe.registerTempTable("on_time_performance")
  carrier_airplane = spark.sql(
    "SELECT DISTINCT Carrier, TailNum FROM on_time_performance"
    )
  
  # Now we need to store a sorted group for each Carrier, along with a fleet count
  airplanes_per_carrier = carrier_airplane.rdd\
    .map(lambda nameTuple: (nameTuple[0], [nameTuple[1]]))\
    .reduceByKey(lambda a, b: a + b)\
    .map(lambda tuple:
        {
          'Carrier': tuple[0], 
          'TailNumbers': sorted(
            filter(
              lambda x: x != '', tuple[1] # empty string tail numbers were getting through
              )
            ),
          'FleetCount': len(tuple[1])
        }
      )
  airplanes_per_carrier.count() # 14
  
  # Save to Mongo in the airplanes_per_carrier relation
  import pymongo_spark
  pymongo_spark.activate()
  airplanes_per_carrier.saveToMongoDB(
    'mongodb://localhost:27017/agile_data_science.airplanes_per_carrier'
  )
  
new_contents: # Load the on-time parquet file
  on_time_dataframe = spark.read.parquet('data/on_time_performance.parquet')
  
  # The first step is easily expressed as SQL: get all unique tail numbers for each airline
  on_time_dataframe.registerTempTable("on_time_performance")
  carrier_airplane = spark.sql(
    "SELECT DISTINCT Carrier, TailNum FROM on_time_performance"
    )
  
  # Now we need to store a sorted group for each Carrier, along with a fleet count
  airplanes_per_carrier = carrier_airplane.rdd\
    .map(lambda nameTuple: (nameTuple[0], [nameTuple[1]]))\
    .reduceByKey(lambda a, b: a + b)\
    .map(lambda tuple:
        {
          'Carrier': tuple[0], 
          'TailNumbers': sorted(
            filter(
              lambda x: x is not None and x != '', tuple[1] # empty string tail numbers were getting through
              )
            ),
          'FleetCount': len(tuple[1])
        }
      )
  airplanes_per_carrier.count() # 14
  
  # Save to Mongo in the airplanes_per_carrier relation
  import pymongo_spark
  pymongo_spark.activate()
  airplanes_per_carrier.saveToMongoDB(
    'mongodb://localhost:27017/agile_data_science.airplanes_per_carrier'
  )
  
subject: Check variable for None value before null string when filtering tail numbers