old_contents: # Scrapy settings for general_spider project
  #
  # For simplicity, this file contains only the most important settings by
  # default. All the other settings are documented here:
  #
  #     http://doc.scrapy.org/en/latest/topics/settings.html
  #
  
  import sys
  import os
  from os.path import dirname
  path = dirname(dirname(os.path.abspath(os.path.dirname(__file__))))
  sys.path.append(path)
  from misc.log import *
  
  BOT_NAME = 'general_spider'
  
  SPIDER_MODULES = ['general_spider.spiders']
  NEWSPIDER_MODULE = 'general_spider.spiders'
  
  # Crawl responsibly by identifying yourself (and your website) on the user-agent
  #USER_AGENT = 'general_spider (+http://www.yourdomain.com)'
  
  DOWNLOADER_MIDDLEWARES = {
     # 'misc.middleware.CustomHttpProxyMiddleware': 400,
      'misc.middleware.CustomUserAgentMiddleware': 401,
  }
  
  ITEM_PIPELINES = {
      'general_spider.pipelines.JsonWithEncodingPipeline': 300,
      #'general_spider.pipelines.TXTWithEncodingPipeline': 301,
      #'general_spider.pipelines.RedisPipeline': 301,
  }
  
  LOG_LEVEL = 'INFO'
  
  DOWNLOAD_DELAY = 1
  DOWNLOAD_TIMEOUT=5
  
new_contents: # Scrapy settings for general_spider project
  #
  # For simplicity, this file contains only the most important settings by
  # default. All the other settings are documented here:
  #
  #     http://doc.scrapy.org/en/latest/topics/settings.html
  #
  
  import sys
  import os
  from os.path import dirname
  path = dirname(dirname(os.path.abspath(os.path.dirname(__file__))))
  sys.path.append(path)
  from misc.log import *
  
  BOT_NAME = 'general_spider'
  
  SPIDER_MODULES = ['general_spider.spiders']
  NEWSPIDER_MODULE = 'general_spider.spiders'
  
  # Crawl responsibly by identifying yourself (and your website) on the user-agent
  #USER_AGENT = 'general_spider (+http://www.yourdomain.com)'
  
  DOWNLOADER_MIDDLEWARES = {
     # 'misc.middleware.CustomHttpProxyMiddleware': 400,
      'misc.middleware.CustomUserAgentMiddleware': 401,
  }
  
  ITEM_PIPELINES = {
      'general_spider.pipelines.JsonWithEncodingPipeline': 300,
      'general_spider.pipelines.TXTWithEncodingPipeline': 301,
      #'general_spider.pipelines.RedisPipeline': 301,
  }
  
  LOG_LEVEL = 'INFO'
  
  DOWNLOAD_DELAY = 1
  DOWNLOAD_TIMEOUT=5
  
subject: Add pipiline.