old_contents: #!/usr/bin/env python
  
  from bs4 import BeautifulSoup
  import csv
  import json
  import os
  import requests
  import sys
  
  #cwd = os.getcwd()  # Get the current working directory (cwd)
  #files = os.listdir(cwd)  # Get all the files in that directory
  #print("Files in '%s': %s" % (cwd, files))
  
  #URI of the data source
  URI = sys.argv[1] if len(sys.argv) > 1 else 'https://arc.lib.montana.edu/ivan-doig/about.php'
  
  def parse_source(uri):
      request = requests.get(uri, headers={'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'})
      #request = requests.get(uri, headers={'User-Agent' : 'jasonclark.info indexing bot'})
  
      #check for HTTP codes other than 200
      if request.status_code != 200:
          print('Status:', request.status_code, 'Problem with the request. Exiting.')
          exit()
  
      soup = BeautifulSoup(request.text, 'html.parser')
      #soup = BeautifulSoup(request, 'html.parser')
      #print soup.findAll('a')
      #print soup.prettify()
      #print(soup.get_text())
  
      pageTitle = soup.title.string
      pageFileName = pageTitle.replace(' ', '-').lower()
      #personName = soup.select(".pv-top-card-section__name")
      #personFileName = personName.replace(' ', '-')
  
      print ('Page Title: \n' + pageTitle)
      #print ('Person: \n' + personName)
      print ('Page URL: \n' + uri)
  
      #set empty list for about json values
      skillList = [] 
  
      for link in soup.find_all(class_='gsc_prf_ila'):
          tagValue = link.string.strip('\r\n\t')
          print('skill data: \n' + tagValue)
          skillList.append({"skill": tagValue, "length": len(tagValue)})
      
      #create json file if it doesn't exist, open and write parsed values into it
      if not os.path.exists(pageTitle+'-skills.json'):
          open(pageFileName+'-skills.json', 'w').close()
      
      with open(pageFileName+'-skills.json', 'r+') as jsonFile:
          json.dump(skillList, jsonFile, indent = 4)
  
      jsonFile.close()
  
      #create csv file if it doesn't exist, open and write parsed values into it
      if not os.path.exists(pageTitle+'-skills.csv'):
          open(pageFileName+'-skills.csv', 'w').close()
  
      with open(pageFileName+'-skills.csv', 'r+') as csvFile:
          writeFile = csv.writer(csvFile)
          writeFile.writerow(skillList)
  
      csvFile.close()
  
  showResult = parse_source(URI)
  print showResult
  print 'JSON and CSV files created.'
  
new_contents: #!/usr/bin/env python
  
  from bs4 import BeautifulSoup
  import csv
  import json
  import os
  import requests
  import sys
  
  #cwd = os.getcwd()  # Get the current working directory (cwd)
  #files = os.listdir(cwd)  # Get all the files in that directory
  #print("Files in '%s': %s" % (cwd, files))
  
  #URI of the data source
  URI = sys.argv[1] if len(sys.argv) > 1 else 'https://scholar.google.com/citations?user=2vZSuAwAAAAJ'
  
  def parse_source(uri):
      request = requests.get(uri, headers={'User-Agent' : 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36'})
      #request = requests.get(uri, headers={'User-Agent' : 'jasonclark.info indexing bot'})
  
      #check for HTTP codes other than 200
      if request.status_code != 200:
          print('Status:', request.status_code, 'Problem with the request. Exiting.')
          exit()
  
      soup = BeautifulSoup(request.text, 'html.parser')
      #soup = BeautifulSoup(request, 'html.parser')
      #print soup.findAll('a')
      #print soup.prettify()
      #print(soup.get_text())
  
      pageTitle = soup.title.string
      pageFileName = pageTitle.replace(' ', '-').lower()
      #personName = soup.select(".pv-top-card-section__name")
      #personFileName = personName.replace(' ', '-')
  
      print ('Page Title: \n' + pageTitle)
      #print ('Person: \n' + personName)
      print ('Page URL: \n' + uri)
  
      #set empty list for about json values
      skillList = [] 
  
      for link in soup.find_all(class_='gsc_prf_ila'):
          tagValue = link.string.strip('\r\n\t')
          print('skill data: \n' + tagValue)
          skillList.append({"skill": tagValue, "length": len(tagValue)})
      
      #create json file if it doesn't exist, open and write parsed values into it
      if not os.path.exists(pageTitle+'-skills.json'):
          open(pageFileName+'-skills.json', 'w').close()
      
      with open(pageFileName+'-skills.json', 'r+') as jsonFile:
          json.dump(skillList, jsonFile, indent = 4)
  
      jsonFile.close()
  
      #create csv file if it doesn't exist, open and write parsed values into it
      if not os.path.exists(pageTitle+'-skills.csv'):
          open(pageFileName+'-skills.csv', 'w').close()
  
      with open(pageFileName+'-skills.csv', 'r+') as csvFile:
          writeFile = csv.writer(csvFile)
          writeFile.writerow(skillList)
  
      csvFile.close()
  
  showResult = parse_source(URI)
  print showResult
  print 'JSON and CSV files created.'
  
subject: set default URL to harvest