import scrapy


class ExpertSpider(scrapy.Spider):
    name = "expert"
    download_delay = 5
    start_urls = [
        # MIT Faculty & Advisors II - Computer Science (Theory)
        'http://people.csail.mit.edu/bab/',
        'http://people.csail.mit.edu/costis',
        'http://erikdemaine.org/',
        'https://www.csail.mit.edu/person/shafi-goldwasser',
        'http://people.csail.mit.edu/indyk',
        'http://people.csail.mit.edu/karger',
        'http://supertech.csail.mit.edu/~cel/',
        'http://theory.csail.mit.edu/~lynch/',
        'http://people.csail.mit.edu/madry/',
        'http://www.csail.mit.edu/user/1327',
        'http://www.csail.mit.edu/user/993',
        'http://www.csail.mit.edu/user/994',
        'http://www.csail.mit.edu/user/1294',
        'http://people.csail.mit.edu/ronitt/',
        'http://www.cs.tau.ac.il/~shanir/',
        'http://people.csail.mit.edu/jsolomon/',
        'http://people.csail.mit.edu/vinodv/',
        'http://www.rle.mit.edu/people/directory/jacob-white/',
        'http://www.csail.mit.edu/user/5000',
        'http://people.csail.mit.edu/rrw/',
        # MIT Faculty & Advisors II - Computer Science (Systems)
        'http://people.csail.mit.edu/alizadeh/',
        'http://www.csail.mit.edu/user/693',
        'http://www.csail.mit.edu/user/849',
        'http://nms.csail.mit.edu/~hari',
        'https://www.csail.mit.edu/person/adam-belay',
        'http://www.w3.org/People/Berners-Lee',
        'http://people.csail.mit.edu/mcarbin/',
        'http://adam.chlipala.net/',
        'http://www.csail.mit.edu/user/1526',
        'http://larch-www.lcs.mit.edu:8001/~corbato/',
        'http://www.csail.mit.edu/user/1552',
        'http://people.csail.mit.edu/devadas',
        'http://people.csail.mit.edu/emer',
        'http://people.csail.mit.edu/ghobadi/',
        'http://people.csail.mit.edu/dnj',
        'https://pdos.csail.mit.edu/~kaashoek/',
        'http://people.csail.mit.edu/dina',
        'http://research.microsoft.com/Lampson/',
        'http://www.pmg.csail.mit.edu/~liskov',
        'http://db.csail.mit.edu/madden',
        'http://www.mit.edu/~rcm',
        'http://www.csail.mit.edu/user/972',
        'http://projects.csail.mit.edu/cgi-bin/wiki/view/LSPgroup/WebHome',
        'http://www.csail.mit.edu/user/724',
        'http://people.csail.mit.edu/sanchez',
        'http://people.csail.mit.edu/asolar/',
        'http://www.csail.mit.edu/user/1547',
        'http://www.rle.mit.edu/people/directory/vivienne-sze/',
        'http://people.csail.mit.edu/cjt/',
        'http://www.csail.mit.edu/user/1517',
        'http://people.csail.mit.edu/nickolai/',
        # MIT Faculty & Advisors II - Computer Science (Artificial Intelligence)
        'http://www.csail.mit.edu/user/1535',
        'http://people.csail.mit.edu/regina/',
        'http://lids.mit.edu/people/faculty/berwick-robert.html',
        'http://people.csail.mit.edu/tbroderick',
        'http://people.csail.mit.edu/~brooks',
        'http://people.csail.mit.edu/people/davis/',
        'http://people.csail.mit.edu/fredo/',
        'https://billf.mit.edu/',
        'http://www.csail.mit.edu/user/989',
        'http://people.csail.mit.edu/welg/',
        'http://nms.csail.mit.edu/~guttag/',
        'http://csail.mit.edu/~bkph',
        'http://people.csail.mit.edu/tommi/tommi.html',
        'http://people.csail.mit.edu/stefje',
        'http://people.csail.mit.edu/people/lpk',
        'http://compbio.mit.edu/',
        'https://www.csail.mit.edu/person/andrew-lo',
        'http://people.csail.mit.edu/tlp',
        'http://people.csail.mit.edu/wojciech/',
        'http://www.csail.mit.edu/user/1000',
        'http://people.csail.mit.edu/rus',
        'http://lids.mit.edu/people/faculty/shah-devavrat.html',
        'http://www.csail.mit.edu/user/1512',
        'http://www.csail.mit.edu/user/1513',
        'http://people.csail.mit.edu/russt/',
        'http://mit.edu/tidor',
        'http://people.csail.mit.edu/torralba',
        'http://people.csail.mit.edu/rweiss/',
        'http://lids.mit.edu/people/faculty/willsky-alan.html',
        'http://people.csail.mit.edu/people/phw/phw.html',
        'http://www.csail.mit.edu/user/1522',
        # Stanford Regular Faculty - Computer Science
        'http://graphics.stanford.edu/~maneesh',
        'http://theory.stanford.edu/~aiken',
        'http://bailis.org/',
        'http://theory.stanford.edu/~barrett',
        'http://bejerano.stanford.edu/',
        'http://hci.stanford.edu/msb/',
        'http://www.stanford.edu/~dabo/',
        'http://cs.cmu.edu/~ebrun',
        'https: // profiles.stanford.edu / moses - charikar',
        'http://cva.stanford.edu/billd_webpage_new.html',
        'http://cs.stanford.edu/people/rondror/',
        'https://zakird.com/',
        'http://www.stanford.edu/~engler/',
        'http://graphics.stanford.edu/~kayvonf',
        'http://physbam.stanford.edu/~fedkiw',
        'http://logic.stanford.edu/people/genesereth/',
        'http://stanford.edu/~ngoodman/',
        'http://geometry.stanford.edu/member/guibas/index.html',
        'http://www-graphics.stanford.edu/~hanrahan/',
        'https://web.stanford.edu/~hennessy/',
        'http://www-vlsi.stanford.edu/~horowitz/',
        'http://graphics.stanford.edu/~djames/',
        'http://www.stanford.edu/~jurafsky/',
        'http://www.stanford.edu/~skatti/',
        'http://robotics.stanford.edu/~ok/',
        'http://www.stanford.edu/~kozyraki',
        'http://anshul.kundaje.net/',
        'http://suif.stanford.edu/~lam',
        'https://profiles.stanford.edu/james-landay',
        'http://cs.stanford.edu/people/jure',
        'http://csl.stanford.edu/~pal',
        'http://ai.stanford.edu/~tengyuma',
        'http://nlp.stanford.edu/~manning/',
        'http://www.scs.stanford.edu/~dm/',
        'http://ee.stanford.edu/~nickm',
        'http://theory.stanford.edu/people/jcm/home.html',
        'http://www.stanford.edu/~subh/',
        'http://ogun.stanford.edu/~kunle/',
        'http://www.stanford.edu/~ouster',
        'http://www.stanford.edu/~balaji',
        'http://cs.stanford.edu/people/chrismre',
        'https://omereingold.wordpress.com/',
        'http://www.stanford.edu/~mendel',
        'http://theory.stanford.edu/~tim',
        'https://dorsa.fyi/',
        'http://robotics.stanford.edu/users/sahami/bio.html',
        'http://cvgl.stanford.edu/silvio/',
        'http://theory.stanford.edu/~liyang/',
        'http://theory.stanford.edu/~valiant/',
        'http://infolab.stanford.edu/~widom',
    ]

    def parse(self, response):
        page = response.url
        name = response.css('head title::text').extract_first()
        details = response.css('body').extract()
        yield {
            'page': page,
            'name': name,
            'details': details,
        }

