from scrapy.crawler import CrawlerProcess
from cricos.spiders.cricos import CricosSpider  # todo: why this not work?
# from cricos.spiders.cricos import CricosSpider
from scrapy.utils.project import get_project_settings

if __name__ == '__main__':
    process = CrawlerProcess({
        "USER_AGENT": "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36"
    })

    # class NewSpider(CricosSpider):
    spider = CricosSpider
    process.crawl(CricosSpider, myurls=[
        'http://cricos.education.gov.au/Institution/InstitutionDetailsOnePage.aspx?ProviderID=1'
    ])
    process.start()
