# -*- coding: utf-8 -*-
import random
import os,db
import time,requests
from joke import *

base = "https://www.ncbi.nlm.nih.gov/pmc/?term=("
keys = [
# "Jennifer+Linden",
# "Troy+Margrie",
# "Professor+Patricia+Salinas",
# "Clare+Stanford",
# "John+Wood",
# "Sophie+Acton",
# "Buzz+Baum",
# "Jonathan+Chubb",
# "Daniel+Cutler",
# "Rob+De+Bruin",
# "Paul+Gissen",
# "Ricardo+Henriques",
# "Robin+Ketteler",
# "Alison+Lloyd",
# "Yanlan+Mao",
# "Mark+Marsh",
# "Jason+Mercer",
# "Sara+Mole",
# "Ewa+Paluch",
# "Franck+Pichaud",
# "Antonella+Riccio",
# "Adolfo+Saiardi",
# "Chris+Stefan",
# "Louise+Cramer",
# "Julie+Pitcher",
# "Nathan+Goehring",
# "David+Ish-Horowicz",
# "Martin+Raff",
# "Graham+Warren",
# "Alessandro+Bossio",
# "Alexander+Agrotis",
# "Apostolos+Papandreou",
# "Betheney+Pennycook",
# "Caron+Jacobs",
# "Christian+Covill-Cooke",
# "Christopher+Minnis",
# "Clare+Rogerson",
# "Corina+Beerli",
# "Davide+Cassani",
# "Edward+Tunnacliffe",
# "Francisca+Nunes+De+Almeida",
# "Gabriel+Tarrason+Risa",
# "Gemma+Wilson",
# "Georgina+Kontou",
# "Hamish+Crerar",
# "Harriet+Mok",
# "Henry+De+Belly",
# "Kalina+Tosheva",
# "Katherine+Massey",
# "Laura+Blackie",
# "Lindsey+Millward",
# "Liza+Malong",
# "Melanie+Krause",
# "Natalie+Kirkland",
# "Noelia+Garcia+Calavia",
# "Olukunbi+Mosaku",
# "Robert+Gray",
# "Sarah+Aldous",
# "Sophie+Cooke",
# "Jerzy+Samolej",
# "Agathe+Chaigne",
# "Catia+Andreassi",
# "Cristina+Azevedo",
# "Dani+Bodor",
# "David+Albrecht",
# "Delyan+Mutavchiev",
# "Eva+Vesela",
# "Irene+Aspalter",
# "Lotte+de+Winde",
# "Michela+Mazzon",
# "Moona+Huttunen",
# "Murielle+Serres",
# "Nargess+Khalilgharibi",
# "Nitya+Ramkumar",
# "Pedro+Pereira",
# "Rob+Tetley",
# "Romain+Laine",
# "Rosalba+Camicia",
# "Sian+Culley",
# "Taki+Nishimura",
# "Victor+Garcia+Martinez",
# "Yann+Desfougeres",
# "Tom+Mrsic-Flogel",
# "Alexandra+Boss",
# "Troy+Margrie",
# "Troy+Margrie",
# "Tiago+Branco",
# "Sonja+Hofer",
# "Yoh+Isogai",
# "Adam+Kampff",
# "John+O'Keefe",
# "Troy+Margrie",
# "Thomas+Mrsic-Flogel",
# "Andy+Murray",
# "Marcus+Stephenson-Jones",
# "Zara+Allardyce",
# "Marius+Bauza",
# "Edward+Bracey",
# "Stephen+Burton",
# "Dario+Campagner",
# "Rob+Campbell",
# "Daniele+Carassiti",
# "Francois+Chabrol",
# "Kelly+Clancy",
# "Lee+Cossell",
# "George+Dimitriadis",
# "Mathew+Edwards",
# "Dominic+Evans",
# "Shohei+Furutachi",
# "Ioana+Gasler",
# "Borgel+Greenaway",
# "Panagiota+Iordanidou",
# "Takahiro+Kanamori",
# "Loukia+Katsouri",
# "Sepiedeh+Keshavarzi",
# "Johannes+Kohl",
# "Yaara+Lefler",
# "Fred+Marbach",
# "Andre+Marques-Smith",
# "Hernando+Martinez",
# "Cristina+Mazuski",
# "Joana+Nogueira",
# "Lennart+Oettl",
# "Jake+Ormond",
# "Zinaida+Perova",
# "Jasper+Poort",
# "Chris+Puhl",
# "Emily+Reader-Harris",
# "Charly+Rousseau",
# "Vanessa+Stempel",
# "Molly+Strom",
# "Chryssanthi+Tsitoura",
# "Victoria+Tung",
# "Mateo+Velez-Fort",
# "Ruth+Wood",
# "Petr+Znamenskiy",
# "Antonin+Blot",
# "Laura+Allum",
# "Samuel+Atack",
# "Mark+Bartlett",
# "Emma+Bennett",
# "Ian+Blaney",
# "Alexandra+Boss",
# "April+Cashin-Garbutt",
# "Mark+Dixon",
# "Yanai+Duran",
# "Brian+Fenelon",
# "Karen+Fergus",
# "Rose+Haigh",
# "Derek+Halpin",
# "Michelle+Li",
# "Graeme+McPhillips",
# "Tina+O'Mahony",
# "David+Smith",
# "Martyn+Stopps",
# "Janet+Watson",
# "IT+Team",
# "Lorenza+Calcaterra",
# "Federico+Claudi",
# "Ruben+do+Vale",
# "Alex+Fratzl",
# "Jesse+Geerts",
# "Francesca+Greenstreet",
# "Mitra+Javadzadeh+No",
# "Danbee+Kim",
# "Anna+Lebedeva",
# "Chiyu+Lee",
# "Stephen+Lenzi",
# "Egzona+Morina",
# "Joana+Neto",
# "Yuuki+Obata",
# "Ivana+Orsolic",
# "Oriol+Pavon+Arocas",
# "Matthew+Phillips",
# "Daniel+Regester",
# "Sara+Rolle-Sonora",
# "Raja+Sekhar+Mannava",
# "Philip+Shamash",
# "Simon+Thompson",
# "Ivan+Voitov",
# "Ruth+Wood",
# "Natalia+Wroblewska",
# "Daniel+Raleigh",
# "Suzanne+Ruddy",
# "Christopher+Taylorson",
# "Jeremy+Brockes",
# "Frances+Brodsky",
# "Kristine+Arnvig",
# "Emmanuel+Boucrot",
# "Kate+Bowers",
# "Jeremy+Brockes",
# "Frances+Brodsky",
# "Alan+Cheung",
# "John+Christodoulou",
# "Snezana+Djordjevic",
# "Francesco+Luigi+Gervasio",
# "Ivan+Gout",
# "Flemming+Hansen",
# "David+Jones",
# "Matilda+Katan",
# "Alan+Lowe",
# "Andrew+Martin",
# "Sylvia+Nagl",
# "Christine+Orengo",
# "Stephen+Perkins",
# "Vitor+Bernardes+Pinheiro",
# "Saul+Purton",
# "Andres+Ramos",
# "Peter+Rich",
# "Joanne+Santini",
# "Elizabeth+Shephard",
# "Surjit+Kaila+Singh+Srai",
# "Konstantinos+Thalassinos",
# "Andrea+Townsend-Nicholson",
# "Cara+Vaughan",
# "Gabriel+Waksman",
# "Finn+Werner",
# "Henning+Walczak",
# "Sibylle+Mittnacht",
# "Hiro+Yamano",
# "Pablo+Rodriguez-Viciana",
# "Kazunori+Tomita",
# "Steen+Ooi",
# "Suzana+Hadjur",
# "Stephan+Beck",
# "Ivana+Bjedov",
# "Richard+Jenner",
# "Simona+Parrinello",
# "Simona+Parrinello",
# "Tariq+Enver",
# "Alexander+Hergovich",
# "Elspeth+Payne",
# "Adele+K",
# "Clare+Bennett",
# "Sergio+A+Quezada",
# "Marc+R+Mansour",
# "Kwee+Yong",
# "Gitendra+Wickramasinghe",
# "Surinder+Sharma",
# "Tim+Meyer",
# "Nicholas+McGranahan",
# "John+HartleyProfessor+Daniel+Hochhauser",
# "Chris+Tape",
# "Bart+Vanhaesebroeck,+PhD+FMedSci",
# "Elina+Vladimirou",
# "Paul+Mulholland",
# "Martin+Forster",
# "Tim+Fenton",
# "Barbara+Pedley",
# "Ricky+Sharma",
# "Gerhardt+Attard",
# "Mark+Linch",
# "Adrienne+Flanagan",
# "Teresa+Marafioti",
# "David+Moore",
# "Nischalan+Pillay",
# "Paul+Crampton",
# "Jane+Dacre",
# "Deborah+Gill",
# "Ann+Griffin",
# "Jayne+Kavanagh",
# "Laura+Knight",
# "Chris+McManus",
# "Miss+Asta+Medisauskaite",
# "Leila+Mehdizadeh",
# "Lorraine+Mary+Noble",
# "Antonia+Rich",
# "Alison+Sturrock",
# "Rowena+Viney",
# "Katherine+Woolf",
# "Laila+Abdullah",
# "Saniath+Akbar",
# "Lisa+Andrews",
# "Zakia+Arfeen",
# "Deanne+Attreed",
# "Jelena+Baburina ",
# "Maggie+Baker",
# "Fahmina+Begum ",
# "Sarah+Bennett",
# "Taylor+Bennie",
# "Kerry+Boardman",
# "Victoria+Clausen-Thue",
# "Vicky+Cooper",
# "Paul+Crampton",
# "Alison+Crook ",
# "Purvangi+Dave ",
# "Jon+Derrick",
# "Petra+Dimmers",
# "Graham+Easton ",
# "Carol+Farguson ",
# "Deborah+Gill",
# "Anjali+Gondhalekar",
# "Ann+Griffin",
# "Janet+Halsey",
# "Lauren+Hammond",
# "David+Harrison",
# "Maresa+Harvey",
# "Jestina+Henry",
# "Samantha+Henry",
# "Melanie+Hill",
# "Jenny+Hopwood",
# "Andrew+Houghton ",
# "Kate+Howe",
# "Izzie+Jay",
# "Gaynor+Jones",
# "Billie+Josef",
# "Sunantha+Kandasamy",
# "Jayne+Kavanagh",
# "Laura+Knight",
# "Miranda+Kronfli",
# "Rupa+Lakhani",
# "Molly+Lavender-Rose",
# "Helen+Lewis",
# "Errol+Lobo",
# "Deborah+Lucas-Georgiou",
# "Natasha+Malik",
# "Gianpaolo+Manalastas",
# "Chloe+Marshall ",
# "Nadine+Mason-Bertrand",
# "Nicola+Mathastein",
# "Jonny+Mayhew",
# "Asta+Medisauskaite",
# "Leila+Mehdizadeh",
# "Tadiwa+Mutongwizo",
# "Jayshree+Naik",
# "John-george+Nicholson",
# "Lorraine+Noble",
# "Adam+North",
# "Tina+Nyazika",
# "Mary+Obemeasor",
# "Michael+Page",
# "Pritpal+Patel",
# "Catherine+Phillips",
# "Fran+Pitman ",
# "Paula+Raftery",
# "Ahmed+Rashid",
# "Khalil+Rawat",
# "Eliot+Rees",
# "Antonia+Rich",
# "Holly+Riches",
# "Marcia+Rigby",
# "Jane+Rowles",
# "Ayana+Sakey",
# "Sabine+Schutte",
# "Halima+Shah",
# "Kanika+Sharma",
# "Sarah+Smith",
# "Lee+Standen",
# "Tara+Stewart",
# "Alison+Sturrock",
# "Emma+Thompsett",
# "Anne+Thurston",
# "Emily+Unwin",
# "Rowena+Viney",
# "Deirdre+Wallace",
# "Stephanie+Woods ",
# "Katherine+Woolf",
# "Tor+Wright",
# "Judith+Breuer",
# "Hans+Stauss",
# "Samantha+Photiades",
# "Carolyn+McElvaney",
# "Arne+Akbar",
# "Siobhan+Burns",
# "Benny+Chain",
# "Ariberto+Fassati",
# "Richard+Goldstein",
# "Paul+Griffiths",
# "Joe+Grove",
# "Ravindra+Gupta",
# "Robert+Heyderman",
# "Clare+Jolly",
# "Mala+Maini",
# "Pierre+Maillard",
# "Victoria+Male",
# "Laura+McCoy",
# "Tim+McHugh",
# "Richard+Milne",
# "Emma+Morris",
# "Mahdad+Noursadeghi",
# "Deenan+Pillay",
# "Gabriele+Pollara",
# "Matthew+Reeves",
# "Helen+Rowe",
# "David+Sansom",
# "Benedict+Seddon",
# "Yasu+Takeuchi",
# "Gillian+Tomlinson",
# "Greg+Towers",
# "Lucy+Walker",
# "Professor+Sir+Alimuddin+Zumla",
# "Doug+King",
# "Jamie+Evans",
# "Eira+Rawlings",
# "Janani+Sivakumaran-Nguyen",
# "Tony+Berry",
# "Stephen+McElvaney",
# "Biljana+Nikolic",
# "Derek+Kempen",
# "Nathan+Buckley",
# "Isabel+Lubeiro",
# "Jade+Donovan",
# "Aimee+Kennedy",
# "Lamlah+Sekweyama",
# "David+Abraham",
# "Kevin+Howell",
# "Gitta+Stockinger+FRS+FMedSci",
# "Alan+Salama+MBBS+PhD+FRCP",
# "Christopher+Denton",
# "Miranda+Rosenthal",
# "Lucy+Walker",
# "JG+Coghlan+MD+FRCP",
# "Benedict+Seddon",
# "David+Sansom",
# "Victoria+Male",
# "Arne+N.+Akbar",
# "Martin+Birchall",
# "Massimiliano+Cerletti",
# "Chris+Mason",
# "Paola+Bonfanti",
# "Ashleigh+Boyd",
# "Mark+W+Lowdell",
# "Douglas+Thorburn",
# "Clare+Bennett",
# "Mark+W+Lowdell",
# "Tim+Meyer",
# "Ronjon+Chakraverty",
# "Emma+Morris",
# "Thomas+Powles",
# "Hans+J+Stauss",
# "Benny+Chain",
# "Sabine+Kinloch",
# "Matthew+Reeves",
# "Joe+Grove",
# "Paul+D.+Griffiths+MD",
# "William+Rosenberg",
# "Siobhan+Burns",
# "med.+B+Grimbacher",
# "Derralynn+Hughes",
# "Uma+Ramaswami+MBBS",
# "Atul+Mehta",
"Amit+Nathwani",
"Hans+Stauss",
"Clare+Bennett",
"Mark+Lowdell",
"Emma+Morris",
"Mark+Lowdell",
"Ronjon+Chakraverty",
"Zaza+Ndhlovu",
"Mark+Siedner",
"Henrik+Kløverpris",
"Deenan+Pillay",
"Ravindra+Gupta+",
"Till+Bärnighausen",
"Nigel+Klein",
"Maryam+Shahmanesh",
"Janet+Seeley",
"Frank+Tanser",
"Alison+Grant",
"Alex+Sigal",
"Philip+Hawkins",
"Sir+Mark+Pepys",
"Vittorio+Bellotti",
"Marianna+Fontana",
"Julian+Gillmore",
"Helen+Lachmann",
"Helen+Lachmann",
"Glenys+Tennent",
"Ashutosh+Wechalekar",
        ]
for key in keys:
    real_url = base +str(key)+'%5BAuthor%5D)+AND+ucl%5BAffiliation%5D'
    # print real_url
    xpath = "//div[@class='aux'][2]/div[@class='links']/a[@class='view'][1]/@href"
    conts = joke(real_url,xpath)
    for cont in conts:
        real_url2 = "https://www.ncbi.nlm.nih.gov"+str(cont)
        # print real_url2
        article = requests.get(real_url2)
        # article.text
        #把网页内容转化为字符串类型
        # article2 = "".join(article)
        article2 = "".join(article)
        # print type(article2)
        mouse_list = [
            "knockout mouse",
            "knockout mice",
            "knockout rat",
            "knockin mouse",
            "knockin mice",
            "knockin rat",
            "conditional knockout mouse",
            "conditional knockout mice",
            "conditional knockout rat",
            "CKO mouse",
            "CKO mice",
            "CKO rat",
            "transgenic mice",
            "transgenic mouse",
            "transgenic rat",
            "cre mouse",
            "cre mice",
            "cre rat",
            "humanized mouse",
            "humanized mice",
            "humanized rat",
            "mouse crispr",
            "mice crispr",
            "rat crispr",
            "mouse model",
            "mice model",
            "rat model",
        ]
        time.sleep(random.randint(3,4))
        for mouse in mouse_list:
            try:
                article3 = article2.split('References', re.IGNORECASE)[0]
                # print article3
                if str(mouse) in article3:
                    #print real_url2
                    # ccont = requests.get(real_url2)
                    titles = re.findall(r'<title>(.*?)</title>',article2,re.S)
                    for title in titles:
                        key = key.replace('+',' ')
                        print real_url2
                        print title
                        print mouse
                        sql = "insert into ucl_detail_3_2(课题组,文献标题,文献链接,文献关键词) values(%s,%s,%s,%s)"
                        r = db.insert(sql, (key,title,real_url2,mouse), flag=True)
                        print("增加了{}条数据到数据库".format(r))
                else:
                    print ("没有{}关键词".format(key))
                    # print real_url2
                    # print mouse
            except Exception as e :
                print e
    #             # print ("-------------------没有{}关键词".format(key))








