'''
# 
# Concept Tag processor @Copyright Vishrut Jha
#
# API Usage: This API is intended to analyze a database feed the same with concept tags for domain names
#   - Define your Mongo Instance and Database name below
#   - Run the Crawler to feed domain names before you run this
#   - Intended for generating and adding concept tags to the domains
# 
'''


from pymongo import MongoClient
from HTMLParser import HTMLParser
import urllib2 # for page retreival
import re # for regex processing

# Mongo Credentials go here

client = MongoClient()
db = client.Crawler
coll = db.Domains

stopwords = {"'tis","'twas","a","aah","aaron","abandon","abandoned","abbott","abby","abe",
"abigail","ability","able","aboard","about","above","absolute","absolutely","absurd","abuse",
"accept","accepted","access","accident","accidentally","accomplish","accomplished","according","account","accounts",
"accuse","accused","accusing","ace","acid","across","act","acted","acting","action",
"actions","activity","actor","actress","acts","actual","actually","ad","adam","add",
"added","addition","address","adjustment","admire","admit","admitted","adopted","adoption","adorable",
"adult","adults","advance","advantage","adventure","advertisement","advice","advise","affair","affairs",
"affect","affected","afford","afraid","africa","after","afternoon","afterwards","again","against",
"age","agency","agenda","agent","agents","ages","ago","agree","agreed","agreement",
"ah","aha","ahead","ahem","ahh","ahhh","aid","aidan","aids","ain't",
"air","airport","aisle","aitoro","al","al's","alan","alarm","albert","alcazar",
"alcazar's","alcohol","alert","alex","alexandra","Alexis","ali","alice","alien","aliens",
"alike","Alison","alistair","alive","all","allergic","alley","allison","allow","allowed",
"ally","almost","alone","along","alonzo","already","alright","also","altar","alternative",
"although","always","am","amanda","amazing","amber","ambulance","amen","america","american",
"americans","among","amount","amp","amusement","amusing","amy","an","ancient","and",
"andie","andrew","andy","angel","angela","angeles","angels","anger","angle","angry",
"animal","animals","anna","anne","annie","anniversary","announce","announcement","annoying","annulment",
"anonymous","another","answer","answered","answering","answers","ant","anthony","anti","antonio",
"antonio's","anxious","any","anya","anybody","anybody's","anyhow","anymore","anyone","anyone's",
"anything","anytime","anyway","anyways","anywhere","apart","apartment","apologize","apology","apparatus",
"apparently","appeal","appear","appearance","appears","appetite","apple","application","apply","appointment",
"appreciate","approach","appropriate","approval","approve","arch","are","area","aren't","argue",
"arguing","argument","arm","armed","arms","army","arnold","around","arrange","arranged",
"arrangement","arrangements","arrest","arrested","arrive","arrived","art","arthur","article","artist",
"as","asa","ash","ashamed","ashes","ashley","aside","ask","asked","asking",
"asks","asleep","ass","assault","asshole","assignment","assistant","associate","assume","assumed",
"assuming","assure","at","ate","atlantic","attached","attack","attacked","attempt","attempted",
"attend","attention","attic","attitude","attorney","attracted","attraction","attractive","auction","audience",
"audition","audrey","aunt","austin","authorities","authority","automatic","available","avatar","average",
"avoid","avoiding","aw","awake","award","aware","away","awesome","awful","awfully",
"awhile","awkward","aww","aye","b","ba","babe","babies","baby","baby's",
"bachelor","back","backed","background","backup","backwards","bacon","bad","badge","badly",
"bag","bags","bail","bait","baker","balance","balcony","bald","baldwin","ball",
"balls","band","bang","bank","banks","bar","barb","barbara","bare","barely",
"bargain","barn","barry","bars","bart","bartender","base","baseball","based","basement",
"basic","basically","basin","basis","basket","basketball","bastard","bastards","bat","bath",
"bathroom","battery","battle","bauer","bay","be","beach","beacon","beans","bear",
"beast","beat","beaten","beating","beats","beautiful","beauty","became","because","become",
"becomes","becoming","bed","bedroom","bee","beef","been","beer","before","beg",
"began","begged","begging","begin","beginning","begins","begun","behalf","behave","behavior",
"behaviour","behind","being","belief","believe","believed","believes","believing","bell","belle",
"belly","belong","belonged","belongs","beloved","below","belt","ben","bench","bend",
"beneath","benefit","benjamin","bennett","benny","bent","berg","bermuda","berry","beside",
"besides","best","bet","beth","betray","betrayed","better","betty","between","beyond",
"bianca","bible","big","bigger","biggest","bike","bill","billie","billion","bills",
"billy","biological","bird","birds","birth","birthday","bit","bitch","bite","bitter",
"bizarre","black","blackmail","blade","blah","blair","blake","blame","blaming","blank",
"blanket","blast","bleeding","bless","blessed","blessing","blew","blind","block","blocks",
"blonde","blood","bloody","blow","blowing","blown","blows","blue","bo","board",
"boarding","boat","bob","bobbie","bobby","bodies","body","boiling","bomb","bond",
"bone","bones","bonnie","boo","book","booked","books","boom","boot","booth",
"boots","booze","border","bored","boring","born","borrow","borrowed","boss","boston",
"both","bother","bothered","bothering","bothers","bottle","bottles","bottom","bought","bound",
"bout","bow","bowl","bowling","box","boxes","boy","boy's","boyfriend","boys",
"bra","bracelet","brad","brady","brain","brains","brake","branch","brand","brandon",
"brandy","brass","brat","brave","bravo","bread","break","breakdown","breakfast","breaking",
"breaks","breast","breasts","breath","breathe","breathing","brenda","brenda's","brian","brick",
"bride","bridge","bridget","brief","briefcase","bright","brilliant","bring","bringing","brings",
"british","brittany","bro","broad","broke","broken","Brooke","brother","brother's","brothers",
"brought","brown","bruce","brush","bryant","bubble","buchanan","buck","bucket","bucks",
"bud","buddies","buddy","budget","buffy","bug","bugs","build","building","built",
"bulb","bull","bulldog","bullet","bullets","bullshit","bum","bump","bunch","bundy",
"bunny","burden","bureau","burger","buried","burn","burned","burning","burns","burst",
"bury","bus","business","bust","busted","buster","busy","but","butt","butter",
"butters","button","buttons","buy","buying","buzz","by","bye","c","c'mon",
"cab","cabin","cabinet","cable","cafe","cafeteria","cage","cake","caleb","california",
"call","called","calling","calls","calm","came","camera","cameras","cameron","camp",
"campaign","campus","can","can't","canada","canadian","cancel","canceled","cancer","candle",
"candles","candy","cannot","canvas","cap","capable","capeside","capital","captain","car",
"card","cards","care","cared","career","careful","carefully","cares","caring","carl",
"carla","Carly","carly's","carmen","carol","caroline","carpet","carriage","carried","carry",
"carrying","cars","cart","carter","cartman","carver","cascade","case","cases","casey",
"cash","casino","cassadine","cassie","cast","castle","casual","cat","catalina","catch",
"catching","catherine","cats","caught","cause","caused","causes","causing","cave","cd",
"celebrate","celebrating","celebration","cell","cells","center","central","cents","century","ceremony",
"certain","certainly","chad","chain","chair","chairs","chalk","challenge","chamber","champagne",
"champion","chance","chances","chandler","change","changed","changes","changing","channel","chaos",
"chapel","chapter","character","charge","charged","charges","charity","charity's","charles","charlie",
"charm","charmed","charming","chase","chasing","chat","cheap","cheat","cheated","cheating",
"check","checked","checking","checks","cheer","cheers","cheese","chef","chemical","chemistry",
"chest","chicago","chick","chicken","chicks","chief","child","child's","childhood","children",
"children's","chili","chill","chin","china","chinese","chip","chips","chloe","chocolate",
"choice","choices","choose","chop","chose","chosen","Chris","christ","christina","christine",
"Christmas","christopher","chuck","church","cia","cigarette","cigarettes","circle","circumstances","circus",
"citizen","city","civil","claim","claims","claire","clark","class","classes","classic",
"claus","clay","clean","cleaned","cleaning","clear","cleared","clearly","clever","click",
"client","clients","cliff","climb","clinic","clock","close","closed","closer","closest",
"closet","closing","cloth","clothes","clothing","cloud","clown","club","clubs","clue",
"co","coach","coal","coast","coat","cocktail","code","coffee","coffin","coincidence",
"coke","colby","cold","cole","coleman","colin","collar","collect","collection","colleen",
"college","collins","colonel","color","colors","colour","com","coma","comb","combination",
"come","comes","comfort","comfortable","comin","coming","command","commander","comment","commercial",
"commission","commissioner","commit","commitment","committed","committee","common","community","companies","company",
"compare","compared","comparison","compassion","competition","complain","complaining","complete","completely","complex",
"complicated","compliment","compromise","computer","computers","con","concentrate","concept","concern","concerned",
"concerns","concert","conclusion","conclusions","condition","conditions","conference","confess","confessed","confession",
"confidence","confident","confirm","confirmed","conflict","confused","confusing","congratulations","congress","connect",
"connected","connection","conscience","conscious","consequences","consider","considered","considering","conspiracy","constable",
"constant","constantly","construction","contact","contest","continue","contract","contrary","control","convenient",
"conversation","convicted","convince","convinced","convincing","cook","cooked","cookie","cookies","cooking",
"cool","cooper","cooperate","cop","copper","cops","copy","cord","cordelia","core",
"corinthos","cork","corky","corn","corner","corporate","correct","cost","costs","costume",
"cottage","cotton","couch","cough","could","could've","couldn't","council","counsel","counselor",
"count","counter","counting","country","counts","county","couple","couples","courage","course",
"court","courtney","courtroom","cousin","cover","covered","covering","covers","cow","coward",
"cowboy","cozy","crack","cracked","craig","crane","crap","crash","crashed","crawl",
"crawling","crazy","cream","create","created","creative","creature","credit","creep","creepy",
"crew","cried","crime","crimes","criminal","criminals","cris","crisis","Cristian","critical",
"cross","crossed","crossing","crowd","cruel","cruise","crush","cry","crying","crystal",
"cue","culture","cup","cure","curious","current","curse","curtain","curve","cushion",
"custody","customer","customers","cut","cute","cuts","cutting","cuz","cynthia","d",
"d'you","da","dad","dad's","daddy","daddy's","dahlia","daily","dallas","damage",
"damaged","dammit","damn","damned","damnit","dan","dana","dance","dancer","dancing",
"danger","dangerous","daniel","Danny","daphne","dare","daria","dark","darkness","darla",
"darling","darn","data","date","dated","dates","dating","daughter","daughter's","daughters",
"dave","David","davidson","davis","dawn","Dawson","day","days","de","deacon",
"dead","deaf","deal","dealer","dealing","deals","dealt","dean","dear","death",
"debate","debt","decent","decide","decided","decides","decision","decisions","deck","deep",
"deeper","deeply","defend","defendant","defending","defense","definitely","degree","degrees","del",
"delay","delia","deliberately","delicate","delicious","deliver","delivered","delivery","demand","demon",
"demons","denial","denied","dentist","deny","department","depend","dependent","depends","deposit",
"depressed","deputy","derek","describe","description","desert","deserve","deserved","deserves","design",
"designed","designer","designs","desire","desk","desperate","desperately","despite","dessert","destiny",
"destroy","destroyed","destroying","destruction","detail","details","detective","determined","devastated","developed",
"development","device","devil","devoted","dial","diamond","diamonds","diana","diane","diary",
"dick","did","didn't","die","died","diego","dies","diet","difference","different",
"differently","difficult","dig","digestion","digging","dignity","dime","dimera","diner","dining",
"dinner","direct","direction","directions","directly","director","dirt","dirty","disagree","disappear",
"disappeared","disappoint","disappointed","disaster","discover","discovered","discovery","discuss","discussed","discussing",
"discussion","disease","disgust","disgusting","dish","dishes","dismissed","distance","distract","distracted",
"distribution","district","disturb","disturbed","disturbing","ditch","dive","division","divorce","divorced",
"dixie","dixon","dizzy","dna","do","doc","doctor","doctor's","doctors","document",
"does","doesn't","dog","dogs","doin","doing","doll","dollar","dollars","don",
"don't","done","donna","donny","donor","doomed","door","doors","dorm","double",
"doubt","doubts","doug","down","downstairs","downtown","doyle","dozen","dr","drag",
"dragged","dragging","drain","drama","dramatic","drank","draw","drawer","drawing","drawn",
"dream","dreamed","dreaming","dreams","dress","dressed","dresses","dressing","drew","drill",
"drink","drinking","drinks","drive","driven","driver","drives","driving","drop","dropped",
"dropping","drops","drove","dru","drug","drugs","drunk","dry","du","duck",
"dude","due","duh","duke","dull","dumb","dump","dumped","dunno","during",
"dust","duty","dying","e","each","eager","ear","earl","earlier","early",
"earn","earned","ears","earth","ease","easier","easily","east","easy","eat",
"eaten","eating","eats","ed","eddie","eden","edge","editor","edmund","education",
"edward","effect","effective","effects","effort","egg","eggs","ego","eh","eight",
"eighteen","eighty","either","el","elaine","elastic","election","electric","elevator","eleven",
"elizabeth","ellen","elliot","ellison","else","else's","em","embarrass","embarrassed","embarrassing",
"emergency","Emily","emma","emotional","emotionally","emotions","empire","employee","employees","empty",
"enchantment","end","ended","ending","ends","enemies","enemy","energy","engaged","engagement",
"engine","england","english","enjoy","enjoyed","enjoying","enormous","enough","enter","entered",
"entering","entertainment","entire","entirely","entitled","entrance","envelope","environment","ephram","episode",
"equal","equipment","er","eric","erica","erica's","error","escape","escaped","escort",
"especially","estate","eternity","Ethan","ethan's","europe","evans","eve","even","evening",
"event","events","eventually","ever","every","everybody","everybody's","everyday","everyone","everyone's",
"everything","everything's","everywhere","evidence","evil","ew","ex","exact","exactly","exam",
"example","excellent","except","exchange","excited","excitement","exciting","excuse","excuses","executive",
"exercise","exhausted","exist","existed","existence","exists","exit","expansion","expect","expected",
"expecting","expensive","experience","experienced","experiment","expert","explain","explained","explaining","explains",
"explanation","explode","explosion","expose","exposed","express","expression","extra","extraordinary","extreme",
"extremely","eye","eyes","f","fabulous","face","faced","faces","facility","facing",
"fact","facts","fail","failed","failure","fair","fairly","fairy","faith","fake",
"faking","fall","fallen","falling","falls","false","familiar","families","family","family's",
"famous","fan","fancy","fans","fantastic","fantasy","far","farm","fascinating","fashion",
"fashioned","fast","faster","fat","fate","father","father's","fathers","fault","favor",
"favorite","favors","favour","favourite","fax","fbi","fear","fears","feather","fed",
"federal","feds","feeble","feed","feeding","feel","feeling","feelings","feels","feet",
"felicia","fell","fella","fellas","fellow","felt","female","fence","fertile","fever",
"few","fiance","fiction","field","fifteen","fifth","fifty","fight","fighter","fighting",
"fights","figure","figured","figures","file","filed","files","fill","filled","filling",
"film","filthy","final","finally","financial","find","finding","finds","fine","finest",
"finger","fingerprints","fingers","finish","finished","fire","fired","firing","firm","first",
"fish","fishing","fit","fits","fitzgerald","five","fix","fixed","flag","flame",
"flash","flat","flattered","flesh","flew","flight","flip","flirting","floating","floor",
"florida","flow","flower","flowers","flu","fly","flying","focus","focused","fold",
"folks","follow","followed","following","fond","food","fool","fooled","foolish","foot",
"football","for","forbid","force","forced","forces","foreign","forest","forever","forget",
"forgetting","forgive","forgiven","forgiveness","forgot","forgotten","fork","form","formal","former",
"forms","forrester","fort","forth","fortune","forty","forward","fought","foul","found",
"foundation","four","fourteen","fourth","fowl","fox","frame","france","francisco","frank",
"frankie","franklin","frankly","fraser","Frasier","fraud","freak","freaked","freaking","freaks",
"fred","frederick","free","freedom","freeze","freezing","french","frequent","fresh","friday",
"fridge","fried","friend","friend's","friendly","friends","friendship","fries","frightened","frog",
"from","front","frozen","fruit","fuck","fucked","fuckin","fucking","fuel","full",
"fully","fun","function","fund","funeral","funny","furious","furniture","further","future",
"g","gabrielle","gain","gallery","game","games","gang","garage","garbage","garden",
"garrison","gary","gas","gate","gather","gave","gay","gear","gee","geez",
"gene","general","generation","generous","genetic","genius","genoa","gentle","gentleman","gentlemen",
"George","german","gesture","get","gets","gettin","getting","ghost","ghosts","gia",
"giant","gift","gifts","gig","giles","gilmore","gimme","gina","ginger","girl",
"girl's","girlfriend","girls","give","given","gives","giving","glad","glass","glasses",
"glen","glory","glove","gloves","glow","go","goa'uld","goal","goat","god",
"god's","goddamn","gods","goes","goin","going","gold","golden","golf","gone",
"gonna","good","goodbye","goodness","goodnight","goods","goose","gordon","gorgeous","gosh",
"got","gotcha","gotta","gotten","government","governor","gown","grab","grabbed","grace",
"grade","grades","graduate","graduation","grain","grams","grand","granddaughter","grandfather","grandma",
"grandmother","grandpa","grandson","grant","granted","grass","grateful","grave","gray","great",
"greater","greatest","greek","green","Greenlee","greg","greta","grew","grey","grief",
"grip","groom","gross","ground","grounded","grounds","group","grow","growing","grown",
"growth","gt","guarantee","guard","guards","guess","guessing","guest","guests","guide",
"guilt","guilty","guitar","gum","gun","guns","gus","gut","guts","guy",
"guy's","guys","gwen","gym","h","ha","habit","had","hadn't","hah",
"hair","hal","half","halfway","hall","halloween","hallway","hammer","hand","handed",
"handle","handled","handling","hands","handsome","handy","hang","hanging","hank","happen",
"happened","happening","happens","happier","happiest","happily","happiness","happy","harbor","harbour",
"hard","harder","hardly","harley","harm","harmony","harold","harris","harry","harsh",
"hart","harvard","harvey","has","hasn't","hat","hate","hated","hates","have",
"haven't","having","hawaii","hayley","hayward","he","he'd","he'll","he's","head",
"headache","headed","heading","heads","heal","healing","health","healthy","hear","heard",
"hearing","hears","heart","hearts","heat","heaven","heaven's","heavy","heck","heels",
"heh","held","helen","helena","helicopter","hell","hell's","hello","help","helped",
"helpful","helping","helpless","helps","henry","her","here","here's","hero","hers",
"herself","hey","hi","hid","hidden","hide","hiding","high","higher","highest",
"highly","highway","hilda","hill","hills","him","himself","hint","hip","hire",
"hired","his","history","hit","hits","hitting","hm","hmm","hmmm","ho",
"hockey","hold","holden","holding","holds","hole","holes","holiday","holidays","hollow",
"holly","hollywood","holy","home","homeless","homer","homework","homicide","hon","honest",
"honestly","honesty","honey","honeymoon","honor","honorable","honored","hoo","hood","hook",
"hooked","hooker","hop","hope","hope's","hoped","hopefully","hopeless","hopes","hoping",
"horn","horrible","horror","horse","horses","hospital","host","hostage","hostile","hot",
"hotel","hour","hours","house","houses","houston","how","how'd","how'll","how's",
"howard","however","hug","huge","hughes","huh","human","humans","humor","humour",
"hundred","hundreds","hung","hungry","hunt","hunting","hurry","hurt","hurting","hurts",
"husband","husband's","I","I'd","i'd","I'll","i'll","I'm","i'm","I've",
"i've","ian","ice","id","idea","ideas","identify","identity","idiot","if",
"ignore","ill","illegal","illness","image","imagination","imagine","imagined","immediate","immediately",
"impact","important","impossible","impress","impressed","impression","impressive","impulse","in","inappropriate",
"inch","incident","include","including","increase","incredible","incredibly","indeed","independence","independent",
"india","indian","industry","infection","influence","information","informed","injured","injury","ink",
"inn","inner","innocent","insane","insanity","insect","inside","insist","insisted","inspector",
"instance","instant","instead","instinct","instincts","instructions","instrument","insult","insurance","integrity",
"intelligence","intelligent","intend","intended","intense","intention","intentions","interest","interested","interesting",
"interests","interfere","internal","international","internet","interrupt","interrupting","interview","intimate","into",
"introduce","introduced","invented","invention","investigating","investigation","investment","invisible","invitation","invite",
"invited","inviting","involve","involved","iron","ironic","is","isaac","isabel","isabella",
"island","isn't","issue","issues","it","it'd","it'll","it's","italian","italy",
"item","its","itself","ivy","j","jabot","jack","jack's","jacket","jackie",
"jacks","jackson","jail","jake","jake's","jam","jamal","James","jamie","jan",
"jane","janine","japanese","Jason","jason's","java","Jax","jay","jealous","jealousy",
"jean","jeez","jeff","jefferson","jeffrey","jelly","jen","jen's","jenkins","jennifer",
"jenny","jerk","Jerry","jersey","jess","jesse","Jessica","jessica's","jessie","Jesus",
"jet","jewel","jewelry","jewish","jill","jim","jimmy","jo","joanna","job",
"jobs","joe","joel","joey","john","johnny","johnson","join","joined","joining",
"joint","joke","jokes","joking","jones","jordan","joseph","josh","joshua","journal",
"journey","joy","judge","judging","judgment","juice","julia","Julian","julian's","julie",
"july","jump","jumped","jumping","june","jungle","junior","junk","jury","just",
"justice","k","kane","karen","kate","katherine","katie","Kay","keep","keeping",
"keeps","keith","kelly","kelso","ken","kendall","kennedy","kenny","kept","keri",
"kettle","kevin","key","keys","kick","kicked","kicking","kid","kid's","kidding",
"kiddo","kidnap","kidnapped","kidnapping","kidney","kids","kill","killed","killer","killing",
"kills","kim","kind","kinda","kindly","kinds","king","kirk","kiss","kissed",
"kissing","kit","kitchen","kitty","knee","knees","knew","knife","knock","knocked",
"knocking","knot","know","knowing","knowledge","known","knows","kraft","kramer","kristen",
"kristina","kyle","l","la","lab","labor","lack","lad","ladies","lady",
"laid","lake","lakeview","lame","lamp","land","landed","landing","lane","language",
"lap","large","larry","las","last","late","lately","later","latest","laugh",
"laughed","laughing","launch","laundry","laura","lauren","laurence","laurie","law","lawn",
"laws","lawsuit","lawyer","lawyers","lay","laying","le","lead","leader","leading",
"leads","leaf","league","leak","lean","leap","learn","learned","learning","least",
"leather","leave","leaves","leaving","lecture","led","lee","leery","left","leg",
"legal","legally","legend","legs","lemme","lenny","leo","leo's","lesbian","leslie",
"less","lesson","lessons","let","let's","lets","letter","letters","letting","level",
"lewis","lexie","liar","libby","liberty","library","license","lie","lied","lies",
"lieutenant","life","life's","lifetime","lift","light","lighten","lightning","lights","like",
"liked","likely","likes","lily","limit","limits","limo","linda","Lindsay","line",
"linen","lines","link","lip","lips","lipstick","liquid","liquor","lisa","list",
"listen","listened","listening","lit","literally","little","liv","live","lived","liver",
"lives","living","livvie","liz","liza","lizzie","llanview","lloyd","lo","load",
"loaded","loan","lobby","local","location","lock","locked","locker","loft","log",
"logan","logical","london","lonely","long","longer","look","looked","lookin","looking",
"looks","loop","loose","lopez","lord","lorelai","lorelei","los","lose","loser",
"losers","loses","losing","loss","lost","lot","lots","lou","loud","louie",
"louis","louise","lousy","love","loved","lovely","lover","lovers","loves","loving",
"low","lower","loyal","loyalty","lt","lucas","lucinda","luck","lucky","lucy",
"lucy's","Luis","luke","luke's","lunatic","lunch","lungs","lying","m","ma",
"ma'am","mac","machine","machines","maciver","mackenzie","macy","mad","madam","maddie",
"made","magazine","magazines","maggie","magic","magical","maid","mail","main","maintain",
"majesty","major","make","makes","makeup","making","malcolm","male","mall","mama",
"man","man's","manage","managed","management","manager","mandy","manhattan","manner","manners",
"manning","mansion","many","map","marah","march","marcus","marcy","margaret","marge",
"margo","maria","marie","marina","maris","marissa","mark","market","marks","marlena",
"marriage","married","marry","marrying","marshall","martha","martin","martini","marty","mary",
"mask","mass","massage","massimo","massive","master","match","matches","mate","mateo",
"material","math","matt","matter","matters","matthew","mature","maureen","max","maxie",
"may","maybe","mayor","me","meal","mean","meaning","means","meant","meantime",
"meanwhile","measure","meat","med","media","medical","medication","medicine","meet","meeting",
"meetings","megan","mel","melissa","member","members","memorial","memories","memory","men",
"men's","mental","mention","mentioned","menu","mercy","merely","merry","mess","message",
"messages","messed","messenger","messing","met","metal","mexico","mia","miami","Michael",
"michael's","michelle","mickey","mid","middle","midnight","might","might've","mightn't","mighty",
"Miguel","mike","mikey","mile","miles","military","milk","mill","miller","million",
"millions","mimi","mind","minds","mine","minimum","minister","minor","minute","minutes",
"miracle","miracles","mirror","miserable","misery","miss","missed","misses","missing","mission",
"missy","mist","mistake","mistaken","mistakes","mister","misunderstanding","misunderstood","mitch","mix",
"mixed","mm","mmm","mob","model","modern","molly","mom","mom's","moment",
"moments","mommy","mommy's","mon","monday","money","monica","monitor","monkey","monster",
"monsters","montgomery","month","months","mood","moon","moral","more","morgan","morning",
"moron","mortal","most","mostly","motel","mother","mother's","motherfucker","mothers","motion",
"motive","motorcycle","mountain","mountains","mouse","mouth","move","moved","moves","movie",
"movies","moving","Mr","mrs","ms","much","mud","muffin","mum","murder",
"murdered","murderer","murphy","muscle","museum","music","must","must've","mustn't","mutual",
"my","myself","mysterious","mystery","n","nah","nail","nails","naive","naked",
"name","name's","named","names","nancy","nanny","nap","narrow","nasty","Natalie",
"natalie's","nate","nathan","nation","national","natural","naturally","nature","naughty","nbsp",
"near","nearly","neat","necessarily","necessary","neck","necklace","ned","need","needed",
"needing","needle","needs","negative","neighbor","neighborhood","neighbors","neil","neither","nephew",
"nerve","nerves","nervous","nest","net","network","never","new","newman","news",
"newspaper","next","nice","nicely","nicholas","nick","nicky","nicole","niece","nigel",
"night","night's","nightmare","nights","niki","nikki","nikolas","Niles","nina","nine",
"ninety","no","noah","noble","nobody","nobody's","noise","non","none","nonsense",
"noon","nope","nor","nora","norma","normal","normally","norman","north","nose",
"not","note","notes","nothin","nothing","nothing's","notice","noticed","now","nowhere",
"nuclear","number","numbers","nurse","nurses","nut","nuts","o","o'clock","o'neill",
"oakdale","oath","object","objection","observation","obsessed","obsession","obvious","obviously","occasion",
"occurred","ocean","odd","odds","of","off","offense","offer","offered","offering",
"office","officer","officers","official","officially","often","oh","ohh","ohhh","oil",
"ok(see also okay)","okay(see also ok)","ol","old","older","oldest","oliver","olivia","on","once",
"one","one's","ones","only","onto","ooh","oooh","oops","opal","open",
"opened","opening","opens","opera","operate","operation","opinion","opportunity","opposite","option",
"options","or","orange","order","ordered","ordering","orders","ordinary","organization","organized",
"original","ornament","oswald","other","other's","others","otherwise","ouch","ought","oughta",
"our","ours","ourselves","out","outfit","outside","outta","oven","over","overheard",
"overnight","overreacting","ow","owe","owes","own","owned","owner","owns","oxygen",
"oz","p","pa","pace","pacey","pack","package","packed","packing","page",
"pages","paid","paige","pain","painful","paint","painted","painting","pair","pal",
"palace","palmer","pan","pancakes","panic","pants","papa","paper","papers","paperwork",
"parade","paradise","parallel","paranoid","parcel","pardon","parent","parents","paris","park",
"parked","parker","parking","parole","part","particular","particularly","parties","partner","partners",
"parts","party","pass","passed","passes","passing","passion","passionate","past","paste",
"pat","patch","path","pathetic","patience","patient","patients","patrick","patrol","pattern",
"patty","Paul","paul's","pay","payback","paying","payment","pays","peace","peaceful",
"peanut","pee","peg","peggy","pen","pencil","penis","penny","penthouse","people",
"people's","per","percent","perfect","perfectly","perform","performance","perhaps","period","perkins",
"permanent","permission","person","personal","personality","personally","personnel","perspective","pet","pete",
"peter","petty","phase","pheebs","phil","philip","phillip","phoebe","phone","phones",
"phony","photo","photograph","photographer","photographs","photos","phyllis","physical","physically","piano",
"pick","picked","picking","picnic","picture","pictures","pie","piece","pieces","pierce",
"pig","pigs","pilar","pile","pill","pillow","pills","pilot","pin","pine",
"pink","pipe","piper","piss","pissed","pit","pitch","pity","pizza","place",
"placed","places","plain","plan","plane","planet","planned","planning","plans","plant",
"planted","plastic","plate","plates","play","played","player","players","playing","plays",
"plea","pleasant","please","pleased","pleasure","plenty","plot","plough","plug","plus",
"pocket","poem","poetry","point","pointing","points","poison","poker","pole","police",
"policy","polish","polite","political","politics","pony","pool","poor","pop","popcorn",
"pops","popular","porch","port","porter","portrait","position","positive","possession","possibilities",
"possibility","possible","possibly","post","posted","pot","potato","potatoes","potential","potion",
"potter","pound","pounds","pour","powder","power","powerful","powers","practical","practically",
"practice","pray","prayer","prayers","praying","pre","precious","precisely","prefer","pregnancy",
"pregnant","premonition","prepare","prepared","presence","present","presents","president","president's","press",
"pressure","pretend","pretended","pretending","pretty","prevent","price","pride","priest","prime",
"prince","princess","principal","print","prints","priority","prison","prisoner","prisoners","privacy",
"private","privilege","prize","pro","probable","probably","problem","problems","procedure","proceed",
"process","produce","producer","product","production","professional","professor","profile","profit","program",
"progress","project","prom","promise","promised","promises","proof","proper","properly","property",
"proposal","propose","proposed","prose","protect","protected","protecting","protection","protective","protest",
"proteus","proud","prove","proved","proven","proves","provide","prue","psychiatrist","psychic",
"psycho","public","publicity","pull","pulled","pulling","pulse","pump","pumpkin","punch",
"punish","punished","punishment","punk","puppy","pure","purple","purpose","purse","push",
"pushed","pushing","pussy","put","puts","putting","q","qualified","quality","quarter",
"quartermaine","quarters","queen","question","questioning","questions","quick","quickly","quiet","quinn",
"quit","quite","quote","r","rabbit","race","rach","rachel","radar","radio",
"rae","rafe","rage","rail","rain","raise","raised","raising","ralph","ramsey",
"ran","random","randy","range","rap","rape","raped","rappaport","rare","rat",
"rate","rather","rats","raul","raw","ray","re","reach","reached","reaching",
"react","reaction","read","reading","ready","real","reality","realize","realized","really",
"realm","reason","reasonable","reasons","rebecca","recall","receipt","receive","received","recent",
"recently","reception","recognize","recognized","recommend","record","records","recover","recovery","red",
"reese","referring","refuse","refused","regarding","reggie","regret","regrets","regular","rehab",
"rehearsal","related","relation","relationship","relationships","relax","release","released","relief","relieved",
"religion","religious","remain","remains","remarkable","remember","remembered","remembering","remembers","remind",
"reminded","reminds","remote","remove","removed","renee","rent","rented","repeat","replace",
"report","reporter","reports","represent","representative","reputation","request","require","required","rescue",
"research","reservation","reservations","resist","resources","respect","respond","response","responsibility","responsible",
"rest","restaurant","restraining","result","results","return","returned","returning","reunion","reva",
"reveal","revenge","reverend","reverse","review","reward","rex","reynolds","rhythm","rianna",
"ric","rice","rich","Richard","richard's","Rick","ricky","rico","rid","ride",
"ridge","ridiculous","riding","right","rights","riley","ring","ringing","rings","riot",
"rip","ripped","rise","risk","risks","risky","river","road","roast","rob",
"robbed","robbery","robbie","robert","robin","rock","rocks","rod","roger","role",
"roll","rolling","roman","romance","romantic","rome","romeo","roof","room","roommate",
"rooms","root","rope","rory","rosanna","Rose","roses","rosie","Ross","rotten",
"rough","round","rounds","route","routine","row","roxanne","roxy","roy","royal",
"roz","rub","rubber","rude","rug","ruin","ruined","ruining","rule","rules",
"rumor","run","running","runs","rush","russell","russian","Ryan","s","sabrina",
"sack","sacred","sacrifice","sad","safe","safely","safer","safety","said","sail",
"saint","sake","sakes","sal","salad","sale","salem","sales","salesman","sally",
"salt","sam","sam's","samantha","same","sami","sample","samples","san","sand",
"sandburg","sandwich","sandwiches","sandy","santa","santos","sara","sarah","sat","satellite",
"satisfied","saturday","sauce","save","saved","saving","saw","say","saying","says",
"scale","scam","scar","scare","scared","scares","scarf","scaring","scary","scenario",
"scene","schedule","scheduled","scheme","school","schools","science","scissors","score","scotch",
"scott","scotty","scout","scratch","scream","screaming","screen","screw","screwed","screwing",
"script","scudder","scum","sd","sea","seal","sealed","sean","search","searching",
"season","seat","seats","seattle","sec","second","seconds","secret","secretary","secrets",
"section","secure","security","seduce","see","seed","seeing","seek","seem","seemed",
"seems","seen","sees","selection","self","selfish","sell","selling","semester","senator",
"send","sending","sends","senior","sense","senses","sensitive","sent","sentence","separate",
"separated","sergeant","serial","series","serious","seriously","servant","serve","served","service",
"services","serving","session","set","seth","sets","setting","settle","settled","setup",
"seven","seventeen","seventh","seventy","several","sex","sexual","sexy","shade","shadow",
"shadows","shake","shaking","shall","shame","shan't","shape","share","shared","sharing",
"shark","sharon","sharp","shave","Shawn","shayne","she","she'd","she'll","she's",
"shed","sheep","sheet","sheets","sheila","shelf","shelter","Sheridan","sheridan's","sheriff",
"sherry","shh","shhh","shift","shine","ship","shirt","shirts","shit","shock",
"shocked","shoe","shoes","shoot","shooting","shop","shopping","short","shortly","shot",
"shots","should","should've","shoulder","shoulders","shouldn't","shove","show","showed","shower",
"showing","shown","shows","shrink","shut","shy","si","sick","sickness","sid",
"side","sides","sidney","sierra","sight","sign","signal","signature","signed","signing",
"signs","silence","silent","silk","silly","silver","similar","simon","simone","simple",
"simply","simpson","sin","since","sing","singer","singing","single","sink","sir",
"sis","sister","sister's","sisters","sit","site","sits","sitting","situation","situations",
"six","sixteen","sixth","sixty","size","skills","skin","skinny","skip","skirt",
"skull","sky","skye","slack","slap","slave","slayer","sleep","sleeping","slept",
"slide","slightly","slip","slipped","sloane","slope","slow","slowly","slut","small",
"smaller","smart","smarter","smash","smell","smells","smile","smiling","smith","smoke",
"smoking","smooth","snake","snap","sneak","sneaking","sneeze","snow","snyder","so",
"soap","sob","social","society","sock","socks","soda","soft","sold","soldier",
"soldiers","solid","solution","solve","solved","some","somebody","somebody's","someday","somehow",
"someone","someone's","someplace","somethin","something","something's","sometime","sometimes","somewhere","son",
"son's","song","songs","sonny","sonny's","sons","sookie","soon","sooner","sore",
"sorel","sorry","sort","sorts","soul","souls","sound","sounded","sounds","soup",
"source","south","spa","space","spade","spanish","spare","spaulding","speak","speaking",
"speaks","special","specific","specifically","speech","speed","spell","spellman","spells","spencer",
"spend","spending","spent","spider","spike","spill","spin","spirit","spirits","spit",
"spite","split","spoil","spoiled","spoke","spoken","sponge","spoon","sport","sports",
"spot","spotted","spread","spring","springfield","spy","spying","squad","square","squeeze",
"st","stab","stabbed","stable","staff","stage","stairs","stake","stalker","stalking",
"stamp","stan","stand","standard","standards","standing","stands","stanley","star","stare",
"staring","starr","stars","start","started","starting","starts","starving","state","statement",
"states","station","statue","status","stavros","stay","stayed","staying","stays","steady",
"steak","steal","stealing","steam","steel","stefan","stefano","stem","stenbeck","step",
"stephanie","stephen","stepped","steps","steve","steven","stewart","stick","sticking","sticks",
"sticky","stiff","still","stink","stinks","stitch","stock","stocking","stole","stolen",
"stomach","stone","stones","stood","stop","stopped","stopping","stops","storage","store",
"stores","stories","storm","story","straight","straighten","strange","stranger","strangers","strategy",
"street","streets","strength","stress","stretch","strictly","strike","strikes","string","strings",
"strip","stroke","strong","stronger","strongly","struck","structure","struggle","stuart","stubborn",
"stuck","student","students","studio","study","studying","stuff","stuffed","stunt","stupid",
"style","sub","subject","substance","subtle","success","successful","such","suck","sucked",
"sucker","sucking","sucks","sudden","suddenly","sue","suffer","suffered","suffering","sugar",
"suggest","suggested","suggesting","suggestion","suicide","suit","suite","suits","summer","sun",
"sunday","sunset","sunshine","super","superior","supplies","supply","support","supportive","suppose",
"supposed","supposedly","sure","surely","surface","surgeon","surgery","surprise","surprised","surprises",
"surrender","surrounded","surveillance","survival","survive","survived","susan","suspect","suspected","suspects",
"suspicious","swallow","swear","sweat","sweater","sweep","sweet","sweetheart","sweetie","swell",
"swim","swimming","swing","switch","switched","sword","swore","sworn","sydney","symbol",
"sympathy","symptoms","system","systems","t","ta","tabby","tabitha","table","tables",
"tad","tag","taggert","tail","take","taken","takes","taking","tale","talent",
"talented","talk","talked","talkin","talking","talks","tall","tammy","tank","tap",
"tape","tapes","target","taste","tastes","tattoo","taught","tax","taxi","taylor",
"tea","teach","teacher","teachers","teaching","teal'c","team","tear","tearing","tears",
"tech","technically","technology","ted","teddy","teenage","teenager","teeth","telephone","television",
"tell","telling","tells","temper","temperature","temporary","ten","tend","tendency","tennis",
"tense","tension","tent","term","terms","terrible","terribly","terrific","terrified","territory",
"tess","test","tested","testify","testimony","testing","tests","texas","th","than",
"thank","thanks","thanksgiving","that","that'd","that'll","that's","the","theater","thee",
"their","them","theme","themselves","then","theory","therapist","therapy","There","there",
"there'll","there's","therefore","Theresa","theresa's","these","they","they'd","they'll","they're",
"they've","thick","thief","thin","thing","thing's","things","think","thinkin","thinking",
"thinks","third","thirsty","thirteen","thirty","this","this'll","thomas","thorne","those",
"thou","though","thought","thoughtful","thoughts","thousand","thousands","thread","threat","threaten",
"threatened","threatening","threats","three","threw","thrilled","throat","through","throw","throwing",
"thrown","thumb","thunder","thursday","thy","ticket","tickets","tie","tied","tiger",
"tight","til","till","tim","time","time's","times","timing","Timmy","timmy's",
"tin","tina","tiny","tip","tips","tire","tired","tis","title","to",
"toast","toby","today","today's","Todd","toe","toes","together","toilet","told",
"tom","tommy","tomorrow","ton","tone","tongue","tonight","tonight's","tons","tony",
"tony's","too","took","tool","tools","tooth","top","topic","tore","torn",
"torture","tory","toss","total","totally","touch","touched","touching","tough","tour",
"toward","towards","towel","towels","tower","town","toy","toys","trace","traci",
"track","tracking","tracks","trade","tradition","traditional","traffic","tragedy","tragic","trail",
"train","trained","training","tramp","transfer","transferred","transplant","transport","trap","trapped",
"trash","trauma","travel","traveling","travers","tray","treasure","treat","treated","treating",
"treatment","tree","trees","trey","trial","tricia","trick","tricked","tricks","tried",
"tries","trigger","trip","trouble","troubles","trousers","troy","truck","true","truly",
"trunk","trust","trusted","trusting","truth","try","tryin","trying","tub","tuesday",
"tune","tunnel","turkey","turn","turned","turning","turns","tv","twas","twelve",
"twenty","twice","twin","twins","twist","twisted","two","tyler","type","types",
"typical","u","ugh","ugly","uh","uhh","uhm","ultimate","um","umbrella",
"umm","un","unbelievable","uncle","uncomfortable","unconscious","under","undercover","underground","underneath",
"understand","understanding","understands","understood","underwear","unexpected","unfair","unfortunate","unfortunately","unhappy",
"uniform","union","unique","unit","united","units","universe","university","unknown","unless",
"unlike","until","unusual","up","upon","upper","ups","upset","upsetting","upside",
"upstairs","urgent","us","use","used","useful","useless","uses","using","usual",
"usually","v","vacation","val","valentine's","valerie","valley","valuable","value","values",
"vampire","vampires","van","vanessa","vanessa's","vanquish","vault","vecchio","vega","vegas",
"vehicle","verse","version","very","vessel","vice","vicious","victim","victims","victor",
"victoria","victory","video","videotape","view","viki","village","vince","violence","violent",
"virgin","virus","vision","visit","visiting","visitors","vodka","voice","voices","volunteer",
"vote","votes","vows","vulnerable","w","wait","waited","waiter","waiting","waitress",
"wake","wakes","waking","walk","walked","walker","walking","walks","wall","wallace",
"wallet","walls","walsh","walter","wanna","want","wanted","wanting","wants","war",
"ward","warehouse","warlock","warm","warn","warned","warning","warrant","warren","warton",
"was","wash","washed","washington","wasn't","waste","wasted","wasting","watch","watched",
"watching","water","wave","wax","way","wayne","ways","we","we'd","we'll",
"we're","we've","weak","weapon","weapons","wear","wearing","wears","weather","web",
"wedding","wednesday","week","weekend","weeks","weight","weird","welcome","well","wendy",
"went","were","weren't","wes","wesley","west","weston","wet","wh","wha",
"what","what'd","what're","what's","whatever","whatever's","whatsoever","wheel","wheelchair","wheels",
"when","when'd","when'll","when's","whenever","where","where'd","where'll","where's","wherever",
"whether","whew","which","while","whip","whistle","white","whitney","who","who'd",
"who'll","who's","whoa","whoever","whole","whom","whoo","whore","whose","why",
"why'd","why'll","why's","wicked","wide","wife","wife's","wild","will","will's",
"william","williams","willie","willing","willow","wilson","win","wind","window","windows",
"wine","wing","wings","winner","winnie","winning","wins","winslow","winter","wipe",
"wiped","wire","wisdom","wise","wish","wished","wishes","wishing","witch","witches",
"with","within","without","witness","witnesses","wives","woke","wolf","woman","woman's",
"women","women's","won","won't","wonder","wondered","wonderful","wondering","woo","wood",
"woods","wool","word","words","wore","work","worked","worker","working","works",
"world","world's","worm","worn","worried","worries","worry","worrying","worse","worst",
"worth","worthless","worthy","would","would've","wouldn't","wound","wounded","wounds","wow",
"wrap","wrapped","wreck","wrestling","write","writer","writing","written","wrong","wrote",
"x","xander","y","y'all","y'know","ya","yacht","yah","yard","yards",
"yay","ye","yeah","year","year's","years","yell","yelling","yellow","yep",
"yes","yesterday","yet","yo","york","you","you'd","you'll","you're","you've",
"young","younger","your","yours","yourself","yourselves","youth","yup","zach","zack",
"zander","zelda","zero","zombie","zone","zoo"}


# The following code removes all the tags from the html file

class MLStripper(HTMLParser):
    def __init__(self):
        self.reset()
        self.fed = []
    def handle_data(self, d):
        self.fed.append(d)
    def get_data(self):
        return ''.join(self.fed)

def strip_tags(html):
    s = MLStripper()
    s.feed(html)
    return s.get_data()


# function to remove the script and style tags of the page
def get_page(url):
    try:
        data = urllib2.urlopen(url).read()
        content = re.subn(r'<(script).*?</\1>(?s)', '', data)[0]
        content = re.subn(r'<(style).*?</\1>(?s)', '', content)[0]
        return content
    except Exception:
	print "generic exception: "+url
	return ""

def processDb():
#   uncomment below to enter URL manually
#   url = raw_input() 
    domains = coll.find()
    for item in domains:
      if not "tags" in item.keys():
	text = get_page("http://"+item["Domain"])
	try:
	    text = unicode(text,'UTF-8')
    	    #print text
    	    striptext = strip_tags(text)
    	    text = striptext
	except Exception, e:
	    print "Decode error at "+item["Domain"]
	    continue
    	keywords = []

    	words = text.split()
    	for w in words:
    	    if not w.lower() in stopwords:
	    	keywords.append(w)

	coll.update({'_id':item["_id"]},{'$set':{"tags":keywords}},False)
    	print keywords

def main():
    processDb()

if __name__ == '__main__':
    main()
