from numpy import *
import re
import feedparser
def loadDataSet():
	postingList=[['my','dog','has','flea','problems','help','please'],
	['maybe','not','take','him','to','dog','park','stupid'],
	['my','dalmation','is','so','cute','I','love','him'],
	['stop','posting','stupid','worthless','garbage'],
	['mr','licks','ate','my','steak','how','to','stop','him'],
	['quit','buying','worthless','dog','food','stupid']]
	classVec=[0,1,0,1,0,1]
	return postingList,classVec

def createVocabList(dataSet):
	vocabSet=set([])
	for document in dataSet:
		vocabSet=vocabSet|set(document)
	return list(vocabSet)

def setOfWords2Vec(vocabList,inputSet):
	returnVec=[0]*len(vocabList)
	for word in inputSet:
		if word in vocabList:
			returnVec[vocabList.index(word)]=1
		else:
			print("the word :%s is not in my Vocabulary!" %word)
	return returnVec

def bagOfWords2VecMN(vocabList,inputSet):
	returnVec=[0]*len(vocabList)
	for word in inputSet:
		if word in vocabList:
			returnVec[vocabList.index(word)]+=1
	return returnVec

def trainNB0(trainMatrix,trainCategory):
	numTrainDocs=len(trainMatrix)
	numWords=len(trainMatrix[0])
	pAbusive=sum(trainCategory)/float(numTrainDocs)
	p0Num=zeros(numWords)
	p1Num=zeros(numWords)
	p0Denom=0.0
	p1Denom=0.0
	for i in range(numTrainDocs):
		if trainCategory[i]==1:
			p1Num+=trainMatrix[i]
			p1Denom+=sum(trainMatrix[i])
		else:
			p0Num+=trainMatrix[i]
			p0Denom+=sum(trainMatrix[i])
	p1Vect=p1Num/p1Denom
	p0Vect=p0Num/p0Denom
	return p0Vect,p1Vect,pAbusive

def classifyNB(vec2Classify,p0Vec,p1Vec,pClass1):
	p1=sum(vec2Classify*p1Vec)+log(pClass1)
	p0=sum(vec2Classify*p0Vec)+log(1.0-pClass1)
	if p1>p0:
		return 1
	else:
		return 0

def testingNB():
	listOPosts,listClasses=loadDataSet()
	myVocabList=createVocabList(listOPosts)
	trainMat=[]
	for postinDoc in listOPosts:
		trainMat.append(setOfWords2Vec(myVocabList,postinDoc))
	p0v,p1v,pAb=trainNB0(trainMat,listClasses)
	testEntry=['love','my','dalmation']
	thisDoc=array(setOfWords2Vec(myVocabList,testEntry))
	print(testEntry)
	print("classified as:")
	print(classifyNB(thisDoc,p0v,p1v,pAb))
	testEntry=['stupid','garbage']
	thisDoc=array(setOfWords2Vec(myVocabList,testEntry))
	print(testEntry)
	print("classifdied as:")
	print(classifyNB(thisDoc,p0v,p1v,pAb))


def textParse(bigString):
	import re
	listOfTokens=re.split(r'\W*',bigString)
	return [tok.lower() for tok in listOfTokens if len(tok)>2]

def spamTest():
	docList=[]
	classList=[]
	fullText=[]
	for i in range(1,26):
		wordList=textParse(open('email/spam/%d.txt' % i,'r',encoding='gbk').read())
		docList.append(wordList)
		fullText.extend(wordList)
		classList.append(1)
		wordList=textParse(open('email/ham/%d.txt' % i,'r',encoding='gbk').read())
		docList.append(wordList)
		fullText.extend(wordList)
		classList.append(0)
	vocabList=createVocabList(docList)
	trainingSet=list(range(50))
	testSet=[]
	for i in range(10):
		randIndex=int(random.uniform(0,len(trainingSet)))
		testSet.append(trainingSet[randIndex])
		del(trainingSet[randIndex])
	trainMat=[];
	trainClasses=[]
	for docIndex in trainingSet:
		trainMat.append(setOfWords2Vec(vocabList,docList[docIndex]))
		trainClasses.append(classList[docIndex])
	p0V,p1V,pSpam=trainNB0(array(trainMat),array(trainClasses))
	errorCount=0
	for docIndex in testSet:
		wordVector=setOfWords2Vec(vocabList,docList[docIndex])
		if classifyNB(array(wordVector),p0V,p1V,pSpam)!=classList[docIndex]:
			errorCount+=1
		print('the error rate is:%s',float(errorCount)/len(testSet))

#testingNB()
#spamTest()

def calcMostFreq(vocabList,fullText):
	import operator
	freqDict={}
	for token in vocabList:
		freqDict[token]=fullText.count(token)
	sortedFreq=sorted(freqDict.items(),key=operator.itemgetter(1),reverse=True)
	return sortedFreq[:30]

def localWords(feed1,feed0):
	import feedparser
	docList=[]
	classList=[]
	fullText=[]
	minLen=min(len(feed1['entries']),len(feed0['entries']))
	for i in range(minLen):
		wordList=textParse(feed1['entries'][i]['summary'])
		docList.append(wordList)
		fullText.extend(wordList)
		classList.append(1)
		wordList=textParse(feed0['entries'][i]['summary'])
		docList.append(wordList)
		fullText.extend(wordList)
		classList.append(0)
	vocabList=createVocabList(docList)
	top30Words=calcMostFreq(vocabList,fullText)
	for pairW in top30Words:
		if pairW[0] in vocabList:
			vocabList.remove(pairW[0])
	trainingSet=list(range(2*minLen))
	testSet=[]
	for i in range(20):
		randIndex=int(random.uniform(0,len(trainingSet)))
		testSet.append(trainingSet[randIndex])
		del(trainingSet[randIndex])
	trainMat=[]
	trainClasses=[]
	for docIndex in trainingSet:
		trainMat.append(bagOfWords2VecMN(vocabList,docList[docIndex]))
		trainClasses.append(classList[docIndex])
	p0V,p1V,pSpam=trainNB0(array(trainMat),array(trainClasses))
	errorCount=0
	for docIndex in testSet:
		wordVector=bagOfWords2VecMN(vocabList,docList[docIndex])
		if classifyNB(array(wordVector),p0V,p1V,pSpam)!=classList[docIndex]:
			errorCount+=1
	print('the error rate is:%s',float(errorCount)/len(testSet))
	return vocabList,p0V,p1V

ny=feedparser.parse('http://newyork.craigslist.org/stp/index.rss')
sf=feedparser.parse('http://sfbay.craigslist.org/stp/index.rss')
vocabList,pSF,pNY=localWords(ny,sf)
vocabList,pSF,pNY=localWords(ny,sf)

def getTopWords(ny,sf):
	import operator
	vocabList,p0V,p1V=localWords(ny,sf)
	topNY=[]
	topSF=[]
	for i in range(len(p0V)):
		if p0V[i]>-6.0:
			topSF.append((vocabList[i],p0V[i]))
		if p1V[i]>-6.0:
			topNY.append((vocabList[i],p1V[i]))
	sortedSF=sorted(topSF,key=lambda pair:pair[],reverse=Tree)
	print("SF**SF**SF")
	for item in sortedSF:
		print(item[0])
	sortedVY=sorted(topNY,key=lambda pair:pair[1],reverse=True)
	print("NY**NY**NY")
	for item in sortedNY:
		print(item[0])