# encoding=utf-8
import sys
import feedparser
import re
import jieba
def getwordcounts(url):
    d = feedparser.parse(url)
    wc = {}
    for e in d.entries:
        if 'summary' in e:
            summary = e.summary
        else:
            summary = e.description
        words = getwords(e.title + ' ' + summary)
        for word in words:
            if(word != ' ' or word != '\n' or word!='' or word!='\t'):
                wc.setdefault(word, 0)
                wc[word] += 1
    return d.feed.title, wc
def getwords(html):
    txt = re.compile(r'<[^>]+>').sub('', html)
    algor = jieba.cut(txt)
    return [tok.lower() for tok in algor if tok != '']

apcount = {}
wordcounts = {}
fp=open("feedlist.txt","r")
feedlist = [line for line in fp.readlines()]
for feedurl in feedlist:
    print(feedurl)
    try:
        title, wc = getwordcounts(feedurl)
        wordcounts[title] = wc
        for word, count in wc.items():
            apcount.setdefault(word, 0)
            if count > 1:
                apcount[word] += 1
    except:
        print ('Failed to parse feed %s' % feedurl)
wordlist = []
for w, bc in apcount.items():
    frac = float(bc) / len(feedlist)
    wordlist.append(w)
out = open('blogdata1.txt', 'w',encoding='utf-8')
out.write('Blog')
for word in wordlist: out.write('\t%s' % word.strip())
out.write('\n')
for blog, wc in wordcounts.items():
    out.write(blog)
    for word in wordlist:
        if word in wc:
            out.write('\t%d' % wc[word])
        else:
            out.write('\t0')
    out.write('\n')