#!/usr/bin/python
# -*- coding: utf-8 -*-

'''
4.13    创建处理数据的管道
Created on 2016年9月2日
@author: wang
'''

import os
import fnmatch
import gzip
import bz2
import re

def gen_find(filepat, top):
    for path, dirlist, filelist in os.walk(top):
        for name in fnmatch.filter(filelist, filepat):
            yield os.path.join(path, name)
def gen_opener(filenames):
    for filename in filenames:
        if filename.endswith('.gz'):
            f = gzip.open(filename, 'rt')
        elif filename.endswith('.bz'):
            f = bz2.open(filename, 'rt')
        else :
            f = open(filename, 'rt')
        yield f
        f.close()
def gen_concatente(iterators):
    for it in iterators:
        yield from it
def gen_grep(pattern, lines):
    pat = re.compile(pattern)
    for line in lines:
        if pat.search(line):
            yield line

lognames = gen_find('access-log*', 'www')
files = gen_opener(lognames)
lines = gen_concatente(files)
pylines = gen_grep('(?i)python', lines)
for line in pylines:
    print(line)
