"""

    作者：大发
    日期：2019/11/19
    内容：利用python进行数据分析 11.4 p320 笔记

"""

import pandas as pd
import numpy as np
from io import StringIO
from lxml import objectify
import requests
import sqlite3
import sqlalchemy as sqla
from io import StringIO
import json
from numpy import nan as NA
import re
import matplotlib.pyplot as plt
from numpy.random import randn
from datetime import datetime
from io import BytesIO
import seaborn as sns
import statsmodels.api as sm
from datetime import datetime
from datetime import timedelta
from dateutil.parser import parse
from pandas.tseries.offsets import Hour,Minute
from pandas.tseries.offsets import Day,MonthEnd
from scipy.stats import percentileofscore

## 11.4  时区处理

## pass 需要的时候再看吧
# 在Python中，时区信息来自第三方库pytz，
# 它使Python可以使用Olson数据库（汇编了世界时区信息）。

## 11.5 时期及其算术运算

# 这个Period对象表示的是从2007年1月1日到2007年12月31日之间的整段时间。
# p = pd.Period(2007 , freq = 'A-DEC')
# p
# Out[54]: Period('2007', 'A-DEC')

# 对Period对象加上或减去一个整数即可达到根据其频率进行位移的效果：
# p +5
# Out[55]: Period('2012', 'A-DEC')
# p-2
# Out[56]: Period('2005', 'A-DEC')

# 如果两个Period对象拥有相同的频率，则它们的差就是它们之间的单位数量：
# pd.Period('2014',freq='A-DEC') - p
# Out[57]: <7 * YearEnds: month=12>

# period_range函数可用于创建规则的时期范围：
# In[58]: rng = pd.period_range('2000-01-01' , '2000-06-30' , freq='M')
# In[59]: rng
# Out[59]: PeriodIndex(['2000-01', '2000-02', '2000-03', '2000-04', '2000-05', '2000-06'], dtype='period[M]', freq='M')

# PeriodIndex类保存了一组Period，它可以在任何pandas数据结构中被用作轴索引：
# In[60]: pd.Series(np.random.randn(6) , index=rng)
# Out[60]:
# 2000-01   -1.139182
# 2000-02    2.388678
# 2000-03    0.062788
# 2000-04    0.514980
# 2000-05   -0.374340
# 2000-06   -0.164902
# Freq: M, dtype: float64

# 如果你有一个字符串数组，你也可以使用PeriodIndex类：
# In[63]: index= pd.PeriodIndex(values , freq='Q-DEC')
# In[64]: index
# Out[64]: PeriodIndex(['2001Q3', '2002Q2', '2003Q1'], dtype='period[Q-DEC]', freq='Q-DEC')


## 11.5.1 区间频率转换

# 一个年度时期，希望将其转换为当年年初或年末的一个月度时期
# p = pd.Period('2007' , freq='A-DEC')
# p
# Out[66]: Period('2007', 'A-DEC')
# p.asfreq('M', how='start')
# Out[67]: Period('2007-01', 'M')
# p.asfreq('M' , how = 'end')
# Out[68]: Period('2007-12', 'M')

# 对于一个不以12月结束的财政年度，月度子时期的归属情况就不一样了：
# p = pd.Period('2007' , freq='A-JUN')
# p
# Out[70]: Period('2007', 'A-JUN')
# p.asfreq('M' , 'start')
# Out[71]: Period('2006-07', 'M')
# p.asfreq('M' , 'end')
# Out[72]: Period('2007-06', 'M')

# 在将高频率转换为低频率时，
# 超时期（superperiod）是由子时期（subperiod）所属的位置决定的。
# 例如，在A-JUN频率中，月份“2007年8 月”实际上是属于周期“2008年”的：
# In[73]: p = pd.Period('Aug-2007' , 'M')
# In[74]: p.asfreq('A-JUN')
# Out[74]: Period('2008', 'A-JUN')
# In[75]: rng = pd.period_range('2006' , '2009' , freq='A-DEC')
# In[76]: ts = pd.Series(np.random.randn(len(rng)) , index = rng)
# In[77]: ts
# Out[77]:
# 2006   -0.390535
# 2007    2.132826
# 2008   -1.148053
# 2009   -1.921745
# Freq: A-DEC, dtype: float64
# In[78]: ts.asfreq('M' , how = 'start')
# Out[78]:
# 2006-01   -0.390535
# 2007-01    2.132826
# 2008-01   -1.148053
# 2009-01   -1.921745
# Freq: M, dtype: float64

# 如果我们想要每年的最后一个工作日，
# 我们可以使用“B”频率，并指明想要该时期的末尾：
# ts.asfreq('B' , how='end')
# Out[79]:
# 2006-12-29   -0.390535
# 2007-12-31    2.132826
# 2008-12-31   -1.148053
# 2009-12-31   -1.921745
# Freq: B, dtype: float64

####  11.5.2 季度区间频率
# pandas支持12种可能的季度型频率，即Q-JAN到Q-DEC：
# p = pd.Period('2012Q4' , freq='Q-JAN')
# p
# Out[81]: Period('2012Q4', 'Q-JAN')

# 2012Q4是从11月到1月（将其转换为日型频率就明白了）
# In[82]: p.asfreq('D' , 'start')
# Out[82]: Period('2011-11-01', 'D')
# In[83]: p.asfreq('D' , 'end')
# Out[83]: Period('2012-01-31', 'D')

# 获取该季度倒数第二个工作日下午4点的时间戳
# In[84]: p4pm = (p.asfreq('B' , 'e')-1).asfreq('T' , 's') + 16*60
# In[85]: p4pm
# Out[85]: Period('2012-01-30 16:00', 'T')
# In[86]: p4pm.to_timestamp()
# Out[86]: Timestamp('2012-01-30 16:00:00')

# 季度型范围的算术运算
# In[87]: rng = pd.period_range('2011Q3' , '2012Q4' , freq='Q-JAN')
# In[88]: ts = pd.Series(np.arange(len(rng)) , index = rng)
# In[89]: ts
# Out[89]:
# 2011Q3    0
# 2011Q4    1
# 2012Q1    2
# 2012Q2    3
# 2012Q3    4
# 2012Q4    5
# Freq: Q-JAN, dtype: int32
# In[94]: new_rng = (rng.asfreq('B' , 'e')-1 ).asfreq('T','s') + 16*60
# In[95]: ts.index = new_rng.to_timestamp()
# In[96]: ts
# Out[96]:
# 2010-10-28 16:00:00    0
# 2011-01-28 16:00:00    1
# 2011-04-28 16:00:00    2
# 2011-07-28 16:00:00    3
# 2011-10-28 16:00:00    4
# 2012-01-30 16:00:00    5
# dtype: int32


##### 11.5.3 将Timestamp(时间戳)转换为Period（区间）（及其反向过程）

# 通过使用to_period方法，可以将由时间戳索引的Series和DataFrame对象
# 转换为以时期索引：

# rng = pd.date_range('2000-01-01' , periods=3 , freq='M')
# ts = pd.Series(np.random.randn(3) , index=rng)
# ts
# Out[98]:
# 2000-01-31    2.333570
# 2000-02-29    1.232524
# 2000-03-31   -0.467164
# Freq: M, dtype: float64

# pts = ts.to_period()
# pts
# Out[100]:
# 2000-01    2.333570
# 2000-02    1.232524
# 2000-03   -0.467164
# Freq: M, dtype: float64

# 新PeriodIndex的频率默认是从时间戳推断而来的，
# 你也可以指定任何别的频率。结果中允许存在重复时期：
# In[101]: rng = pd.date_range('1/29/2000' , periods=6 , freq='D')
# In[102]: ts2 = pd.Series(np.random.randn(6) , index=rng)
# In[103]: ts2
# Out[103]:
# 2000-01-29   -0.078066
# 2000-01-30   -2.602328
# 2000-01-31    2.103791
# 2000-02-01   -0.526704
# 2000-02-02   -1.657575
# 2000-02-03   -1.072117
# Freq: D, dtype: float64
# In[104]: ts2.to_period('M')
# Out[104]:
# 2000-01   -0.078066
# 2000-01   -2.602328
# 2000-01    2.103791
# 2000-02   -0.526704
# 2000-02   -1.657575
# 2000-02   -1.072117
# Freq: M, dtype: float64

# 要转换回时间戳，使用to_timestamp即可：
# In[105]: pts = ts2.to_period()
# In[106]: pts
# Out[106]:
# 2000-01-29   -0.078066
# 2000-01-30   -2.602328
# 2000-01-31    2.103791
# 2000-02-01   -0.526704
# 2000-02-02   -1.657575
# 2000-02-03   -1.072117
# Freq: D, dtype: float64
# In[107]: pts.to_timestamp(how = 'end')
# Out[107]:
# 2000-01-29 23:59:59.999999999   -0.078066
# 2000-01-30 23:59:59.999999999   -2.602328
# 2000-01-31 23:59:59.999999999    2.103791
# 2000-02-01 23:59:59.999999999   -0.526704
# 2000-02-02 23:59:59.999999999   -1.657575
# 2000-02-03 23:59:59.999999999   -1.072117
# Freq: D, dtype: float64


### 11.5.4 通过数组创建PeriodIndex

# In[108]: data = pd.read_csv('D:\data_py\pydata-book-2nd-edition\examples\macrodata.csv')
# In[109]: data.head(5)
# Out[109]:
#      year  quarter   realgdp  realcons  ...  unemp      pop  infl  realint
# 0  1959.0      1.0  2710.349    1707.4  ...    5.8  177.146  0.00     0.00
# 1  1959.0      2.0  2778.801    1733.7  ...    5.1  177.830  2.34     0.74
# 2  1959.0      3.0  2775.488    1751.8  ...    5.3  178.657  2.74     1.09
# 3  1959.0      4.0  2785.204    1753.7  ...    5.6  179.386  0.27     4.06
# 4  1960.0      1.0  2847.699    1770.5  ...    5.2  180.007  2.31     1.19
#
# [5 rows x 14 columns]


# In[110]: data.year
# Out[110]:
# 0      1959.0
# 1      1959.0
# 2      1959.0
# 3      1959.0
# 4      1960.0
# 5      1960.0
# 6      1960.0
# 7      1960.0
# 8      1961.0
# 9      1961.0
# 10     1961.0
# 11     1961.0
# 12     1962.0
# 13     1962.0
# 14     1962.0
# 15     1962.0
# 16     1963.0
# 17     1963.0
# 18     1963.0
# 19     1963.0
# 20     1964.0
# 21     1964.0
# 22     1964.0
# 23     1964.0
# 24     1965.0
# 25     1965.0
# 26     1965.0
# 27     1965.0
# 28     1966.0
# 29     1966.0
#         ...
# 173    2002.0
# 174    2002.0
# 175    2002.0
# 176    2003.0
# 177    2003.0
# 178    2003.0
# 179    2003.0
# 180    2004.0
# 181    2004.0
# 182    2004.0
# 183    2004.0
# 184    2005.0
# 185    2005.0
# 186    2005.0
# 187    2005.0
# 188    2006.0
# 189    2006.0
# 190    2006.0
# 191    2006.0
# 192    2007.0
# 193    2007.0
# 194    2007.0
# 195    2007.0
# 196    2008.0
# 197    2008.0
# 198    2008.0
# 199    2008.0
# 200    2009.0
# 201    2009.0
# 202    2009.0
# Name: year, Length: 203, dtype: float64

# In[111]: data.quarter
# Out[111]:
# 0      1.0
# 1      2.0
# 2      3.0
# 3      4.0
# 4      1.0
# 5      2.0
# 6      3.0
# 7      4.0
# 8      1.0
# 9      2.0
# 10     3.0
# 11     4.0
# 12     1.0
# 13     2.0
# 14     3.0
# 15     4.0
# 16     1.0
# 17     2.0
# 18     3.0
# 19     4.0
# 20     1.0
# 21     2.0
# 22     3.0
# 23     4.0
# 24     1.0
# 25     2.0
# 26     3.0
# 27     4.0
# 28     1.0
# 29     2.0
#       ...
# 173    2.0
# 174    3.0
# 175    4.0
# 176    1.0
# 177    2.0
# 178    3.0
# 179    4.0
# 180    1.0
# 181    2.0
# 182    3.0
# 183    4.0
# 184    1.0
# 185    2.0
# 186    3.0
# 187    4.0
# 188    1.0
# 189    2.0
# 190    3.0
# 191    4.0
# 192    1.0
# 193    2.0
# 194    3.0
# 195    4.0
# 196    1.0
# 197    2.0
# 198    3.0
# 199    4.0
# 200    1.0
# 201    2.0
# 202    3.0
# Name: quarter, Length: 203, dtype: float64

# 通过将这些数组以及一个频率传入PeriodIndex，
# 就可以将它们合并成DataFrame的一个索引：

# index = pd.PeriodIndex(year = data.year , quarter = data.quarter ,
#                        freq='Q-DEC')
# In[113]: index
# Out[113]:
# PeriodIndex(['1959Q1', '1959Q2', '1959Q3', '1959Q4', '1960Q1', '1960Q2',
#              '1960Q3', '1960Q4', '1961Q1', '1961Q2',
#              ...
#              '2007Q2', '2007Q3', '2007Q4', '2008Q1', '2008Q2', '2008Q3',
#              '2008Q4', '2009Q1', '2009Q2', '2009Q3'],
#             dtype='period[Q-DEC]', length=203, freq='Q-DEC')

# In[114]: data.index = index
# In[115]: data.infl
# Out[115]:
# 1959Q1    0.00
# 1959Q2    2.34
# 1959Q3    2.74
# 1959Q4    0.27
# 1960Q1    2.31
# 1960Q2    0.14
# 1960Q3    2.70
# 1960Q4    1.21
# 1961Q1   -0.40
# 1961Q2    1.47
# 1961Q3    0.80
# 1961Q4    0.80
# 1962Q1    2.26
# 1962Q2    0.13
# 1962Q3    2.11
# 1962Q4    0.79
# 1963Q1    0.53
# 1963Q2    2.75
# 1963Q3    0.78
# 1963Q4    2.46
# 1964Q1    0.13
# 1964Q2    0.90
# 1964Q3    1.29
# 1964Q4    2.05
# 1965Q1    1.28
# 1965Q2    2.54
# 1965Q3    0.89
# 1965Q4    2.90
# 1966Q1    4.99
# 1966Q2    2.10
#           ...
# 2002Q2    1.56
# 2002Q3    2.66
# 2002Q4    3.08
# 2003Q1    1.31
# 2003Q2    1.09
# 2003Q3    2.60
# 2003Q4    3.02
# 2004Q1    2.35
# 2004Q2    3.61
# 2004Q3    3.58
# 2004Q4    2.09
# 2005Q1    4.15
# 2005Q2    1.85
# 2005Q3    9.14
# 2005Q4    0.40
# 2006Q1    2.60
# 2006Q2    3.97
# 2006Q3   -1.58
# 2006Q4    3.30
# 2007Q1    4.58
# 2007Q2    2.75
# 2007Q3    3.45
# 2007Q4    6.38
# 2008Q1    2.82
# 2008Q2    8.53
# 2008Q3   -3.16
# 2008Q4   -8.79
# 2009Q1    0.94
# 2009Q2    3.37
# 2009Q3    3.56
# Freq: Q-DEC, Name: infl, Length: 203, dtype: float64


####  11.6 重采样及频率转换

# resample有一个类似于groupby的API，调用resample可以分组数据，
# 然后会调用一个聚合函数：
# In[117]: rng = pd.date_range('2000-01-01' , periods=100 , freq='D')
# In[118]: ts = pd.Series(np.random.randn(len(rng)) , index=rng)
# In[119]: ts
# Out[119]:
# 2000-01-01    1.871264
# 2000-01-02   -0.645876
# 2000-01-03    1.743281
# 2000-01-04    1.793549
# 2000-01-05    1.118646
# 2000-01-06   -1.014249
# 2000-01-07    0.517270
# 2000-01-08    0.563543
# 2000-01-09    0.854410
# 2000-01-10   -0.209021
# 2000-01-11    0.312114
# 2000-01-12    1.658985
# 2000-01-13    1.346582
# 2000-01-14   -0.909061
# 2000-01-15   -0.737225
# 2000-01-16   -1.330148
# 2000-01-17   -1.057828
# 2000-01-18    0.906004
# 2000-01-19    1.876974
# 2000-01-20   -0.708598
# 2000-01-21   -0.936277
# 2000-01-22   -0.894562
# 2000-01-23   -0.726804
# 2000-01-24    0.464354
# 2000-01-25    0.015691
# 2000-01-26   -1.301010
# 2000-01-27   -0.917954
# 2000-01-28   -0.186757
# 2000-01-29    0.296659
# 2000-01-30    0.007226
#                 ...
# 2000-03-11    1.007964
# 2000-03-12   -1.304735
# 2000-03-13    0.644216
# 2000-03-14    0.010659
# 2000-03-15   -1.062434
# 2000-03-16   -1.624091
# 2000-03-17   -1.448267
# 2000-03-18    0.005752
# 2000-03-19   -0.035676
# 2000-03-20   -1.024155
# 2000-03-21   -0.467312
# 2000-03-22   -1.978395
# 2000-03-23   -0.473411
# 2000-03-24   -0.456118
# 2000-03-25   -0.327860
# 2000-03-26    0.809823
# 2000-03-27    0.391165
# 2000-03-28   -0.808820
# 2000-03-29   -1.389184
# 2000-03-30   -0.990508
# 2000-03-31    0.700348
# 2000-04-01   -1.568796
# 2000-04-02   -0.355836
# 2000-04-03   -1.520903
# 2000-04-04   -0.502874
# 2000-04-05   -1.049903
# 2000-04-06   -0.995756
# 2000-04-07    1.121450
# 2000-04-08   -0.627745
# 2000-04-09   -0.447108
# Freq: D, Length: 100, dtype: float64

# In[120]: ts.resample('M' , kind='period').mean()
# Out[120]:
# 2000-01    0.102908
# 2000-02   -0.019095
# 2000-03   -0.386598
# 2000-04   -0.660830
# Freq: M, dtype: float64

### 11.6.1 降采样
# In[121]: rng = pd.date_range('2000-01-01' , periods=12 , freq='T')
# In[122]: ts = pd.Series(np.arange(12) , index=rng)
# In[123]: ts
# Out[123]:
# 2000-01-01 00:00:00     0
# 2000-01-01 00:01:00     1
# 2000-01-01 00:02:00     2
# 2000-01-01 00:03:00     3
# 2000-01-01 00:04:00     4
# 2000-01-01 00:05:00     5
# 2000-01-01 00:06:00     6
# 2000-01-01 00:07:00     7
# 2000-01-01 00:08:00     8
# 2000-01-01 00:09:00     9
# 2000-01-01 00:10:00    10
# 2000-01-01 00:11:00    11
# Freq: T, dtype: int32

# 通过求和的方式将这些数据聚合到“5分钟”块中：
## closed 表示哪一段是闭合的
# In[124]: ts.resample('5min' , closed='right').sum()
# Out[124]:
# 1999-12-31 23:55:00     0
# 2000-01-01 00:00:00    15
# 2000-01-01 00:05:00    40
# 2000-01-01 00:10:00    11
# Freq: 5T, dtype: int32
# 默认情况下，面元的右边界是包含的，
# 因此00:00到00:05的区间中是包含00:05的。

# 传入closed='left'会让区间以左边界闭合：
# In[125]: ts.resample('5min' , closed='right').sum()
# Out[125]:
# 1999-12-31 23:55:00     0
# 2000-01-01 00:00:00    15
# 2000-01-01 00:05:00    40
# 2000-01-01 00:10:00    11
# Freq: 5T, dtype: int32

# 如你所见，最终的时间序列是以各面元右边界的时间戳进行标记的。
# 传入label='right'即可用面元的邮编界对其进行标记：
# In[126]: ts.resample('5min' , closed='right' , label='right').sum()
# Out[126]:
# 2000-01-01 00:00:00     0
# 2000-01-01 00:05:00    15
# 2000-01-01 00:10:00    40
# 2000-01-01 00:15:00    11
# Freq: 5T, dtype: int32

# 最后，你可能希望对结果索引做一些位移，比如从右边界减去一秒以便更容
# 易明白该时间戳到底表示的是哪个区间。只需通过loffset设置一个字符串或
# 日期偏移量即可实现这个目的：
# ts.resample('5min' , closed='right' , label='right' , loffset='-1s').sum()
# Out[127]:
# 1999-12-31 23:59:59     0
# 2000-01-01 00:04:59    15
# 2000-01-01 00:09:59    40
# 2000-01-01 00:14:59    11
# Freq: 5T, dtype: int32

# ts.resample('5min' , closed='right' ,
#             label='right' , loffset='-1s').sum()
# Out[128]:
# 1999-12-31 23:59:59     0
# 2000-01-01 00:04:59    15
# 2000-01-01 00:09:59    40
# 2000-01-01 00:14:59    11
# Freq: 5T, dtype: int32

### 11.6.1.1 OHLC重采样
# 第一个值（open，开盘）、最后一个值（close，收盘）、最大值（high，最高）以及最小值（low，最低）
# 传入how='ohlc'即可得到一个含有这四种聚合值的DataFrame
# ts.resample('5min').ohlc()
# Out[129]:
#                      open  high  low  close
# 2000-01-01 00:00:00     0     4    0      4
# 2000-01-01 00:05:00     5     9    5      9
# 2000-01-01 00:10:00    10    11   10     11

### 11.6.2 升采样和插值
# 升采样：将数据从低频率转换到高频率
# frame = pd.DataFrame(np.random.randn(2,4) ,
#                      index=pd.date_range('1/1/2000',periods=2,freq='W-WED'),
#                      columns=['Colorado' , 'Texas' , 'New York' , 'Ohio'])
# In[131]: frame
# Out[131]:
#             Colorado     Texas  New York      Ohio
# 2000-01-05  0.341068  0.684468  0.298451  0.913267
# 2000-01-12 -2.987119 -0.001058 -0.198974 -1.998187

# 当你对这个数据进行聚合，每组只有一个值，这样就会引入缺失值。
# 我们使用asfreq方法转换成高频，不经过聚合：
# In[132]: df_daily = frame.resample('D').asfreq()
# In[133]: df_daily
# Out[133]:
#             Colorado     Texas  New York      Ohio
# 2000-01-05  0.341068  0.684468  0.298451  0.913267
# 2000-01-06       NaN       NaN       NaN       NaN
# 2000-01-07       NaN       NaN       NaN       NaN
# 2000-01-08       NaN       NaN       NaN       NaN
# 2000-01-09       NaN       NaN       NaN       NaN
# 2000-01-10       NaN       NaN       NaN       NaN
# 2000-01-11       NaN       NaN       NaN       NaN
# 2000-01-12 -2.987119 -0.001058 -0.198974 -1.998187

# 假设你想要用前面的周型值填充“非星期三”。
# resampling的填充和插值方式跟fillna和reindex的一样：
# In[134]: frame.resample('D').ffill()
# Out[134]:
#             Colorado     Texas  New York      Ohio
# 2000-01-05  0.341068  0.684468  0.298451  0.913267
# 2000-01-06  0.341068  0.684468  0.298451  0.913267
# 2000-01-07  0.341068  0.684468  0.298451  0.913267
# 2000-01-08  0.341068  0.684468  0.298451  0.913267
# 2000-01-09  0.341068  0.684468  0.298451  0.913267
# 2000-01-10  0.341068  0.684468  0.298451  0.913267
# 2000-01-11  0.341068  0.684468  0.298451  0.913267
# 2000-01-12 -2.987119 -0.001058 -0.198974 -1.998187

# 同样，这里也可以只填充指定的时期数（目的是限制前面的观测值的持续使用距离）：
# In[135]: frame.resample('D').ffill(limit=2)
# Out[135]:
#             Colorado     Texas  New York      Ohio
# 2000-01-05  0.341068  0.684468  0.298451  0.913267
# 2000-01-06  0.341068  0.684468  0.298451  0.913267
# 2000-01-07  0.341068  0.684468  0.298451  0.913267
# 2000-01-08       NaN       NaN       NaN       NaN
# 2000-01-09       NaN       NaN       NaN       NaN
# 2000-01-10       NaN       NaN       NaN       NaN
# 2000-01-11       NaN       NaN       NaN       NaN
# 2000-01-12 -2.987119 -0.001058 -0.198974 -1.998187

# 注意，新的日期索引完全没必要跟旧的重叠：
# In[136]: frame.resample('W-THU').ffill()
# Out[136]:
#             Colorado     Texas  New York      Ohio
# 2000-01-06  0.341068  0.684468  0.298451  0.913267
# 2000-01-13 -2.987119 -0.001058 -0.198974 -1.998187

###  11.6.3 使用区间进行重采样
# frame = pd.DataFrame(np.random.randn(24,4),
#                      index=pd.period_range('1-2000' , '12-2001' , freq='M') ,
#                      columns=['Colorado' , 'Texas' , 'New York' , 'Ohio'])
# In[138]: frame[:5]
# Out[138]:
#          Colorado     Texas  New York      Ohio
# 2000-01 -1.310158  0.875443 -0.538754  0.712522
# 2000-02  0.359522 -0.945807 -0.718091 -0.496742
# 2000-03 -0.476879 -0.487266  0.328465  0.273803
# 2000-04 -2.711463 -0.343555 -0.900624 -0.739197
# 2000-05 -0.528240  0.742256  0.871689  1.040381

# In[139]: annual_frame = frame.resample('A-DEC').mean()
# In[140]: annual_frame
# Out[140]:
#       Colorado     Texas  New York      Ohio
# 2000 -0.453147 -0.246840  0.083828 -0.231396
# 2001 -0.250636  0.040315 -0.402323 -0.268332

# 升采样要稍微麻烦一些，因为你必须决定在新频率中各区间的哪端用于放置
# 原来的值，就像asfreq方法那样。
# convention参数默认为'start'，也可设置为'end'：
# In[141]: annual_frame.resample('Q-DEC').ffill()
# Out[141]:
#         Colorado     Texas  New York      Ohio
# 2000Q1 -0.453147 -0.246840  0.083828 -0.231396
# 2000Q2 -0.453147 -0.246840  0.083828 -0.231396
# 2000Q3 -0.453147 -0.246840  0.083828 -0.231396
# 2000Q4 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q1 -0.250636  0.040315 -0.402323 -0.268332
# 2001Q2 -0.250636  0.040315 -0.402323 -0.268332
# 2001Q3 -0.250636  0.040315 -0.402323 -0.268332
# 2001Q4 -0.250636  0.040315 -0.402323 -0.268332

# In[142]: annual_frame.resample('Q-DEC' , convention='end').ffill()
# Out[142]:
#         Colorado     Texas  New York      Ohio
# 2000Q4 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q1 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q2 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q3 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q4 -0.250636  0.040315 -0.402323 -0.268332

# In[143]: annual_frame.resample('Q-MAR').ffill()
# Out[143]:
#         Colorado     Texas  New York      Ohio
# 2000Q4 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q1 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q2 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q3 -0.453147 -0.246840  0.083828 -0.231396
# 2001Q4 -0.250636  0.040315 -0.402323 -0.268332
# 2002Q1 -0.250636  0.040315 -0.402323 -0.268332
# 2002Q2 -0.250636  0.040315 -0.402323 -0.268332
# 2002Q3 -0.250636  0.040315 -0.402323 -0.268332


####  11.7 移动窗口函数

# 加载一些时间序列数据，将其重采样为工作日频率：
# close_px_all = pd.read_csv('D:\data_py\pydata-book-2nd-edition\examples\stock_px_2.csv',
#                            parse_dates=True , index_col=0)
# close_px = close_px_all[['AAAP' , 'MSFT' , 'XOM']]
# close_px = close_px.resample('B').ffill()

# 引入rolling运算符，它与resample和groupby很像。
# 可以在TimeSeries或DataFrame以及一个window（表示期数，见图11-4）上调用它：
# close_px.AAPL.plot()
# Backend Qt5Agg is interactive backend. Turning interactive mode on.
# Out[146]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b1e7f160>
# close_px.AAPL.rolling(250).mean().plot()
# Out[147]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b1e7f160>

# 默认情况下，rolling函数需要窗口中所有的值为非NA值。可以修改该行为以
# 解决缺失数据的问题。其实，在时间序列开始处尚不足窗口期的那些数据就
# 是个特例（见图11-5）：
# In[148]: appl_std250 = close_px.AAPL.rolling(250 , min_periods = 10).std()
# In[149]: appl_std250[5:12]
# Out[149]:
# 2003-01-09         NaN
# 2003-01-10         NaN
# 2003-01-13         NaN
# 2003-01-14         NaN
# 2003-01-15    0.077496
# 2003-01-16    0.074760
# 2003-01-17    0.112368
# Freq: B, Name: AAPL, dtype: float64

# apple_std250时间序列的扩展窗口平均如下所示：
# expanding_mean = appl_std250.expanding().mean()
# 对DataFrame调用rolling_mean（以及与之类似的函数）会将转换应用到所有的列上（见图11-6）：
# close_px.rolling(60).mean().plot(logy=True)
# Out[153]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b58a5780>
# close_px.rolling('20D').mean()
# Out[154]:
#                   AAPL       MSFT        XOM
# 2003-01-02    7.400000  21.110000  29.220000
# 2003-01-03    7.425000  21.125000  29.230000
# 2003-01-06    7.433333  21.256667  29.473333
# 2003-01-07    7.432500  21.425000  29.342500
# 2003-01-08    7.402000  21.402000  29.240000
# 2003-01-09    7.391667  21.490000  29.273333
# 2003-01-10    7.387143  21.558571  29.238571
# 2003-01-13    7.378750  21.633750  29.197500
# 2003-01-14    7.370000  21.717778  29.194444
# 2003-01-15    7.355000  21.757000  29.152000
# 2003-01-16    7.350909  21.756364  29.129091
# 2003-01-17    7.325833  21.628333  29.085000
# 2003-01-20    7.304615  21.520000  29.047692
# 2003-01-21    7.283571  21.423571  28.968571
# 2003-01-22    7.250714  21.347143  28.851429
# 2003-01-23    7.225000  21.304286  28.728571
# 2003-01-24    7.203333  21.190000  28.608667
# 2003-01-27    7.160000  20.980000  28.316429
# 2003-01-28    7.160714  20.827857  28.178571
# 2003-01-29    7.170000  20.662143  28.067143
# 2003-01-30    7.155714  20.446429  27.948571
# 2003-01-31    7.157333  20.326667  27.960667
# 2003-02-03    7.147857  19.959286  27.846429
# 2003-02-04    7.153571  19.707857  27.828571
# 2003-02-05    7.147143  19.472143  27.772143
# 2003-02-06    7.159286  19.358571  27.720000
# 2003-02-07    7.153333  19.288000  27.716000
# 2003-02-10    7.172857  19.110714  27.647857
# 2003-02-11    7.190000  18.982857  27.654286
# 2003-02-12    7.197857  18.819286  27.625714
#                 ...        ...        ...
# 2011-09-05  375.711429  25.276429  72.607857
# 2011-09-06  375.661429  25.295000  72.392857
# 2011-09-07  376.938571  25.390000  72.586429
# 2011-09-08  378.946429  25.545000  72.802143
# 2011-09-09  378.848667  25.558000  72.682667
# 2011-09-12  380.902143  25.754286  72.731429
# 2011-09-13  381.505000  25.835714  72.596429
# 2011-09-14  382.617857  25.973571  72.658571
# 2011-09-15  383.287857  26.097857  72.756429
# 2011-09-16  384.435333  26.166000  72.876000
# 2011-09-19  385.585714  26.259286  72.772143
# 2011-09-20  387.630000  26.286429  72.771429
# 2011-09-21  389.852143  26.270714  72.662857
# 2011-09-22  391.835714  26.217857  72.455714
# 2011-09-23  392.666667  26.140667  72.246000
# 2011-09-26  395.670000  26.160000  72.294286
# 2011-09-27  396.765000  26.136429  72.241429
# 2011-09-28  397.684286  26.090714  72.187857
# 2011-09-29  398.619286  26.070000  72.392857
# 2011-09-30  397.466000  25.991333  72.408667
# 2011-10-03  398.002143  25.890714  72.413571
# 2011-10-04  396.802143  25.807857  72.427143
# 2011-10-05  395.751429  25.729286  72.422857
# 2011-10-06  394.099286  25.673571  72.375714
# 2011-10-07  392.479333  25.712000  72.454667
# 2011-10-10  389.351429  25.602143  72.527857
# 2011-10-11  388.505000  25.674286  72.835000
# 2011-10-12  388.531429  25.810000  73.400714
# 2011-10-13  388.826429  25.961429  73.905000
# 2011-10-14  391.038000  26.048667  74.185333
# [2292 rows x 3 columns]

## 11.7.1 指数加权函数

# 下面这个例子对比了苹果公司股价的30日移动平均和span=30的指数加权移动平均：
# aapl_px = close_px.AAPL['2006':'2007']
# ma60 = aapl_px.rolling(30 , min_periods=20).mean()
# ewma60 = aapl_px.ewm(span=30).mean()
# ma60.plot(style='k--' , label='Simple MA')
# Out[156]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b5e10470>
# ewma60.plot(style='k-' , label = 'EW MA')
# Out[157]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b5e10470>
# plt.legend()

###  11.7.2 二元移动窗口函数

# 先计算我们感兴趣的时间序列的百分数变化：

# spx_px = close_px_all['SPX']
# spx_rets = spx_px.pct_change()
# returns = close_px.pct_change()

# 调用rolling之后，corr聚合函数开始计算与spx_rets滚动相关系数
# corr = returns.AAPL.rolling(125 , min_periods=100).corr(spx_rets)
# corr.plot()
# Out[165]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b5e10470>
# corr.plot()
# Out[166]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7b5e1bc18>


## 11.7.3 用户自定义的移动窗口函数
# rolling_apply函数使你能够在移动窗口上应用自己设计的数组函数。
# 唯一要求的就是：该函数要能从数组的各个片段中产生单个值（即约简）。

# 比如说，当我们用rolling(...).quantile(q)计算样本分位数时，可能对样本中特定值
# 的百分等级感兴趣。scipy.stats.percentileofscore函数就能达到这个目的
# score_at_2percent = lambda x: percentileofscore(x,0.02)
# In[173]: result = returns.AAPL.rolling(250).apply(score_at_2percent)
#
# In[174]: result.plot()
# Out[174]: <matplotlib.axes._subplots.AxesSubplot at 0x1a7bbee61d0>






