数据聚合与分组运算

GroupBy技术

import numpy as np
import pandas as pd
from pandas import DataFrame,Series

df = DataFrame({'key1':['a','a','b','b','a'],
               'key2':['one','two','one','two','one'],
               'data1':np.random.randn(5),
               'data2':np.random.randn(5)})
df

#根据key1进行分组，并计算data1列的平均值。
grouped = df['data1'].groupby(df['key1'])
grouped

<pandas.core.groupby.generic.SeriesGroupBy object at 0x000001CCD8450910>

grouped.mean()

key1
a    0.090859
b    0.804812
Name: data1, dtype: float64

means = df['data1'].groupby([df['key1'],df['key2']]).mean()
means

key1  key2
a     one    -0.037649
      two     0.347874
b     one     0.399766
      two     1.209857
Name: data1, dtype: float64

means.unstack()

states = np.array(['Ohio','California','California','Ohio','Ohio'])
years = np.array([2005,2005,2006,2005,2006])
df['data1'].groupby([states,years]).mean()

California  2005    0.347874
            2006    0.399766
Ohio        2005    0.567867
            2006   -0.001175
Name: data1, dtype: float64

df.groupby('key1').mean()

df.groupby(['key1','key2']).mean()

#GroupBy的size方法，可以返回一个含有分组大小的Series。目前，分组键中的任何缺失值都会被排除在结果之外。
df.groupby(['key1','key2']).size()

key1  key2
a     one     2
      two     1
b     one     1
      two     1
dtype: int64

对分组进行迭代

for name,group in df.groupby('key1'):
    print(name)
    print(group)

a
  key1 key2     data1     data2
0    a  one -0.074122 -0.571432
1    a  two  0.347874 -0.794645
4    a  one -0.001175  0.180895
b
  key1 key2     data1     data2
2    b  one  0.399766 -0.596056
3    b  two  1.209857 -0.266257

for (k1,k2),group in df.groupby(['key1','key2']):
    print(k1,k2)
    print(group)

a one
  key1 key2     data1     data2
0    a  one -0.074122 -0.571432
4    a  one -0.001175  0.180895
a two
  key1 key2     data1     data2
1    a  two  0.347874 -0.794645
b one
  key1 key2     data1     data2
2    b  one  0.399766 -0.596056
b two
  key1 key2     data1     data2
3    b  two  1.209857 -0.266257

pieces = dict(list(df.groupby('key1')))
pieces['b']

df.dtypes

key1      object
key2      object
data1    float64
data2    float64
dtype: object

grouped = df.groupby(df.dtypes,axis=1)
dict(list(grouped))

{dtype('float64'):       data1     data2
 0 -0.074122 -0.571432
 1  0.347874 -0.794645
 2  0.399766 -0.596056
 3  1.209857 -0.266257
 4 -0.001175  0.180895,
 dtype('O'):   key1 key2
 0    a  one
 1    a  two
 2    b  one
 3    b  two
 4    a  one}

选取一个或一组列

df.groupby('key1')['data1']
df.groupby('key1')[['data2']]

<pandas.core.groupby.generic.DataFrameGroupBy object at 0x000001CCCF3F55A0>

df.groupby(['key1','key2'])[['data2']].mean()

s_grouped = df.groupby(['key1','key2'])['data2']
s_grouped

<pandas.core.groupby.generic.SeriesGroupBy object at 0x000001CCD8452DA0>

s_grouped.mean()

key1  key2
a     one    -0.195268
      two    -0.794645
b     one    -0.596056
      two    -0.266257
Name: data2, dtype: float64

通过字典或Series进行分组

people = DataFrame(np.random.randn(5,5),
                  columns=['a','b','c','d','e'],
                  index=['Joe','Steve','Wes','Jim','Travis'])
people.loc[2:3,['b','c']] = np.nan#添加几个NA值
people

C:\windows\ FutureWarning: Slicing a positional slice with .loc is not supported, and will raise TypeError in a future version.  Use .loc with labels or .iloc with positions instead.
  people.loc[2:3,['b','c']] = np.nan#添加几个NA值

mapping = {'a':'red','b':'red','c':'blue','d':'blue','e':'red','f':'orange'}
by_column = people.groupby(mapping,axis=1)
by_column.sum()

map_series = Series(mapping)
map_series

a       red
b       red
c      blue
d      blue
e       red
f    orange
dtype: object

people.groupby(map_series,axis=1).count()

通过函数进行分组

people.groupby(len).sum()

key_list = ['one','one','one','two','two']
people.groupby([len,key_list]).min()

根据索引级别分组

columns = pd.MultiIndex.from_arrays([['US','US','US','JP','JP'],
                                    [1,3,5,1,3]],names=['cty','tenor'])
hier_df = DataFrame(np.random.randn(4,5),columns=columns)
hier_df

hier_df.groupby(level='cty',axis=1).count()

数据聚合

grouped = df.groupby('key1')
#如果传入的百分位上没有值，则quantile会进行线性插值
grouped['data1'].quantile(0.9)

key1
a    0.278064
b    1.128848
Name: data1, dtype: float64

def peak_to_peak(arr):
    return arr.max() - arr.min()
grouped.agg(peak_to_peak)

C:\windows\TFutureWarning: ['key2'] did not aggregate successfully. If any error is raised this will raise in a future version of pandas. Drop these columns/ops to avoid this warning.
  grouped.agg(peak_to_peak)

grouped.describe()

#有个知识点
import matplotlib.pyplot as plt
from pylab import *
img = plt.imread('经过优化的GroupBy的方法.png')
imshow(img)

tips = pd.read_csv("E:\\python_study_files\\python\\pydata-book-2nd-edition\\examples\\tips.csv")
#添加“小费占总额百分比”的列
tips['tip_pct'] = tips['tip']/tips['total_bill']
tips[:6]

面向列的多函数应用

grouped = tips.groupby(['day','smoker'])
grouped_pct = grouped['tip_pct']
grouped_pct.agg('mean')

day   smoker
Fri   No        0.151650
      Yes       0.174783
Sat   No        0.158048
      Yes       0.147906
Sun   No        0.160113
      Yes       0.187250
Thur  No        0.160298
      Yes       0.163863
Name: tip_pct, dtype: float64

grouped_pct.agg(['mean','std',peak_to_peak])

#由（name,function)组成的列表，第一个元素会被用作列名
grouped_pct.agg([('foo','mean'),('bar',np.std)])

functions = ['count','mean','max']
result = grouped['tip_pct','total_bill'].agg(functions)
result

C:\windowFutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
  result = grouped['tip_pct','total_bill'].agg(functions)

result['tip_pct']

ftuples = [('Durchschnitt','mean'),('Abweichung',np.var)]
grouped['tip_pct','total_bill'].agg(ftuples)

C:\windowsFutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.
  grouped['tip_pct','total_bill'].agg(ftuples)

#对不同的列应用不同的函数
grouped.agg({'tip':np.max,'size':'sum'})

grouped.agg({'tip_pct':['min','max','mean','std'],
            'size':'sum'})

以“无索引”的形式返回聚合数据

tips.groupby(['day','smoker'],as_index=False).mean()

分组级运算和转换

#添加一个用于存放各索引分组平均值的列
k1_means = df.groupby('key1').mean().add_prefix('mean_')
k1_means

pd.merge(df,k1_means,left_on='key1',right_index=True)

key = ['one','two','one','two','one']
people.groupby(key).mean()

#transform会将一个函数应用到各个分组
people.groupby(key).transform(np.mean)

#从各组中减去平均值
def demean(arr):
    return arr-arr.mean()
demeaned = people.groupby(key).transform(demean)
demeaned

demeaned.groupby(key).mean()

apply:一般性的“拆分——应用——合并”

#在指定列找到最大值，然后把这个值所在的行选取出来
#将sort_index()改为sort_values()即可
def top(df,n=5,column='tip_pct'):
    return df.sort_values(by=column)[-n:]
top(tips,n=6)

tips.groupby('smoker').apply(top)

tips.groupby(['smoker','day']).apply(top,n=1,column='total_bill')

result = tips.groupby('smoker')['tip_pct'].describe()
result

result.unstack('smoker')

       smoker
count  No        151.000000
       Yes        93.000000
mean   No          0.159328
       Yes         0.163196
std    No          0.039910
       Yes         0.085119
min    No          0.056797
       Yes         0.035638
25%    No          0.136906
       Yes         0.106771
50%    No          0.155625
       Yes         0.153846
75%    No          0.185014
       Yes         0.195059
max    No          0.291990
       Yes         0.710345
dtype: float64

当调用describe之类的方法时，实际上只是应用了以下两条代码的快捷方式：
f = lambda x: x.describe()
grouped.apply(f)

禁止分组键

tips.groupby('smoker',group_keys=False).apply(top)

分位数和桶分析

frame = DataFrame({'data1':np.random.randn(1000),
                  'data2':np.random.randn(1000)})
factor = pd.cut(frame.data1,4)
factor[:10]

0    (-1.448, 0.107]
1    (-1.448, 0.107]
2    (-1.448, 0.107]
3    (-1.448, 0.107]
4     (0.107, 1.663]
5     (0.107, 1.663]
6     (0.107, 1.663]
7    (-1.448, 0.107]
8    (-1.448, 0.107]
9     (0.107, 1.663]
Name: data1, dtype: category
Categories (4, interval[float64, right]): [(-3.01, -1.448] < (-1.448, 0.107] < (0.107, 1.663] < (1.663, 3.218]]

def get_stats(group):
    return {'min':group.min(),'max':group.max(),
           'count':group.count(),'mean':group.mean()}
grouped = frame.data2.groupby(factor)
grouped.apply(get_stats).unstack()
#区间大小相等

#数据点数量相等,使用qcut
#返回分位数编号
grouping = pd.qcut(frame.data1,10,labels=False)
grouped = frame.data2.groupby(grouping)
grouped.apply(get_stats).unstack()

示例：用特定于分组的值填充缺失值

#用平均值填充NA值
s = Series(np.random.randn(6))
s[::2] = np.nan
s

0         NaN
1    0.209858
2         NaN
3    1.379023
4         NaN
5   -0.743300
dtype: float64

s.fillna(s.mean())

0    0.281860
1    0.209858
2    0.281860
3    1.379023
4    0.281860
5   -0.743300
dtype: float64

states = ['Ohio','New York','Vermont','Florida',
         'Oregon','Nevada','California','Idaho']
group_key = ['East']*4+['West']*4
data = Series(np.random.randn(8),index=states)
data[['Vermont','Nevada','Idaho']] = np.nan
data

Ohio          0.155978
New York     -0.133767
Vermont            NaN
Florida      -0.765162
Oregon        0.682524
Nevada             NaN
California    0.730390
Idaho              NaN
dtype: float64

data.groupby(group_key).mean()

East   -0.247650
West    0.706457
dtype: float64

fill_mean = lambda g:g.fillna(g.mean())
data.groupby(group_key).apply(fill_mean)

Ohio          0.155978
New York     -0.133767
Vermont      -0.247650
Florida      -0.765162
Oregon        0.682524
Nevada        0.706457
California    0.730390
Idaho         0.706457
dtype: float64

fill_values = {'East':0.5,'West':-1}
fill_func = lambda g:g.fillna(fill_values[g.name])
data.groupby(group_key).apply(fill_func)

Ohio          0.155978
New York     -0.133767
Vermont       0.500000
Florida      -0.765162
Oregon        0.682524
Nevada       -1.000000
California    0.730390
Idaho        -1.000000
dtype: float64

示例：随机采样和排列

抽取的一个办法：选取np.random.permutation(N)的前K个元素，其中N为完整数据的大小，K为期望的样本大小。

#红桃（Hearts)、黑桃（Spades)、梅花（Clubs)、方片（Diamonds)
suits = ['H','S','C','D']
#python2中，range()返回的是list，可以将两个range()直接相加，如range(5)+range(10) ;python3中，range()成了一个class
card_val = (list(range(1,11))+ [10] * 3) * 4
base_names = ['A'] + list(range(2,11)) + ['J','K','Q']
cards = []
for suit in ['H','S','C','D']:
    cards.extend(str(num) + suit for num in base_names)
deck = Series(card_val,index=cards)
deck[:13]

AH      1
2H      2
3H      3
4H      4
5H      5
6H      6
7H      7
8H      8
9H      9
10H    10
JH     10
KH     10
QH     10
dtype: int64

def draw(deck,n=5):
    return deck.take(np.random.permutation(len(deck))[:n])
draw(deck)

7H     7
4D     4
8H     8
QC    10
4S     4
dtype: int64

#从每种花色中随机抽取两张牌
get_suit = lambda card:card[-1]#只要最后一个字母
deck.groupby(get_suit).apply(draw,n=2)

C  AC      1
   JC     10
D  5D      5
   8D      8
H  10H    10
   JH     10
S  9S      9
   5S      5
dtype: int64

#另一种办法
deck.groupby(get_suit,group_keys=False).apply(draw,n=2)

10C    10
AC      1
KD     10
10D    10
3H      3
9H      9
5S      5
8S      8
dtype: int64

示例：分组加权平均数和相关系数

df = DataFrame({'category':['a','a','a','a','b','b','b','b'],
               'data':np.random.randn(8),
               'weights':np.random.randn(8)})
df

grouped = df.groupby('category')
get_wavg = lambda g: np.average(g['data'],weights=g['weights'])
grouped.apply(get_wavg)

category
a   -0.723088
b   -0.453212
dtype: float64

close_px = pd.read_csv("E:\python_study_files\python\pydata-book-2nd-edition\examples\stock_px.csv",parse_dates=True,index_col=0)
close_px

5472 rows × 9 columns

close_px[-4:]

#计算日收益率与SPX之间的年度相关系数
rets = close_px.pct_change().dropna()
spx_corr = lambda x:x.corrwith(x['SPX'])
by_year = rets.groupby(lambda x:x.year)
by_year.apply(spx_corr)

#苹果和微软的年度相关系数
by_year.apply(lambda g: g['AAPL'].corr(g['MSFT']))

1990    0.408271
1991    0.266807
1992    0.450592
1993    0.236917
1994    0.361638
1995    0.258642
1996    0.147539
1997    0.196144
1998    0.364106
1999    0.329484
2000    0.275298
2001    0.563156
2002    0.571435
2003    0.486262
2004    0.259024
2005    0.300093
2006    0.161735
2007    0.417738
2008    0.611901
2009    0.432738
2010    0.571946
2011    0.581987
dtype: float64

示例：面向分组的线性回归

import statsmodels.api as sm
def regress(data,yvar,xvars):
    Y = data[yvar]
    X = data[xvars]
    X['intercept'] = 1.
    result = sm.OLS(Y,X).fit()
    return result.params
by_year.apply(regress,'AAPL',['SPX'])

E:\python_study_files\python_pip\.venvs\lpthw\lib\site-packages\statsmodels\compat\pandas.py:65: FutureWarning: pandas.Int64Index is deprecated and will be removed from pandas in a future version. Use pandas.Index with the appropriate dtype instead.
  from pandas import Int64Index as NumericIndex

透视表和交叉表

tips.pivot_table(index=['day','smoker'])

tips.pivot_table(['tip_pct','size'],index=['time','day'],columns='smoker')

tips.pivot_table(['tip_pct','size'],index=['time','day'],
                columns='smoker',margins=True)

tips.pivot_table('tip_pct',index=['time','smoker'],columns='day',
                aggfunc=len,margins=True)

tips.pivot_table('size',index=['time','smoker'],
                columns='day',aggfunc='sum',fill_value=0)

#有个知识点
import matplotlib.pyplot as plt
from pylab import *
img = plt.imread('pivot_table的参数.png')
imshow(img)

rows改为index,cols改为columns

交叉表：crosstab

pd.crosstab([tips.time,tips.day],tips.smoker,margins=True)

示例：2012联邦选举委员会数据库

fec =pd.read_csv("E:\\python_study_files\\python\\pydata-book-2nd-edition\\datasets\\fec\\P00000001-ALL.csv")
fec

C:\windowsDtypeWarning: Columns (6) have mixed types. Specify dtype option on import or set low_memory=False.
  fec =pd.read_csv("E:\\python_study_files\\python\\pydata-book-2nd-edition\\datasets\\fec\\P00000001-ALL.csv")

1001731 rows × 16 columns

fec.loc[123456]

cmte_id                             C00431445
cand_id                             P80003338
cand_nm                         Obama, Barack
contbr_nm                         ELLMAN, IRA
contbr_city                             TEMPE
contbr_st                                  AZ
contbr_zip                          852816719
contbr_employer      ARIZONA STATE UNIVERSITY
contbr_occupation                   PROFESSOR
contb_receipt_amt                        50.0
contb_receipt_dt                    01-DEC-11
receipt_desc                              NaN
memo_cd                                   NaN
memo_text                                 NaN
form_tp                                 SA17A
file_num                               772372
Name: 123456, dtype: object

unique_cands = fec.cand_nm.unique()
unique_cands

array(['Bachmann, Michelle', 'Romney, Mitt', 'Obama, Barack',
       "Roemer, Charles E. 'Buddy' III", 'Pawlenty, Timothy',
       'Johnson, Gary Earl', 'Paul, Ron', 'Santorum, Rick',
       'Cain, Herman', 'Gingrich, Newt', 'McCotter, Thaddeus G',
       'Huntsman, Jon', 'Perry, Rick'], dtype=object)

unique_cands[2]

'Obama, Barack'

parties = {'Bachmann, Michelle':'Republican',
          'Cain, Herman':'Republican',
          'Gingrich, Newt':'Republican',
          'Huntsman, Jon':'Republican',
          'Johnson, Gary Earl':'Republican',
          'McCotter, Thaddeus G':'Republican',
          'Obama, Barack':'Democrat',
          'Paul, Ron':'Republican',
          'Pawlenty, Timothy':'Republican',
          'Perry, Rick':'Republican',
          "Roemer, Charles E. 'Buddy' Ⅲ":'Republican',
          'Romney, Mitt':'Republican',
          'Santorum, Rick':'Republican'}
fec.cand_nm[123456:123461]

125611    Obama, Barack
125612    Obama, Barack
125613    Obama, Barack
125614    Obama, Barack
125615    Obama, Barack
Name: cand_nm, dtype: object

fec.cand_nm[123456:123461].map(parties)

125611    Democrat
125612    Democrat
125613    Democrat
125614    Democrat
125615    Democrat
Name: cand_nm, dtype: object

#添加一个新列
fec['party'] = fec.cand_nm.map(parties)
fec['party'].value_counts()

Democrat      589127
Republican    396504
Name: party, dtype: int64

(fec.contb_receipt_amt>0).value_counts()

True    991475
Name: contb_receipt_amt, dtype: int64

fec = fec[fec.contb_receipt_amt>0]
fec_mrbo = fec[fec.cand_nm.isin(['Obama, Barack','Romney, Mitt'])]

雇主职业和雇主统计赞助信息

fec.contbr_occupation.value_counts()[:10]

RETIRED         233990
NOT PROVIDED     56245
ATTORNEY         34286
HOMEMAKER        29931
PHYSICIAN        23432
ENGINEER         14334
TEACHER          13990
CONSULTANT       13273
PROFESSOR        12555
NOT EMPLOYED      9828
Name: contbr_occupation, dtype: int64

occ_mapping = {'INFORMATION REQUESTED PER BEST EFFORTS':'NOT PROVIDED',
              'INFORMATION REQUESTED':'NOT PROVIDED',
              'INFORMATIO REQUESTED (BEST EFFORTS)':'NOT PROVIDED',
              'C.E.O':'CEO'}
#如果没有提供相关映射，则返回X
f = lambda x:occ_mapping.get(x,x)
fec.contbr_occupation = fec.contbr_occupation.map(f)
emp_mapping = {'INFORMATION REQUESTED PER BEST EFFORTS':'NOT PROVIDED',
              'INFORMATION REQUESTED':'NOT PROVIDED',
               'SELF':'SELF-EMPLOYED',
               'SELF EMPLOYED':'SELF-EMPLOTED'
}
f = lambda x:emp_mapping.get(x,x)
fec.contbr_employer = fec.contbr_employer.map(f)
by_occupation = fec.pivot_table('contb_receipt_amt',index='contbr_occupation',columns='party',aggfunc='sum')
over_2mm = by_occupation[by_occupation.sum(1)>2000000]
over_2mm

over_2mm.plot(kind='barh')

def get_top_amounts(group,key,n=5):
    totals = group.groupby(key)['contb_receipt_amt'].sum()
    #根据key对totals进行降序排列
    return totals.sort_values(ascending=False)[n:]
grouped = fec_mrbo.groupby('cand_nm')
grouped.apply(get_top_amounts,'contbr_occupation',n=7)

cand_nm        contbr_occupation                   
Obama, Barack  PROFESSOR                               2165071.08
               CEO                                     2074284.79
               PRESIDENT                               1878509.95
               NOT EMPLOYED                            1709188.20
               EXECUTIVE                               1355161.05
                                                          ...    
Romney, Mitt   INDEPENDENT PROFESSIONAL                      3.00
               IFC CONTRACTING SOLUTIONS                     3.00
               REMODELER & SEMI RETIRED                      3.00
               AFFORDABLE REAL ESTATE DEVELOPER              3.00
               3RD GENERATION FAMILY BUSINESS OWNER          3.00
Name: contb_receipt_amt, Length: 35973, dtype: float64

grouped.apply(get_top_amounts,'contbr_employer',n=10)

cand_nm        contbr_employer         
Obama, Barack  REFUSED                     149516.07
               DLA PIPER                   148235.00
               HARVARD UNIVERSITY          131368.94
               IBM                         128490.93
               GOOGLE                      125302.88
                                             ...    
Romney, Mitt   UN                               3.00
               UPTOWN CHEAPSKATE                3.00
               WILL MERRIFIELD                  3.00
               INDEPENDENT PROFESSIONAL         3.00
               HONOLD COMMUNICTAIONS            3.00
Name: contb_receipt_amt, Length: 95890, dtype: float64

对出资额分组

bins = np.array([0,1,10,100,1000,10000,100000,1000000,10000000])
labels = pd.cut(fec_mrbo.contb_receipt_amt,bins)
labels

411         (10, 100]
412       (100, 1000]
413       (100, 1000]
414         (10, 100]
415         (10, 100]
             ...     
701381      (10, 100]
701382    (100, 1000]
701383        (1, 10]
701384      (10, 100]
701385    (100, 1000]
Name: contb_receipt_amt, Length: 694282, dtype: category
Categories (8, interval[int64, right]): [(0, 1] < (1, 10] < (10, 100] < (100, 1000] < (1000, 10000] < (10000, 100000] < (100000, 1000000] < (1000000, 10000000]]

grouped = fec_mrbo.groupby(['cand_nm',labels])
grouped.size().unstack(0)

bucket_sums = grouped.contb_receipt_amt.sum().unstack(0)
bucket_sums

normed_sums = bucket_sums.div(bucket_sums.sum(axis=1),axis=0)
normed_sums

normed_sums[:2].plot(kind='barh',stacked=True)

根据州统计赞助信息

grouped = fec_mrbo.groupby(['cand_nm','contbr_st'])
totals = grouped.contb_receipt_amt.sum().unstack(0).fillna(0)
totals = totals[totals.sum(1)>100000]
totals[:10]

percent = totals.div(totals.sum(1),axis=0)
percent[:10]

from mpl_toolkits.basemap import Basemap, cm
import numpy as np
from matplotlib import rcParams
from matplotlib.collections import LineCollection
import matplotlib.pyplot as plt
#from shapelib import ShapeFile
import pyshp
import dbflib

---------------------------------------------------------------------------

ModuleNotFoundError                       Traceback (most recent call last)

      5 import matplotlib.pyplot as plt
      6 #from shapelib import ShapeFile
----> 7 import pyshp
      8 import dbflib


ModuleNotFoundError: No module named 'pyshp'