数据分析——从入门到精通(二十)

城市气候与海洋的关系研究

import numpy as np
import pandas as pd 
from pandas import Series,DataFrame

import matplotlib.pyplot as plt
%matplotlib inline

plt.rcParams['font.sans-serif'] = ['SimSun']
plt.rcParams['axes.unicode_minus'] = False

导入数据各个海滨城市数据

• os.listdir(‘data3’)
• filter() 过滤出所有的.csv文件，返回一个可迭代的Filter对象
• df.append()以行的方式追加，返回一个新的DataFrame

import os

for csv_file in filter(lambda filename: filename.endswith('.csv'),
                      os.listdir('data3')):
    print(csv_file)

asti_150715.csv
asti_250715.csv
asti_270615.csv
bologna_150715.csv
bologna_250715.csv
bologna_270615.csv
cesena_150715.csv
cesena_250715.csv
cesena_270615.csv
faenza_150715.csv
faenza_250715.csv
faenza_270615.csv
ferrara_150715.csv
ferrara_250715.csv
ferrara_270615.csv
mantova_150715.csv
mantova_250715.csv
mantova_270615.csv
milano_150715.csv
milano_250715.csv
milano_270615.csv
piacenza_150715.csv
piacenza_250715.csv
piacenza_270615.csv
ravenna_150715.csv
ravenna_250715.csv
ravenna_270615.csv
torino_150715.csv
torino_250715.csv
torino_270615.csv

for csv_file in filter(lambda filename: filename.endswith('.csv'),
                      os.listdir('data3')):
    df= pd.read_csv(f'data3/{csv_file}')
    display(df.head())

full_df =None
for csv_file in filter(lambda filename: filename.endswith('.csv'),
                      os.listdir('data3')):
    df= pd.read_csv(f'data3/{csv_file}')
    if full_df is None:
        full_df = df
    else:
        full_df = full_df.append(df)
full_df.head()

full_df.shape

(675, 11)

pd.read_csv('data3/asti_150715.csv')

pd.read_csv('data3/asti_150715.csv').shape

(24, 11)

• 查看列数

full_df.columns

Index(['Unnamed: 0', 'temp', 'humidity', 'pressure', 'description', 'dt',
       'wind_speed', 'wind_deg', 'city', 'day', 'dist'],
      dtype='object')

• 去除没用的列 Unnamed:0

# del all_['Unnamed:0'] 删除一列

# 方法二
full_df.drop(columns='Unnamed: 0',inplace=True)
full_df.columns

Index(['temp', 'humidity', 'pressure', 'description', 'dt', 'wind_speed',
       'wind_deg', 'city', 'day', 'dist'],
      dtype='object')

显示最高温度与力海远近的关系

• 按city分组，计算temp和dist的最大值
• 按dist排序，df.sort_values()

full_df.head()

full_df.dtypes

temp           float64
humidity         int64
pressure       float64
description     object
dt               int64
wind_speed     float64
wind_deg       float64
city            object
day             object
dist             int64
dtype: object

dist_temp = full_df.groupby('city')['temp','dist'].max().sort_values('dist')
dist_temp

画图显示关系

• scatter()显示距离与温度的散布图
• plot()显示距离与温度的线性图

plt.scatter(dist_temp.dist,dist_temp.temp,s=50,c='r')
plt.plot(dist_temp.dist,dist_temp.temp,ls=':',c='b')
plt.show()

读取Astil城市某一天的温度变化数据，并画出线型图和散点图，如2015-7-14这一天的天气变化

• 处理day时间格式
• 转成年、月、日的date日期
• 或转成时、分、秒的字符串
• 将绘制的画布保存成一张png图片

full_df.city.unique()

array(['Asti', 'Bologna', 'Cesena', 'Faenza', 'Ferrara', 'Mantova',
       'Milano', 'Piacenza', 'Ravenna', 'Torino'], dtype=object)

asti = full_df.query('city=="Asti"')
asti.head()

def covert_date_hour(item):
    return Series({'date':item[:10],'hour':item[11:13]})

asti['day'].map(covert_date_hour)

0     date    2015-07-14
hour            10
dtype: o...
1     date    2015-07-14
hour            11
dtype: o...
2     date    2015-07-14
hour            12
dtype: o...
3     date    2015-07-14
hour            13
dtype: o...
4     date    2015-07-14
hour            14
dtype: o...
                            ...                        
15    date    2015-06-28
hour            04
dtype: o...
16    date    2015-06-28
hour            05
dtype: o...
17    date    2015-06-28
hour            06
dtype: o...
18    date    2015-06-28
hour            07
dtype: o...
19    date    2015-06-28
hour            08
dtype: o...
Name: day, Length: 68, dtype: object

def covert_date(item):
    return item[:10]
def covert_hour(item):
    return item[11:13]

asti['day'].map(covert_date)

0     2015-07-14
1     2015-07-14
2     2015-07-14
3     2015-07-14
4     2015-07-14
         ...    
15    2015-06-28
16    2015-06-28
17    2015-06-28
18    2015-06-28
19    2015-06-28
Name: day, Length: 68, dtype: object

# 添加date列，hour列
asti['date'] =  asti['day'].map(covert_date)
asti['hour'] =  asti['day'].map(covert_hour)
asti.head()

D:\yingyong\Anaconda3\lib\site-packages\ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  
D:\yingyong\Anaconda3\lib\site-packages\ipykernel_launcher.py:3: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  This is separate from the ipykernel package so we can avoid doing imports until

asti.query('date == "2015-07-14"')[['hour','temp']]

asti1 = asti.query('date == "2015-07-14"')[['hour','temp']]
plt.plot(asti1.hour,asti1.temp)

[<matplotlib.lines.Line2D at 0x9541270>]

查看asti的一周的湿度变化的数据，并画出散点与线型图

• 按天进行分组
• 将分组后的data设置为列

asti.head()

asti.date.unique()

array(['2015-07-14', '2015-07-15', '2015-07-24', '2015-07-25',
       '2015-06-27', '2015-06-28'], dtype=object)

asti.columns

Index(['temp', 'humidity', 'pressure', 'description', 'dt', 'wind_speed',
       'wind_deg', 'city', 'day', 'dist', 'date', 'hour'],
      dtype='object')

asti.groupby('date')['humidity'].mean()

date
2015-06-27    54.083333
2015-06-28    75.250000
2015-07-14    55.571429
2015-07-15    73.400000
2015-07-24    55.692308
2015-07-25    73.272727
Name: humidity, dtype: float64

asti.groupby('date')['humidity'].mean().reset_index(drop=False)

asti2 = asti.groupby('date')['humidity'].mean().reset_index(drop=False)
asti2

plt.scatter(asti2.date,asti2.humidity)

<matplotlib.collections.PathCollection at 0xabab650>

plt.plot(asti2.date,asti2.humidity)

[<matplotlib.lines.Line2D at 0xabab730>]

观察发现，离海近的可以形成一条直线，离海远的也能形成一条直线

分别以100公里和50公里为分界点，划分为离海近和离海远的两组数据

• city_temp[‘dist’] > 50 # 远距离
• city_temp[‘dist’] <100 # 近距离

dist_temp

dist_temp.query('dist >50')

dist_temp.query('dist < 100')

dt1 = dist_temp[dist_temp.dist > 50]
dt2 = dist_temp[dist_temp.dist < 100]
display(dt1,dt2)

• 画出远、近的距离与温度的散布图(散点图)

plt.scatter(dt1.dist,dt1.temp,c='r',s =50)
plt.scatter(dt2.dist,dt2.temp,c='b',s = 100)
plt.show()

使用线性回归预测给定距离的温度

• 机器学习的线性回归模型LinearRegression
• scikit-learn 模块中线性模型
• sklearn.linear_model
• 创建回归模型对象
• linear = LinearRegression()
• 训练模型
• fit(x,y)
• x 是实际数据的自变量，必须是二维数组
• y 实际数据的因变量
• 预测模型
• predict(x)根据预测函数，计算x的结果
• x也是一个二维数组

from sklearn.linear_model import LinearRegression

# 线性回归模型：有标签(目标)的机器学习,(有监督)
linear = LinearRegression()

# reshape(-1,1) 其中的-1表示为任意数量的行，1表示1列
linear.fit(dt1.dist.values.reshape(-1,1),dt1.temp)  # 训练

LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)

linear.intercept_  # b

33.733483439733746

linear.coef_    # w

array([0.00255639])

• 线性函数：y = w*x + b

• 创建线性回归模型对象

训练模型

• 一个被训练过的模型，就是一个预测函数表达式f(x)=wx+b

预测模型

# 生成预测数据
x1 = np.linspace(50,400,num=50)  # 远距离的预测数据
x2 = np.linspace(0,100,num=50)  # 近距离的预测数据

y1 = linear.predict(x1.reshape(-1,1))

linear = LinearRegression()
linear.fit(dt2.dist.values.reshape(-1,1),dt2.temp)
y2 = linear.predict(x2.reshape(-1,1))

plt.scatter(dt1.dist,dt1.temp,c='r',s =50)
plt.plot(x1,y1,color='r')
plt.scatter(dt2.dist,dt2.temp,c='b',s = 100)
plt.plot(x2,y2,color='b')
plt.show()

风向与风速的关系

• 查看Asti城市的风向wind_deg与风速wind_speed
• 按wind_deg风向排序
• 重新创建索引 reset_index()

asti

68 rows × 12 columns

asti_ds = asti[['wind_deg','wind_speed']]
asti_ds.sort_values('wind_deg',inplace=True)
asti_ds

D:\yingyong\Anaconda3\lib\site-packages\ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy

68 rows × 2 columns

asti_ds.reset_index(inplace=True)
asti_ds

68 rows × 3 columns

asti_ds.drop(columns='index',inplace=True)
asti_ds

68 rows × 2 columns

画出风向与风俗的线性图

• 按wind_deg分组，并计算wind_speed的平均风速，最后再plot()画出线形图

asti_ds_mean = asti_ds.groupby('wind_deg').mean()
asti_ds_mean

plt.plot(asti_ds_mean)

[<matplotlib.lines.Line2D at 0xc3b2170>]

在子图中，同时比较风向与湿度和风力的关系

• 查看humidity列的数据类型
• 通过pd.to_numeric()将字符类型转成数值型

asti_dsht = asti[['wind_deg','wind_speed','humidity','temp']]
asti_dsht

68 rows × 4 columns

asti_dsht = asti_dsht.sort_values('wind_deg').reset_index(drop=True)
asti_dsht

68 rows × 4 columns

asti_dsht_mean = asti_dsht.pivot_table(index='wind_deg')
asti_dsht_mean

# 通过三个子画布绘制风向和风力、温度、湿度的关系

plt.figure(figsize=(15,6))
for i,column in enumerate(('wind_speed','humidity','temp')):
    plt.subplot(1,3,i+1) # index 从1开始
    plt.plot(asti_dsht_mean[column],c=np.random.random(size=3))
    plt.title('风向  Vs '+ column,size=20)
    
plt.show()

角度 转成 弧度

• 角度/180 = 弧度/pi， 弧度 = 角度 * pi / 180
• asti_wind[(asti_wind[‘wind_deg’] >= deg) & (asti_wind[‘wind_deg’] < 45+deg)][‘wind_deg’]

由于风向是360度，我们可以考虑使用玫瑰图(极坐标条形图)

首先自定义一个画图函数

用numpy创建一个直方图，将360度划分为8个面元，将数据分类到这8个面元中

x = np.arange(360,step=45)
x

array([  0,  45,  90, 135, 180, 225, 270, 315])

asti_ds_mean

ds = asti_ds_mean.reset_index()
ds

x = np.arange(360,step=45)
ds[(ds.wind_deg >=0) & (ds.wind_deg < 45)]

x = np.arange(360,step=45)
speeds = [ds[(ds.wind_deg >=deg) & (ds.wind_deg < 45+deg)]['wind_speed'].mean()
           for deg in x
         ]
speeds

[2.133111111111111,
 2.2583333333333333,
 2.0833333333333335,
 3.0866666666666664,
 1.0,
 1.82,
 2.2600000000000002,
 1.9626666666666666]

x1 = x*np.pi /180
x1

array([0.        , 0.78539816, 1.57079633, 2.35619449, 3.14159265,
       3.92699082, 4.71238898, 5.49778714])

# 画极坐标轴
plt.subplot(1,1,1,polar=True)
plt.bar(x1,speeds,
        color=np.random.random(size=(8,3)),
        align='edge')

<BarContainer object of 8 artists>

# 设置宽度
plt.subplot(1,1,1,polar=True)
plt.bar(x1,speeds,
        color=np.random.random(size=(8,3)),
        width=0.7,
        align='edge')

<BarContainer object of 8 artists>

plt.subplot(1,1,1,polar=True)
plt.bar(x1,speeds,
        color=np.random.random(size=(8,3)),
        width=0.7,
        align='edge')
# 设置刻度
plt.xticks(x1,['东','东北','北','西北','西','西南','南','东南'])

([<matplotlib.projections.polar.ThetaTick at 0xaf8e930>,
  <matplotlib.projections.polar.ThetaTick at 0xaedf590>,
  <matplotlib.projections.polar.ThetaTick at 0xae54650>,
  <matplotlib.projections.polar.ThetaTick at 0xae544b0>,
  <matplotlib.projections.polar.ThetaTick at 0xade56d0>,
  <matplotlib.projections.polar.ThetaTick at 0xade5af0>,
  <matplotlib.projections.polar.ThetaTick at 0xade52f0>,
  <matplotlib.projections.polar.ThetaTick at 0xade5030>],
 <a list of 8 Text xticklabel objects>)

练习：计算米兰各个方向上的风速

full_df.city.unique()

array(['Asti', 'Bologna', 'Cesena', 'Faenza', 'Ferrara', 'Mantova',
       'Milano', 'Piacenza', 'Ravenna', 'Torino'], dtype=object)

def draw_polar(full_df,city,compare_colname,title=None):
    # 获取指定城市的 风向与某一列比较的信息
    all_ = full_df.copy()
    city_wind = all_[all_['city'] == city][['wind_deg',compare_colname]]
    city_wind.sort_values('wind_deg',inplace=True) # 按方向排序
    city_wind.reset_index(drop=True,inplace=True)
    
    # 8个方向角度
    degs = np.arange(0,360,step=45)
    
    # 获取每个方向上的数据的平均值
    heights  = [city_wind[compare_colname][(city_wind['wind_deg'] >= deg) &
                                          (city_wind['wind_deg'] < 45+deg)].mean() for deg in degs]
    
    # 将角度转为弧度：角度/360 =弧度/2*pi
    x = degs*np.pi/180
    
    plt.figure(figsize=(6,6))
    plt.axes(polar = True, frameon = True)
    
    plt.bar(x,height=heights,
            align='edge',
            bottom=0.1,
            width=0.75,
            color=np.random.random(size=(8,3)))
    
    plt.xticks(x,('东','东北','北','西北','西','西南','南','东南'),
               fontsize=14)
    
    if title:
        plt.title(city+'城市的'+ title,fontsize=20,color='r',
                 loc='center',position=(0.5,1.1))
        
    plt.show()

draw_polar(full_df,'Milano','wind_speed','风向与风速的关系')

draw_polar(full_df,'Milano','temp','风向与温度的关系')