1、DataFrame返回最大行并求這行的平均值和總平均值
- 用pandas里面的DataFrame生成數據
import pandas as pd
# Subway ridership for 5 stations on 10 different days
ridership_df = pd.DataFrame(
data=[[ 0, 0, 2, 5, 0],
[1478, 3877, 3674, 2328, 2539],
[1613, 4088, 3991, 6461, 2691],
[1560, 3392, 3826, 4787, 2613],
[1608, 4802, 3932, 4477, 2705],
[1576, 3933, 3909, 4979, 2685],
[ 95, 229, 255, 496, 201],
[ 2, 0, 1, 27, 0],
[1438, 3785, 3589, 4174, 2215],
[1342, 4043, 4009, 4665, 3033]],
index=['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
'05-06-11', '05-07-11', '05-08-11', '05-09-11', '05-10-11'],
columns=['R003', 'R004', 'R005', 'R006', 'R007']
)
- 求總平均值和最大行平均值的函數
def mean_riders_for_max_station(ridership):
'''
Fill in this function to find the station with the maximum riders on the
first day, then return the mean riders per day for that station. Also
return the mean ridership overall for comparsion.
This is the same as a previous exercise, but this time the
input is a Pandas DataFrame rather than a 2D NumPy array.
'''
max_station = ridership.iloc[0].argmax()
mean_for_max = ridership[max_station].mean()
overall_mean = ridership.values.mean()
return (overall_mean, mean_for_max)
mean_riders_for_max_station(ridership_df)
2、array返回最大行并求這行的平均值和總平均值
import numpy as np
# Subway ridership for 5 stations on 10 different days
ridership = np.array([
[ 0, 0, 2, 5, 0],
[1478, 3877, 3674, 2328, 2539],
[1613, 4088, 3991, 6461, 2691],
[1560, 3392, 3826, 4787, 2613],
[1608, 4802, 3932, 4477, 2705],
[1576, 3933, 3909, 4979, 2685],
[ 95, 229, 255, 496, 201],
[ 2, 0, 1, 27, 0],
[1438, 3785, 3589, 4174, 2215],
[1342, 4043, 4009, 4665, 3033]
])
def mean_riders_for_max_station(ridership):
'''
Fill in this function to find the station with the maximum riders on the
first day, then return the mean riders per day for that station. Also
return the mean ridership overall for comparsion.
Hint: NumPy's argmax() function might be useful:
http://docs.scipy.org/doc/numpy/reference/generated/numpy.argmax.html
'''
max_station = ridership[0,:].argmax()
mean_for_max = ridership[:,max_station].mean()
overall_mean = ridership.mean()
return (overall_mean, mean_for_max)
- 以上輸出結果都是：
(2342.5999999999999, 3239.9)
3、DataFrame向量化運算
# --- Quiz ---
# Cumulative entries and exits for one station for a few hours.
entries_and_exits = pd.DataFrame({
'ENTRIESn': [3144312, 3144335, 3144353, 3144424, 3144594,
3144808, 3144895, 3144905, 3144941, 3145094],
'EXITSn': [1088151, 1088159, 1088177, 1088231, 1088275,
1088317, 1088328, 1088331, 1088420, 1088753]
})
#計算每小時進出人數的函數
def get_hourly_entries_and_exits(entries_and_exits):
return entries_and_exits - entries_and_exits.shift(1)
get_hourly_entries_and_exits(entries_and_exits)
- 輸出結果：
ENTRIESn EXITSn
0 NaN NaN
1 23.0 8.0
2 18.0 18.0
3 71.0 54.0
4 170.0 44.0
5 214.0 42.0
6 87.0 11.0
7 10.0 3.0
8 36.0 89.0
9 153.0 333.0
4、DataFrame applymap
- 使用示例
import pandas as pd
if True:
df = pd.DataFrame({
'a': [1, 2, 3],
'b': [10, 20, 30],
'c': [5, 10, 15]
})
def add_one(x):
return x + 1
print(df.applymap(add_one))
- 輸出結果：
a b c
0 2 11 6
1 3 21 11
2 4 31 16
- 把分數轉化為等級
The conversion rule is:
90-100 -> A
80-89 -> B
70-79 -> C
60-69 -> D
0-59 -> F
- 實現函數：
grades_df = pd.DataFrame(
data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio',
'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)
def convert_grade(grade):
if grade >= 90:
return 'A'
elif grade >= 80:
return 'B'
elif grade >= 70:
return 'C'
elif grade >= 60:
return 'D'
else:
return 'F'
def convert_grades(grades):
return grades.applymap(convert_grade)
print(grades_df)
convert_grades(grades_df)
- 輸出結果：
exam1 exam2
Andre 43 24
Barry 81 63
Chris 78 56
Dan 75 56
Emilio 89 67
Fred 70 51
Greta 91 79
Humbert 65 46
Ivan 98 72
James 87 60
----------------------------
exam1 exam2
Andre F F
Barry B D
Chris C F
Dan C F
Emilio B D
Fred C F
Greta A C
Humbert D F
Ivan A C
James B D
5、DataFrame apply
案例1：
import pandas as pd
grades_df = pd.DataFrame(
data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio',
'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)
# Change False to True for this block of code to see what it does
# DataFrame apply()
if True:
def convert_grades_curve(exam_grades):
# Pandas has a bult-in function that will perform this calculation
# This will give the bottom 0% to 10% of students the grade 'F',
# 10% to 20% the grade 'D', and so on. You can read more about
# the qcut() function here:
# http://pandas.pydata.org/pandas-docs/stable/generated/pandas.qcut.html
return pd.qcut(exam_grades,
[0, 0.1, 0.2, 0.5, 0.8, 1],
labels=['F', 'D', 'C', 'B', 'A'])
# qcut() operates on a list, array, or Series. This is the
# result of running the function on a single column of the
# DataFrame.
# qcut() does not work on DataFrames, but we can use apply()
# to call the function on each column separately
def standardize(df):
'''
Fill in this function to standardize each column of the given
DataFrame. To standardize a variable, convert each value to the
number of standard deviations it is above or below the mean.
'''
return df.apply(standardize_column)
def standardize_column(column):
return (column-column.mean())/column.std()
-
輸出 exam1的等級：
print(convert_grades_curve(grades_df['exam1']))
Andre F
Barry B
Chris C
Dan C
Emilio B
Fred C
Greta A
Humbert D
Ivan A
James B
Name: exam1, dtype: category
Categories (5, object): [F < D < C < B < A]
- 把grades_df分數轉化為等級：
print(grades_df.apply(convert_grades_curve))
exam1 exam2
Andre F F
Barry B B
Chris C C
Dan C C
Emilio B B
Fred C C
Greta A A
Humbert D D
Ivan A A
James B B
- 標準化：
standardize(grades_df)
exam1 exam2
Andre -2.196525 -2.186335
Barry 0.208891 0.366571
Chris 0.018990 -0.091643
Dan -0.170911 -0.091643
Emilio 0.715295 0.628408
Fred -0.487413 -0.418938
Greta 0.841896 1.413917
Humbert-0.803916 -0.746234
Ivan 1.284999 0.955703
James 0.588694 0.170194
案例2：
- 1、輸出每列中的最大值和平均值：
import numpy as np
import pandas as pd
df = pd.DataFrame({
'a': [4, 5, 3, 1, 2],
'b': [20, 10, 40, 50, 30],
'c': [25, 20, 5, 15, 10]
})
# Change False to True for this block of code to see what it does
# DataFrame apply() - use case 2
if True:
print(df.apply(np.mean))
print(df.apply(np.max))
- 輸出結果：
a 3.0
b 30.0
c 15.0
dtype: float64
a 5
b 50
c 25
dtype: int64
- 2、輸出每列中的第二大值
def second_largest_in_column(column):
sorted_column = column.sort_values(ascending = False)
return sorted_column.iloc[1]
def second_largest(df):
'''
Fill in this function to return the second-largest value of each
column of the input DataFrame.
'''
return df.apply(second_largest_in_column)
second_largest(df)
- 輸出結果：
a 4
b 40
c 20
dtype: int64
6、向Series中添加DataFrame
- 直接相加
import pandas as pd
# Adding using +
if True:
s = pd.Series([1, 2, 3, 4])
df = pd.DataFrame({
0: [10, 20, 30, 40],
1: [50, 60, 70, 80],
2: [90, 100, 110, 120],
3: [130, 140, 150, 160]
})
print(df)
print('') # Create a blank line between outputs
print(df + s)
- 輸出
0 1 2 3
0 10 50 90 130
1 20 60 100 140
2 30 70 110 150
3 40 80 120 160
0 1 2 3
0 11 52 93 134
1 21 62 103 144
2 31 72 113 154
3 41 82 123 164
- 按index相加
# Adding with axis='index'
if True:
s = pd.Series([1, 2, 3, 4])
df = pd.DataFrame({
0: [10, 20, 30, 40],
1: [50, 60, 70, 80],
2: [90, 100, 110, 120],
3: [130, 140, 150, 160]
})
print(df)
print('') # Create a blank line between outputs
print(df.add(s, axis='index'))
# The functions sub(), mul(), and div() work similarly to add()
- 輸出：
0 1 2 3
0 10 50 90 130
1 20 60 100 140
2 30 70 110 150
3 40 80 120 160
0 1 2 3
0 11 51 91 131
1 22 62 102 142
2 33 73 113 153
3 44 84 124 164
- 按column相加
# Adding with axis='columns'
s = pd.Series([1,2,3,4])
df = pd.DataFrame({
0: [10, 20, 30, 40],
1: [50, 60, 70, 80],
2: [90, 100, 110, 120],
3: [130, 140, 150, 160]
})
print (df)
print ('') # Create a blank line between outputs
print (df.add(s, axis='columns'))
# The functions sub(), mul(), and div() work similarly to add()
- 輸出：
0 1 2 3
0 10 50 90 130
1 20 60 100 140
2 30 70 110 150
3 40 80 120 160
0 1 2 3
0 11 52 93 134
1 21 62 103 144
2 31 72 113 154
3 41 82 123 164
7、標準化DateFrame的行
- 數據
grades_df = pd.DataFrame(
data={'exam1': [43, 81, 78, 75, 89, 70, 91, 65, 98, 87],
'exam2': [24, 63, 56, 56, 67, 51, 79, 46, 72, 60]},
index=['Andre', 'Barry', 'Chris', 'Dan', 'Emilio',
'Fred', 'Greta', 'Humbert', 'Ivan', 'James']
)
-
grades_df輸出：
exam1 exam2
Andre 43 24
Barry 81 63
Chris 78 56
Dan 75 56
Emilio 89 67
Fred 70 51
Greta 91 79
Humbert 65 46
Ivan 98 72
James 87 60
-
grades_df.mean()的輸出：
# 默認輸出的是按index計算的平均值
exam1 77.7
exam2 57.4
dtype: float64
-
grades_df.mean(axis='columns')的輸出：
# 指定按columns輸出平均值
Andre 33.5
Barry 72.0
Chris 67.0
Dan 65.5
Emilio 78.0
Fred 60.5
Greta 85.0
Humbert 55.5
Ivan 85.0
James 73.5
dtype: float64
- 計算每人的兩次成績與兩次成績平均值的偏差并標準化:
mean_diffs =grades_df.sub(grades_df.mean(axis='columns'),axis='index'
exam1 exam2
Andre 9.5 -9.5
Barry 9.0 -9.0
Chris 11.0 -11.0
Dan 9.5 -9.5
Emilio 11.0 -11.0
Fred 9.5 -9.5
Greta 6.0 -6.0
Humbert 9.5 -9.5
Ivan 13.0 -13.0
James 13.5 -13.5
mean_diffs.div(grades_df.std(axis='columns'),axis='index')
exam1 exam2
Andre 0.707107 -0.707107
Barry 0.707107 -0.707107
Chris 0.707107 -0.707107
Dan 0.707107 -0.707107
Emilio 0.707107 -0.707107
Fred 0.707107 -0.707107
Greta 0.707107 -0.707107
Humbert 0.707107 -0.707107
Ivan 0.707107 -0.707107
James 0.707107 -0.707107
8、DataFrame中groupby的使用
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
values = np.array([1, 3, 2, 4, 1, 6, 4])
example_df = pd.DataFrame({
'value': values,
'even': values % 2 == 0,
'above_three': values > 3
}, index=['a', 'b', 'c', 'd', 'e', 'f', 'g'])
-
print (example_df)輸出結果：
above_three even value
a False False 1
b False False 3
c False True 2
d True True 4
e False False 1
f True True 6
g True True 4
- 按
even分組：
grouped_data = example_df.groupby('even')
# The groups attribute is a dictionary mapping keys to lists of row indexes
print(grouped_data.groups)
- 輸出結果：
{False: ['a', 'b', 'e'], True: ['c', 'd', 'f', 'g']}
- 按
even和above_three分組：
grouped_data = example_df.groupby(['even', 'above_three'])
print(grouped_data.groups)
- 輸出結果：
{(True, False): ['c'], (False, False): ['a', 'b', 'e'], (True, True): ['d', 'f', 'g']}
- 求每個
group的和
grouped_data = example_df.groupby('even')
print(grouped_data.sum())
- 輸出：
above_three value
even
False 0.0 5
True 3.0 16
- 按columns計算和
grouped_data = example_df.groupby('even')
# You can take one or more columns from the result DataFrame
print(grouped_data.sum()['value'])
print ('\n') # Blank line to separate results
print(grouped_data['value'].sum())
- 以上兩個
print計算結果一樣:
even
False 5
True 16
Name: value, dtype: int32
- 用
group實現分組后的標準化和求第二大的值
import numpy as np
import pandas as pd
values = np.array([1, 3, 2, 4, 1, 6, 4])
example_df = pd.DataFrame({
'value': values,
'even': values % 2 == 0,
'above_three': values > 3
}, index=['a', 'b', 'c', 'd', 'e', 'f', 'g'])
# Change False to True for each block of code to see what it does
# Standardize each group
if True:
def standardize(xs):
return (xs - xs.mean()) / xs.std()
grouped_data = example_df.groupby('even')
print(grouped_data.groups)
print(grouped_data['value'].apply(standardize))
if True:
def second_largest(xs):
sorted_xs = xs.sort(inplace=False, ascending=False)
return sorted_xs.iloc[1]
grouped_data = example_df.groupby('even')
print(grouped_data['value'].apply(second_largest))
- 輸出：
# print按even分組
{False: ['a', 'b', 'e'], True: ['c', 'd', 'f', 'g']}
# print標準化
a -0.577350
b 1.154701
c -1.224745
d 0.000000
e -0.577350
f 1.224745
g 0.000000
Name: value, dtype: float64
# print第二大值
even
False 1
True 4
Name: value, dtype: int64
- 每小時入站和出站數
ridership_df = pd.DataFrame({
'UNIT': ['R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051', 'R079', 'R051'],
'TIMEn': ['00:00:00', '02:00:00', '04:00:00', '06:00:00', '08:00:00', '10:00:00', '12:00:00', '14:00:00', '16:00:00'],
'ENTRIESn': [3144312, 8936644, 3144335, 8936658, 3144353, 8936687, 3144424, 8936819, 3144594],
'EXITSn': [1088151, 13755385, 1088159, 13755393, 1088177, 13755598, 1088231, 13756191, 1088275]
})
def hours_for_group(entries_and_exits):
return entries_and_exits-entries_and_exits.shift(1)
ridership_df.groupby('UNIT')[['ENTRIESn','EXITSn']].apply(hours_for_group)
- 輸出結果：
ENTRIESn EXITSn
0 NaN NaN
1 NaN NaN
2 23.0 8.0
3 14.0 8.0
4 18.0 18.0
5 29.0 205.0
6 71.0 54.0
7 132.0 593.0
8 170.0 44.0
9、DataFrame合并
import pandas as pd
subway_df = pd.DataFrame({
'UNIT': ['R003', 'R003', 'R003', 'R003', 'R003', 'R004', 'R004', 'R004',
'R004', 'R004'],
'DATEn': ['05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11',
'05-01-11', '05-02-11', '05-03-11', '05-04-11', '05-05-11'],
'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'ENTRIESn': [ 4388333, 4388348, 4389885, 4391507, 4393043, 14656120,
14656174, 14660126, 14664247, 14668301],
'EXITSn': [ 2911002, 2911036, 2912127, 2913223, 2914284, 14451774,
14451851, 14454734, 14457780, 14460818],
'latitude': [ 40.689945, 40.689945, 40.689945, 40.689945, 40.689945,
40.69132 , 40.69132 , 40.69132 , 40.69132 , 40.69132 ],
'longitude': [-73.872564, -73.872564, -73.872564, -73.872564, -73.872564,
-73.867135, -73.867135, -73.867135, -73.867135, -73.867135]
})
weather_df = pd.DataFrame({
'DATEn': ['05-01-11', '05-01-11', '05-02-11', '05-02-11', '05-03-11',
'05-03-11', '05-04-11', '05-04-11', '05-05-11', '05-05-11'],
'daten': ['05-01-11', '05-01-11', '05-02-11', '05-02-11', '05-03-11',
'05-03-11', '05-04-11', '05-04-11', '05-05-11', '05-05-11'],
'hour': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'latitude': [ 40.689945, 40.69132 , 40.689945, 40.69132 , 40.689945,
40.69132 , 40.689945, 40.69132 , 40.689945, 40.69132 ],
'longitude': [-73.872564, -73.867135, -73.872564, -73.867135, -73.872564,
-73.867135, -73.872564, -73.867135, -73.872564, -73.867135],
'pressurei': [ 30.24, 30.24, 30.32, 30.32, 30.14, 30.14, 29.98, 29.98,
30.01, 30.01],
'fog': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'rain': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
'tempi': [ 52. , 52. , 48.9, 48.9, 54. , 54. , 57.2, 57.2, 48.9, 48.9],
'wspdi': [ 8.1, 8.1, 6.9, 6.9, 3.5, 3.5, 15. , 15. , 15. , 15. ]
})
subway_df.merge(weather_df,on =['DATEn','hour','latitude','longitude'],how = 'inner')
subway_df.merge(weather_df,left_on =['DATEn','hour','latitude','longitude'],right_on =['daten','hour','latitude','longitude'],how = 'inner')
-
輸出結果：
print1
print2
