pandas

10 minutes to pandas

详细参考 http://pandas.pydata.org/pandas-docs/stable/getting_started/10min.html

1. 导入

import numpy as np
import pandas as pd

2. 创建DataFrame

s = pd.Series([1, 3, 5, np.nan, 6, 8])
dates = pd.date_range('20130101', periods=6)
df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list('ABCD'))
df2 = pd.DataFrame({'A': 1.,'B':pd.Timestamp('20130102'),
 'C': pd.Series(1, index=list(range(4)), dtype='float32'), 'D': np.array([3] * 4, dtype='int32'),
 'E': pd.Categorical(["test", "train", "test", "train"]), 'F': 'foo'})

• 查看DataFrame数据类型 df2.dtypes

3. 查看DataFrame

df.head(3)
df.tail(3)
df.index
df.to_numpy()
df.describe()
df.info()
df.T
df.sort_index(axis=1, ascending=False)
df.sort_values(by='B')

4. 选择DataFrame

df['A']
df[0:3]
#Selection by label(行，列）
df.loc[dates[0]]
df.loc[:, ['A', 'B']]
df.loc['20130102':'20130104', ['A', 'B']]
df.loc['20130102', ['A', 'B']]
df.loc[dates[0], 'A']
df.at[dates[0], 'A']  # access to a scalar 值
#Selection by position
df.iloc[3]
df.iloc[3:5, 0:2]
df.iloc[[1, 2, 4], [0, 2]]
df.iloc[1:3, :]
df.iloc[:, 1:3] #1-3列，不包含索引
df.iloc[1, 1]
df.iat[1, 1] #access to a scalar

# Boolean indexing
df[df.A > 0]
df[df > 0]
df2[df2['E'].isin(['two', 'four'])]  # in('','')

5. 赋值

s1 = pd.Series([1, 2, 3, 4, 5, 6], index=pd.date_range('20130102', periods=6))
df.at[dates[0], 'A'] = 0
df.iat[0, 1] = 0
df.loc[:, 'D'] = np.array([5] * len(df))
df2 = df.copy() #复制副本

6. 缺失数据处理

df1.dropna(how='any') #丢失空数据
df1.fillna(value=5) #填充空数据
pd.isna(df1) #判断数据哪些为空
df['price'].isnull()  #检查特定列空值
#df3中有重复的行数，我们如何将重复的行数去除？
df3.drop_duplicates(
    subset=['收盘价', '交易日期'],  # subset参数用来指定根据哪类类数据来判断是否重复。若不指定，则用全部列的数据来判断是否重复
    keep='first',  # 在去除重复值的时候，我们是保留上面一行还是下面一行？first保留上面一行，last保留下面一行，False就是一行都不保留
    inplace=True
df['city'].drop_duplicates(keep='last')  #删除先出现的重复值
df['city'].drop_duplicates()   #删除后出现的重复值

7. Operations

df.mean()
df.mean(1)
s = pd.Series([1, 3, 5, np.nan, 6, 8], index=dates).shift(2)
df.sub(s, axis='index')

# Apply
df.apply(np.cumsum)
df.apply(lambda x: x.max() - x.min())

# value_counts 统计值出现频次
s = pd.Series(np.random.randint(0, 7, size=10))
s.value_counts()

#String Methods
s = pd.Series(['A', 'B', 'C', 'Aaba', 'Baca', np.nan, 'CABA', 'dog', 'cat'])
s.str.lower()

8. Merge(类似表连接）

# Concat
df = pd.DataFrame(np.random.randn(10, 4))
pieces = [df[:3], df[3:7], df[7:]]
pd.concat(pieces)

# Join
left = pd.DataFrame({'key': ['foo', 'bar'], 'lval': [1, 2]})
right = pd.DataFrame({'key': ['foo', 'bar'], 'rval': [4, 5]})
pd.merge(left, right, on='key')

9. 添加DataFrame

df = pd.DataFrame(np.random.randn(8, 4), columns=['A', 'B', 'C', 'D'])
s = df.iloc[3]
df.append(s, ignore_index=True) #添加s行到最后

10. groupby 分组

df = pd.DataFrame({'A': ['foo', 'bar', 'foo', 'bar','foo', 'bar', 'foo', 'foo'],
'B': ['one', 'one', 'two', 'three','two', 'two', 'one', 'three'],
'C': np.random.randn(8),'D': np.random.randn(8)})
# groupby
df.groupby('A').sum()
df.groupby(['A', 'B']).sum()

11. Pivot tables 透视表

df = pd.DataFrame({'A': ['one', 'one', 'two', 'three'] * 3, 'B': ['A', 'B', 'C'] * 4,
'C': ['foo', 'foo', 'foo', 'bar', 'bar', 'bar'] * 2, 'D': np.random.randn(12),  'E': np.random.randn(12)})

pd.pivot_table(df, values='D', index=['A', 'B'], columns=['C'])

12. 其它

# Time series
rng = pd.date_range('1/1/2012', periods=100, freq='S')
ts = pd.Series(np.random.randint(0, 500, len(rng)), index=rng)
ts.resample('5Min').sum()
# 更改Series 数据类型
df["grade"] = df["raw_grade"].astype("category")
# 分类数据提取
df["grade"].cat.categories = ["very good", "good", "very bad"]
df.sort_values(by="grade")
df.groupby("grade").size()
# 绘图 plot
df = pd.DataFrame(np.random.randn(1000, 4), index=ts.index,columns=['A', 'B', 'C', 'D'])
df = df.cumsum()
plt.figure()
df.plot()
plt.legend(loc='best')

13. 输入输出

# CSV
df.to_csv('foo.csv')
pd.read_csv('foo.csv')

# Excel
df.to_excel('foo.xlsx', sheet_name='Sheet1')
pd.read_excel('foo.xlsx', 'Sheet1', index_col=None, na_values=['NA'])