【pandas、numpy】缺失值填充和重复值处理练习

import numpy as np
import pandas as pd

df = pd.DataFrame([[np.nan, 2, np.nan, 0]
                 ,[3, 4, np.nan, 1]
                 ,[np.nan, np.nan, np.nan, 4]
                 ,[np.nan, 3, np.nan, 4]]
                 , columns = list('ABCD'))
df

	A	B	C	D
0	NaN	2.0	NaN	0
1	3.0	4.0	NaN	1
2	NaN	NaN	NaN	4
3	NaN	3.0	NaN	4

df.fillna(33)

	A	B	C	D
0	33.0	2.0	33.0	0
1	3.0	4.0	33.0	1
2	33.0	33.0	33.0	4
3	33.0	3.0	33.0	4

df

	A	B	C	D
0	NaN	2.0	NaN	0
1	3.0	4.0	NaN	1
2	NaN	NaN	NaN	4
3	NaN	3.0	NaN	4

# 向前填充
df.fillna(method='ffill')

	A	B	C	D
0	NaN	2.0	NaN	0
1	3.0	4.0	NaN	1
2	3.0	4.0	NaN	4
3	3.0	3.0	NaN	4

df

	A	B	C	D
0	NaN	2.0	NaN	0
1	3.0	4.0	NaN	1
2	NaN	NaN	NaN	4
3	NaN	3.0	NaN	4

# 按照指定值进行填充
values = {'A':0, 'B':1, 'C':2, 'D':3}
df.fillna(value=values, limit=1)

	A	B	C	D
0	0.0	2.0	2.0	0
1	3.0	4.0	NaN	1
2	NaN	1.0	NaN	4
3	NaN	3.0	NaN	4

df.fillna(value=values)  # 对比上面，limit参数可以限制填充前limit个缺失值

	A	B	C	D
0	0.0	2.0	2.0	0
1	3.0	4.0	2.0	1
2	0.0	1.0	2.0	4
3	0.0	3.0	2.0	4

df.fillna(value=values, inplace=True)

	A	B	C	D
0	0.0	2.0	2.0	0
1	3.0	4.0	2.0	1
2	0.0	1.0	2.0	4
3	0.0	3.0	2.0	4

# 发现重复值
data = pd.read_csv('data1.csv')
data.head()

	orderid	bikeid	userid	start_time	start_location_x	start_location_y	end_time	end_location_x	end_location_y	track
0	78387	158357	10080	2016-08-20 06:57	121.348	31.389	2016-08-20 07:04	121.357	31.388	121.347,31.392#121.348,31.389#121.349,31.390#1...
1	891333	92776	6605	2016-08-29 19:09	121.508	31.279	2016-08-29 19:31	121.489	31.271	121.489,31.270#121.489,31.271#121.490,31.270#1...
2	1106623	152045	8876	2016-08-13 16:17	121.383	31.254	2016-08-13 16:36	121.405	31.248	121.381,31.251#121.382,31.251#121.382,31.252#1...
3	1389484	196259	10648	2016-08-23 21:34	121.484	31.320	2016-08-23 21:43	121.471	31.325	121.471,31.325#121.472,31.325#121.473,31.324#1...
4	188537	78208	11735	2016-08-16 07:32	121.407	31.292	2016-08-16 07:41	121.418	31.288	121.407,31.291#121.407,31.292#121.408,31.291#1...

data.duplicated().sum()

# subset : column label or sequence of labels, optional
#    Only consider certain columns for identifying duplicates, by
#    default use all of the columns

0

# 显示重复列
data[data.duplicated()]

	orderid	bikeid	userid	start_time	start_location_x	start_location_y	end_time	end_location_x	end_location_y	track

# 删除重复值
data.drop_duplicates()

	orderid	bikeid	userid	start_time	start_location_x	start_location_y	end_time	end_location_x	end_location_y	track
0	78387	158357	10080	2016-08-20 06:57	121.348	31.389	2016-08-20 07:04	121.357	31.388	121.347,31.392#121.348,31.389#121.349,31.390#1...
1	891333	92776	6605	2016-08-29 19:09	121.508	31.279	2016-08-29 19:31	121.489	31.271	121.489,31.270#121.489,31.271#121.490,31.270#1...
2	1106623	152045	8876	2016-08-13 16:17	121.383	31.254	2016-08-13 16:36	121.405	31.248	121.381,31.251#121.382,31.251#121.382,31.252#1...
3	1389484	196259	10648	2016-08-23 21:34	121.484	31.320	2016-08-23 21:43	121.471	31.325	121.471,31.325#121.472,31.325#121.473,31.324#1...
4	188537	78208	11735	2016-08-16 07:32	121.407	31.292	2016-08-16 07:41	121.418	31.288	121.407,31.291#121.407,31.292#121.408,31.291#1...
5	537030	66346	10335	2016-08-07 21:00	121.488	31.224	2016-08-07 21:20	121.468	31.233	121.468,31.233#121.469,31.233#121.470,31.233#1...
6	517706	99631	11258	2016-08-29 13:39	121.445	31.176	2016-08-29 13:54	121.447	31.192	121.445,31.176#121.446,31.175#121.446,31.176#1...
7	270836	63136	11361	2016-08-29 09:21	121.369	31.252	2016-08-29 13:43	121.376	31.233	121.367,31.238#121.367,31.239#121.368,31.240#1...
8	441362	331921	11705	2016-08-31 08:16	121.529	31.324	2016-08-31 08:19	121.532	31.323	121.529,31.324#121.530,31.324#121.531,31.323#1...
9	76435	347335	8135	2016-08-30 12:49	121.467	31.320	2016-08-30 13:03	121.447	31.318	121.447,31.317#121.447,31.318#121.448,31.314#1...
10	903987	13745	3117	2016-08-09 19:51	121.514	31.307	2016-08-09 19:59	121.510	31.304	121.510,31.304#121.510,31.305#121.510,31.306#1...
11	1749105	250190	8174	2016-08-26 17:57	121.385	31.166	2016-08-26 18:43	121.439	31.193	121.384,31.166#121.385,31.166#121.386,31.166#1...
12	760441	274835	17177	2016-08-31 09:53	121.419	31.292	2016-08-31 10:04	121.442	31.297	121.419,31.292#121.420,31.292#121.420,31.293#1...
13	163153	129925	9763	2016-08-09 08:33	121.551	31.279	2016-08-09 08:40	121.558	31.272	121.551,31.278#121.551,31.279#121.552,31.277#1...
14	1718308	291555	9867	2016-08-24 16:32	121.528	31.309	2016-08-24 16:45	121.523	31.321	121.521,31.316#121.521,31.320#121.522,31.315#1...
15	113197	277976	1499	2016-08-31 06:51	121.413	31.358	2016-08-31 07:08	121.432	31.356	121.412,31.359#121.413,31.359#121.414,31.359#1...
16	1660785	74378	3637	2016-08-06 22:30	121.590	31.243	2016-08-06 22:41	121.584	31.246	121.585,31.246#121.586,31.245#121.586,31.246#1...
17	1796188	210920	4156	2016-08-24 22:52	121.483	31.278	2016-08-24 23:02	121.473	31.273	121.473,31.273#121.474,31.273#121.475,31.273#1...
18	1725437	94034	10712	2016-08-31 21:31	121.442	31.264	2016-08-31 21:36	121.437	31.263	121.437,31.262#121.437,31.263#121.438,31.261#1...
19	974312	87103	4921	2016-08-08 17:31	121.437	31.294	2016-08-08 17:44	121.427	31.303	121.427,31.303#121.428,31.303#121.429,31.298#1...
20	1016344	319980	557	2016-08-29 20:20	121.466	31.185	2016-08-29 21:01	121.450	31.197	121.450,31.196#121.450,31.197#121.451,31.195#1...
21	4156	57862	2395	2016-08-09 02:34	121.384	31.188	2016-08-09 02:37	121.383	31.187	121.383,31.187#121.383,31.188#121.384,31.188
22	1013785	364751	14595	2016-08-30 12:17	121.388	31.268	2016-08-30 12:28	121.368	31.272	121.368,31.272#121.369,31.272#121.369,31.273#1...
23	420053	162491	13745	2016-08-20 14:25	121.432	31.180	2016-08-20 14:31	121.427	31.184	121.426,31.182#121.427,31.182#121.427,31.183#1...
24	737601	58198	6349	2016-08-01 17:57	121.513	31.302	2016-08-01 18:19	121.541	31.311	121.513,31.302#121.513,31.303#121.514,31.302#1...
25	1663013	9935	8929	2016-08-30 20:15	121.382	31.246	2016-08-30 20:22	121.380	31.251	121.378,31.249#121.378,31.250#121.378,31.251#1...
26	399789	292393	10700	2016-08-24 05:49	121.364	31.248	2016-08-24 05:58	121.367	31.237	121.361,31.244#121.362,31.243#121.362,31.244#1...
27	1399630	44728	5944	2016-08-22 18:27	121.506	31.329	2016-08-22 18:33	121.501	31.335	121.499,31.336#121.499,31.337#121.500,31.336#1...
28	773687	263024	15033	2016-08-24 07:45	121.464	31.193	2016-08-24 07:53	121.470	31.190	121.464,31.192#121.464,31.193#121.465,31.192#1...
29	466460	240822	12012	2016-08-21 22:13	121.533	31.318	2016-08-21 22:42	121.502	31.309	121.502,31.308#121.502,31.309#121.503,31.308#1...
...	...	...	...	...	...	...	...	...	...	...
102331	752209	264648	5474	2016-08-19 11:10	121.349	31.259	2016-08-19 11:16	121.354	31.261	121.349,31.258#121.349,31.259#121.350,31.257#1...
102332	1177538	94648	2319	2016-08-12 14:07	121.499	31.279	2016-08-12 14:11	121.502	31.275	121.499,31.278#121.499,31.279#121.499,31.280#1...
102333	194189	71476	7647	2016-08-10 08:47	121.403	31.130	2016-08-10 08:55	121.412	31.124	121.402,31.128#121.402,31.129#121.403,31.127#1...
102334	1612820	129498	9547	2016-08-06 19:21	121.501	31.275	2016-08-06 20:12	121.546	31.289	121.501,31.275#121.502,31.275#121.503,31.276#1...
102335	1676331	314831	15930	2016-08-28 20:23	121.460	31.328	2016-08-28 20:29	121.458	31.335	121.458,31.335#121.459,31.328#121.459,31.329#1...
102336	734652	333917	3905	2016-08-31 09:41	121.295	31.271	2016-08-31 09:53	121.304	31.272	121.295,31.271#121.296,31.271#121.297,31.271#1...
102337	947568	146806	9711	2016-08-27 14:47	121.495	31.170	2016-08-27 15:22	121.505	31.181	121.494,31.170#121.494,31.171#121.494,31.177#1...
102338	34563	203064	3038	2016-08-25 11:50	121.551	31.279	2016-08-25 11:55	121.554	31.273	121.551,31.275#121.551,31.279#121.552,31.274#1...
102339	375055	119804	6027	2016-08-01 11:27	121.532	31.315	2016-08-01 11:35	121.538	31.313	121.532,31.314#121.532,31.315#121.533,31.314#1...
102340	172833	66527	10097	2016-08-15 07:12	121.525	31.317	2016-08-15 07:19	121.531	31.321	121.525,31.317#121.526,31.318#121.526,31.319#1...
102341	405791	180713	5788	2016-08-20 14:01	121.405	31.235	2016-08-20 14:09	121.410	31.231	121.404,31.232#121.405,31.231#121.405,31.232#1...
102342	998890	175676	508	2016-08-23 13:14	121.432	31.259	2016-08-23 13:28	121.453	31.270	121.432,31.259#121.433,31.259#121.434,31.259#1...
102343	893197	214737	9990	2016-08-25 18:31	121.541	31.318	2016-08-25 18:39	121.532	31.314	121.533,31.314#121.534,31.314#121.535,31.314#1...
102344	1648715	11728	8836	2016-08-06 21:33	121.438	31.228	2016-08-06 21:40	121.430	31.234	121.430,31.234#121.431,31.232#121.431,31.233#1...
102345	48162	92143	3885	2016-08-08 00:19	121.436	31.320	2016-08-08 00:26	121.438	31.310	121.436,31.318#121.436,31.319#121.436,31.320#1...
102346	665032	6743	17715	2016-08-29 16:06	121.443	31.183	2016-08-29 16:08	121.441	31.178	121.442,31.179#121.443,31.179#121.443,31.180#1...
102347	1191602	301472	4041	2016-08-23 17:37	121.507	31.249	2016-08-23 17:58	121.515	31.269	121.506,31.250#121.506,31.251#121.507,31.249#1...
102348	1539900	185638	71	2016-08-31 18:09	121.422	31.171	2016-08-31 18:19	121.425	31.163	121.421,31.171#121.422,31.171#121.423,31.171#1...
102349	1449488	349625	414	2016-08-27 19:39	121.451	31.279	2016-08-27 20:17	121.425	31.295	121.425,31.295#121.426,31.295#121.427,31.295#1...
102350	1303672	80655	7520	2016-08-17 17:16	121.532	31.331	2016-08-17 17:22	121.531	31.323	121.528,31.331#121.531,31.328#121.532,31.324#1...
102351	657156	59970	5497	2016-08-31 09:09	121.372	31.119	2016-08-31 09:23	121.375	31.107	121.372,31.118#121.372,31.119#121.373,31.117#1...
102352	186351	24298	11196	2016-08-11 07:42	121.422	31.201	2016-08-11 07:46	121.420	31.197	121.419,31.200#121.419,31.201#121.420,31.199#1...
102353	1168582	322536	15738	2016-08-26 09:03	121.473	31.193	2016-08-26 09:10	121.471	31.202	121.473,31.193#121.473,31.194#121.473,31.196#1...
102354	182261	16353	2628	2016-08-03 10:05	121.498	31.288	2016-08-03 10:13	121.508	31.281	121.498,31.287#121.498,31.288#121.499,31.287#1...
102355	640008	329621	10791	2016-08-29 15:39	121.518	31.311	2016-08-29 15:48	121.523	31.322	121.518,31.311#121.519,31.311#121.519,31.312#1...
102356	1479550	214335	1423	2016-08-28 18:03	121.478	31.297	2016-08-28 18:09	121.481	31.304	121.478,31.297#121.479,31.297#121.480,31.297#1...
102357	1478273	160487	3067	2016-08-14 20:22	121.320	31.238	2016-08-14 20:28	121.312	31.235	121.312,31.235#121.313,31.235#121.313,31.236#1...
102358	367733	179530	12746	2016-08-27 09:54	121.391	31.307	2016-08-27 10:06	121.398	31.306	121.390,31.307#121.391,31.307#121.392,31.307#1...
102359	64915	167419	837	2016-08-20 06:04	121.515	31.269	2016-08-20 06:10	121.510	31.272	121.510,31.272#121.510,31.273#121.511,31.273#1...
102360	326987	63172	2547	2016-08-09 10:51	121.521	31.261	2016-08-09 11:01	121.518	31.274	121.518,31.274#121.519,31.270#121.519,31.271#1...

102361 rows × 10 columns