pycharm处理csv文件重复值

1、获取csv文件表头

import csv

filename = "F:\sj\\一个月数据.csv"
with open(filename) as f:
    render = csv.reader(f)  # reader迭代器
    # 取表头
    header_row = next(render)
    print(header_row)

2、统计文件行数

import csv
import pandas as pd
user_info = pd.read_csv("F:\sj\\一个月数据.csv", iterator=True)
count = 0
for i in range(300):
    try:
        user = user_info.get_chunk(10000)
        count += user.shape[0]
    except StopIteration:
        break
print(count)

3、统计文件列数

import pandas as pd

data = pd.read_csv('F:\sj\\一个月数据.csv', encoding='utf-8')    # [2678400 rows x 389 columns]
name = data.columns
print(len(name))
print()

4、对csv每一列进行重复的值以及重复次数统计，并取出重复次数最多的值

import pandas as pd

data = pd.read_csv('F:\sj\\一个月数据.csv', encoding='utf-8')    # [2678400 rows x 389 columns]
name = data.columns

for i in range(1, len(name)):
    data_1 = name[i]
    data[data_1] = pd.to_numeric(data[data_1])
    data_sort = data[data_1].value_counts()  # 降序
    print(data_sort.head(1))
    print()