csv数据处理

提取某列合并并去除重复的元素

import pandas as pd
import glob
import ast
import numpy as np


def hebing():#合并
    csv_list = glob.glob('D:/pythonProject/python入门/pandas/*.csv')
    print(u'共发现%s个CSV文件' % len(csv_list))
    print(u'正在处理............')
    for i in csv_list:
        fr = open(i, 'r').read()
        with open('D:/pythonProject/python入门/pandas/hebing.csv', 'a') as f:
            f.write(fr)
    print(u'合并完毕！')


def quchong(file):#去重
    df = pd.read_csv(file, header=0)
    datalist = df.drop_duplicates()
    datalist.to_csv(file)


if __name__ == '__main__':
    #   hebing()
    #   quchong("D:/pythonProject/python入门/pandas/hebing.csv")

    f = pd.read_csv('D:/pythonProject/python入门/pandas/hebing.csv', header=None)

    lis = f.iloc[:, -1].tolist()
    all=[]
    for i in lis:
        i=i.replace(']','')
        i=i.replace('[','')
        i.strip('\"')
        i=i.replace("\'",'')
        i=i.split(",")
        all+=i
    all=sorted(set(all), key=all.index)
    print(all)



```python
在这里插入代码片