开房数据的数据清洗

find使用

举例子:

pos="hellos world".find("gogogo")#找不到=-1
pos="hellos world".find("llo")# 2
初始h的位置为-1，e为0，l为1以此类推find找到符合的位置后的第一个字符开始算

接下来对开房的数据进行数据进行切割，在windows里有两个快捷键一个table可以查看所有数据，一个more可以看当前数据的百分比

针对下面这个图里面的曹阳这个信息进行切割

mystr="曹阳,32010619720506042x,F,19720506,-,210005,13770848687,025-842019149,-,cy_qing@163.com,0"
mylist=mystr.split(",") #切割，返回列表
print(mylist) #打印列表
for  data in mylist: #循环列表
    print(data)

数据清洗

readfilepath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1\kaifangX.txt"
writefilegoodpath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1down\kaifanggood.txt"
writefilebadpath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1down\kaifangbad.txt"

readfile=open(readfilepath,"r",errors="ignore")
goodwritefile=open(writefilegoodpath,"w",errors="ignore") #打开文件
badwritefile=open(writefilebadpath,"w",errors="ignore") #打开文件

while True:
    line=readfile.readline()
    mylist=line.split(",")#数据切割。
    if  len(mylist)>2:#判断长度
        if  len(mylist[1])==18:#判断身份证号码合法
            goodwritefile.write(line)
        else:
            badwritefile.write(line)
    else:
        badwritefile.write(line)
    if  not line:
        break
readfile.close()#关闭文件
goodwritefile.close() #关闭
badwritefile.close() #关闭

但是身份证的信息是有区域的划分的，因此

area=[[1,"华北"],[2,"东北"],[3,"华东"],[4,"中南"],[5,"西南"],[6,"西北"]]
filelist=[]#列表存储文件对象
for  areatext  in area:
    savepath="C:\\Users\\Tsinghua-yincheng\\Desktop\\yinchengDay1down\\区域\\"
    writefilepath=savepath+areatext[1]+".txt" #生成路径，6个路径
    writefile=open(writefilepath,"w")#创建一个文件对象
    filelist.append(writefile) #加入文件对象列表


readfilepath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1down\kaifanggood.txt"
readfile=open(readfilepath,"r",errors="ignore")
alllist=readfile.readlines()#全部读取到内存
for  line in alllist:#遍历每一行
    linelist=line.split(",")#切割
    #linelist[1]身份证
    #linelist[1][0]#取出的身份证第一位
    for i   in  range(len(area)): #遍历区域，1 华北，2，东北
        if str(area[i][0])==linelist[1][0]:
            filelist[i].write(line) #写入
            break
readfile.close()#关闭文件
for  file in filelist: #批量关闭文件
    file.close()

但是对于内存消耗太大，改成硬盘存储，就一行的差别

area=[[1,"华北"],[2,"东北"],[3,"华东"],[4,"中南"],[5,"西南"],[6,"西北"]]
filelist=[]#列表存储文件对象
for  areatext  in area:
    savepath="C:\\Users\\Tsinghua-yincheng\\Desktop\\yinchengDay1down\\区域disk\\"
    writefilepath=savepath+areatext[1]+".txt" #生成路径，6个路径
    writefile=open(writefilepath,"w")#创建一个文件对象
    filelist.append(writefile) #加入文件对象列表


readfilepath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1down\kaifanggood.txt"
readfile=open(readfilepath,"r",errors="ignore")
while  True:
    line=readfile.readline()
    if  not line: #处理最后一行
        break


    linelist = line.split(",")  # 切割
    # linelist[1]身份证
    # linelist[1][0]#取出的身份证第一位
    for i in range(len(area)):  # 遍历区域，1 华北，2，东北
        if str(area[i][0]) == linelist[1][0]:
            filelist[i].write(line)  # 写入
            break

readfile.close()#关闭文件
for  file in filelist: #批量关闭文件
    file.close()

因此还可以加上省份的划分

area=[[11,"北京"],[12,"天津"],[13,"河北"],[14,"山西"],[15,"内蒙古"],[21,"辽宁"],[22,"吉林"],[23,"黑龙江"],[31,"上海"],[32,"江苏"],[33,"浙江"],[34, "安徽"],[35,"福建"],[36,"江西"],[37,"山东"],[41,"河南"],[42,"湖北"],
         [43, "湖南"],[44,"广东"],[45,"广西"],[46,"海南"],[50,"重庆"],
         [51, "四川"],[52,"贵州"],[53,"云南"],[54,"西藏"],[61,"陕西"],
         [62, "甘肃"],[63,"青海"],[64,"宁夏"],[65,"新疆"],[71,"台湾"],
         [81, "香港"],[82,"澳门"]]

filelist=[]#列表存储文件对象
for  areatext  in area:
    savepath="C:\\Users\\Tsinghua-yincheng\\Desktop\\yinchengDay1down\\省份\\"
    writefilepath=savepath+areatext[1]+".txt" #生成路径，6个路径
    writefile=open(writefilepath,"w")#创建一个文件对象
    filelist.append(writefile) #加入文件对象列表


readfilepath=r"C:\Users\Tsinghua-yincheng\Desktop\yinchengDay1down\kaifanggood.txt"
readfile=open(readfilepath,"r",errors="ignore")
alllist=readfile.readlines()#全部读取到内存
for  line in alllist:#遍历每一行
    linelist=line.split(",")#切割
    #linelist[1]身份证
    #linelist[1][0]#取出的身份证第一位
    for i   in  range(len(area)): #遍历区域，1 华北，2，东北
        if str(area[i][0])==linelist[1][0:2]:
            filelist[i].write(line) #写入
            break

readfile.close()#关闭文件
for  file in filelist: #批量关闭文件
    file.close()

因此按照年龄、月份、某天都是可行的。