python读取文件的时候去重_对python读写文件去重、RE、set的使用详解

如下所示：

# -*- coding:utf-8 -*-

from datetime import datetime

import re

def main():

sourcr_dir = '/data/u_lx_data/fudan/muying/muying_11yue_all.txt'

target_dir = '/data/u_lx_data/fudan/muying/python/uid_regular_get.txt'

uset = set() #去重

print("开始。。。。。")

print(datetime.now().strftime('%y-%m-%d %h:%m:%s'))

with open(target_dir, 'w+') as f_write:

with open(sourcr_dir, 'r') as f_scorce:

for line in f_scorce:

line = line.strip().split("\t")

# 宝宝树

if line[2] == 'babytree.com':

uidlist = re.findall(r'.*nl=u%02(u\d+)', line[3], re.i)

if uidlist:

# 去重代码

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

print("宝宝树已完成")

# 柚宝宝

elif line[2] == 'youzibuy.com':

if line[4].find("yunqi.youzibuy.com/tae_top_notify") != -1:

uidlist = re.findall(r'.*myuid=(\d+)', line[4], re.i)

if uidlist:

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

print("柚宝宝已完成")

# 妈妈帮

elif line[2] == 'mmbang.com':

uidlist = re.findall(r'.*uid=(\d+)', line[3], re.i)

if uidlist:

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

print("妈妈帮已完成")

# 妈妈网

elif line[2] == 'mama.cn':

if line[4].find("mapi.mama.cn/feed/users/show") != -1:

uidlist = re.findall(r'.*friend_uid=(\d+)', line[4], re.i)

if uidlist:

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

if line[4].find("mamaquan/mmq_thread") != -1:

uidlist = re.findall(r'.*uid=(\d+)', line[4], re.i)

if uidlist:

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

print("妈妈网已完成")

# 育儿网

elif line[2] == 'ci123.com':

uidlist = re.findall(r'.*ci123js=([a-za-z]+\d+)', line[3], re.i)

if uidlist:

if uidlist[0] not in uset:

f_write.write(uidlist[0] + "\n")

uset.add(uidlist[0])

print("育儿网已完成")

print("完成。。。。。")

print(datetime.now().strftime('%y-%m-%d %h:%m:%s'))

if __name__ == "__main__":

main()

以上这篇对python读写文件去重、re、set的使用详解就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持萬仟网。

如您对本文有疑问或者有任何想说的，请点击进行留言回复，万千网友为您解惑！