文本预处理小结

#coding=gbk
'''
Created on 2016-01-14 08:56:27

@author: heng
'''
import re
import datetime
start = datetime.datetime.now()
file1 = open('I:\\test1\\8. Tweet and Retweets with indexed content\\weibocontents\\Retweet_content.txt','r')
file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','w')
rule3 = '^@.*(\n)$'
rule2 = '^(retweet).*\n'
rule1 = '^(link).*(\n)$'
compile_rule1 = re.compile(rule1)
compile_rule2 = re.compile(rule2)
compile_rule3 = re.compile(rule3)
for i in file1:
    i1 = ''.join(i)
    i2 = compile_rule1.sub('',i1)
    i3 = compile_rule2.sub('',i2)
    i4 = compile_rule3.sub('',i3)
    file2.write(str(i4))
file1.close()
file2.close()
end = datetime.datetime.now()
print end-start
print 'program is ended!!!!!!!!'

#coding=gbk
'''
Created on 2016-01-14 15:08:49
抽取转发微博相关数据
@author: heng
'''
import datetime

def retweet_content(file1,file2):
    j = 0;k=0
    for i in file1:
        j = j+1
        i1  = ''.join(i)
        i2 = i1.strip('\n').split('\t')
        if len(i2)==4:
            str_0 = ''.join(i2[0]) 
        if len(i2)==3:
            file2.write('\t'.join(i2)+'\t'+str_0)
            k = j
        if  j==k+1:
            file2.write('\t'+str(i1))
    file1.close()
    file2.close()


def time_processing(file3,file4):
    for i in file3:
        i1=''.join(i)
        i2 = i1.split('\t')
        i3 = ''.join(i2[1])
        i3 = i3[0:10]+' '+i3[11:19]
        i2[1] = i3
        i4 = '\t'.join(i2)        
        file4.write(str(i4))
    file3.close()
    file4.close()

start = datetime.datetime.now()
file1  = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test_final1.txt','r')
file2 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','w')
retweet_content(file1, file2)
file3 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test3.txt','r')
file4 = open('F:\\test\\sina_user_profile_test_Jan_12\\Retweet_content_test4.txt','w')
time_processing(file3,file4)
end = datetime.datetime.now()
print end-start
print 'program is ended!!!!!!!!'