自然语言处理2---jieba分词

# -*- encoding:utf-8 -*-
import re,HTMLParser
import sys
reload(sys)
import os
import jieba
from sklearn.feature_extraction.text import TfidfVectorizer
#sys.setdefaultencoding('utf-8')#输出的内容（爬下来的）是utf-8的形式
def file_jieba(file_dir):  # 得到文本.txt的路径.\Fractal\Interne
    for root, dirs, files in os.walk(file_dir):
        for f in files:
            tmp_dir = '.\Reduced_internet_preprocess' + '\\' + f  # 加上标签后的文本
            text_init_dir = file_dir + '\\' + f  # 原始文本
            # print text_init_dir
            file_source = open(text_init_dir, 'r')



            ok_file = open(tmp_dir, 'w+')
            line_content = file_source.readlines()

            for lines in line_content:
                #file_source_decode = lines.decode('GBK')
                file_source_cut = jieba.cut(lines, cut_all=False)
                result = ' '.join(file_source_cut)
                result = result.encode('utf-8')
                ok_file.write(result)

            file_source.close()
            ok_file.close()
            print 'ok'