python去除停用词_如何用python对一个文件夹下的多个txt文本进行去停用词。

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58#encoding=utf-8

import sys

import re

import codecs

import os

import shutil

import jieba

import jieba.analyse

#导入自定义词典

#jieba.load_userdict("dict_baidu.txt")

#Read file and cut

def read_file_cut():

#create path

stopwords = {}.fromkeys([ line.strip() for line in open('stopword.txt') ])

path = "Lon\"

respath = "Lon_Result\"

if os.path.isdir(respath): #如果respath这个路径存在

shutil.rmtree(respath, True) #则递归移除这个路径

os.makedirs(respath) #重新建立一个respath目录

num = 1

while num<=20:

name = "%d" % num

fileName = path + str(name) + ".txt"

resName = respath + str(name) + ".txt"

source = open(fileName, 'r')

if os.path.exists(resName):

os.remove(resName)

result = codecs.open(resName, 'w', 'utf-8')

line = source.readline()

line = line.rstrip('\n')

while line!="":

line = unicode(line, "utf-8")

output=''

seglist = jieba.cut(line,cut_all=False)

for seg in seglist:

seg=seg.encode('utf-8')

if seg not in stopwords:

output+=seg

output = ' '.join(list(seglist))#空格拼接

print output

result.write(output + '\r\n')

line = source.readline()

else:

print 'End file: ' + str(num)

source.close()

result.close()

num = num + 1

else:

print 'End All'

#Run function

if __name__ == '__main__':

read_file_cut()