1 文本加载
文本处理函数学习:
re.sub():正则表达式替换函数,对于输入的一个字符串,利用正则表达式(的强大的字符串处理功能),去实现(相对复杂的)字符串替换处理,然后返回被替换后的字符串,实现比普通字符串的replace更加强大的替换功能。
import re
s= "大家好,我是一个小白。I 'm so glad to introduce myself, and I’m 18 years old. Today is 2020/01/01. It is a wonderful DAY!"
re.sub(r'[a-z]', '*', s) # 匹配单一小写字母,并替换为*
re.sub(r'[A-Z]', '*', s) # 表示只匹配单一大写字母,并替换为*
re.sub(r'[A-Za-z]', '*', s) # 表示只匹配单一字母,并替换为*
re.sub(r'[0-9]+', '*', s) # 表示匹配多个连续的数字,并替换为*
re.sub(r'[A-Za-z]+', '*', s) # 表示匹配多个连续的字母,并替换为*
re.sub(r'[^a-z]', '*', s) # 表示匹配单个非小写字母
strip():想去掉字符串里面的哪些字符,那么你就把这些字符当参数传入。此函数只会删除头和尾的字符,中间的不会删除。)
如果strip()的参数为空,那么会默认删除字符串头和尾的空白字符(包括\n,\r,\t这些
词元化:
将列表化的句子一个一个拆成word或者str,形成token(词元)
2 构建语料库(corpus)
2.3 示例代码
import d2I.torch1 as d2l
import torch
import collections
import re
# 载入文本数据,并通过正则化去点除字母外的文字,并且转化为小写
def load_machine_time():
with open(r'../data/txt/timemachine.txt') as f:
lines = f.readlines()
new = [re.sub(r'[^A-Za-z]+',' ',line).strip().lower() for line in lines]
return new
# 将得到的data,根据词元的要求转化为词元
def tokenize(lines,token='word'):
if token == 'word':
return [line.split() for line in lines]
elif token == 'char':
return [list(line) for line in lines]
else:
print('Error'+token)
# 构建语料库corpus:
class corpus:
def __init__(self,min_freqs=0,token=None,reversed_token=None):
self.min_freqs = min_freqs
if token is None:
token = []
if reversed_token is None:
reversed_token = []
freqs = count_freqs(token)
self.token_freq = sorted(freqs.items(),key=lambda x:x[1],reverse=True)
self.idx_to_token = ['<unk>']+reversed_token
self.token_to_idx = {token:idx for idx,token in enumerate(self.idx_to_token)}
for token,freq in self.token_freq:
if freq < self.min_freqs:
break
if token not in self.idx_to_token:
self.idx_to_token.append(token)
self.token_to_idx[token] = len(self.idx_to_token) - 1
def __getitem__(self, tokens):
if not isinstance(tokens,(list,tuple)):
return self.token_to_idx.get(tokens,self.unk)
return [self.__getitem__(token) for token in tokens]
def __len__(self):
return len(self.idx_to_token)
def to_token(self,indices):
if not isinstance(indices,(list,tuple)):
return self.idx_to_token[indices]
return [self.idx_to_token[indice] for indice in indices]
@property
def get_freq(self):
return self.token_freq
@property
def unk(self):
return 0
# 统计词语出现的频率
def count_freqs(tokens):
new = None
if len(tokens)==0 or isinstance(tokens,list):
new = [token for line in tokens for token in line]
freqs = collections.Counter(new)
return freqs
def create_time_machine_corpus(max_slices=-1):
data = load_machine_time()
tokens = tokenize(data, token='char')
vocab = corpus(token=tokens)
'''
这里的vocab[]:就是调用的vocab里面的__getitem__方法
'''
corpuss = [vocab[token] for list in tokens for token in list]
if max_slices > 0:
corpuss = corpuss[:max_slices]
return vocab,corpuss
if __name__ == '__main__':
data = load_machine_time()
print('文本长度:',len(data))
for i in range(10):
print(data[i])
tokens = tokenize(data,token='word')
for i in range(10):
print(tokens[i])
vocab = corpus(token=tokens)
print(list(vocab.token_to_idx.items())[:10])
for i in range(10):
print('文本:',tokens[i])
print('索引:',vocab[tokens[i]])
vocab,corpuss = create_time_machine_corpus(max_slices=-1)
print(len(vocab),len(corpuss))