2021-08-03

最近两天在看<<自然语言处理实践>>,学习上面的代码进行分词,感觉还不错,讲解得很详细.(隔两天继续写,争取在开学前看完这本书,嘻嘻)
<<自然语言处理实践>>:作者Hobson Lane et al.译者:史亮等人

# -*-coding:utf-8-*-
import pandas as pd
import numpy as np
import re #正则表达式
from nltk.tokenize import RegexpTokenizer
from nltk.tokenize import TreebankWordTokenizer
from nltk.tokenize.casual import casual_tokenize #用于专门处理不规范文本的分词

#用NLTK包进行分词
sentence = '[yh is - the [b]est one. haha!! [biaoqing.,\t'
#使用TreebankWordTokenizer
tokenizer = TreebankWordTokenizer()
print(tokenizer.tokenize(sentence))
#使用RegexpTokenizer分词器
tokenizer= RegexpTokenizer(r'\w+|$[0-9.]+|\s+')
print(tokenizer.tokenize(sentence))
exit()
# 标点符合的处理
sentence = '[yh is - the [b]est one. haha!! [biaoqing.,\t'
tokens = re.split(r'[-\s.,;!?\[\]]+', sentence)#如果句子中有[]如何去掉呢？(用\[\])这里的+号是说明至少里面的符号出现一次
print(tokens)
pattern = re.compile(r"[-\s.,;!?\]\[]+")#不太理解为什么会有个空格?应该是因为用符号分开，什么都没有的用“”来代替
token = pattern.split(sentence)
print(token)
#去掉上述所提到的空白符
print(list(x for x in token if x and x not in '-\t\n. ,;!?'))
#使用lambda表达式
print(list(filter(lambda x:x if x and x not in '-\t\n ,;.!?' else None, token)))


#掌握点积
v1 = pd.np.array([1,2,3])
v2 = pd.np.array([2,3,4])
print(v1.dot(v2))
print((v1 * v2).sum())
# 矩阵乘积：
print(v1.reshape(-1,1).shape)#(3,1)
v1 = v1.reshape(-1,1).T
print(v1.shape)#(1,3)
v2 = v2.reshape(-1,1)#维度为3的矩阵转换为(3,1)的矩阵
print(v1@v2)#矩阵乘积用@操作符
print(np.matmul(v1,v2))#矩阵乘积用np.matmul()函数


#多个文本构建词袋向量：
sentence = 'liu ya hui is the best one, she is a hero\n'
sentence += 'I do not know if I can do it, but My mom believes me. \n'
sentence += 'ok, I will'
#sentence.splitlines()用于分割\n
corpus = {}
for i, sent in enumerate(sentence.split('\n')):
    corpus['sent{}'.format(i)] = dict([(token, 1) for token in sent.split()])
df = pd.DataFrame.from_records(corpus).fillna(0).astype(int).T
print(df[df.columns[:10]])#只输出前10列

#用上述的点积计算句子之间的重合程度（词之间的重合度可以作为句子相似度的一种度量方法）
df = df.T
print(df.shape)
print(df.sent0.dot(df.sent1))#df.senti是个向量，向量是竖着的
print(df.sent1.dot(df.sent2))
print((df.sent1 & df.sent2).items())#句子1和句子2向量的and操作并打包
print([(k, v) for (k, v) in (df.sent1 & df.sent2).items() if v])#输出两个句子重复的词




#构建词袋向量：用词典pandas中的Series便于将字典进行封装到DataFrame
sentence_bow = {}
sentence = 'liu ya hui is the best one, she is a hero'
for token  in sentence.split():
    sentence_bow[token] = 1
print(len(sentence_bow))
print(sorted(sentence_bow.items()))
print(sorted(dict([(token, 1) for token in sentence.split()])))
df = pd.DataFrame(pd.Series(dict([(token, 1) for token in sorted(sentence.split())])), columns=["sentence"]).T
print(df)

#分词

sentence = 'liu ya hui is the best one, she is a hero'
token_sentence = str.split(sentence)
vocab = sorted(set(token_sentence))#去重排序
# print(", ".join(vocab))
num_token = len(token_sentence)
vocab_size = len(vocab)#词袋
onehot_vectors = np.zeros((num_token, vocab_size), int)
for i, word in enumerate(token_sentence):
    onehot_vectors[i, vocab.index(word)] = 1#独热向量
# 另一种独热向量(不能用于机器学习流水线中，会导致数学计算上的混乱)显示美观
df = pd.DataFrame(onehot_vectors, columns=vocab)
print(df)
df[df == 0] = " "
print(df)


# 正则表达式, []指定某个字符类，*表示前面的字符再出现0次或多次的情况下都可以匹配
r = "(hi|hello|hey) [ ] * ([a-z]*)"
print(re.match(r, "Hello  Rosa", flags=re.IGNORECASE))

s = "Find textbooks with titles containing 'NLP', or 'natural' and 'language'."
print(len(s.split()))
# np.prod()函数用来计算所有元素的乘积,对于有多个维度的数组可以指定轴,如axis=1指定计算每一行的乘积。
print(np.arange(1, 10+1).prod())#计算排列组合的和

# 词序 全排列
from itertools import permutations
print([" ".join(combo) for combo in permutations("liu ya hui".split(), 3)])

# 计数
from collections import Counter
print(Counter("liu yahui is the best one !".split()))