Bert使用之一_基本使用

Bert使用参考：
https://blog.csdn.net/c991262331/article/details/89381972
https://zhuanlan.zhihu.com/p/50773178
https://www.jiqizhixin.com/articles/2018-11-01-9

一 Demo

注意：
BertTokenizer.from_pretrained中参数要指定到vocab的文件路径，而BertModel.from_pretrained中参数要指定到pytorch_model.bin和config.json所在的目录路径，不是要指定到pytorch_model.bin的文件路径

from pytorch_pretrained_bert import BertTokenizer,BertModel
import torch

# bert_tokenizer = BertTokenizer.from_pretrained("bert-base-chinese")
bert_tokenizer = BertTokenizer.from_pretrained(r'C:\XXX\bert-base-chinese-vocab.txt')
a = "张三和李四都住在村头"
a_token = bert_tokenizer.tokenize(a)
print(a_token)
a_seq_ids = bert_tokenizer.convert_tokens_to_ids(a_token)
print(a_seq_ids)

# bert_model = BertModel.from_pretrained("bert-base-chinese")
bert_model = BertModel.from_pretrained(r'C:\XXX\bert-base-chinese')
batch_data = torch.Tensor(a_seq_ids).long().view((1,-1))
out,_ = bert_model(batch_data)
print(out[0].shape)

['张', '三', '和', '李', '四', '都', '住', '在', '村', '头']
[2476, 676, 1469, 3330, 1724, 6963, 857, 1762, 3333, 1928]
torch.Size([1, 10, 768])