手搓bert(还没搓完)

今天有点搓不动了，明天继续

import torch
import torch.nn as nn
import torch.nn.functional as F
import math
# BertForMaskedLM - BertModel,BertOnlyMLMHead

# BertModel - BertEmbedding, BertEncoder(no BertPooler)
# BertOnlyMLMHead - BertLMPredictionHead

# BertEncoder - BertLayer
# BertLayer - BertAttention, BertIntermediate, BertOutput

class Config(object):
    def __init__(self):
        self.vocab_size = 1000
        self.hidden_size = 768
        self.max_position_embeddings = 32  # 输入句子的最大长度
        self.hidden_dropout_prob = 0.5
        self.layer_norm_eps = 1e-6  # layer_norm中防止分母为0
        self.type_vocab_size = 1  # 输入句子的类别，例如问和答
        self.pad_token_id = 0
        self.num_attention_heads = 12
        self.attention_probs_dropout_prob = 0.5
        self.position_embedding_type = "absolute"
        self.intermediate_size = 512
        self.hidden_act = "relu"


class BertEmbedding(nn.Module):
    def __init__(self, config):
        super(BertEmbedding, self).__init__()
        # 可学习的三个embeddings
        self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id)
        self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size)  # 一个段落中最长的position
        self.token_type_embeddings = nn.Embedding(config.type_vocab_size, config.hidden_size)  # token_type分别句子的类别
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
        self.dropout = nn.Dropout(config.hidden_dropout_prob)
        self.position_embeddings_type = getattr(config, "position_embeddings_type", "absolute")
        self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1)), persistent=False)
        # 注册到内存缓冲区中，参与模型训练，但参数不更新，不保存和恢复，只在每次创建模型实例时动态生成
        self.register_buffer("token_type_ids