目录
赛题描述及数据说明
大赛地址：https://tianchi.aliyun.com/s/3bd272d942f97725286a8e44f40f3f74
代码实践
Step 1:环境准备
import pandas as pd
import codecs, gc
import numpy as np
from sklearn.model_selection import KFold
from keras_bert import load_trained_model_from_checkpoint, Tokenizer
from keras.metrics import top_k_categorical_accuracy
from keras.layers import *
from keras.callbacks import *
from keras.models import Model
import keras.backend as K
from keras.optimizers import Adam
from keras.utils import to_categorical
from sklearn.preprocessing import LabelEncoder
google colab上运行代码，需要先将数据上传至driver上。执行以下代码挂在driver并配置相关环境.
from google.colab import drive
drive.mount('/content/drive')
'''
路径说明：
../code #保存代码
../data #保存数据
../subs #保存数据
../chinese_roberta_wwm_large_ext_L-24_H-1024_A-16 #bert路径
'''
pip install keras-bert
Step 2：数据读取
# 将ocnli中content1[0:maxlentext1]+content2作为ocnli任务的content
times_train = pd.read_csv('/data/TNEWS_train1128.csv', sep='\t', header=None, names=('id', 'content', 'label')).astype(str)
ocemo_train = pd.read_csv('/data/OCEMOTION_train1128.csv',sep='\t', header=None, names=('id', 'content', 'label')).astype(str)
ocnli_train = pd.read_csv('/data/OCNLI_train1128.csv', sep='\t', header=None, names=('id', 'content1', 'content2', 'label')).astype(str)
ocnli_train['content'] = ocnli_train['content1'] + ocnli_train['content2'] # .apply( lambda x: x[:maxlentext1] )
times_testa = pd.read_csv('/data/TNEWS_a.csv', sep='\t', header=None, names=('id', 'content')).astype(str)
ocemo_testa = pd.read_csv('/data/OCEMOTION_a.csv',sep='\t', header=None, names=('id', 'content')).astype(str)
ocnli_testa = pd.read_csv('/data/OCNLI_a.csv', sep='\t', header=None, names=('id', 'content1', 'content2')).astype(str)
ocnli_testa['content'] = ocnli_testa['content1'] + ocnli_testa['content2'] # .apply( lambda x: x[:maxlentext1] )
1) 数据集合并
分别将三个任务的content、label列按行concat在一起作为训练集和标签、测试集，以此简单地将三任务转化为单任务。
# 合并三个任务的训练、测试数据
train_df = pd.concat([times_train, ocemo_train, ocnli_train[['id','content', 'label']]], axis=0).copy()
testa_df = pd.concat([times_testa, ocemo_testa, ocnli_testa[['id', 'content']]], axis=0).copy()
2)标签编码
# LabelEncoder处理标签，因为bert输入的label需要从0开始
# LabelEncoder(): Encode labels with value between 0 and n_classes-1.
encode_label = LabelEncoder()
train_df['label'] = encode_label.fit_transform(train_df['label'].apply(str))
3) 数据信息查看
train_df.info()
'''
<class 'pandas.core.frame.DataFrame'>
Int64Index: 147453 entries, 0 to 48777
Data columns (total 3 columns):
# Column Non-Null Count Dtype
--- ------ -------------- -----
0 id 147453 non-null object
1 content 147453 non-null object
2 label 147453 non-null int64
dtypes: int64(1), object(2)
memory usage: 4.5+ MB
'''
Step 3: 数据分析（EDA）
1) 子句长度统计分析
统计子句长度主要用于设置输入bert的序列长度。
times_train['content'].str.len().describe(percentiles=[.95, .98, .99])\
,ocemo_train['content'].str.len().describe(percentiles=[.95, .98, .99])\
,ocnli_train['content1'].str.len().describe(percentiles=[.95, .98, .99])\
,ocnli_train['content2'].str.len().describe(percentiles=[.95, .98, .99])
'''
(count 63360.000000
mean 22.171086
std 7.334206
min 2.000000
50% 22.000000
95% 33.000000
98% 37.000000
99% 39.000000
max 145.000000
Name: content, dtype: float64, count 35315.000000
mean 48.214328
std 84.391942
min 3.000000
50% 34.000000
95% 134.000000
98% 138.000000
99% 142.000000
max 12326.000000
Name: content, dtype: float64, count 48778.000000
mean 24.174607
std 11.515428
min 8.000000
50% 22.000000
95% 46.000000
98% 49.000000
99% 50.000000
max 50.000000
Name: content1, dtype: float64, count 48778.000000
mean 15.828529
std 977.396848
min 2.000000
50% 10.000000
95% 21.000000
98% 24.000000
99% 27.000000
max 215874.000000
Name: content2, dtype: float64)
'''
从上可以看出，当设置bert序列长度为142时即可覆盖约99%子句的全部内容。
2)统计标签的基本分布信息
train_df['label'].value_counts() / train_df.shape[0]
'''
1 0.113467
0 0.109940
17 0.107397
23 0.084603
21 0.060318
10 0.047771
6 0.041749
4 0.039918
13 0.039036
8 0.033292
3 0.032668
5 0.032268
11 0.029487
19 0.029481
9 0.027690
18 0.027588
12 0.027541
16 0.027460
22 0.027412
15 0.022923
7 0.016853
2 0.008993
24 0.006097
20 0.004001
14 0.002048
Name: label, dtype: float64
'''
Step 4: 预训练模型选择
1) 模型选择
在众多nlp预训练模型中，本文baseline选择了哈工大与讯飞联合发布的基于全词遮罩（Whole Word Masking）技术的中文预训练模型：RoBERTa-wwm-ext-large。点击以下链接了解更多详细信息：
论文地址：https://arxiv.org/abs/1906.08101
开源模型地址：https://github.com/ymcui/Chinese-BERT-wwm
哈工大讯飞联合实验室的项目介绍：https://mp.weixin.qq.com/s/EE6dEhvpKxqnVW_bBAKrnA
2) 调优参数配置
为方便调优，在同一代码块中配置调优的参数。
#一些调优参数
er_patience = 2 # early_stopping patience
lr_patience = 5 # ReduceLROnPlateau patience
max_epochs = 2 # epochs
lr_rate = 2e-6 # learning rate
batch_sz = 4 # batch_size
maxlen = 256 # 设置序列长度为，base模型要保证序列长度不超过512
lr_factor = 0.85 # ReduceLROnPlateau factor
maxlentext1 = 200 # 选择ocnli子句一的长度
n_folds = 10 # 设置验证集的占比:1/n_folds
Step 5: 模型构建
1) 切分数据集（Train,Val）进行模型训练、评价
采用StratifiedKFold分层抽样抽取10%的训练数据作为验证集。
### 采用分层抽样的方式，从训练集中抽取10%作为验证机
from sklearn.model_selection import StratifiedKFold
skf = StratifiedKFold(n_splits=n_folds, shuffle=True, random_state=222)
X_trn = pd.DataFrame()
X_val = pd.DataFrame()
for train_index, test_index in skf.split(train_df.copy(), train_df['label']):
X_trn, X_val = train_df.iloc[train_index], train_df.iloc[test_index]
break#不能多折训练
采用f1值做为评价指标，当评价指标不在提升时，降低学习率。
from keras import backend as K
def f1(y_true, y_pred):
def recall(y_true, y_pred):
"""Recall metric.
Only computes a batch-wise average of recall.
Computes the recall, a metric for multi-label classification of
how many relevant items are selected.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
def precision(y_true, y_pred):
"""Precision metric.
Only computes a batch-wise average of precision.
Computes the precision, a metric for multi-label classification of
how many selected items are relevant.
"""
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
precision = precision(y_true, y_pred)
recall = recall(y_true, y_pred)
return2*((precision*recall)/(precision+recall+K.epsilon()))
keras.backend.clip(x, min_value, max_value)
逐元素clip（将超出指定范围的数强制变为边界值）
参数
x: 张量或变量。
min_value: Python 浮点或整数。
max_value: Python 浮点或整数。
返回一个张量。
keras.backend.round()
将小数取整：四舍六入五取偶
keras.backend.epsilon()
进行除法运算时，通常将其添加到分母中以防止被零除误差。Epsilon是一个很小的值（在TensorFlow Core v2.2.0中为1e-07），与分母的值几乎没有区别，但可确保它不等于零。
keras.backend.round()
将小数取整：四舍六入五取偶
keras.backend.epsilon()
进行除法运算时，通常将其添加到分母中以防止被零除误差。Epsilon是一个很小的值（在TensorFlow Core v2.2.0中为1e-07），与分母的值几乎没有区别，但可确保它不等于零。
2) 构造输入bert的数据格式
# 标签类别个数
n_cls = len( train_df['label'].unique() )
# 训练数据、测试数据和标签转化为模型输入格式
# 训练集每行的content、label转为tuple存入list，再转为numpy array
TRN_LIST = []
for data_row in X_trn.iloc[:].itertuples():
TRN_LIST.append((data_row.content, to_categorical(data_row.label, n_cls)))
TRN_LIST = np.array(TRN_LIST)
# 验证集每行的content、label转为tuple存入list，再转为numpy array
VAL_LIST = []
for data_row in X_val.iloc[:].itertuples():
VAL_LIST.append((data_row.content, to_categorical(data_row.label, n_cls)))
VAL_LIST = np.array(VAL_LIST)
#测试集每行的content、label转为tuple存入list，再转为numpy array，其中label全为0
DATA_LIST_TEST = []
for data_row in testa_df.iloc[:].itertuples():
DATA_LIST_TEST.append((data_row.content, to_categorical(0, n_cls)))
DATA_LIST_TEST = np.array(DATA_LIST_TEST)
to_categorical
to_categorical就是将类别向量转换为二进制（只有0和1）的矩阵类型表示。其表现为将原有的类别向量转换为独热编码的形式。
3) 模型搭建
在bert后接一层Lambda层取出[CLS]对应的向量，再接一层Dense层用于分类输出。
#bert模型设置
def build_bert(nclass):
global lr_rate
bert_model = load_trained_model_from_checkpoint(config_path, checkpoint_path, seq_len=None) # 加载预训练模型
for l in bert_model.layers:
l.trainable = True
x1_in = Input(shape=(None,))
x2_in = Input(shape=(None,))
x = bert_model([x1_in, x2_in])
x = Lambda(lambda x: x[:, 0])(x) # 取出[CLS]对应的向量用来做分类
p = Dense(nclass, activation='softmax')(x) # 直接dense层softmax输出
model = Model([x1_in, x2_in], p)
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr_rate), #选择优化器并设置学习率
metrics=['accuracy', f1])
print(model.summary())
return model
4) 模型训练
使用google colab 上的V100卡训练一个epoch需要约1.5小时，跑两个epoch即可。
#模型训练函数
def run_nocv(nfold, trn_data, val_data, data_labels, data_test, n_cls):
global er_patience
global lr_patience
global max_epochs
global f1metrics
global lr_factor
test_model_pred = np.zeros((len(data_test), n_cls))
model = build_bert(n_cls)
# 下行代码用于加载保存的权重继续训练
# model.load_weights(path + '/subs/model.epoch01_val_loss0.9911_val_acc0.6445_val_f10.6276.hdf5')
early_stopping = EarlyStopping(monitor="val_f1", patience=er_patience) # 早停法，防止过拟合 #'val_accuracy'
plateau = ReduceLROnPlateau(monitor="val_f1", verbose=1, mode='max', factor=lr_factor, patience=lr_patience) # 当评价指标不在提升时，降低学习率
checkpoint = ModelCheckpoint(path + "/subs/model.epoch{epoch:02d}_val_loss{val_loss:.4f}_val_acc{val_accuracy:.4f}_val_f1{val_f1:.4f}.hdf5", monitor="val_f1", verbose=2, save_best_only=True, mode='max', save_weights_only=True) #保存val_f1最好的模型权重
#训练跟验证集可shuffle打乱，测试集不可打乱（否则在生成结果文件的时候没法跟ID对应上）
train_D = data_generator(trn_data, shuffle=True)
valid_D = data_generator(val_data, shuffle=True)
test_D = data_generator(data_test, shuffle=False)
#模型训练
model.fit_generator(
train_D.__iter__(),
steps_per_epoch=len(train_D),
epochs=max_epochs,
validation_data=valid_D.__iter__(),
validation_steps=len(valid_D),
callbacks=[early_stopping, plateau, checkpoint],
)
#模型预测
test_model_pred = model.predict_generator(test_D.__iter__(), steps=len(test_D), verbose=1)
train_model_pred = test_model_pred # model.predict(train_D.__iter__(), steps=len(train_D), verbose=1)
del model
gc.collect() #清理内存
K.clear_session() #clear_session就是清除一个session
return test_model_pred, train_model_pred
调用上述函数进行训练与预测。
cvs = 1
#输出为numpy array格式的25列概率
test_model_pred, train_model_pred = run_nocv(cvs, TRN_LIST, VAL_LIST, None, DATA_LIST_TEST, n_cls)
5) 输出结果
#将结果转为DataFrame格式
preds_tst_df = pd.DataFrame(test_model_pred)
#再将range(0,25)做encode_label逆变换作为该DataFrame的列名
preds_col_names = encode_label.inverse_transform( range(0,n_cls) )
preds_tst_df.columns = preds_col_names
#从每个任务对应的概率标签列中找出最大的概率对应的列名作为预测结果
'''
如ocnli任务的预测结果只能为0、1、2，那么从preds_tst_df中选择0-1-2三列中每行概率最大的列名作为ocnli任务的测试集预测结果，其它两个任务依此类推。
'''
times_preds = preds_tst_df.head(times_testa.shape[0])[times_train['label'].unique().tolist()]
times_preds = times_preds.eq(times_preds.max(1), axis=0).dot(times_preds.columns)
ocemo_preds = preds_tst_df.head(times_testa.shape[0] + ocemo_testa.shape[0]).tail(ocemo_testa.shape[0])[ocemo_train['label'].unique().tolist()]
ocemo_preds = ocemo_preds.eq(ocemo_preds.max(1), axis=0).dot(ocemo_preds.columns)
ocnli_preds = preds_tst_df.tail(ocnli_testa.shape[0])[ocnli_train['label'].unique().tolist()]
ocnli_preds = ocnli_preds.eq(ocnli_preds.max(1), axis=0).dot(ocnli_preds.columns)
#输出任务tnews的预测结果
times_sub = times_testa[['id']].copy()
times_sub['label'] = times_preds.values
times_sub.to_json(path + "/subs/tnews_predict.json", orient='records', lines=True)
#输出任务ocemo的预测结果
ocemo_sub = ocemo_testa[['id']].copy()
ocemo_sub['label'] = ocemo_preds.values
ocemo_sub.to_json(path + "/subs/ocemotion_predict.json", orient='records', lines=True)
#输出任务ocnli的预测结果
ocnli_sub = ocnli_testa[['id']].copy()
ocnli_sub['label'] = ocnli_preds.values
ocnli_sub.to_json(path + "/subs/ocnli_predict.json", orient='records', lines=True)
参考链接：https://blog.csdn.net/nanke_4869/article/details/113658868
1294
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
