【功能模块】完整代码在附件,数据集需要的话也可以提供
class EmbeddingImagenet(nn.Cell): def __init__(self,emb_size,cifar_flag=False): super(EmbeddingImagenet, self).__init__() # set size self.hidden = 64 self.last_hidden = self.hidden * 25 if not cifar_flag else self.hidden * 4 self.emb_size = emb_size self.out_dim = emb_size # set layers self.conv_1 = nn.SequentialCell(nn.Conv2d(in_channels=3, out_channels=self.hidden, kernel_size=3, padding=1, pad_mode='pad', has_bias=False), nn.BatchNorm2d(num_features=self.hidden), nn.MaxPool2d(kernel_size=2,stride=2), nn.LeakyReLU(alpha=0.2)) self.conv_2 = nn.SequentialCell(nn.Conv2d(in_channels=self.hidden, out_channels=int(self.hidden*1.5), kernel_size=3, padding=1, pad_mode='pad', has_bias=False), nn.BatchNorm2d(num_features=int(self.hidden*1.5)), nn.MaxPool2d(kernel_size=2,stride=2), nn.LeakyReLU(alpha=0.2)) self.conv_3 = nn.SequentialCell(nn.Conv2d(in_channels=int(self.hidden*1.5), out_channels=self.hidden*2, kernel_size=3, padding=1, pad_mode='pad', has_bias=False), nn.BatchNorm2d(num_features=self.hidden * 2), nn.MaxPool2d(kernel_size=2,stride=2), nn.LeakyReLU(alpha=0.2), nn.Dropout(0.6)) self.conv_4 = nn.SequentialCell(nn.Conv2d(in_channels=self.hidden*2, out_channels=self.hidden*4, kernel_size=3, padding=1, pad_mode='pad', has_bias=False), nn.BatchNorm2d(num_features=self.hidden * 4), # 16 * 64 * (5 * 5) nn.MaxPool2d(kernel_size=2,stride=2), nn.LeakyReLU(alpha=0.2), nn.Dropout(0.5)) # self.layer_last = nn.SequentialCell(nn.Dense(in_channels=self.last_hidden * 4, # out_channels=self.emb_size, has_bias=True), # nn.BatchNorm1d(self.emb_size)) self.layer_last = nn.Dense(in_channels=self.last_hidden * 4,out_channels=self.emb_size, has_bias=True) #self.bn = nn.BatchNorm1d(self.emb_size) def construct(self, input_data): #print("img:",input_data[0]) x = self.conv_1(input_data) x = self.conv_2(x) x = self.conv_3(x) x = self.conv_4(x) #x = ops.Reshape()(x,(x.shape[0],-1)) print("feat:", input_data[0]) #x = self.layer_last(x) x = self.layer_last(x.view(x.shape[0],-1)) print("last--------------------------------:",x[0]) return x
class NodeUpdateNetwork(nn.Cell): def __init__(self, in_features, num_features, ratio=[2, 1], dropout=0.0): super(NodeUpdateNetwork, self).__init__() # set size self.in_features = in_features self.num_features_list = [num_features * r for r in ratio] self.dropout = dropout self.eye = ops.Eye() self.bmm = ops.BatchMatMul() self.cat = ops.Concat(-1) self.split = ops.Split(1,2) self.repeat = ops.Tile() self.unsqueeze = ops.ExpandDims() self.squeeze = ops.Squeeze() self.transpose = ops.Transpose() # layers layer_list = OrderedDict() for l in range(len(self.num_features_list)): layer_list['conv{}'.format(l)] = nn.Conv2d( in_channels=self.num_features_list[l - 1] if l > 0 else self.in_features * 3, out_channels=self.num_features_list[l], kernel_size=1, has_bias=False) layer_list['norm{}'.format(l)] = nn.BatchNorm2d(num_features=self.num_features_list[l],) layer_list['relu{}'.format(l)] = nn.LeakyReLU(alpha=1e-2) if self.dropout > 0 and l == (len(self.num_features_list) - 1): layer_list['drop{}'.format(l)] = nn.Dropout(keep_prob=1-self.dropout) self.network = nn.SequentialCell(layer_list) def construct(self, node_feat, edge_feat): # get size num_tasks = node_feat.shape[0] num_data = node_feat.shape[1] # get eye matrix (batch_size x 2 x node_size x node_size) diag_mask = 1.0 - self.repeat(self.unsqueeze(self.unsqueeze(self.eye(num_data,num_data,ms.float32),0),0),(num_tasks,2,1,1)) # set diagonal as zero and normalize 原论文是l1归一化 # edge_feat = edge_feat * diag_mask # edge_feat = edge_feat / ops.clip_by_value(ops.ReduceSum(keep_dims=True)(ops.Abs()(edge_feat), -1),Tensor(0,ms.float32),Tensor(num_data,ms.float32)) edge_feat = ops.L2Normalize(-1)(edge_feat * diag_mask) # compute attention and aggregate aggr_feat = self.bmm(self.squeeze(ops.Concat(2)(self.split(edge_feat))),node_feat) node_feat = self.cat([node_feat,self.cat(ops.Split(1, 2)(aggr_feat))]).swapaxes(1,2) #node_feat = self.transpose(self.cat([node_feat,self.cat(ops.Split(1, 2)(aggr_feat))]),(0,2,1)) node_feat = self.network(self.unsqueeze(node_feat,(-1))).swapaxes(1,2).squeeze() #node_feat = self.squeeze(self.transpose(self.network(self.unsqueeze(node_feat,(-1))),(0,2,1,3))) return node_feat class EdgeUpdateNetwork(nn.Cell): def __init__(self, in_features, num_features, ratio=[2, 2, 1, 1], separate_dissimilarity=False, dropout=0.0): super(EdgeUpdateNetwork, self).__init__() # set size self.in_features = in_features self.num_features_list = [num_features * r for r in ratio] self.separate_dissimilarity = separate_dissimilarity self.dropout = dropout self.eye = ops.Eye() self.repeat = ops.Tile() self.unsqueeze = ops.ExpandDims() # layers layer_list = OrderedDict() for l in range(len(self.num_features_list)): # set layer layer_list['conv{}'.format(l)] = nn.Conv2d(in_channels=self.num_features_list[l-1] if l > 0 else self.in_features, out_channels=self.num_features_list[l], kernel_size=1, has_bias=False) layer_list['norm{}'.format(l)] = nn.BatchNorm2d(num_features=self.num_features_list[l], ) layer_list['relu{}'.format(l)] = nn.LeakyReLU(alpha=1e-2) if self.dropout > 0: layer_list['drop{}'.format(l)] = nn.Dropout(keep_prob=1-self.dropout) layer_list['conv_out'] = nn.Conv2d(in_channels=self.num_features_list[-1], out_channels=1, kernel_size=1) self.sim_network = nn.SequentialCell(layer_list) def construct(self, node_feat, edge_feat): # compute abs(x_i, x_j) x_i = ops.ExpandDims()(node_feat,2) x_j = x_i.swapaxes(1,2) #x_j = ops.Transpose()(x_i,(0,2,1,3)) #x_ij = (x_i-x_j)**2 x_ij = ops.Abs()(x_i-x_j) #print("x_ij:",x_ij[0,0,:,:]) x_ij = ops.Transpose()(x_ij,(0,3,2,1)) sim_val = self.sim_network(x_ij) sim_val = ops.Sigmoid()(sim_val) #print("sim_val", sim_val[0, 0, :, :]) dsim_val = 1.0 - sim_val diag_mask = 1.0 - self.repeat(self.unsqueeze(self.unsqueeze(self.eye(node_feat.shape[1],node_feat.shape[1],ms.float32),0),0),(node_feat.shape[0],2,1,1)) edge_feat = edge_feat * diag_mask merge_sum = ops.ReduceSum(keep_dims=True)(edge_feat,-1) # set diagonal as zero and normalize # edge_feat = ops.Concat(1)([sim_val,dsim_val])*edge_feat # edge_feat = edge_feat / ops.clip_by_value((ops.ReduceSum(keep_dims=True)(ops.Abs()(edge_feat), -1)),Tensor(0,ms.float32),Tensor(num_data,ms.float32)) # edge_feat = edge_feat*merge_sum edge_feat = ops.L2Normalize(-1)(ops.Concat(1)([sim_val,dsim_val])*edge_feat)*merge_sum force_edge_feat = self.repeat(self.unsqueeze(ops.Concat(0)([self.unsqueeze(self.eye(node_feat.shape[1],node_feat.shape[1],ms.float32),0),self.unsqueeze(ops.Zeros()((node_feat.shape[1],node_feat.shape[1]),ms.float32),0)]),0),(node_feat.shape[0],1,1,1)) edge_feat = edge_feat + force_edge_feat edge_feat = edge_feat + 1e-6 #print("sum_edge",self.repeat(self.unsqueeze(ops.ReduceSum()(edge_feat,1),1),(1,2,1,1))[0,0]) edge_feat = edge_feat / self.repeat(self.unsqueeze(ops.ReduceSum()(edge_feat,1),1),(1,2,1,1)) return edge_feat class GraphNetwork(nn.Cell): def __init__(self, in_features, node_features, edge_features, num_layers, dropout=0.0 ): super(GraphNetwork, self).__init__() # set size self.in_features = in_features self.node_features = node_features self.edge_features = edge_features self.num_layers = num_layers self.dropout = dropout self.layers = nn.CellList() # for each layer for l in range(self.num_layers): # set edge to node edge2node_net = NodeUpdateNetwork(in_features=self.in_features if l == 0 else self.node_features, num_features=self.node_features, dropout=self.dropout if l < self.num_layers-1 else 0.0) # set node to edge node2edge_net = EdgeUpdateNetwork(in_features=self.node_features, num_features=self.edge_features, separate_dissimilarity=False, dropout=self.dropout if l < self.num_layers-1 else 0.0) self.layers.append(nn.CellList([edge2node_net,node2edge_net])) # forward def construct(self, node_feat, edge_feat): # for each layer edge_feat_list = [] #print("node_feat---------------------------------------------------------- -1", node_feat[0, 0, :]) for l in range(self.num_layers): # (1) edge to node node_feat = self.layers[l][0](node_feat, edge_feat) # (2) node to edge edge_feat = self.layers[l][1](node_feat, edge_feat) # save edge feature edge_feat_list.append(edge_feat) return edge_feat_list
【操作步骤&问题现象】
我们代码主要功能是用4层卷积加一层全连接层提取图片特征,之后将图片的特征当成图网络每个节点,用GNN。(代码在附件上)
1、在训练了很多个batch之后,提取出来的特征(经过了4层卷积层和全连接层)出现了很大很大的值,之后几个batch后出现NAN,而在没有经过全连接层的时候,特征数字还是正常的
2、
【截图信息】
这是代码输出的特征
last--------------------------------: [ 1.918492 -0.8280923 2.0575197 0.3089749 -1.0514854 0.5368729
0.14135109 1.5270222 -1.4794292 -1.4336827 1.0335447 -0.7093582
-0.41919574 -0.5667086 -0.3535831 1.5567536 0.5002996 -1.4093596
0.9674009 -0.18156137 0.14888959 0.6358457 1.406878 -0.03820777
-0.24577822 -0.25783274 0.5756687 -1.4558431 -1.1002262 0.68062806
-1.6467474 0.88712454 0.3551372 -1.3449378 -1.7011788 -0.8629771
-0.92482185 0.9867192 -1.5548937 1.340383 -2.299356 -0.3421743
1.3239275 -1.3792732 -0.31955895 -0.58364254 -3.7381008 -1.2121737
-0.75104207 -0.7562581 0.04980466 0.45131734 -1.2448095 -0.33418307
0.86268485 -1.3601649 1.2753168 2.469506 -1.7358601 -2.9104383
-0.07392117 -0.73263663 0.11657254 -0.05724781 0.34374043 -0.31884825
0.13456154 2.3561432 -0.18908082 0.5410311 1.7249999 0.9508886
-0.30631644 1.6836481 1.1513023 -0.33672807 -0.889638 -0.76715356
-0.7316199 1.597606 -1.6586273 0.4502733 0.5224928 -3.5851111
-2.906651 -1.5284328 0.83426046 1.354644 -1.4453334 2.0504599
-1.3200179 -0.50427496 0.97681373 0.30048305 0.17170379 0.8179815
-0.92994857 1.333491 -1.2931286 -0.3569969 2.7953048 -3.352736
1.878619 2.018083 -1.1191074 -1.1341975 1.4532931 -0.66957355
2.3269157 -0.4198427 0.7148121 0.5458231 -1.3050007 -0.34666243
2.519589 0.804219 0.91191477 1.3088121 0.6767241 2.1667008
0.24471135 1.2600335 -1.8683847 2.5641935 -0.9636249 -1.0340385
-0.32570755 -1.7694132 ]
------------------------------------------
------------------------------------------------------------------------------- 1 0.7806913
---------------------------------------------
feat: [[[-1.6726604 -1.6897851 -1.7069099 ... 0.43368444 0.46793392
0.41655967]
[-1.7069099 -1.7069099 -1.7069099 ... 0.5364329 0.5193082
0.4850587 ]
[-1.7240347 -1.7240347 -1.7069099 ... 0.60493195 0.5535577
0.4850587 ]
...
[-0.6622999 -0.8335474 -0.8677969 ... -0.02868402 0.00556549
-0.02868402]
[-0.6622999 -0.69654936 -0.69654936 ... -0.11430778 -0.11430778
-0.14855729]
[-0.95342064 -0.8335474 -0.78217316 ... -0.26843056 -0.30268008
-0.31980482]]
[[-1.7556022 -1.7731092 -1.7906162 ... -0.617647 -0.582633
-0.635154 ]
[-1.7906162 -1.7906162 -1.7906162 ... -0.512605 -0.512605
-0.565126 ]
[-1.8081232 -1.8081232 -1.7906162 ... -0.460084 -0.495098
-0.565126 ]
...
[-0.28501397 -0.37254897 -0.40756297 ... -1.0028011 -0.9677871
-1.0203081 ]
[-0.26750696 -0.33753496 -0.32002798 ... -1.12535 -1.1428571
-1.160364 ]
[-0.53011197 -0.53011197 -0.44257697 ... -1.2829131 -1.317927
-1.317927 ]]
[[-1.68244 -1.6998693 -1.7172985 ... -1.490719 -1.4558606
-1.490719 ]
[-1.7172985 -1.7172985 -1.7172985 ... -1.4732897 -1.4384314
-1.4732897 ]
[-1.7347276 -1.7347276 -1.7172985 ... -1.4558606 -1.4732897
-1.5255773 ]
...
[-1.3338562 -1.4210021 -1.4210021 ... -1.6127234 -1.5430065
-1.5604358 ]
[-1.2815686 -1.3512855 -1.3338562 ... -1.6127234 -1.5952941
-1.6127234 ]
[-1.5081482 -1.4732897 -1.4210021 ... -1.5778649 -1.6127234
-1.6301525 ]]]
last--------------------------------: [-9.7715964e+37 -1.3229437e+37 -1.5262715e+38 -2.5811514e+38
3.2964988e+38 -7.1266450e+37 -7.2963347e+37 -3.0699307e+38
-1.6108344e+38 5.8011444e+37 -3.9925391e+37 -9.5891957e+37
-1.7783365e+38 2.2280316e+38 -4.4186918e+37 3.4825655e+37
5.8457292e+37 7.2160006e+37 1.4259578e+38 9.4037617e+37
7.4650717e+37 1.8146209e+37 -2.5143476e+38 2.4387442e+38
-7.5397363e+37 1.4157064e+38 -1.1084308e+38 1.9522180e+38
2.5864164e+37 -8.5381704e+37 3.3140050e+36 -1.2379668e+38
-3.3449897e+37 1.6203643e+38 1.4627435e+38 6.6909600e+37
6.0661751e+37 -1.2335753e+38 1.3377397e+38 -3.7530971e+37
3.5314601e+37 -1.4393099e+37 -inf -6.0411279e+37
-7.0721061e+37 1.5951782e+38 9.0163464e+37 1.3680580e+37
-1.2254094e+37 1.0919689e+38 -1.5229139e+37 -3.4862508e+36
-8.9739065e+37 2.8713203e+38 9.4768839e+37 7.8658815e+37
-2.6619306e+38 -7.8224467e+37 6.8780734e+37 inf
-9.8889302e+37 -1.9009123e+38 -1.4562352e+38 -4.5324568e+37
-2.6728082e+38 1.0300855e+38 -5.7767852e+37 1.3662499e+37
-4.0048543e+37 -3.1911765e+37 -1.9702732e+38 -6.5395945e+37
1.0223747e+38 -2.8775531e+38 -1.1156091e+38 -1.8772822e+38
1.2472896e+38 1.2465860e+38 -6.7286062e+37 -8.9167649e+37
-2.8327554e+37 -2.7379526e+37 -1.5994879e+37 1.1577176e+38
1.1864721e+38 1.7089999e+38 -1.5323652e+37 -1.5374746e+38
1.2187025e+38 -8.9546139e+37 1.7550813e+38 -5.7048014e+37
-8.5996788e+37 -5.2310546e+36 -1.4450948e+37 -1.9950120e+37
4.2429252e+37 -1.4849557e+38 1.0697206e+38 -7.6313524e+37
-inf 1.7437526e+38 -1.0569269e+38 -1.5577321e+38
-7.8117285e+37 6.4801082e+37 -3.3032475e+37 -6.4655517e+36
-2.3770844e+38 1.0880277e+38 3.6430118e+37 -6.9370110e+37
8.5146681e+37 1.1550550e+38 -2.5614073e+38 -2.1489826e+38
-8.3233807e+37 2.7233982e+37 -1.3777926e+38 -9.6201629e+37
-2.1125345e+38 -1.4252791e+36 3.6633845e+37 2.6106833e+37
9.6643025e+37 -1.4538810e+37 -1.3660478e+38 1.9220696e+38]
1 采用warmup调整一下学习率,最大学习率设置为0.01;
2 采用梯度剪裁方法进行保护;
3 检查最后是否进行归一处理,估计可能取值范围不在0-1之间。