1 注意力机制的诞生
注意力机制,起初是作为自然语言处理中的工作为大家熟知,文章Attention is all you need
详细介绍了“什么是注意力机制”,有兴趣的小伙伴可以下载原文看看。 SeNet-Squeeze-and-Excitation Networks
是注意力机制在计算机视觉中应用的早期工作之一,并获得了2017年的imagenet,也就是最后一届imagenet比赛的冠军。
2 介绍SeNet模型及Pytorch代码实现
import numpy as np
import torch
from torch import nn
from torch. nn import init
class SEAttention ( nn. Module) :
def __init__ ( self, channel= 512 , reduction= 16 ) :
super ( ) . __init__( )
self. avg_pool = nn. AdaptiveAvgPool2d( 1 )
self. fc = nn. Sequential(
nn. Linear( channel, channel // reduction, bias= False ) ,
nn. ReLU( inplace= True ) ,
nn. Linear( channel // reduction, channel, bias= False ) ,
nn. Sigmoid( )
)
def init_weights ( self) :
for m in self. modules( ) :
print ( m)
if isinstance ( m, nn. Conv2d) :
init. kaiming_normal_( m. weight, mode= 'fan_out' )
if m. bias is not None :
init. constant_( m. bias, 0 )
elif isinstance ( m, nn. BatchNorm2d) :
init. constant_( m. weight, 1 )
init. constant_( m. bias, 0 )
elif isinstance ( m, nn. Linear) :
init. normal_( m. weight, std= 0.001 )
if m. bias is not None :
init. constant_( m. bias, 0 )
def forward ( self, x) :
b, c, _, _ = x. size( )
y = self. avg_pool( x) . view( b, c)
y = self. fc( y) . view( b, c, 1 , 1 )
return x * y. expand_as( x)
if __name__ == '__main__' :
input = torch. randn( 50 , 512 , 7 , 7 )
se = SEAttention( channel= 512 , reduction= 8 )
output = se( input )
print ( output. shape)
也有说SeNet的模型是这样的:(我觉得过于简单了)
class SELayer ( nn. Module) :
def __init__ ( self, channel, reduction= 16 ) :
super ( SELayer, self) . __init__( )
self. avg_pool = nn. AdaptiveAvgPool2d( 1 )
self. fc = nn. Sequential(
nn. Linear( channel, channel // reduction, bias= False ) ,
nn. ReLU( inplace= True ) ,
nn. Linear( channel // reduction, channel, bias= False ) ,
nn. Sigmoid( )
)
def forward ( self, x) :
b, c, _, _ = x. size( )
y = self. avg_pool( x) . view( b, c)
y = self. fc( y) . view( b, c, 1 , 1 )
return x * y. expand_as( x)