整体网络结构介绍
视频-音频交互方式
def tpavi_va(self, x, audio, stage):
# x: visual, [B*T, C=256, H, W]
# audio: [B*T, 128]
# ra_flag: return audio feature list or not
tpavi_b = getattr(self, f'tpavi_b{stage+1}')
audio = audio.view(-1, 5, audio.shape[-1]) # [B, T, 128]
x = self.pre_reshape_for_tpavi(x) # [B, C, T, H, W]
x, a = tpavi_b(x, audio) # [B, C, T, H, W], [B, T, C]
x = self.post_reshape_for_tpavi(x) # [B*T, C, H, W]
return x, a
实验效果