# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import sys import numpy as np import math import copy import paddle import paddle.nn as nn import paddle.nn.functional as F from paddle.nn import (Conv2D, BatchNorm2D, Linear, Dropout) from paddle.nn.initializer import Constant, Normal from ...utils.save_load import load_ckpt from ..registry import BACKBONES from ..weight_init import weight_init_ ACT2FN = {"gelu": F.gelu, "relu": F.relu, "swish": F.swish} class BertEmbeddings(nn.Layer): """Construct the embeddings from word, position and token_type embeddings. """ def __init__(self, vocab_size, max_position_embeddings, type_vocab_size, hidden_size, hidden_dropout_prob): super(BertEmbeddings, self).__init__() self.word_embeddings = nn.Embedding(vocab_size, hidden_size, padding_idx=0) self.position_embeddings = nn.Embedding(max_position_embeddings, hidden_size) self.token_type_embeddings = nn.Embedding(type_vocab_size, hidden_size) self.LayerNorm = nn.LayerNorm(hidden_size, epsilon=1e-12) self.dropout = nn.Dropout(hidden_dropout_prob) def forward(self, input_ids, token_type_ids=None): seq_length = input_ids.shape[1] position_ids = paddle.arange(end=seq_length, dtype="int64") position_ids = position_ids.unsqueeze(0).expand_as(input_ids) if token_type_ids is None: token_type_ids = paddle.zeros_like(input_ids) words_embeddings = self.word_embeddings(input_ids) #8,36 -> 8,36,768 position_embeddings = self.position_embeddings( position_ids) #8,36 -> 8,36,768 token_type_embeddings = self.token_type_embeddings( token_type_ids) #8,36 -> 8,36,768 embeddings = words_embeddings + position_embeddings + token_type_embeddings embeddings = self.LayerNorm(embeddings) embeddings = self.dropout(embeddings) return embeddings class BertImageEmbeddings(nn.Layer): def __init__(self, v_feature_size, v_hidden_size, v_hidden_dropout_prob): super(BertImageEmbeddings, self).__init__() self.image_embeddings = nn.Linear(v_feature_size, v_hidden_size) self.image_location_embeddings = nn.Linear(5, v_hidden_size) self.LayerNorm = nn.LayerNorm(v_hidden_size, epsilon=1e-12) self.dropout = nn.Dropout(v_hidden_dropout_prob) def forward(self, input_ids, input_loc): img_embeddings = self.image_embeddings( input_ids) #8,37,2048 -> 8,37,1024 loc_embeddings = self.image_location_embeddings( input_loc) #8,37,5 -> 8,37,1024 embeddings = self.LayerNorm(img_embeddings + loc_embeddings) embeddings = self.dropout(embeddings) return embeddings # shape: bs*seq_len*hs class BertActionEmbeddings(nn.Layer): def __init__(self, a_feature_size, a_hidden_size, a_hidden_dropout_prob): super(BertActionEmbeddings, self).__init__() self.action_embeddings = nn.Linear(a_feature_size, a_hidden_size) self.LayerNorm = nn.LayerNorm(a_hidden_size, epsilon=1e-12) self.dropout = nn.Dropout(a_hidden_dropout_prob) def forward(self, input_ids): action_embeddings = self.action_embeddings( input_ids) #8,5,2048 -> 8,5,768 embeddings = self.LayerNorm(action_embeddings) embeddings = self.dropout(embeddings) return embeddings class BertSelfAttention(nn.Layer): def __init__(self, hidden_size, num_attention_heads, attention_probs_dropout_prob): super(BertSelfAttention, self).__init__() if hidden_size % num_attention_heads != 0: raise ValueError( "The hidden size (%d) is not a multiple of the number of attention " "heads (%d)" % (hidden_size, num_attention_heads)) self.num_attention_heads = num_attention_heads self.attention_head_size = int(hidden_size / num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size self.query = nn.Linear(hidden_size, self.all_head_size) self.key = nn.Linear(hidden_size, self.all_head_size) self.value = nn.Linear(hidden_size, self.all_head_size) self.dropout = nn.Dropout(attention_probs_dropout_prob) def transpose_for_scores(self, x): new_x_shape = x.shape[:-1] + [ self.num_attention_heads, self.attention_head_size, ] x = x.reshape(new_x_shape) return x.transpose((0, 2, 1, 3)) def forward(self, hidden_states, attention_mask): mixed_query_layer = self.query(hidden_states) mixed_key_layer = self.key(hidden_states) mixed_value_layer = self.value(hidden_states) query_layer = self.transpose_for_scores(mixed_query_layer) key_layer = self.transpose_for_scores(mixed_key_layer) value_layer = self.transpose_for_scores(mixed_value_layer) # Take the dot product between "query" and "key" to get the raw attention scores. attention_scores = paddle.matmul(query_layer, key_layer.transpose((0, 1, 3, 2))) attention_scores = attention_scores / math.sqrt( self.attention_head_size) # Apply the attention mask is (precomputed for all layers in BertModel forward() function) attention_scores = attention_scores + attention_mask # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(axis=-1)(attention_scores) # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. attention_probs = self.dropout(attention_probs) context_layer = paddle.matmul(attention_probs, value_layer) context_layer = context_layer.transpose((0, 2, 1, 3)) new_context_layer_shape = context_layer.shape[:-2] + [ self.all_head_size ] context_layer = context_layer.reshape(new_context_layer_shape) return context_layer, attention_probs class BertSelfOutput(nn.Layer): def __init__(self, hidden_size, hidden_dropout_prob): super(BertSelfOutput, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) self.LayerNorm = nn.LayerNorm(hidden_size, epsilon=1e-12) self.dropout = nn.Dropout(hidden_dropout_prob) def forward(self, hidden_states, input_tensor): hidden_states = self.dense(hidden_states) hidden_states = self.dropout(hidden_states) hidden_states = self.LayerNorm(hidden_states + input_tensor) return hidden_states class BertAttention(nn.Layer): def __init__(self, hidden_size, hidden_dropout_prob, num_attention_heads, attention_probs_dropout_prob): super(BertAttention, self).__init__() self.self = BertSelfAttention(hidden_size, num_attention_heads, attention_probs_dropout_prob) self.output = BertSelfOutput(hidden_size, hidden_dropout_prob) def forward(self, input_tensor, attention_mask): self_output, attention_probs = self.self(input_tensor, attention_mask) attention_output = self.output(self_output, input_tensor) return attention_output, attention_probs class BertIntermediate(nn.Layer): def __init__(self, hidden_size, intermediate_size, hidden_act): super(BertIntermediate, self).__init__() self.dense = nn.Linear(hidden_size, intermediate_size) if isinstance(hidden_act, str) or (sys.version_info[0] == 2 and isinstance(hidden_act, str)): self.intermediate_act_fn = ACT2FN[hidden_act] else: self.intermediate_act_fn = hidden_act def forward(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.intermediate_act_fn(hidden_states) return hidden_states class BertOutput(nn.Layer): def __init__(self, intermediate_size, hidden_size, hidden_dropout_prob): super(BertOutput, self).__init__() self.dense = nn.Linear(intermediate_size, hidden_size) self.LayerNorm = nn.LayerNorm(hidden_size, epsilon=1e-12) self.dropout = nn.Dropout(hidden_dropout_prob) def forward(self, hidden_states, input_tensor): hidden_states = self.dense(hidden_states) hidden_states = self.dropout(hidden_states) hidden_states = self.LayerNorm(hidden_states + input_tensor) return hidden_states class BertEntAttention(nn.Layer): """Core mudule of tangled transformer. """ def __init__( self, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, bi_num_attention_heads, ): super(BertEntAttention, self).__init__() if bi_hidden_size % bi_num_attention_heads != 0: raise ValueError( "The hidden size (%d) is not a multiple of the number of attention " "heads (%d)" % (bi_hidden_size, bi_num_attention_heads)) self.num_attention_heads = bi_num_attention_heads self.attention_head_size = int(bi_hidden_size / bi_num_attention_heads) self.all_head_size = self.num_attention_heads * self.attention_head_size # self attention layers for vision input self.query1 = nn.Linear(v_hidden_size, self.all_head_size) self.key1 = nn.Linear(v_hidden_size, self.all_head_size) self.value1 = nn.Linear(v_hidden_size, self.all_head_size) self.dropout1 = nn.Dropout(v_attention_probs_dropout_prob) # self attention layers for text input self.query2 = nn.Linear(hidden_size, self.all_head_size) self.key2 = nn.Linear(hidden_size, self.all_head_size) self.value2 = nn.Linear(hidden_size, self.all_head_size) self.dropout2 = nn.Dropout(attention_probs_dropout_prob) # self attention layers for action input self.query3 = nn.Linear(a_hidden_size, self.all_head_size) self.key3 = nn.Linear(a_hidden_size, self.all_head_size) self.value3 = nn.Linear(a_hidden_size, self.all_head_size) self.dropout3 = nn.Dropout(a_attention_probs_dropout_prob) # self attention layers for action_text self.key_at = nn.Linear(bi_hidden_size, self.all_head_size) self.value_at = nn.Linear(bi_hidden_size, self.all_head_size) self.dropout_at = nn.Dropout(av_attention_probs_dropout_prob) # self attention layers for action_vision self.key_av = nn.Linear(bi_hidden_size, self.all_head_size) self.value_av = nn.Linear(bi_hidden_size, self.all_head_size) self.dropout_av = nn.Dropout(at_attention_probs_dropout_prob) def transpose_for_scores(self, x): new_x_shape = x.shape[:-1] + [ self.num_attention_heads, self.attention_head_size, ] x = x.reshape(new_x_shape) return x.transpose((0, 2, 1, 3)) def forward( self, input_tensor1, attention_mask1, input_tensor2, attention_mask2, input_tensor3, attention_mask3, ): # for vision input. mixed_query_layer1 = self.query1(input_tensor1) mixed_key_layer1 = self.key1(input_tensor1) mixed_value_layer1 = self.value1(input_tensor1) query_layer1 = self.transpose_for_scores(mixed_query_layer1) key_layer1 = self.transpose_for_scores(mixed_key_layer1) value_layer1 = self.transpose_for_scores(mixed_value_layer1) # for text input: mixed_query_layer2 = self.query2(input_tensor2) mixed_key_layer2 = self.key2(input_tensor2) mixed_value_layer2 = self.value2(input_tensor2) query_layer2 = self.transpose_for_scores(mixed_query_layer2) key_layer2 = self.transpose_for_scores(mixed_key_layer2) value_layer2 = self.transpose_for_scores(mixed_value_layer2) # for action input: mixed_query_layer3 = self.query3(input_tensor3) mixed_key_layer3 = self.key3(input_tensor3) mixed_value_layer3 = self.value3(input_tensor3) query_layer3 = self.transpose_for_scores(mixed_query_layer3) key_layer3 = self.transpose_for_scores(mixed_key_layer3) value_layer3 = self.transpose_for_scores(mixed_value_layer3) def do_attention(query_layer, key_layer, value_layer, attention_mask, dropout): """ compute attention """ attention_scores = paddle.matmul(query_layer, key_layer.transpose((0, 1, 3, 2))) attention_scores = attention_scores / math.sqrt( self.attention_head_size) attention_scores = attention_scores + attention_mask # Normalize the attention scores to probabilities. attention_probs = nn.Softmax(axis=-1)(attention_scores) # This is actually dropping out entire tokens to attend to, which might # seem a bit unusual, but is taken from the original Transformer paper. attention_probs = dropout(attention_probs) context_layer = paddle.matmul(attention_probs, value_layer) context_layer = context_layer.transpose((0, 2, 1, 3)) new_context_layer_shape = context_layer.shape[:-2] + [ self.all_head_size ] context_layer = context_layer.reshape(new_context_layer_shape) return context_layer context_av = do_attention(query_layer3, key_layer1, value_layer1, attention_mask1, self.dropout_av) context_at = do_attention(query_layer3, key_layer2, value_layer2, attention_mask2, self.dropout_at) context_key_av = self.key_av(context_av).transpose((0, 2, 1)) # interpolate only support 4-D tensor now. context_key_av = F.interpolate(context_key_av.unsqueeze(-1), size=(key_layer2.shape[2], 1)).squeeze(-1) context_key_av = self.transpose_for_scores( context_key_av.transpose((0, 2, 1))) key_layer2 = key_layer2 + context_key_av context_key_at = self.key_at(context_at).transpose((0, 2, 1)) context_key_at = F.interpolate(context_key_at.unsqueeze(-1), size=(key_layer1.shape[2], 1)).squeeze(-1) context_key_at = self.transpose_for_scores( context_key_at.transpose((0, 2, 1))) key_layer1 = key_layer1 + context_key_at context_val_av = self.value_at(context_av).transpose((0, 2, 1)) context_val_av = F.interpolate(context_val_av.unsqueeze(-1), size=(value_layer2.shape[2], 1)).squeeze(-1) context_val_av = self.transpose_for_scores( context_val_av.transpose((0, 2, 1))) value_layer2 = value_layer2 + context_val_av context_val_at = self.value_at(context_at).transpose((0, 2, 1)) context_val_at = F.interpolate(context_val_at.unsqueeze(-1), size=(value_layer1.shape[2], 1)).squeeze(-1) context_val_at = self.transpose_for_scores( context_val_at.transpose((0, 2, 1))) value_layer1 = value_layer1 + context_val_at context_layer1 = do_attention(query_layer1, key_layer1, value_layer1, attention_mask1, self.dropout1) context_layer2 = do_attention(query_layer2, key_layer2, value_layer2, attention_mask2, self.dropout2) context_layer3 = do_attention(query_layer3, key_layer3, value_layer3, attention_mask3, self.dropout3) return context_layer1, context_layer2, context_layer3 # vision, text, action class BertEntOutput(nn.Layer): def __init__( self, bi_hidden_size, hidden_size, v_hidden_size, v_hidden_dropout_prob, hidden_dropout_prob, ): super(BertEntOutput, self).__init__() self.dense1 = nn.Linear(bi_hidden_size, v_hidden_size) self.LayerNorm1 = nn.LayerNorm(v_hidden_size, epsilon=1e-12) self.dropout1 = nn.Dropout(v_hidden_dropout_prob) self.dense2 = nn.Linear(bi_hidden_size, hidden_size) self.LayerNorm2 = nn.LayerNorm(hidden_size, epsilon=1e-12) self.dropout2 = nn.Dropout(hidden_dropout_prob) self.dense3 = nn.Linear(bi_hidden_size, hidden_size) self.LayerNorm3 = nn.LayerNorm(hidden_size, epsilon=1e-12) self.dropout3 = nn.Dropout(hidden_dropout_prob) def forward( self, hidden_states1, input_tensor1, hidden_states2, input_tensor2, hidden_states3, input_tensor3, ): context_state1 = self.dense1(hidden_states1) context_state1 = self.dropout1(context_state1) context_state2 = self.dense2(hidden_states2) context_state2 = self.dropout2(context_state2) context_state3 = self.dense3(hidden_states3) context_state3 = self.dropout3(context_state3) hidden_states1 = self.LayerNorm1(context_state1 + input_tensor1) hidden_states2 = self.LayerNorm2(context_state2 + input_tensor2) hidden_states3 = self.LayerNorm3(context_state3 + input_tensor3) return hidden_states1, hidden_states2, hidden_states3 class BertLayer(nn.Layer): def __init__(self, hidden_size, intermediate_size, hidden_act, hidden_dropout_prob, num_attention_heads, attention_probs_dropout_prob): super(BertLayer, self).__init__() self.attention = BertAttention(hidden_size, hidden_dropout_prob, num_attention_heads, attention_probs_dropout_prob) self.intermediate = BertIntermediate(hidden_size, intermediate_size, hidden_act) self.output = BertOutput(intermediate_size, hidden_size, hidden_dropout_prob) def forward(self, hidden_states, attention_mask): attention_output, attention_probs = self.attention( hidden_states, attention_mask) intermediate_output = self.intermediate(attention_output) layer_output = self.output(intermediate_output, attention_output) return layer_output, attention_probs class BertConnectionLayer(nn.Layer): def __init__(self, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, bi_num_attention_heads, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob): super(BertConnectionLayer, self).__init__() self.ent_attention = BertEntAttention( hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, bi_num_attention_heads, ) self.ent_output = BertEntOutput( bi_hidden_size, hidden_size, v_hidden_size, v_hidden_dropout_prob, hidden_dropout_prob, ) self.v_intermediate = BertIntermediate(v_hidden_size, v_intermediate_size, v_hidden_act) self.v_output = BertOutput(v_intermediate_size, v_hidden_size, v_hidden_dropout_prob) self.t_intermediate = BertIntermediate(hidden_size, intermediate_size, hidden_act) self.t_output = BertOutput(intermediate_size, hidden_size, hidden_dropout_prob) self.a_intermediate = BertIntermediate(a_hidden_size, a_intermediate_size, a_hidden_act) self.a_output = BertOutput(a_intermediate_size, a_hidden_size, a_hidden_dropout_prob) def forward( self, input_tensor1, attention_mask1, input_tensor2, attention_mask2, input_tensor3, attention_mask3, ): ent_output1, ent_output2, ent_output3 = self.ent_attention( input_tensor1, attention_mask1, input_tensor2, attention_mask2, input_tensor3, attention_mask3) attention_output1, attention_output2, attention_output3 = self.ent_output( ent_output1, input_tensor1, ent_output2, input_tensor2, ent_output3, input_tensor3) intermediate_output1 = self.v_intermediate(attention_output1) layer_output1 = self.v_output(intermediate_output1, attention_output1) intermediate_output2 = self.t_intermediate(attention_output2) layer_output2 = self.t_output(intermediate_output2, attention_output2) intermediate_output3 = self.a_intermediate(attention_output3) layer_output3 = self.a_output(intermediate_output3, attention_output3) return layer_output1, layer_output2, layer_output3 class BertEncoder(nn.Layer): """ ActBert Encoder, consists 3 pathway of multi-BertLayers and BertConnectionLayer. """ def __init__( self, v_ent_attention_id, t_ent_attention_id, a_ent_attention_id, fixed_t_layer, fixed_v_layer, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, num_attention_heads, v_num_attention_heads, a_num_attention_heads, bi_num_attention_heads, num_hidden_layers, v_num_hidden_layers, a_num_hidden_layers, ): super(BertEncoder, self).__init__() self.v_ent_attention_id = v_ent_attention_id self.t_ent_attention_id = t_ent_attention_id self.a_ent_attention_id = a_ent_attention_id self.fixed_t_layer = fixed_t_layer self.fixed_v_layer = fixed_v_layer layer = BertLayer(hidden_size, intermediate_size, hidden_act, hidden_dropout_prob, num_attention_heads, attention_probs_dropout_prob) v_layer = BertLayer(v_hidden_size, v_intermediate_size, v_hidden_act, v_hidden_dropout_prob, v_num_attention_heads, v_attention_probs_dropout_prob) a_layer = BertLayer(a_hidden_size, a_intermediate_size, a_hidden_act, a_hidden_dropout_prob, a_num_attention_heads, a_attention_probs_dropout_prob) connect_layer = BertConnectionLayer( hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, bi_num_attention_heads, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob) self.layer = nn.LayerList( [copy.deepcopy(layer) for _ in range(num_hidden_layers)]) #12 self.v_layer = nn.LayerList( [copy.deepcopy(v_layer) for _ in range(v_num_hidden_layers)]) #2 self.a_layer = nn.LayerList( [copy.deepcopy(a_layer) for _ in range(a_num_hidden_layers)]) #3 self.c_layer = nn.LayerList([ copy.deepcopy(connect_layer) for _ in range(len(v_ent_attention_id)) ] #2 [0,1] ) def forward( self, txt_embedding, image_embedding, action_embedding, txt_attention_mask, image_attention_mask, action_attention_mask, output_all_encoded_layers=True, ): v_start, a_start, t_start = 0, 0, 0 count = 0 all_encoder_layers_t = [] all_encoder_layers_v = [] all_encoder_layers_a = [] for v_layer_id, a_layer_id, t_layer_id in zip(self.v_ent_attention_id, self.a_ent_attention_id, self.t_ent_attention_id): v_end = v_layer_id a_end = a_layer_id t_end = t_layer_id assert self.fixed_t_layer <= t_end assert self.fixed_v_layer <= v_end ### region embedding for idx in range(v_start, self.fixed_v_layer): #两次训练，这个循环都没有进去 #前面的层固定住 with paddle.no_grad(): image_embedding, image_attention_probs = self.v_layer[idx]( image_embedding, image_attention_mask) v_start = self.fixed_v_layer for idx in range(v_start, v_end): image_embedding, image_attention_probs = self.v_layer[idx]( image_embedding, image_attention_mask) ### action embedding for idx in range(a_start, a_end): action_embedding, action_attention_probs = self.a_layer[idx]( action_embedding, action_attention_mask) ### text embedding for idx in range(t_start, self.fixed_t_layer): with paddle.no_grad(): txt_embedding, txt_attention_probs = self.layer[idx]( txt_embedding, txt_attention_mask) t_start = self.fixed_t_layer for idx in range(t_start, t_end): txt_embedding, txt_attention_probs = self.layer[idx]( txt_embedding, txt_attention_mask) image_embedding, txt_embedding, action_embedding = self.c_layer[ count](image_embedding, image_attention_mask, txt_embedding, txt_attention_mask, action_embedding, action_attention_mask) v_start = v_end t_start = t_end a_start = a_end count += 1 if output_all_encoded_layers: all_encoder_layers_t.append(txt_embedding) all_encoder_layers_v.append(image_embedding) all_encoder_layers_a.append(action_embedding) for idx in range(v_start, len(self.v_layer)): # 1 image_embedding, image_attention_probs = self.v_layer[idx]( image_embedding, image_attention_mask) for idx in range(a_start, len(self.a_layer)): action_embedding, action_attention_probs = self.a_layer[idx]( action_embedding, action_attention_mask) for idx in range(t_start, len(self.layer)): txt_embedding, txt_attention_probs = self.layer[idx]( txt_embedding, txt_attention_mask) # add the end part to finish. if not output_all_encoded_layers: all_encoder_layers_t.append(txt_embedding) #8, 36, 768 all_encoder_layers_v.append(image_embedding) #8, 37, 1024 all_encoder_layers_a.append(action_embedding) #8, 5, 768 return all_encoder_layers_t, all_encoder_layers_v, all_encoder_layers_a class BertPooler(nn.Layer): """ "Pool" the model by simply taking the hidden state corresponding to the first token. """ def __init__(self, hidden_size, bi_hidden_size): super(BertPooler, self).__init__() self.dense = nn.Linear(hidden_size, bi_hidden_size) self.activation = nn.ReLU() def forward(self, hidden_states): first_token_tensor = hidden_states[:, 0] #8, 768 pooled_output = self.dense(first_token_tensor) pooled_output = self.activation(pooled_output) return pooled_output class BertModel(nn.Layer): def __init__( self, vocab_size, max_position_embeddings, type_vocab_size, v_feature_size, a_feature_size, num_hidden_layers, v_num_hidden_layers, a_num_hidden_layers, v_ent_attention_id, t_ent_attention_id, a_ent_attention_id, fixed_t_layer, fixed_v_layer, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, num_attention_heads, v_num_attention_heads, a_num_attention_heads, bi_num_attention_heads, ): super(BertModel, self).__init__() # initilize word embedding self.embeddings = BertEmbeddings(vocab_size, max_position_embeddings, type_vocab_size, hidden_size, hidden_dropout_prob) # initlize the region embedding self.v_embeddings = BertImageEmbeddings(v_feature_size, v_hidden_size, v_hidden_dropout_prob) # initlize the action embedding self.a_embeddings = BertActionEmbeddings(a_feature_size, a_hidden_size, a_hidden_dropout_prob) self.encoder = BertEncoder( v_ent_attention_id, t_ent_attention_id, a_ent_attention_id, fixed_t_layer, fixed_v_layer, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, num_attention_heads, v_num_attention_heads, a_num_attention_heads, bi_num_attention_heads, num_hidden_layers, v_num_hidden_layers, a_num_hidden_layers) self.t_pooler = BertPooler(hidden_size, bi_hidden_size) self.v_pooler = BertPooler(v_hidden_size, bi_hidden_size) self.a_pooler = BertPooler(a_hidden_size, bi_hidden_size) def forward( self, text_ids, action_feat, image_feat, image_loc, token_type_ids=None, text_mask=None, image_mask=None, action_mask=None, output_all_encoded_layers=False, ): """ text_ids: input text ids. Shape: [batch_size, seqence_length] action_feat: input action feature. Shape: [batch_size, action_length, action_feature_dim] image_feat: input image feature. Shape: [batch_size, region_length, image_feature_dim]] image_loc: input region location. Shape: [batch_size, region_length, region_location_dim] token_type_ids: segment ids of each video clip. Shape: [batch_size, seqence_length] text_mask: text mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, seqence_length] image_mask: image mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, region_length] action_mask: action mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, action_length] output_all_encoded_layers: is output encoded layers feature or not. Type: Bool. """ if text_mask is None: text_mask = paddle.ones_like(text_ids) if token_type_ids is None: token_type_ids = paddle.zeros_like(text_ids) if image_mask is None: image_mask = paddle.ones(image_feat.shape[0], image_feat.shape[1]).astype(text_ids.dtype) if action_mask is None: action_mask = paddle.ones(action_feat.shape[0], action_feat.shape[1]).astype( text_ids.dtype) # We create a 3D attention mask from a 2D tensor mask. # Sizes are [batch_size, 1, 1, to_seq_length] # So we can broadcast to [batch_size, num_heads, from_seq_length, to_seq_length]. extended_text_mask = text_mask.unsqueeze(1).unsqueeze(2) extended_image_mask = image_mask.unsqueeze(1).unsqueeze(2) extended_action_mask = action_mask.unsqueeze(1).unsqueeze(2) # Since attention_mask is 1.0 for positions we want to attend and 0.0 for # masked positions, this operation will create a tensor which is 0.0 for # positions we want to attend and -10000.0 for masked positions. # Since we are adding it to the raw scores before the softmax, this is # effectively the same as removing these entirely. def set_mask(extended_attention_mask): extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 return extended_attention_mask extended_text_mask = set_mask(extended_text_mask) extended_image_mask = set_mask(extended_image_mask) extended_action_mask = set_mask(extended_action_mask) t_embedding_output = self.embeddings(text_ids, token_type_ids) v_embedding_output = self.v_embeddings(image_feat, image_loc) a_embedding_output = self.a_embeddings(action_feat) # var = [t_embedding_output, v_embedding_output, a_embedding_output] # import numpy as np # for i, item in enumerate(var): # np.save('tmp/' + str(i)+'.npy', item.numpy()) encoded_layers_t, encoded_layers_v, encoded_layers_a = self.encoder( t_embedding_output, v_embedding_output, a_embedding_output, extended_text_mask, extended_image_mask, extended_action_mask, output_all_encoded_layers=output_all_encoded_layers, ) sequence_output_t = encoded_layers_t[-1] #get item from list sequence_output_v = encoded_layers_v[-1] sequence_output_a = encoded_layers_a[-1] pooled_output_t = self.t_pooler(sequence_output_t) pooled_output_v = self.v_pooler(sequence_output_v) pooled_output_a = self.a_pooler(sequence_output_a) if not output_all_encoded_layers: encoded_layers_t = encoded_layers_t[-1] encoded_layers_v = encoded_layers_v[-1] encoded_layers_a = encoded_layers_a[-1] return encoded_layers_t, encoded_layers_v, encoded_layers_a, \ pooled_output_t, pooled_output_v, pooled_output_a # For Head class BertPredictionHeadTransform(nn.Layer): def __init__(self, hidden_size, hidden_act): super(BertPredictionHeadTransform, self).__init__() self.dense = nn.Linear(hidden_size, hidden_size) if isinstance(hidden_act, str) or (sys.version_info[0] == 2 and isinstance(hidden_act, str)): self.transform_act_fn = ACT2FN[hidden_act] else: self.transform_act_fn = hidden_act self.LayerNorm = nn.LayerNorm(hidden_size, epsilon=1e-12) def forward(self, hidden_states): hidden_states = self.dense(hidden_states) hidden_states = self.transform_act_fn(hidden_states) hidden_states = self.LayerNorm(hidden_states) return hidden_states class BertLMPredictionHead(nn.Layer): def __init__(self, hidden_size, hidden_act, bert_model_embedding_weights): super(BertLMPredictionHead, self).__init__() self.transform = BertPredictionHeadTransform(hidden_size, hidden_act) # The output weights are the same as the input embeddings, but there is # an output-only bias for each token. assert bert_model_embedding_weights.shape[1] == hidden_size vocab_size = bert_model_embedding_weights.shape[0] # another implementation which would create another big params: # self.decoder = nn.Linear(hidden_size, vocab_size) # NOTE bias default: constant 0.0 # self.decoder.weight = self.create_parameter(shape=[hidden_size, vocab_size], # default_initializer=nn.initializer.Assign( # bert_model_embedding_weights.t())) # transpose self.decoder_weight = bert_model_embedding_weights self.decoder_bias = self.create_parameter( shape=[vocab_size], dtype=bert_model_embedding_weights.dtype, is_bias=True) # NOTE bias default: constant 0.0 def forward(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = paddle.tensor.matmul( hidden_states, self.decoder_weight, transpose_y=True) + self.decoder_bias return hidden_states class BertImageActionPredictionHead(nn.Layer): def __init__(self, hidden_size, hidden_act, target_size): super(BertImageActionPredictionHead, self).__init__() self.transform = BertPredictionHeadTransform(hidden_size, hidden_act) self.decoder = nn.Linear(hidden_size, target_size) def forward(self, hidden_states): hidden_states = self.transform(hidden_states) hidden_states = self.decoder(hidden_states) return hidden_states class BertPreTrainingHeads(nn.Layer): def __init__(self, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, hidden_act, v_hidden_act, a_hidden_act, v_target_size, a_target_size, fusion_method, bert_model_embedding_weights): super(BertPreTrainingHeads, self).__init__() self.predictions = BertLMPredictionHead(hidden_size, hidden_act, bert_model_embedding_weights) self.seq_relationship = nn.Linear(bi_hidden_size, 2) self.imagePredictions = BertImageActionPredictionHead( v_hidden_size, v_hidden_act, v_target_size) # visual class number self.actionPredictions = BertImageActionPredictionHead( a_hidden_size, a_hidden_act, a_target_size) # action class number self.fusion_method = fusion_method self.dropout = nn.Dropout(0.1) def forward(self, sequence_output_t, sequence_output_v, sequence_output_a, pooled_output_t, pooled_output_v, pooled_output_a): if self.fusion_method == 'sum': pooled_output = self.dropout(pooled_output_t + pooled_output_v + pooled_output_a) elif self.fusion_method == 'mul': pooled_output = self.dropout(pooled_output_t * pooled_output_v + pooled_output_a) else: assert False prediction_scores_t = self.predictions( sequence_output_t) # 8， 36 ，30522 seq_relationship_score = self.seq_relationship(pooled_output) # 8, 2 prediction_scores_v = self.imagePredictions( sequence_output_v) # 8, 37, 1601 prediction_scores_a = self.actionPredictions( sequence_output_a) # 8, 5, 401 return prediction_scores_t, prediction_scores_v, prediction_scores_a, seq_relationship_score @BACKBONES.register() class BertForMultiModalPreTraining(nn.Layer): """BERT model with multi modal pre-training heads. """ def __init__( self, vocab_size=30522, max_position_embeddings=512, type_vocab_size=2, v_target_size=1601, a_target_size=700, v_feature_size=2048, a_feature_size=2048, num_hidden_layers=12, v_num_hidden_layers=2, a_num_hidden_layers=3, t_ent_attention_id=[10, 11], v_ent_attention_id=[0, 1], a_ent_attention_id=[0, 1], fixed_t_layer=0, fixed_v_layer=0, hidden_size=768, v_hidden_size=1024, a_hidden_size=768, bi_hidden_size=1024, intermediate_size=3072, v_intermediate_size=1024, a_intermediate_size=3072, hidden_act="gelu", v_hidden_act="gelu", a_hidden_act="gelu", hidden_dropout_prob=0.1, v_hidden_dropout_prob=0.1, a_hidden_dropout_prob=0.1, attention_probs_dropout_prob=0.1, v_attention_probs_dropout_prob=0.1, a_attention_probs_dropout_prob=0.1, av_attention_probs_dropout_prob=0.1, at_attention_probs_dropout_prob=0.1, num_attention_heads=12, v_num_attention_heads=8, a_num_attention_heads=12, bi_num_attention_heads=8, fusion_method="mul", pretrained=None, ): """ vocab_size: vocabulary size. Default: 30522. max_position_embeddings: max position id. Default: 512. type_vocab_size: max segment id. Default: 2. v_target_size: class number of visual word. Default: 1601. a_target_size: class number of action word. Default: 700. v_feature_size: input visual feature dimension. Default: 2048. a_feature_size: input action feature dimension. Default: 2048. num_hidden_layers: number of BertLayer in text transformer. Default: 12. v_num_hidden_layers: number of BertLayer in visual transformer. Default: 2. a_num_hidden_layers: number of BertLayer in action transformer. Default:3. t_ent_attention_id: index id of BertConnectionLayer in text transformer. Default: [10, 11]. v_ent_attention_id: index id of BertConnectionLayer in visual transformer. Default:[0, 1]. a_ent_attention_id: index id of BertConnectionLayer in action transformer. Default:[0, 1]. fixed_t_layer: index id of fixed BertLayer in text transformer. Default: 0. fixed_v_layer: index id of fixed BertLayer in visual transformer. Default: 0. hidden_size: hidden size in text BertLayer. Default: 768. v_hidden_size: hidden size in visual BertLayer. Default: 1024. a_hidden_size: hidden size in action BertLayer. Default: 768. bi_hidden_size: hidden size in BertConnectionLayer. Default: 1024, intermediate_size: intermediate size in text BertLayer. Default: 3072. v_intermediate_size: intermediate size in visual BertLayer. Default: 1024. a_intermediate_size: intermediate size in text BertLayer. Default: 3072. hidden_act: hidden activation function in text BertLayer. Default: "gelu". v_hidden_act: hidden activation function in visual BertLayer. Default: "gelu". a_hidden_act: hidden activation function in action BertLayer. Default: "gelu". hidden_dropout_prob: hidden dropout probability in text Embedding Layer. Default: 0.1 v_hidden_dropout_prob: hidden dropout probability in visual Embedding Layer. Default: 0.1 a_hidden_dropout_prob: hidden dropout probability in action Embedding Layer. Default: 0.1 attention_probs_dropout_prob: attention dropout probability in text BertLayer. Default: 0.1 v_attention_probs_dropout_prob: attention dropout probability in visual BertLayer. Default: 0.1 a_attention_probs_dropout_prob: attention dropout probability in action BertLayer. Default: 0.1 av_attention_probs_dropout_prob: attention dropout probability in action-visual BertConnectionLayer. Default: 0.1 at_attention_probs_dropout_prob: attention dropout probability in action-text BertConnectionLayer. Default: 0.1 num_attention_heads: number of heads in text BertLayer. Default: 12. v_num_attention_heads: number of heads in visual BertLayer. Default: 8. a_num_attention_heads: number of heads in action BertLayer. Default: 12. bi_num_attention_heads: number of heads in BertConnectionLayer. Default: 8. fusion_method: methods of fusing pooled output from 3 transformer. Default: "mul". """ super(BertForMultiModalPreTraining, self).__init__() self.pretrained = pretrained self.vocab_size = vocab_size self.a_target_size = a_target_size self.bert = BertModel( vocab_size, max_position_embeddings, type_vocab_size, v_feature_size, a_feature_size, num_hidden_layers, v_num_hidden_layers, a_num_hidden_layers, v_ent_attention_id, t_ent_attention_id, a_ent_attention_id, fixed_t_layer, fixed_v_layer, hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, intermediate_size, v_intermediate_size, a_intermediate_size, hidden_act, v_hidden_act, a_hidden_act, hidden_dropout_prob, v_hidden_dropout_prob, a_hidden_dropout_prob, attention_probs_dropout_prob, v_attention_probs_dropout_prob, a_attention_probs_dropout_prob, av_attention_probs_dropout_prob, at_attention_probs_dropout_prob, num_attention_heads, v_num_attention_heads, a_num_attention_heads, bi_num_attention_heads, ) self.cls = BertPreTrainingHeads( hidden_size, v_hidden_size, a_hidden_size, bi_hidden_size, hidden_act, v_hidden_act, a_hidden_act, v_target_size, a_target_size, fusion_method, self.bert.embeddings.word_embeddings.weight) def init_weights(self): """Initiate the parameters. """ if isinstance(self.pretrained, str) and self.pretrained.strip() != "": load_ckpt(self, self.pretrained) elif self.pretrained is None or self.pretrained.strip() == "": for layer in self.sublayers(): if isinstance(layer, (nn.Linear, nn.Embedding)): weight_init_(layer, 'Normal', std=0.02) elif isinstance(layer, nn.LayerNorm): weight_init_(layer, 'Constant', value=1) def forward( self, text_ids, #8,36 action_feat, #8,5,2048 image_feat, #8,37,2048 image_loc, #8,37,5 token_type_ids=None, #8,36 text_mask=None, #8,36 image_mask=None, #8,37 action_mask=None, #8,5 ): """ text_ids: input text ids. Shape: [batch_size, seqence_length] action_feat: input action feature. Shape: [batch_size, action_length, action_feature_dim] image_feat: input image feature. Shape: [batch_size, region_length+1, image_feature_dim]], add 1 for image global feature. image_loc: input region location. Shape: [batch_size, region_length+1, region_location_dim], add 1 for image global feature location. token_type_ids: segment ids of each video clip. Shape: [batch_size, seqence_length] text_mask: text mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, seqence_length] image_mask: image mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, region_length] action_mask: action mask, 1 for real tokens and 0 for padding tokens. Shape: [batch_size, action_length] """ sequence_output_t, sequence_output_v, sequence_output_a, \ pooled_output_t, pooled_output_v, pooled_output_a = self.bert( text_ids, action_feat, image_feat, image_loc, token_type_ids, text_mask, image_mask, action_mask, output_all_encoded_layers=False, ) prediction_scores_t, prediction_scores_v, prediction_scores_a, seq_relationship_score = self.cls( sequence_output_t, sequence_output_v, sequence_output_a, pooled_output_t, pooled_output_v, pooled_output_a) return prediction_scores_t, prediction_scores_v, prediction_scores_a, seq_relationship_score