FullyConnectedLayer?

[lcxsnow]

I wonder why you don't use the nn.Linear and how is the z as long as the w_dim?

[lcxsnow]

In the FeedForwardBlock, there isn't any forward function and return, is it right?
Then in the GeneratorTransformerEncoderBlock, FeedForwardBlock won't return, so the hidden won't be change, is it right?

class FeedForwardBlock(nn.Sequential):
def init(self, emb_size, expansion=4, drop_p=0., bias=False):
super().init(
FullyConnectedLayer(emb_size, expansion * emb_size, activation='gelu', bias=False),
nn.Dropout(drop_p),
FullyConnectedLayer(expansion * emb_size, emb_size, bias=False),
)

class GeneratorTransformerEncoderBlock(nn.Module):
def init(self,
hidden_size=384,
sln_paremeter_size=384,
drop_p=0.,
forward_expansion=4,
forward_drop_p=0.,
**kwargs):
super().init()
self.sln = SLN(hidden_size, parameter_size=sln_paremeter_size)
self.msa = MultiHeadAttention(hidden_size, **kwargs)
self.dropout = nn.Dropout(drop_p)
self.feed_forward = FeedForwardBlock(hidden_size, expansion=forward_expansion, drop_p=forward_drop_p)

def forward(self, hidden, w):
    res = hidden
    hidden = self.sln(hidden, w)
    hidden = self.msa(hidden)
    hidden = self.dropout(hidden)
    hidden += res

    res = hidden
    hidden = self.sln(hidden, w)
    self.feed_forward(hidden)
    hidden = self.dropout(hidden)
    hidden += res
    return hidden