How to implement custom layer with multiple input in Keras

Question

I need to implement a custom layer like this:

class MaskedDenseLayer(Layer):
    def __init__(self, output_dim, activation, **kwargs):
        self.output_dim = output_dim
        super(MaskedDenseLayer, self).__init__(**kwargs)
        self._activation = activations.get(activation)
    def build(self, input_shape):

        # Create a trainable weight variable for this layer.
        self.kernel = self.add_weight(name='kernel', 
                                  shape=(input_shape[0][1], self.output_dim),
                                  initializer='glorot_uniform',
                                  trainable=True)
        super(MaskedDenseLayer, self).build(input_shape)  

    def call(self, l):
        self.x = l[0]
        self._mask = l[1][1]
        print('kernel:', self.kernel)
        masked = Multiply()([self.kernel, self._mask])
        self._output = K.dot(self.x, masked)
        return self._activation(self._output)


    def compute_output_shape(self, input_shape):
    return (input_shape[0][0], self.output_dim)

This is just like the way Keras API introduces to implement custom layers. And I need to give two inputs to this layer like this:

def main():
    with np.load('datasets/simple_tree.npz') as dataset:
        inputsize = dataset['inputsize']
        train_length = dataset['train_length']
        train_data = dataset['train_data']
        valid_length = dataset['valid_length']
        valid_data = dataset['valid_data']
        test_length = dataset['test_length']
        test_data = dataset['test_data']
        params = dataset['params']

    num_of_all_masks = 20
    num_of_hlayer = 6
    hlayer_size = 5
    graph_size = 4

    all_masks = generate_all_masks(num_of_all_masks, num_of_hlayer, hlayer_size, graph_size)

    input_layer = Input(shape=(4,))

    mask_1 = Input( shape = (graph_size , hlayer_size) )
    mask_2 = Input( shape = (hlayer_size , hlayer_size) )
    mask_3 = Input( shape = (hlayer_size , hlayer_size) )
    mask_4 = Input( shape = (hlayer_size , hlayer_size) )
    mask_5 = Input( shape = (hlayer_size , hlayer_size) )
    mask_6 = Input( shape = (hlayer_size , hlayer_size) )
    mask_7 = Input( shape = (hlayer_size , graph_size) )


    hlayer1 = MaskedDenseLayer(hlayer_size, 'relu')( [input_layer, mask_1] )
    hlayer2 = MaskedDenseLayer(hlayer_size, 'relu')( [hlayer1, mask_2] )
    hlayer3 = MaskedDenseLayer(hlayer_size, 'relu')( [hlayer2, mask_3] )
    hlayer4 = MaskedDenseLayer(hlayer_size, 'relu')( [hlayer3, mask_4] )
    hlayer5 = MaskedDenseLayer(hlayer_size, 'relu')( [hlayer4, mask_5] )
    hlayer6 = MaskedDenseLayer(hlayer_size, 'relu')( [hlayer5, mask_6] )
    output_layer = MaskedDenseLayer(graph_size, 'sigmoid')( [hlayer6, mask_7] )

    autoencoder = Model(inputs=[input_layer, mask_1, mask_2, mask_3,
                    mask_4, mask_5, mask_6, mask_7], outputs=[output_layer])

    autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
    #reassign_mask = ReassignMask()

    for i in range(0, num_of_all_masks):
        state = np.random.randint(0,20)
        autoencoder.fit(x=[train_data, 
                      np.tile(all_masks[state][0], [300, 1, 1]),
                      np.tile(all_masks[state][1], [300, 1, 1]),
                      np.tile(all_masks[state][2], [300, 1, 1]),
                      np.tile(all_masks[state][3], [300, 1, 1]),
                      np.tile(all_masks[state][4], [300, 1, 1]),
                      np.tile(all_masks[state][5], [300, 1, 1]),
                      np.tile(all_masks[state][6], [300, 1, 1])],
                    y=[train_data],
                    epochs=1,
                    batch_size=20,
                    shuffle=True,
                    #validation_data=(valid_data, valid_data),
                    #callbacks=[reassign_mask],
                    verbose=1)

Unfortunately when i run this code i get the following error:

TypeError: can only concatenate tuple (not "int") to tuple

What i need is a way to implement a custom layer with two inputs containing previous layer and a mask matrix. Here the all_mask variable is a list containing some pre-generated masks for all layers.

Can anyone help? What's wrong here with my code.

Update

Some parameters:

train data: (300, 4)

number of hidden layers: 6

hidden layer units: 5

mask: (size of previous layer, size of current layer)

And here is my model summary:

__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
input_361 (InputLayer)          (None, 4)            0                                            
__________________________________________________________________________________________________
input_362 (InputLayer)          (None, 4, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_281 (MaskedD (None, 5)            20          input_361[0][0]                  
                                                                 input_362[0][0]                  
__________________________________________________________________________________________________
input_363 (InputLayer)          (None, 5, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_282 (MaskedD (None, 5)            25          masked_dense_layer_281[0][0]     
                                                                 input_363[0][0]                  
__________________________________________________________________________________________________
input_364 (InputLayer)          (None, 5, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_283 (MaskedD (None, 5)            25          masked_dense_layer_282[0][0]     
                                                                 input_364[0][0]                  
__________________________________________________________________________________________________
input_365 (InputLayer)          (None, 5, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_284 (MaskedD (None, 5)            25          masked_dense_layer_283[0][0]     
                                                                 input_365[0][0]                  
__________________________________________________________________________________________________
input_366 (InputLayer)          (None, 5, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_285 (MaskedD (None, 5)            25          masked_dense_layer_284[0][0]     
                                                                 input_366[0][0]                  
__________________________________________________________________________________________________
input_367 (InputLayer)          (None, 5, 5)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_286 (MaskedD (None, 5)            25          masked_dense_layer_285[0][0]     
                                                                 input_367[0][0]                  
__________________________________________________________________________________________________
input_368 (InputLayer)          (None, 5, 4)         0                                            
__________________________________________________________________________________________________
masked_dense_layer_287 (MaskedD (None, 4)            20          masked_dense_layer_286[0][0]     
                                                                 input_368[0][0]                  
==================================================================================================
Total params: 165
Trainable params: 165
Non-trainable params: 0

Answer 1

Your input_shape is a list of tuples.

input_shape:  [(None, 4), (None, 4, 5)]

You can't simply use input_shape[0] or input_shape[1]. If you want to use the actual values, you have to choose which tuple, then which value. Example:

self.kernel = self.add_weight(name='kernel', 

                              #here: 
                              shape=(input_shape[0][1], self.output_dim), 


                              initializer='glorot_uniform',
                              trainable=True)

The same would be necessary (following your own shape rules) in the method compute_output_shape, where it seems what you want is to concatenate tuples:

return input_shape[0] + (self.output_dim,)

Don't forget to uncomment the super(MaskedDenseLayer, self).build(input_shape) line.