Graph produced by keras.utils.plot_model is not correct

Question

I have a model consisting of multiple smaller models. I want to make sure that everything is hooked up correctly and looked at the graph produced by keras.utils.plot_model. There I stumbled on some parts that don't look correct to me. Some connections are missing and at the inputs some additional connections are drawn (probably because of the stacking of the models).

Why are there connections missing? Is the model correct? Is the graph correct? What is the best way to check if the information flow is as expected?

from tensorflow.keras.utils import plot_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense, concatenate, Add


n_input_a0 = 10
# A -> n_input_b0 = n_input_c0

n_input_b0 = 20
n_input_b1 = 21
# B -> n_input_c1, n_input_c1

n_input_c0 = 20
n_input_c1 = 31
# C -> n_output

n_output = 3

# A
ip_a0 = Input(shape=(n_input_a0,), name='ip_a0')
x_a = Dense(units=10)(ip_a0)
x_a = Dense(units=n_input_b0)(x_a)
model_a = Model(inputs=ip_a0, outputs=x_a, name='model_a')

# B
ip_b0 = Input(shape=(n_input_b0,), name='ip_b0')
ip_b1 = Input(shape=(n_input_b1,), name='ip_b1')
ip_b0b1 = concatenate([ip_b0, ip_b1])
x_b = Dense(units=10)(ip_b0b1)
x_b = Dense(units=n_output)(x_b)
x_b_left = Dense(units=n_input_c1)(x_b)
x_b_right = Dense(units=n_input_c1)(x_b)
model_b = Model(inputs=[ip_b0, ip_b1], outputs=[x_b_left, x_b_right], name='model_b')

# C
ip_c0 = Input(shape=(n_input_c0,), name='ip_c0')
ip_c1 = Input(shape=(n_input_c1,), name='ip_c1')
ip_c0c1 = concatenate([ip_c0, ip_c1])
x_c = Dense(units=10)(ip_c0c1)
x_c = Dense(units=n_output)(x_c)
model_c = Model(inputs=[ip_c0, ip_c1], outputs=[x_c], name='model_c')


# Combined Model
ip_a0_external = Input(shape=(n_input_a0,), name='ip_a0_external')
ip_b1_external = Input(shape=(n_input_b1,), name='ip_b1_external')

x_a2 = model_a(ip_a0_external)
x_b_left2, x_b_right2 = model_b([x_a2, ip_b1_external])
x_b2 = Add()([x_b_left2, x_b_right2])
x_c2 = model_c([x_a2, x_b2])

model_total = Model(inputs=[ip_a0_external, ip_b1_external], outputs=[x_c2], name='model_total')
plot_model(model_total, expand_nested=True, show_shapes=True, to_file='model.png', dpi=80)

I also looked at the output of TensorBoard. The results are a little bit better but the addition of x_b_left + x_b_right is missing as well.

import numpy as np
import tensorflow as tf
tf.compat.v1.disable_eager_execution()
with tf.compat.v1.Session() as sess:
    writer = tf.compat.v1.summary.FileWriter('logs', sess.graph)
    model_total([np.ones((1, n_input_a0)), np.ones((1, n_input_b1))])
    writer.close()

Answer 1

I think you corrupted the models when you passed the same input to them twice. This is a strange thing to do.

• It's here: x_a2 = model_a(ip_a0)
• And here: x_b_left2, x_b_right2 = model_b([x_a2, ip_b1])

The inputs ip_a0 and ip_b1 are already part of the models and should not be called again.

I'd do it this way (didn't check if the graph gets corrected)

Original code:

from tensorflow.keras.utils import plot_model
from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input, Dense, concatenate, Add


n_input_a0 = 10
# A -> n_input_b0 = n_input_c0

n_input_b0 = 20
n_input_b1 = 21
# B -> n_input_c1, n_input_c1

n_input_c0 = 20
n_input_c1 = 31
# C -> n_output

n_output = 3

# A
ip_a0 = Input(shape=(n_input_a0,), name='ip_a0')
x_a = Dense(units=10)(ip_a0)
x_a = Dense(units=n_input_b0)(x_a)

model_a = Model(inputs=ip_a0, outputs=x_a, name='model_a')

Change here:

#don't call the model on its own inputs!!!!
x_a2 = model_a.output

#alternatively, following the same pattern as followed for model B
#ip_a0_external = Input(shape=(n_input_a0,), name='ip_a0_external')
#x_a2 = model_a(ip_a0_external)

Original code:

# B
ip_b0 = Input(shape=(n_input_b0,), name='ip_b0')
ip_b1 = Input(shape=(n_input_b1,), name='ip_b1')
ip_b0b1 = concatenate([ip_b0, ip_b1])
x_b = Dense(units=10)(ip_b0b1)
x_b_temp = Dense(units=n_output)(x_b)
x_b_left = Dense(units=n_input_c1)(x_b_temp)
x_b_right = Dense(units=n_input_c1)(x_b_temp)

model_b = Model(inputs=[ip_b0, ip_b1], outputs=[x_b_left, x_b_right], name='model_b')

Change here:

#don't call the model on its own inputs!!
#create a new input for this
ip_b1_external = Input(shape=(n_input_b1,), name='ip_b1_external')
x_b_left2, x_b_right2 = model_b([x_a2, ip_b1_external])

Original code:

x_b2 = Add()([x_b_left2, x_b_right2])

# C
ip_c0 = Input(shape=(n_input_c0,), name='ip_c0')
ip_c1 = Input(shape=(n_input_c1,), name='ip_c1')
ip_c0c1 = concatenate([ip_c0, ip_c1])
x_c = Dense(units=10)(ip_c0c1)
x_c = Dense(units=n_output)(x_c)

model_c = Model(inputs=[ip_c0, ip_c1], outputs=[x_c], name='model_c')

#ok, both inputs are from outside
x_c2 = model_c([x_a2, x_b2])

Change here:

# Combined Model
# keep track of what inputs were actual inputs for this model 
# and what inputs were only helpers for the submodels:

model_total = Model(inputs=[model_a.input, ip_b1_external], outputs=[x_c2], name='model_total', )
#alternatively
#model_total = Model(inputs=[ip_a0_external, ip_b1_external], outputs=[x_c2])
plot_model(model_total, expand_nested=True, show_shapes=True, to_file='model.png', dpi=64)