[FIXED] Merge two sequential models on Keras for hybrid model

Issue

I want to combine two sequential models for a hybrid model (with Keras 2.6.0). The first model is a succession of dense layer of a set of 4 parameters, and the second is a succession of 2D convolution of an image ((32,32)). The goal is to predict a curve of 128 points.

My actual model:

def get_model_v2(params_shape, img_shape):
    params_model = models.Sequential()
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
    params_model.add(layers.Dense(256, name='Output'))
    
    img_model = models.Sequential()
    img_model.add(layers.Input(img_shape, name='InputLayer2'))
    img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Flatten())
    
    concat = tf.keras.layers.concatenate([params_model, img_model])

    model = models.Sequential()
    model.add(layers.Input(concat, name='InputLayer3'))
    model.add(layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1'))
    model.add(layers.Dense(128, name='Output'))
    
    model.compile(optimizer = 'adam', 
                  loss      = 'mse', 
                  metrics   = ['mae', 'mse'])
    return model


model = get_model_v2 ( (4,),  (32, 32, 1) )

My problem is when I have to combine the two models, I don't know what to use, with this "concatenate" example I have an error like: TypeError: 'NoneType' object is not subscriptable. I understand the problem, but I can't find an other solution...

Solution

Few issues here,

• You are not using params_shape for your params_model (which comes out with an undefined shape).
• As you understood, you can't concatenate models with a concatenation layer
• The final model needs to through the Functional API
• You got a bunch of layers with same name - you cannot have the same name for two layers in the same model

import tensorflow.keras.layers as layers
import tensorflow.keras.models as models
import tensorflow.keras.regularizers as regularizers
import tensorflow as tf

def get_model_v2(params_shape, img_shape):
    params_model = models.Sequential()
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n1', input_shape=params_shape))
    params_model.add(layers.Dense(512, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n2'))
    params_model.add(layers.Dense(256, name='Output'))
    
    img_model = models.Sequential()
    img_model.add(layers.Input(img_shape, name='InputLayer2'))
    img_model.add(layers.Conv2D(64, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Conv2D(16, kernel_size=4, strides=2, padding="same"))
    img_model.add(layers.LeakyReLU(alpha=0.2))
    img_model.add(layers.Flatten())
    
    param_out = params_model.outputs[0]
    img_out = img_model.outputs[0]

    concat_out = tf.keras.layers.concatenate([param_out, img_out])

    full_dense_out = layers.Dense(256, kernel_regularizer=regularizers.l2(0.001), activation='relu', name='Dense_n3')(concat_out)
    final_out = layers.Dense(128, name='Output_final')(full_dense_out)
    
    model = models.Model(inputs=[params_model.inputs, img_model.inputs], outputs=final_out)

    model.summary()

    model.compile(optimizer = 'adam', 
                  loss      = 'mse', 
                  metrics   = ['mae', 'mse'])
    return model


model = get_model_v2 ( (4,),  (32, 32, 1) )

Answered By - thushv89