[FIXED] Keras multiple input model fails to match inputs from generator

Issue

I'm completely new to Keras and AI. I have Keras 2.9 with Python 3.8.10 under Ubuntu 20.04. I have a 2-input model which uses synthetic data generated by a C++ function. I call the C++ function using Pybind11. The function returns an 512 by 512 grayscale image and a number. I give these alongside a generated parameter number to the model, the 2 numbers in a vector with repetitions. Training the model gives this error message:

[INFO] training model...
Epoch 1/10
2022-08-22 18:36:27.276873: W tensorflow/core/framework/op_kernel.cc:1733] INVALID_ARGUMENT: TypeError: `generator` yielded an element that did not match the expected structure. The expected structure was ((tf.float32, tf.float32), tf.float32), but the yielded element was [[array([0.47688578, 0.47688578, 0.53283023, 0.53283023]), array([[0.56156078, 0.56156078, 0.56291341, ..., 0.64667391, 0.64674161,
        0.64741869],
          ...,
       [0.42745098, 0.43529412, 0.41568627, ..., 0.48235294, 0.45882353,
        0.45098039]])], array([0.64286654])].

While printing model branch inputs and outputs (see code below) gives this:

KerasTensor(type_spec=TensorSpec(shape=(None, 4), dtype=tf.float32, name='input_1'), name='input_1', description="created by layer 'input_1'")
KerasTensor(type_spec=TensorSpec(shape=(None, 512, 512, 1), dtype=tf.float32, name='input_2'), name='input_2', description="created by layer 'input_2'")
KerasTensor(type_spec=TensorSpec(shape=(None, 4), dtype=tf.float32, name=None), name='dense/Relu:0', description="created by layer 'dense'")
KerasTensor(type_spec=TensorSpec(shape=(None, 4), dtype=tf.float32, name=None), name='activation_5/Relu:0', description="created by layer 'activation_5'")

The code is:

def generate(aBatchSize:int=32, aRepeatParameter:int=2):
  dim = (512, 512)
  paraShape = (aRepeatParameter * 2,)
  def generator():
    xParameter = numpy.empty(paraShape, dtype=float)
    xImage     = numpy.empty(dim, dtype=float)
    y          = numpy.empty((1), dtype=float)
# populate variables
    xImage = randomLandscape(dist, height, tempAmb, tempBase) # Pybind11 call
    for i in range(1, aRepeatParameter):
      xParameter[i] = xParameter[0]
      xParameter[aRepeatParameter + i] = xParameter[aRepeatParameter]
    y[0]          = (tempBase - tempAmb) / 5
    yield [[xParameter, xImage], y]    # This was already yield {"parameters": xParameter, "image": xImage}, y -- no luck
  
  dataset = tensorflow.data.Dataset.from_generator(generator,
    output_signature=(
      (tensorflow.TensorSpec(shape=paraShape, dtype=tensorflow.float32, name="parameters"),
      tensorflow.TensorSpec(shape=dim, dtype=tensorflow.float32, name="image")),
      tensorflow.TensorSpec(shape=(1), dtype=tensorflow.float32, name="y")
            ))
  dataset = dataset.batch(aBatchSize)
  return dataset

def createMlp(aRepeatParameter:int):
  vectorSize = aRepeatParameter * 2
  inputs = Input(shape=(vectorSize,))
  x = inputs
  x = Dense(vectorSize, activation="relu")(x)
  return Model(inputs, x)

def createCnn(): 
  filters=(8, 4, 2, 1)
  inputShape = (512, 512, 1)
  chanDim = -1
  inputs = Input(shape=inputShape)
  x = inputs 
  for (i, f) in enumerate(filters):
    x = Conv2D(f, (3, 3), padding="same")(x)
    x = Activation("relu")(x)
    x = BatchNormalization(axis=chanDim)(x)
    x = MaxPooling2D(pool_size=(2, 2))(x)
  x = Flatten()(x)
  x = Dense(16)(x)
  x = Activation("relu")(x)
  x = BatchNormalization(axis=chanDim)(x)
  x = Dropout(0.5)(x)
  x = Dense(4)(x)
  x = Activation("relu")(x)
  return Model(inputs, x)

repeatParameter:int = 2
mlp = createMlp(repeatParameter)
cnn = createCnn()

print(mlp.input)
print(cnn.input)
print(mlp.output)
print(cnn.output)

combinedInput = Concatenate(axis=1)([mlp.output, cnn.output])
x = Dense(4, activation="relu")(combinedInput)
x = Dense(1, activation="linear")(x)
model = Model(inputs=[mlp.input, cnn.input], outputs=x)

opt = Adam(learning_rate=1e-3, decay=1e-3 / 200)
model.compile(loss="mean_absolute_percentage_error", optimizer=opt)

batchSize = 32
model.fit(landscapeGenerator.generate(batchSize, repeatParameter), validation_data=landscapeGenerator.generate(batchSize, repeatParameter),
  epochs=10, steps_per_epoch=10, validation_split=0.3,
  use_multiprocessing=True, workers=2)

Solution

It turned out my generator function was no real Python generator. Here is the correct form:

def generate(aBatchSize:int=32, aRepeatParameter:int=2):
  dim = (512, 512)
  paraShape = (aRepeatParameter * 2,)
  def generator():
    while True:
      xParameter = numpy.empty(paraShape, dtype=float)
      xImage     = numpy.empty(dim, dtype=float)
      y          = numpy.empty((1), dtype=float)
# populate variables
      xImage = randomLandscape(dist, height, tempAmb, tempBase)
      for i in range(1, aRepeatParameter):
        xParameter[i] = xParameter[0]
        xParameter[aRepeatParameter + i] = xParameter[aRepeatParameter]
      y[0]          = (tempBase - tempAmb) / 5
      yield ((xParameter, xImage), y)

  dataset = tensorflow.data.Dataset.from_generator(generator,
    output_signature=(
      (tensorflow.TensorSpec(shape=paraShape, dtype=tensorflow.float32),
      tensorflow.TensorSpec(shape=dim, dtype=tensorflow.float32)),
      tensorflow.TensorSpec(shape=(1), dtype=tensorflow.float32)
            ))
  dataset = dataset.batch(aBatchSize)
  return dataset

Answered By - Balázs Bámer