[FIXED] Python: Issue defining grid search parameters for neural network

Issue

I am trying to perform a grid search on several parameters of a neural network by using the code below:

def create_network(optimizer='rmsprop'):
    
    # Start Artificial Neural Network
    network = Sequential()
    
    # Adding the input layer and the first hidden layer
    # units = neurons
    network.add(Dense(units = 16, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the second hidden layer
    network.add(Dense(units = 16, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the third hidden layer
    network.add(Dense(units = 16, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the output layer
    network.add(Dense(units = 1))
    
    # Compile NN
    network.compile(optimizer = optimizer, 
                loss = 'mean_squared_error', 
                metrics=['mae', tf.keras.metrics.RootMeanSquaredError()])
    
    # Return compiled network
    return network

# Wrap Keras model so it can be used by scikit-learn
ann = KerasRegressor(build_fn=create_network, verbose=0)

# Create hyperparameter space
epoch_values = [10, 25, 50, 100, 150, 200]
batches = [10, 20, 30, 40, 50, 100, 1000]
optimizers = ['rmsprop', 'adam', 'SGD']
neurons = [16, 32, 64, 128, 256]
lr_values = [0.001, 0.01, 0.1, 0.2, 0.3]

# Create hyperparameter options
hyperparameters = dict(optimizer=optimizers, epochs=epoch_values, batch_size=batches, units=neurons,learning_rate=lr_values)

# Create grid search
# cv=5 is the default 5-fold
grid = GridSearchCV(estimator=ann, cv=5, param_grid=hyperparameters)

# Fit grid search
grid_result = grid.fit(X, y)

But I am getting the error:

learning_rate is not a legal parameter

Only the optimizers, epochs, and batch_size work... the other parameters are not recognized in the search.

How can I fix this?

In case it is relevant to know, I will also like to add more parameters to the grid search.

Solution

Currently, you're not instructing the network to use a learning rate, so the scikit-learn grid search doesn't know how to change it. Explicitly tell the optimizer how to change the learning rate in your create_network function (same goes for neurons or any other parameter). Something like this should work:

def create_network(optimizer='rmsprop', neurons=16, learning_rate=0.001):
    
    # Start Artificial Neural Network
    network = Sequential()
    
    # Adding the input layer and the first hidden layer
    network.add(Dense(units = neurons, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the second hidden layer
    network.add(Dense(units = neurons, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the third hidden layer
    network.add(Dense(units = neurons, 
                  activation = tf.keras.layers.LeakyReLU(alpha=0.3)))

    # Adding the output layer
    network.add(Dense(units = 1))

    ###############################################
    # Add optimizer with learning rate
    if optimizer == 'rmsprop':
        opt = tf.keras.optimizers.RMSprop(learning_rate=learning_rate)
    elif optimizer == 'adam':
        opt = tf.keras.optimizers.Adam(learning_rate=learning_rate)
    elif optimizer == 'SGD':
        opt = tf.keras.optimizers.SGD(learning_rate=learning_rate)
    else:
        raise ValueError('optimizer {} unrecognized'.format(optimizer))
    ##############################################    

    # Compile NN
    network.compile(optimizer = opt, 
                loss = 'mean_squared_error', 
                metrics=['mae', tf.keras.metrics.RootMeanSquaredError()])
    
    # Return compiled network
    return network

# Wrap Keras model so it can be used by scikit-learn
ann = KerasRegressor(build_fn=create_network, verbose=0)

# Create hyperparameter space
epoch_values = [10, 25, 50, 100, 150, 200]
batches = [10, 20, 30, 40, 50, 100, 1000]
optimizers = ['rmsprop', 'adam', 'SGD']
neuron_list = [16, 32, 64, 128, 256]
lr_values = [0.001, 0.01, 0.1, 0.2, 0.3]

# Create hyperparameter options
hyperparameters = dict(
    epochs=epoch_values, 
    batch_size=batches, 
    optimizer=optimizers, 
    neurons=neuron_list,
    learning_rate=lr_values)

# Create grid search
# cv=5 is the default 5-fold
grid = GridSearchCV(estimator=ann, cv=5, param_grid=hyperparameters)

# Fit grid search
grid_result = grid.fit(X, y)

A similar modification can be made for neurons as well or any other parameters related to the network structure. Make sure to match the name of the argument to create_network with the key in hyperparameters.

Answered By - TC Arlen