[FIXED] How can I define the activation function as a hyperparameter in PyTorch through RAY.Tune?

Issue

This is the link to the main page of the topic I want.

https://docs.ray.io/en/latest/tune/examples/tune-pytorch-cifar.html#tune-pytorch-cifar-ref

But unfortunately, there is no good documentation to answer all the questions. Also, if you know how I can define nested cross validation in this environment, please tell me.

Solution

I solved this problem as follows.

n_samples,n_features=X_train.shape
class NeuralNetwork (nn.Module):
def __init__(self,n_input_features,l1, l2,l3,config):
    super (NeuralNetwork, self).__init__()
    self.config = config
    self.linear1=nn.Linear(n_input_features,4*math.floor(n_input_features/2)+l1)
    self.linear2=nn.Linear(l1+4*math.floor(n_input_features/2),math.floor(n_input_features/2)+l2)
    self.linear3=nn.Linear(math.floor(n_input_features/2)+l2,math.floor(n_input_features/3)+l3)
    self.linear4=nn.Linear(math.floor(n_input_features/3)+l3,math.floor(n_input_features/6))
    self.linear5=nn.Linear(math.floor(n_input_features/6),1)

    self.a1 = self.config.get("a1")
    self.a2 = self.config.get("a2")
    self.a3 = self.config.get("a3")
    self.a4 = self.config.get("a4") 
@staticmethod
def activation_func(act_str):
    if act_str=="tanh" or act_str=="sigmoid":
        return eval("torch."+act_str)
    elif act_str=="silu" or act_str=="relu" or act_str=="leaky_relu" or act_str=="gelu":   
        return eval("torch.nn.functional."+act_str)
def forward(self,x):
    out=self.linear1(x)
    out=self.activation_func(self.a1)(out.float())
    out=self.linear2(out)
    out=self.activation_func(self.a2)(out.float())
    out=self.linear3(out)
    out=self.activation_func(self.a3)(out.float())
    out=self.linear4(out)
    out=self.activation_func(self.a3)(out.float())
    out=torch.sigmoid(self.linear5(out))
    y_predicted=out
    return y_predicted

Answered By - Arash Sajjadi