[FIXED] Accuracy measure in Optuna/LightGBM

Issue

Trying to use Optuna Gridsearch to tune hyperparameters of my LightGBM model. However instead of returning log loss I want to just return the average accuracy in each trial.

If I add print('Testing accuracy {:.4f}'.format(model.score(X_test,y_test))) I can see the accuracy of each CV-fold in the iteration but I want the model to average them and return this measure INSTEAD of the log loss value. Is this possible?

This is my current code:

def objective(trial, X, y):
    param_grid = {
        # "device_type": trial.suggest_categorical("device_type", ['gpu']),
        "n_estimators": trial.suggest_categorical("n_estimators", [10000]),
        "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.05),
        "num_leaves": trial.suggest_int("num_leaves", 2, 50, step=2),
        "max_depth": trial.suggest_int("max_depth", 1, 5),
        "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 5, 100, step=5),
        "lambda_l1": trial.suggest_int("lambda_l1", 0, 100, step=5),
        "lambda_l2": trial.suggest_int("lambda_l2", 0, 100, step=5),
        "min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 15),
        "bagging_fraction": trial.suggest_float(
            "bagging_fraction", 0.2, 0.90, step=0.1
        ),
        "bagging_freq": trial.suggest_categorical("bagging_freq", [1]),
        "feature_fraction": trial.suggest_float(
            "feature_fraction", 0.2, 0.90, step=0.1
        ),
    }

    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=1121218)

    cv_scores = np.empty(5)

    for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
        X_train, X_test = X[train_idx], X[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]

        model = lg.LGBMClassifier(objective="multiclass",num_classes=3, **param_grid)
        model.fit(
            X_train,
            y_train,
            eval_set=[(X_test, y_test)],
            eval_metric="multi_logloss",
            early_stopping_rounds=100,
            callbacks=[
                LightGBMPruningCallback(trial, "multi_logloss")
            ],  # Add a pruning callback
        )
        preds = model.predict_proba(X_test)
        cv_scores[idx] = log_loss(y_test, preds)
        print('Testing accuracy {:.4f}'.format(model.score(X_test,y_test)))


    return np.mean(cv_scores)

study = optuna.create_study(direction="minimize", study_name="LGBM Classifier")
func = lambda trial: objective(trial, X_train, y_train)
study.optimize(func, n_trials=20)

Solution

1. Replace

cv_scores[idx] = log_loss(y_test, preds)

with

cv_scores[idx] = accuracy_score(y_test, preds))

2. Change direction to direction="maximize" as you want to maximize your accuracy not minimize as in the case of log_loss. Or you can return negative value -accuracy and set direction to minimize

3. You need to make sure the metric of optuna.integration.LightGBMPruningCallback is consistent with a direction of a study.

Complete example:

import lightgbm as lg
import numpy as np
import optuna
from optuna.integration import LightGBMPruningCallback
from sklearn.datasets import load_digits
from sklearn.metrics import accuracy_score
from sklearn.model_selection import StratifiedKFold


def objective(trial, X, y):
    param_grid = {
        # "device_type": trial.suggest_categorical("device_type", ['gpu']),
        "n_estimators": trial.suggest_categorical("n_estimators", [3, 10, 20]),
        "learning_rate": trial.suggest_float("learning_rate", 0.01, 0.05),
        "num_leaves": trial.suggest_int("num_leaves", 2, 50, step=2),
        "max_depth": trial.suggest_int("max_depth", 1, 5),
        "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 5, 100, step=5),
        "lambda_l1": trial.suggest_int("lambda_l1", 0, 100, step=5),
        "lambda_l2": trial.suggest_int("lambda_l2", 0, 100, step=5),
        "min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 15),
        "bagging_fraction": trial.suggest_float(
            "bagging_fraction", 0.2, 0.90, step=0.1
        ),
        "bagging_freq": trial.suggest_categorical("bagging_freq", [1]),
        "feature_fraction": trial.suggest_float(
            "feature_fraction", 0.2, 0.90, step=0.1
        ),
    }

    cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=1121218)
    cv_scores = np.empty(5)

    for idx, (train_idx, test_idx) in enumerate(cv.split(X, y)):
        X_train, X_test = X[train_idx], X[test_idx]
        y_train, y_test = y[train_idx], y[test_idx]

        model = lg.LGBMClassifier(objective="multiclass", num_classes=10, **param_grid)
        model.fit(
            X_train,
            y_train,
            eval_set=[(X_test, y_test)],
            eval_metric="auc_mu",
            early_stopping_rounds=100,
            callbacks=[
                LightGBMPruningCallback(trial, "auc_mu")
            ],  # Add a pruning callback
        )
        preds = model.predict(X_test)
        cv_scores[idx] = accuracy_score(y_test, preds)
        print("Testing accuracy {:.4f}".format(cv_scores[idx]))

    return np.mean(cv_scores)


X, y = load_digits(return_X_y=True)
study = optuna.create_study(direction="maximize", study_name="LGBM Classifier")
func = lambda trial: objective(trial, X, y)
study.optimize(func, n_trials=20)

Or you can replace these paramters:

• return accuracy; LightGBMPruningCallback(metric=auc_mu); direction=maximize

with

• return -accuracy; LightGBMPruningCallback(metric=multi_error);direction=minimize

Also you can find official examples here: https://github.com/optuna/optuna-examples/blob/main/lightgbm/lightgbm_integration.py

Answered By - u1234x1234