From 00cd50876b7af6c590f9bb04e7ec683460e1f21f Mon Sep 17 00:00:00 2001 From: Daniel Comaduran Date: Tue, 12 Aug 2025 11:17:21 -0600 Subject: [PATCH 1/2] Changed metrics calculation in training --- .../erp_rg_classifier_hyperparamgridsearch.py | 95 +++++++++++-------- 1 file changed, 54 insertions(+), 41 deletions(-) diff --git a/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py b/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py index f4466d2d..99a6c921 100644 --- a/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py +++ b/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py @@ -154,45 +154,28 @@ def fit( # Optimize hyperparameters with cross-validation self.__optimize_hyperparameters() - # Fit the model with the complete dataset and optimized hyperparameters - self.clf.fit(self.X, self.y) - - # Get predictions for final model - y_pred_proba = self.clf.predict_proba(self.X)[:, 1] - - # Calculate estimate of training metrics of final model - # TODO: Implement proper training metrics calculation, using cross validation. - # self.offline_accuracy = sum(y_pred == self.y) / len(self.y) - # self.offline_precision = precision_score(self.y, y_pred) - # self.offline_recall = recall_score(self.y, y_pred) + # Calculate proper training metrics using cross-validation with optimized hyperparameters + self.__calculate_training_metrics() - try: - roc_auc = roc_auc_score(self.y, y_pred_proba) - logger.info(f"ROC AUC Score: {roc_auc:0.3f}") - except Exception as e: - logger.warning(f"Could not calculate ROC AUC score: {e}") + # Fit the final model with the complete dataset and optimized hyperparameters + self.clf.fit(self.X, self.y) # Display training confusion matrix - # self.offline_cm = confusion_matrix(self.y, y_pred) if plot_cm: disp = ConfusionMatrixDisplay(confusion_matrix=self.offline_cm) disp.plot() - plt.title("Training confusion matrix") + plt.title("Training confusion matrix (Cross-Validation)") + plt.show() if plot_roc: - # TODO Implementation missing - pass + logger.error("ROC plot has not been implemented yet") # Log training metrics - logger.info("Final model training performance metrics:") - logger.info(f"Accuracy: {self.offline_accuracy:0.3f} - MAY NOT BE ACCURATE") - logger.info(f"Precision: {self.offline_precision:0.3f} - MAY NOT BE ACCURATE") - logger.info(f"Recall: {self.offline_recall:0.3f} - MAY NOT BE ACCURATE") - logger.info(f"Confusion Matrix:\n{self.offline_cm} ") - logger.warning( - "Note: Training metrics may not be accurate due to the use of " - "cross-validation and resampling methods. Use with caution." - ) + logger.info("Training performance metrics (from cross-validation):") + logger.info(f"Accuracy: {self.offline_accuracy:.3f}") + logger.info(f"Precision: {self.offline_precision:.3f}") + logger.info(f"Recall: {self.offline_recall:.3f}") + logger.info(f"Confusion Matrix:\n{self.offline_cm}") def predict(self, X): """Predict the class of the data @@ -304,23 +287,53 @@ def __optimize_hyperparameters(self): best_params = grid_search.best_params_ best_score = grid_search.best_score_ - # Report training metrics: TODO: Verify this is the right way to calculate training metrics - self.offline_accuracy = grid_search.best_estimator_.score(self.X, self.y) - self.offline_cm = confusion_matrix( - self.y, grid_search.best_estimator_.predict(self.X) - ) - self.offline_precision = precision_score( - self.y, grid_search.best_estimator_.predict(self.X) - ) - self.offline_recall = recall_score( - self.y, grid_search.best_estimator_.predict(self.X) - ) - # Update classifier with best parameters self.clf.set_params(**best_params) logger.info(f"Best parameters found: {best_params}") logger.info(f"Best CV score: {best_score:0.3f}") + def __calculate_training_metrics(self): + """Calculate proper training metrics using cross-validation with optimized hyperparameters. + + This method performs cross-validation with the optimized hyperparameters to get + unbiased estimates of the training performance metrics. + + Returns + ------- + None + Sets the offline metrics (accuracy, precision, recall, confusion matrix). + """ + logger.info("Calculating training metrics using cross-validation with optimized hyperparameters") + + # Define the strategy for cross validation + cv = StratifiedKFold( + n_splits=self.n_splits, shuffle=True, random_state=self.random_seed + ) + + # Initialize predictions array + cv_preds = np.zeros(len(self.y)) + + # Perform cross-validation with the optimized hyperparameters + for train_idx, test_idx in cv.split(self.X, self.y): + X_train, X_test = self.X[train_idx], self.X[test_idx] + y_train, y_test = self.y[train_idx], self.y[test_idx] + + # Create a copy of the classifier with optimized hyperparameters + cv_clf = Pipeline(self.clf.steps) + cv_clf.set_params(**self.clf.get_params()) + + # Fit and predict + cv_clf.fit(X_train, y_train) + cv_preds[test_idx] = cv_clf.predict(X_test) + + # Calculate metrics from cross-validation predictions + self.offline_accuracy = sum(cv_preds == self.y) / len(cv_preds) + self.offline_precision = precision_score(self.y, cv_preds) + self.offline_recall = recall_score(self.y, cv_preds) + self.offline_cm = confusion_matrix(self.y, cv_preds) + + logger.info("Training metrics calculation completed") + def _valid_roc_auc(self, y_true, y_pred, **kwargs): """Calculate the ROC AUC score for the classifier. This method is used because the stock `roc_auc_score` function From aa4558016f93ad4fc822608bd61626834b9d882b Mon Sep 17 00:00:00 2001 From: Daniel Comaduran Date: Tue, 12 Aug 2025 11:41:35 -0600 Subject: [PATCH 2/2] Blacked --- .../erp_rg_classifier_hyperparamgridsearch.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py b/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py index 99a6c921..0f35856a 100644 --- a/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py +++ b/bci_essentials/classification/erp_rg_classifier_hyperparamgridsearch.py @@ -294,17 +294,19 @@ def __optimize_hyperparameters(self): def __calculate_training_metrics(self): """Calculate proper training metrics using cross-validation with optimized hyperparameters. - - This method performs cross-validation with the optimized hyperparameters to get + + This method performs cross-validation with the optimized hyperparameters to get unbiased estimates of the training performance metrics. - + Returns ------- None Sets the offline metrics (accuracy, precision, recall, confusion matrix). """ - logger.info("Calculating training metrics using cross-validation with optimized hyperparameters") - + logger.info( + "Calculating training metrics using cross-validation with optimized hyperparameters" + ) + # Define the strategy for cross validation cv = StratifiedKFold( n_splits=self.n_splits, shuffle=True, random_state=self.random_seed @@ -317,21 +319,21 @@ def __calculate_training_metrics(self): for train_idx, test_idx in cv.split(self.X, self.y): X_train, X_test = self.X[train_idx], self.X[test_idx] y_train, y_test = self.y[train_idx], self.y[test_idx] - + # Create a copy of the classifier with optimized hyperparameters cv_clf = Pipeline(self.clf.steps) cv_clf.set_params(**self.clf.get_params()) - + # Fit and predict cv_clf.fit(X_train, y_train) cv_preds[test_idx] = cv_clf.predict(X_test) - + # Calculate metrics from cross-validation predictions self.offline_accuracy = sum(cv_preds == self.y) / len(cv_preds) self.offline_precision = precision_score(self.y, cv_preds) self.offline_recall = recall_score(self.y, cv_preds) self.offline_cm = confusion_matrix(self.y, cv_preds) - + logger.info("Training metrics calculation completed") def _valid_roc_auc(self, y_true, y_pred, **kwargs):