Understanding Accuracy and Evaluation Metrics in Machine Learning

Classification Metrics:
  → Accuracy, Precision, Recall, F1-Score, AUC-ROC

Regression Metrics:
  → MAE, MSE, RMSE, R², MAPE

                      Predicted: YES        Predicted: NO
Actual: YES    |   True Positive (TP)  | False Negative (FN) |
Actual: NO     |   False Positive (FP) | True Negative (TN)  |

from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay
import matplotlib.pyplot as plt
import numpy as np

# Actual values (ground truth)
y_true = [1, 0, 1, 1, 0, 1, 0, 0, 1, 0]

# Model predictions
y_pred = [1, 0, 1, 0, 0, 1, 1, 0, 1, 0]

# Confusion matrix banao
cm = confusion_matrix(y_true, y_pred)
print("Confusion Matrix:")
print(cm)

# print kitne sahi hai kiya kitne galat hai
tn, fp, fn, tp = cm.ravel()
print(f"True Negatives: {tn}")
print(f"False Positives: {fp}")
print(f"False Negatives: {fn}")
print(f"True Positives: {tp}")

Confusion Matrix:
[[4 1]
 [1 4]]

TN=4, FP=1
FN=1, TP=4

Accuracy = (TP + TN) / (TP + TN + FP + FN)

from sklearn.metrics import accuracy_score

y_true = [1, 0, 1, 1, 0, 1, 0, 0, 1, 0]
y_pred = [1, 0, 1, 0, 0, 1, 1, 0, 1, 0]

accuracy = accuracy_score(y_true, y_pred)
print(f"Accuracy: {accuracy:.2%}")  # 80.00%

Precision = TP / (TP + FP)

from sklearn.metrics import precision_score

precision = precision_score(y_true, y_pred)
print(f"Precision: {precision:.2%}")  # 80.00%

Recall = TP / (TP + FN)

from sklearn.metrics import recall_score

recall = recall_score(y_true, y_pred)
print(f"Recall: {recall:.2%}")  # 80.00%

High Precision → Model bahut cautious hai, sirf confident hone pe YES bolta hai
                  → FP kam, lekin FN badh sakta hai

High Recall    → Model aggressive hai, jyada cases ko YES bolta hai
                  → FN kam, lekin FP badh sakta hai

F1 = 2 × (Precision × Recall) / (Precision + Recall)

from sklearn.metrics import f1_score

f1 = f1_score(y_true, y_pred)
print(f"F1-Score: {f1:.2%}")

# Weighted F1 for multi-class
f1_weighted = f1_score(y_true, y_pred, average='weighted')

F_beta = (1 + β²) × (Precision × Recall) / (β² × Precision + Recall)

from sklearn.metrics import fbeta_score

# Beta=2 means Recall 2x more important than Precision
f2 = fbeta_score(y_true, y_pred, beta=2)
print(f"F2-Score: {f2:.4f}")

# Beta=0.5 means Precision 2x more important
f05 = fbeta_score(y_true, y_pred, beta=0.5)
print(f"F0.5-Score: {f05:.4f}")

Specificity = TN / (TN + FP)

# Manual calculation
tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
specificity = tn / (tn + fp)
print(f"Specificity: {specificity:.2%}")

MCC = (TP×TN - FP×FN) / √((TP+FP)(TP+FN)(TN+FP)(TN+FN))

from sklearn.metrics import matthews_corrcoef

mcc = matthews_corrcoef(y_true, y_pred)
print(f"MCC: {mcc:.4f}")

from sklearn.metrics import cohen_kappa_score

kappa = cohen_kappa_score(y_true, y_pred)
print(f"Cohen's Kappa: {kappa:.4f}")

Log Loss = -1/N × Σ [y×log(p) + (1-y)×log(1-p)]

from sklearn.metrics import log_loss

# y_prob = probability of class 1
y_prob = [0.9, 0.1, 0.8, 0.3, 0.2, 0.95, 0.7, 0.05, 0.85, 0.15]
y_true = [1,   0,   1,   1,   0,   1,    0,   0,    1,    0  ]

loss = log_loss(y_true, y_prob)
print(f"Log Loss: {loss:.4f}")  # Lower is better

from sklearn.metrics import hamming_loss

# Multi-label example
y_true_ml = [[1, 0, 1], [0, 1, 1]]
y_pred_ml = [[1, 0, 0], [0, 1, 1]]

h_loss = hamming_loss(y_true_ml, y_pred_ml)
print(f"Hamming Loss: {h_loss:.4f}")  # Lower is better

MAE = (1/n) × Σ |y_actual - y_predicted|

from sklearn.metrics import mean_absolute_error

y_true_reg = [100, 200, 300, 400, 500]
y_pred_reg = [110, 190, 320, 380, 510]

mae = mean_absolute_error(y_true_reg, y_pred_reg)
print(f"MAE: {mae:.2f}")  # 18.0

MSE = (1/n) × Σ (y_actual - y_predicted)²

from sklearn.metrics import mean_squared_error

mse = mean_squared_error(y_true_reg, y_pred_reg)
print(f"MSE: {mse:.2f}")  # 460.0

RMSE = √MSE

import numpy as np

rmse = np.sqrt(mse)
# OR
rmse = mean_squared_error(y_true_reg, y_pred_reg, squared=False)
print(f"RMSE: {rmse:.2f}")  # 21.45

R² = 1 - (SS_residual / SS_total)
SS_residual = Σ(y - ŷ)²
SS_total    = Σ(y - ȳ)²

from sklearn.metrics import r2_score

r2 = r2_score(y_true_reg, y_pred_reg)
print(f"R² Score: {r2:.4f}")

Adjusted R² = 1 - [(1-R²)(n-1) / (n-k-1)]

def adjusted_r2(r2, n, k):
    return 1 - (1 - r2) * (n - 1) / (n - k - 1)

n = len(y_true_reg)
k = 3  # number of features
adj_r2 = adjusted_r2(r2, n, k)
print(f"Adjusted R²: {adj_r2:.4f}")

MAPE = (1/n) × Σ |((y_actual - y_predicted) / y_actual)| × 100

def mape(y_true, y_pred):
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100

result = mape(y_true_reg, y_pred_reg)
print(f"MAPE: {result:.2f}%")

# Sklearn version (available from sklearn 0.24+)
from sklearn.metrics import mean_absolute_percentage_error
mape_sk = mean_absolute_percentage_error(y_true_reg, y_pred_reg) * 100
print(f"MAPE (sklearn): {mape_sk:.2f}%")

from sklearn.linear_model import HuberRegressor
import numpy as np

# Huber loss for training (outlier-robust regression)
X = np.array([[1], [2], [3], [4], [5]])
y = np.array([1.5, 2.1, 2.9, 4.2, 5.1])

model = HuberRegressor()
model.fit(X, y)

from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_classification

# Imbalanced dataset banao
X, y = make_classification(
    n_samples=1000,
    n_features=10,
    weights=[0.95, 0.05],  # 95% class 0, 5% class 1
    random_state=42
)

# class_weight='balanced' automatically adjust karta hai
model = LogisticRegression(class_weight='balanced', random_state=42)
model.fit(X, y)

# pip install imbalanced-learn
from imblearn.over_sampling import SMOTE
from collections import Counter

print(f"Before SMOTE: {Counter(y)}")

smote = SMOTE(random_state=42)
X_resampled, y_resampled = smote.fit_resample(X, y)

print(f"After SMOTE: {Counter(y_resampled)}")

from sklearn.metrics import balanced_accuracy_score

# Regular accuracy imbalanced data mein mislead karti hai
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

model.fit(X_train, y_train)
y_pred_imb = model.predict(X_test)

regular_acc = accuracy_score(y_test, y_pred_imb)
balanced_acc = balanced_accuracy_score(y_test, y_pred_imb)

print(f"Regular Accuracy: {regular_acc:.2%}")
print(f"Balanced Accuracy: {balanced_acc:.2%}")

from sklearn.metrics import classification_report

print(classification_report(y_test, y_pred_imb, target_names=['Normal', 'Fraud']))

              precision    recall  f1-score   support

      Normal       0.97      0.98      0.98       190
       Fraud       0.67      0.60      0.63        10

    accuracy                           0.96       200
   macro avg       0.82      0.79      0.80       200
weighted avg       0.96      0.96      0.96       200

TPR (Y-axis) = Recall = TP / (TP + FN)
FPR (X-axis) = FP / (FP + TN)  [1 - Specificity]

from sklearn.metrics import roc_curve, auc, roc_auc_score
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

# Dataset
X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# Models
lr = LogisticRegression(random_state=42)
lr.fit(X_train, y_train)

# Probabilities
y_prob_lr = lr.predict_proba(X_test)[:, 1]

# ROC Curve
fpr, tpr, thresholds = roc_curve(y_test, y_prob_lr)
roc_auc = auc(fpr, tpr)

# Plot
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, color='darkorange', lw=2, label=f'ROC Curve (AUC = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--', label='Random Classifier')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate (Recall)')
plt.title('ROC Curve')
plt.legend(loc='lower right')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

print(f"AUC-ROC Score: {roc_auc:.4f}")

from sklearn.metrics import precision_recall_curve, average_precision_score

precision_vals, recall_vals, pr_thresholds = precision_recall_curve(y_test, y_prob_lr)
avg_precision = average_precision_score(y_test, y_prob_lr)

plt.figure(figsize=(8, 6))
plt.plot(recall_vals, precision_vals, color='green', lw=2,
         label=f'PR Curve (AP = {avg_precision:.2f})')
plt.xlabel('Recall')
plt.ylabel('Precision')
plt.title('Precision-Recall Curve')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

# Youden's J Statistic: Best threshold on ROC Curve
fpr, tpr, thresholds = roc_curve(y_test, y_prob_lr)
j_scores = tpr - fpr
best_idx = np.argmax(j_scores)
best_threshold = thresholds[best_idx]

print(f"Optimal Threshold (Youden's J): {best_threshold:.4f}")
print(f"At this threshold → TPR: {tpr[best_idx]:.4f}, FPR: {fpr[best_idx]:.4f}")

# Apply optimal threshold
y_pred_optimal = (y_prob_lr >= best_threshold).astype(int)
print(f"\nWith optimal threshold:")
print(classification_report(y_test, y_pred_optimal))

from sklearn.datasets import load_iris
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import (classification_report, confusion_matrix,
                              roc_auc_score, accuracy_score)
from sklearn.preprocessing import label_binarize

# Multi-class dataset
iris = load_iris()
X_train, X_test, y_train, y_test = train_test_split(
    iris.data, iris.target, test_size=0.2, random_state=42
)

rf = RandomForestClassifier(n_estimators=100, random_state=42)
rf.fit(X_train, y_train)
y_pred_mc = rf.predict(X_test)
y_prob_mc = rf.predict_proba(X_test)

# Classification Report
print(classification_report(y_test, y_pred_mc, target_names=iris.target_names))

# Multi-class AUC (One-vs-Rest)
auc_ovr = roc_auc_score(y_test, y_prob_mc, multi_class='ovr', average='macro')
print(f"AUC-ROC (OvR, Macro): {auc_ovr:.4f}")

# Multi-class AUC (One-vs-One)
auc_ovo = roc_auc_score(y_test, y_prob_mc, multi_class='ovo', average='macro')
print(f"AUC-ROC (OvO, Macro): {auc_ovo:.4f}")

# Cancer detection: Recall sabse important
# Even if Precision suffers, we want to catch ALL cancer cases

from sklearn.svm import SVC

svc = SVC(probability=True, class_weight='balanced', random_state=42)
svc.fit(X_train, y_train)
y_prob_svc = svc.predict_proba(X_test)[:, 1]

# Lower threshold → Higher Recall
y_pred_conservative = (y_prob_svc >= 0.3).astype(int)  # Threshold = 0.3

print("Conservative threshold (0.3) - High Recall:")
print(classification_report(y_test, y_pred_conservative))

# Higher threshold → Higher Precision
y_pred_strict = (y_prob_lr >= 0.7).astype(int)  # Threshold = 0.7

print("Strict threshold (0.7) - High Precision:")
print(classification_report(y_test, y_pred_strict))

# Complete fraud detection evaluation
def fraud_model_report(y_true, y_pred, y_prob, model_name="Model"):
    print(f"\n{'='*50}")
    print(f"  {model_name} — Complete Evaluation")
    print(f"{'='*50}")
    
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    
    print(f"TP: {tp}, TN: {tn}, FP: {fp}, FN: {fn}")
    print(f"Accuracy:          {accuracy_score(y_true, y_pred):.4f}")
    print(f"Balanced Accuracy: {balanced_accuracy_score(y_true, y_pred):.4f}")
    print(f"Precision:         {precision_score(y_true, y_pred):.4f}")
    print(f"Recall:            {recall_score(y_true, y_pred):.4f}")
    print(f"F1-Score:          {f1_score(y_true, y_pred):.4f}")
    print(f"F2-Score:          {fbeta_score(y_true, y_pred, beta=2):.4f}")
    print(f"AUC-ROC:           {roc_auc_score(y_true, y_prob):.4f}")
    print(f"MCC:               {matthews_corrcoef(y_true, y_pred):.4f}")
    print(f"Log Loss:          {log_loss(y_true, y_prob):.4f}")

# Usage
from sklearn.metrics import balanced_accuracy_score, precision_score, recall_score
fraud_model_report(y_test, y_pred_optimal, y_prob_lr, "Logistic Regression")

from sklearn.ensemble import GradientBoostingRegressor
from sklearn.datasets import fetch_california_housing

housing = fetch_california_housing()
X_h, y_h = housing.data, housing.target

X_train_h, X_test_h, y_train_h, y_test_h = train_test_split(
    X_h, y_h, test_size=0.2, random_state=42
)

gbr = GradientBoostingRegressor(n_estimators=100, random_state=42)
gbr.fit(X_train_h, y_train_h)
y_pred_h = gbr.predict(X_test_h)

# Complete regression evaluation
mae_h = mean_absolute_error(y_test_h, y_pred_h)
rmse_h = np.sqrt(mean_squared_error(y_test_h, y_pred_h))
r2_h = r2_score(y_test_h, y_pred_h)
mape_h = mean_absolute_percentage_error(y_test_h, y_pred_h) * 100

print(f"MAE:   {mae_h:.4f}")
print(f"RMSE:  {rmse_h:.4f}")
print(f"R²:    {r2_h:.4f}")
print(f"MAPE:  {mape_h:.2f}%")

import pandas as pd
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.metrics import (accuracy_score, precision_score, recall_score,
                              f1_score, roc_auc_score, matthews_corrcoef,
                              balanced_accuracy_score, log_loss)
import warnings
warnings.filterwarnings('ignore')

# Dataset banao
X, y = make_classification(
    n_samples=2000, n_features=20, n_informative=15,
    n_redundant=3, weights=[0.85, 0.15], random_state=42
)

X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y
)

# Models
models = {
    'Logistic Regression': LogisticRegression(class_weight='balanced', random_state=42, max_iter=1000),
    'Decision Tree':       DecisionTreeClassifier(class_weight='balanced', random_state=42),
    'Random Forest':       RandomForestClassifier(class_weight='balanced', random_state=42),
    'Gradient Boosting':   GradientBoostingClassifier(random_state=42),
    'SVM':                 SVC(probability=True, class_weight='balanced', random_state=42),
    'KNN':                 KNeighborsClassifier(),
    'Naive Bayes':         GaussianNB(),
}

# Evaluate all models
results = []

for name, model in models.items():
    model.fit(X_train, y_train)
    y_pred_m = model.predict(X_test)
    y_prob_m = model.predict_proba(X_test)[:, 1]
    
    results.append({
        'Model': name,
        'Accuracy':       round(accuracy_score(y_test, y_pred_m), 4),
        'Bal. Accuracy':  round(balanced_accuracy_score(y_test, y_pred_m), 4),
        'Precision':      round(precision_score(y_test, y_pred_m), 4),
        'Recall':         round(recall_score(y_test, y_pred_m), 4),
        'F1-Score':       round(f1_score(y_test, y_pred_m), 4),
        'AUC-ROC':        round(roc_auc_score(y_test, y_prob_m), 4),
        'MCC':            round(matthews_corrcoef(y_test, y_pred_m), 4),
        'Log Loss':       round(log_loss(y_test, y_prob_m), 4),
    })

# Display as DataFrame
df_results = pd.DataFrame(results).set_index('Model')
print(df_results.sort_values('AUC-ROC', ascending=False).to_string())

from sklearn.model_selection import cross_validate

scoring = {
    'accuracy':          'accuracy',
    'precision':         'precision',
    'recall':            'recall',
    'f1':                'f1',
    'roc_auc':           'roc_auc',
    'balanced_accuracy': 'balanced_accuracy',
    'neg_log_loss':      'neg_log_loss',
}

rf_model = RandomForestClassifier(class_weight='balanced', n_estimators=100, random_state=42)

cv_results = cross_validate(
    rf_model, X, y,
    cv=5,
    scoring=scoring,
    return_train_score=True
)

print("5-Fold Cross-Validation Results:")
print("-" * 40)
for metric in ['accuracy', 'precision', 'recall', 'f1', 'roc_auc']:
    test_scores = cv_results[f'test_{metric}']
    print(f"{metric.capitalize():20s}: {test_scores.mean():.4f} ± {test_scores.std():.4f}")

import matplotlib.pyplot as plt
from sklearn.metrics import (confusion_matrix, roc_curve, precision_recall_curve,
                              ConfusionMatrixDisplay, auc)

def full_evaluation_dashboard(model, X_test, y_test, model_name="Model"):
    """
    Complete visual evaluation dashboard for a binary classifier.
    """
    y_pred_d = model.predict(X_test)
    y_prob_d = model.predict_proba(X_test)[:, 1]
    
    fig, axes = plt.subplots(2, 3, figsize=(18, 10))
    fig.suptitle(f'{model_name} — Complete Evaluation Dashboard', fontsize=16, fontweight='bold')
    
    # 1. Confusion Matrix
    cm_d = confusion_matrix(y_test, y_pred_d)
    ConfusionMatrixDisplay(cm_d).plot(ax=axes[0, 0], cmap='Blues')
    axes[0, 0].set_title('Confusion Matrix')
    
    # 2. ROC Curve
    fpr_d, tpr_d, _ = roc_curve(y_test, y_prob_d)
    roc_auc_d = auc(fpr_d, tpr_d)
    axes[0, 1].plot(fpr_d, tpr_d, 'darkorange', lw=2, label=f'AUC = {roc_auc_d:.3f}')
    axes[0, 1].plot([0, 1], [0, 1], 'navy', linestyle='--')
    axes[0, 1].set_xlabel('FPR'); axes[0, 1].set_ylabel('TPR')
    axes[0, 1].set_title('ROC Curve'); axes[0, 1].legend()
    axes[0, 1].grid(True, alpha=0.3)
    
    # 3. Precision-Recall Curve
    prec_d, rec_d, _ = precision_recall_curve(y_test, y_prob_d)
    ap_d = average_precision_score(y_test, y_prob_d)
    axes[0, 2].plot(rec_d, prec_d, 'green', lw=2, label=f'AP = {ap_d:.3f}')
    axes[0, 2].set_xlabel('Recall'); axes[0, 2].set_ylabel('Precision')
    axes[0, 2].set_title('Precision-Recall Curve'); axes[0, 2].legend()
    axes[0, 2].grid(True, alpha=0.3)
    
    # 4. Metrics Bar Chart
    metrics_d = {
        'Accuracy': accuracy_score(y_test, y_pred_d),
        'Precision': precision_score(y_test, y_pred_d),
        'Recall': recall_score(y_test, y_pred_d),
        'F1-Score': f1_score(y_test, y_pred_d),
        'Bal. Acc': balanced_accuracy_score(y_test, y_pred_d),
        'AUC-ROC': roc_auc_d,
    }
    colors_d = ['#2196F3', '#4CAF50', '#FF9800', '#9C27B0', '#F44336', '#00BCD4']
    bars = axes[1, 0].bar(metrics_d.keys(), metrics_d.values(), color=colors_d, alpha=0.8)
    axes[1, 0].set_ylim(0, 1.1)
    axes[1, 0].set_title('All Metrics Overview')
    for bar_d, val_d in zip(bars, metrics_d.values()):
        axes[1, 0].text(bar_d.get_x() + bar_d.get_width()/2., bar_d.get_height() + 0.01,
                        f'{val_d:.2f}', ha='center', va='bottom', fontsize=9)
    
    # 5. Probability Distribution
    axes[1, 1].hist(y_prob_d[y_test == 0], bins=30, alpha=0.6, color='blue', label='Class 0 (Negative)')
    axes[1, 1].hist(y_prob_d[y_test == 1], bins=30, alpha=0.6, color='red', label='Class 1 (Positive)')
    axes[1, 1].set_xlabel('Predicted Probability')
    axes[1, 1].set_ylabel('Count')
    axes[1, 1].set_title('Probability Distribution')
    axes[1, 1].legend()
    axes[1, 1].grid(True, alpha=0.3)
    
    # 6. Threshold Analysis
    thresholds_range = np.arange(0.1, 0.9, 0.05)
    precisions_t, recalls_t, f1s_t = [], [], []
    
    for thresh in thresholds_range:
        y_pred_t = (y_prob_d >= thresh).astype(int)
        precisions_t.append(precision_score(y_test, y_pred_t, zero_division=0))
        recalls_t.append(recall_score(y_test, y_pred_t, zero_division=0))
        f1s_t.append(f1_score(y_test, y_pred_t, zero_division=0))
    
    axes[1, 2].plot(thresholds_range, precisions_t, 'b-', label='Precision')
    axes[1, 2].plot(thresholds_range, recalls_t, 'r-', label='Recall')
    axes[1, 2].plot(thresholds_range, f1s_t, 'g-', label='F1-Score')
    axes[1, 2].set_xlabel('Threshold')
    axes[1, 2].set_ylabel('Score')
    axes[1, 2].set_title('Threshold vs Metrics')
    axes[1, 2].legend()
    axes[1, 2].grid(True, alpha=0.3)
    
    plt.tight_layout()
    plt.show()
    
    return metrics_d

# Usage
rf_final = RandomForestClassifier(class_weight='balanced', n_estimators=100, random_state=42)
rf_final.fit(X_train, y_train)
metrics = full_evaluation_dashboard(rf_final, X_test, y_test, "Random Forest")

# Problem: Jab koi class predict hi nahi ki
from sklearn.exceptions import UndefinedMetricWarning
import warnings

y_pred_all_zero = np.zeros(len(y_test))  # Sab zero predict kiya

# Warning aayega
with warnings.catch_warnings(record=True) as w:
    p = precision_score(y_test, y_pred_all_zero)
    if w:
        print(f"Warning: {w[0].message}")

# Fix: zero_division parameter use karo
p_safe = precision_score(y_test, y_pred_all_zero, zero_division=0)
print(f"Precision (safe): {p_safe}")

# Problem: Binary metric function multi-class pe call karna
y_mc_true = [0, 1, 2, 0, 1, 2]
y_mc_pred = [0, 2, 1, 0, 1, 2]

# WRONG — binary default
# precision_score(y_mc_true, y_mc_pred)  # ValueError!

# CORRECT — specify average
print(precision_score(y_mc_true, y_mc_pred, average='macro'))
print(precision_score(y_mc_true, y_mc_pred, average='micro'))
print(precision_score(y_mc_true, y_mc_pred, average='weighted'))

# WRONG — Test data pe fit karna
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()

# DO NOT DO THIS:
X_scaled_wrong = scaler.fit_transform(X)  # Puri data pe fit!
X_train_w, X_test_w, y_train_w, y_test_w = train_test_split(X_scaled_wrong, y, test_size=0.2)

# CORRECT — Pipeline use karo
from sklearn.pipeline import Pipeline

pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('classifier', RandomForestClassifier(class_weight='balanced', random_state=42))
])

X_train_c, X_test_c, y_train_c, y_test_c = train_test_split(X, y, test_size=0.2, random_state=42)
pipeline.fit(X_train_c, y_train_c)
y_pred_pipe = pipeline.predict(X_test_c)
print(f"F1-Score (No Leakage): {f1_score(y_test_c, y_pred_pipe):.4f}")

# WRONG — Imbalanced data mein regular split
X_train_w, X_test_w, y_train_w, y_test_w = train_test_split(X, y, test_size=0.2)

# CORRECT — Stratified split
X_train_s, X_test_s, y_train_s, y_test_s = train_test_split(
    X, y, test_size=0.2, random_state=42, stratify=y  # stratify=y is key!
)

from collections import Counter
print(f"Test set class distribution (Stratified): {Counter(y_test_s)}")

# Har baar train aur test metrics dono check karo
rf_check = RandomForestClassifier(n_estimators=100, random_state=42)
rf_check.fit(X_train, y_train)

train_f1 = f1_score(y_train, rf_check.predict(X_train))
test_f1 = f1_score(y_test, rf_check.predict(X_test))

print(f"Train F1: {train_f1:.4f}")
print(f"Test F1:  {test_f1:.4f}")
print(f"Gap:      {train_f1 - test_f1:.4f}")

if train_f1 - test_f1 > 0.1:
    print("⚠️ Warning: Model is OVERFITTING!")

Step 1: Dataset balanced hai?
   → Yes → Accuracy, F1 use karo
   → No  → Balanced Accuracy, MCC use karo

Step 2: Cost of errors kya hai?
   → FN costly (Cancer/Fraud miss) → Recall focus
   → FP costly (Wrong alarm)       → Precision focus
   → Both costly                   → F1 / F-beta

Step 3: Probability output chahiye?
   → Yes → AUC-ROC, Log Loss use karo
   → No  → Threshold-based metrics

# Never rely on single metric
# Always evaluate: Accuracy + F1 + AUC + Confusion Matrix

# ML metric ko business impact se connect karo
# Example: Fraud Detection
# 1 FN (missed fraud) = Rs. 50,000 loss
# 1 FP (false alarm) = Rs. 500 investigation cost

def business_cost(y_true, y_pred, fn_cost=50000, fp_cost=500):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    total_cost = (fn * fn_cost) + (fp * fp_cost)
    return total_cost, fn, fp

cost, fn_count, fp_count = business_cost(y_test, y_pred_optimal)
print(f"Total Business Cost: Rs. {cost:,}")
print(f"Missed Frauds: {fn_count}, False Alarms: {fp_count}")

from scipy import stats

# Do alag models ke scores compare karo
scores_model1 = cross_val_score(
    RandomForestClassifier(random_state=42), X, y, cv=10, scoring='f1'
)
scores_model2 = cross_val_score(
    GradientBoostingClassifier(random_state=42), X, y, cv=10, scoring='f1'
)

t_stat, p_value = stats.ttest_rel(scores_model1, scores_model2)
print(f"t-statistic: {t_stat:.4f}, p-value: {p_value:.4f}")
if p_value < 0.05:
    print("Difference is STATISTICALLY SIGNIFICANT")
else:
    print("Difference is NOT statistically significant")

from sklearn.calibration import calibration_curve, CalibratedClassifierCV

# Model ki probabilities reliable hain?
prob_true, prob_pred = calibration_curve(y_test, y_prob_lr, n_bins=10)

plt.figure(figsize=(8, 6))
plt.plot(prob_pred, prob_true, marker='o', linewidth=1, label='Logistic Regression')
plt.plot([0, 1], [0, 1], linestyle='--', label='Perfectly Calibrated')
plt.xlabel('Mean Predicted Probability')
plt.ylabel('Fraction of Positives')
plt.title('Calibration Curve (Reliability Diagram)')
plt.legend()
plt.grid(True, alpha=0.3)
plt.show()

Data Collection
     ↓
Data Preprocessing
     ↓
Feature Engineering
     ↓
Model Training
     ↓
Model Evaluation  ← YE STEP HAI HAMARA
     ↓
Hyperparameter Tuning
     ↓
Final Evaluation on Test Set
     ↓
Model Deployment
     ↓
Monitoring (Production Evaluation)

from sklearn.model_selection import GridSearchCV

param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 7, None],
    'min_samples_split': [2, 5, 10],
}

# AUC-ROC pe optimize karo
grid_search = GridSearchCV(
    RandomForestClassifier(class_weight='balanced', random_state=42),
    param_grid,
    cv=5,
    scoring='roc_auc',   # Ya 'f1', 'recall', 'precision'
    n_jobs=-1,
    verbose=1
)

grid_search.fit(X_train, y_train)
print(f"Best Params: {grid_search.best_params_}")
print(f"Best AUC-ROC: {grid_search.best_score_:.4f}")

from sklearn.metrics import make_scorer

# Business-specific metric: Rs. mein loss minimize karo
def business_loss_scorer(y_true, y_pred, fn_cost=50000, fp_cost=500):
    tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
    total_cost = -(fn * fn_cost + fp * fp_cost)  # Negative kyunki GridSearch maximize karta hai
    return total_cost

custom_scorer = make_scorer(business_loss_scorer)

grid_search_custom = GridSearchCV(
    RandomForestClassifier(class_weight='balanced', random_state=42),
    {'n_estimators': [50, 100], 'max_depth': [3, 5]},
    cv=5,
    scoring=custom_scorer,
    n_jobs=-1
)

grid_search_custom.fit(X_train, y_train)
print(f"Best Params for min. business loss: {grid_search_custom.best_params_}")

# Production mein model drift detect karna
def monitor_model_performance(y_true_prod, y_pred_prod, baseline_f1=0.80):
    current_f1 = f1_score(y_true_prod, y_pred_prod)
    
    print(f"Baseline F1: {baseline_f1:.4f}")
    print(f"Current F1:  {current_f1:.4f}")
    print(f"Drift:       {baseline_f1 - current_f1:.4f}")
    
    if baseline_f1 - current_f1 > 0.05:
        print("🚨 ALERT: Significant model drift detected! Retraining needed.")
    else:
        print("✅ Model performance stable.")

# Simulate production predictions
y_prod_true = np.random.choice([0, 1], size=500, p=[0.85, 0.15])
y_prod_pred = np.random.choice([0, 1], size=500, p=[0.85, 0.15])

monitor_model_performance(y_prod_true, y_prod_pred)

Mera dataset balanced hai?
├── YES → Accuracy + F1-Score
└── NO  → MCC + Balanced Accuracy + AUC-ROC

FN miss karna costly hai?
├── YES → Recall maximize karo (Cancer, Fraud)
└── NO  → Precision focus karo (Spam filter)

Model probability output karta hai?
├── YES → AUC-ROC + Log Loss + Calibration
└── NO  → Threshold-based: F1, Precision, Recall

Regression problem hai?
├── Outliers hain? → Huber Loss / MAE
├── Large errors costly? → RMSE
└── Model comparison? → R² / Adjusted R²