AUC-ROC

graph TD
    A[AUC-ROC] --> B[Area Calculation]
    A --> C[Interpretation]
    A --> D[Properties]
    A --> E[Applications]

    B --> B1[Integral of ROC Curve]
    B --> B2[0 to 1 Range]

    C --> C1[Probability Interpretation]
    C --> C2[Performance Metric]

    D --> D1[Threshold Independent]
    D --> D2[Class Imbalance Resistant]
    D --> D3[0.5 to 1 Range]

    E --> E1[Model Comparison]
    E --> E2[Threshold Selection]
    E --> E3[Performance Assessment]

    style A fill:#f9f,stroke:#333
    style B fill:#cfc,stroke:#333
    style C fill:#fcc,stroke:#333

import numpy as np
import matplotlib.pyplot as plt
from sklearn.metrics import roc_auc_score, roc_curve, auc
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression

# Generate synthetic data
X, y = make_classification(n_samples=1000, n_classes=2, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Train model
model = LogisticRegression()
model.fit(X_train, y_train)

# Get predicted probabilities
y_scores = model.predict_proba(X_test)[:, 1]

# Calculate AUC
auc_score = roc_auc_score(y_test, y_scores)
print(f"AUC-ROC Score: {auc_score:.4f}")

# Compute ROC curve
fpr, tpr, thresholds = roc_curve(y_test, y_scores)
roc_auc = auc(fpr, tpr)

# Plot ROC curve with AUC
plt.figure(figsize=(8, 6))
plt.plot(fpr, tpr, color='darkorange', lw=2,
         label=f'ROC curve (AUC = {roc_auc:.2f})')
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('ROC Curve with AUC')
plt.legend(loc="lower right")
plt.grid(True)
plt.show()

from sklearn.preprocessing import label_binarize
from sklearn.multiclass import OneVsRestClassifier
from itertools import cycle

# Generate multi-class data
X, y = make_classification(n_samples=1000, n_classes=3, n_informative=5,
                          random_state=42)
y = label_binarize(y, classes=[0, 1, 2])
n_classes = y.shape[1]

# Split data
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

# Train model
classifier = OneVsRestClassifier(LogisticRegression())
classifier.fit(X_train, y_train)

# Get predicted probabilities
y_score = classifier.predict_proba(X_test)

# Compute AUC for each class
auc_scores = dict()
for i in range(n_classes):
    auc_scores[i] = roc_auc_score(y_test[:, i], y_score[:, i])

# Compute micro-average AUC
micro_auc = roc_auc_score(y_test, y_score, average='micro')

# Compute macro-average AUC
macro_auc = roc_auc_score(y_test, y_score, average='macro')

print("Class-specific AUC scores:")
for i, score in auc_scores.items():
    print(f"Class {i}: {score:.4f}")

print(f"\nMicro-average AUC: {micro_auc:.4f}")
print(f"Macro-average AUC: {macro_auc:.4f}")

# Plot ROC curves
plt.figure(figsize=(8, 6))
colors = cycle(['aqua', 'darkorange', 'cornflowerblue'])
for i, color in zip(range(n_classes), colors):
    fpr, tpr, _ = roc_curve(y_test[:, i], y_score[:, i])
    roc_auc = auc(fpr, tpr)
    plt.plot(fpr, tpr, color=color, lw=2,
             label=f'Class {i} (AUC = {roc_auc:.2f})')

plt.plot([0, 1], [0, 1], 'k--', lw=2)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Multi-Class ROC Curves')
plt.legend(loc="lower right")
plt.grid(True)
plt.show()

from sklearn.utils import resample
from scipy.stats import norm

def bootstrap_auc(y_true, y_scores, n_bootstraps=1000, alpha=0.95):
    """Calculate AUC with confidence intervals using bootstrapping"""
    n_samples = len(y_true)
    auc_scores = []

    # Bootstrap sampling
    for _ in range(n_bootstraps):
        # Sample with replacement
        indices = resample(np.arange(n_samples), replace=True)
        y_true_sample = y_true[indices]
        y_scores_sample = y_scores[indices]

        # Calculate AUC
        if len(np.unique(y_true_sample)) < 2:
            continue  # Skip if only one class in sample

        auc = roc_auc_score(y_true_sample, y_scores_sample)
        auc_scores.append(auc)

    # Calculate statistics
    mean_auc = np.mean(auc_scores)
    std_auc = np.std(auc_scores)

    # Confidence interval
    lower = np.percentile(auc_scores, (1 - alpha) / 2 * 100)
    upper = np.percentile(auc_scores, (1 + alpha) / 2 * 100)

    # Normal approximation
    z = norm.ppf((1 + alpha) / 2)
    margin = z * std_auc / np.sqrt(n_bootstraps)
    lower_norm = mean_auc - margin
    upper_norm = mean_auc + margin

    return {
        'auc': mean_auc,
        'std': std_auc,
        'ci_lower': lower,
        'ci_upper': upper,
        'ci_lower_norm': lower_norm,
        'ci_upper_norm': upper_norm,
        'all_scores': auc_scores
    }

# Example usage
results = bootstrap_auc(y_test, y_scores)
print(f"AUC: {results['auc']:.4f}")
print(f"95% Confidence Interval: [{results['ci_lower']:.4f}, {results['ci_upper']:.4f}]")
print(f"Standard Deviation: {results['std']:.4f}")

# Plot bootstrap distribution
plt.figure(figsize=(8, 6))
plt.hist(results['all_scores'], bins=30, alpha=0.7, color='skyblue')
plt.axvline(results['auc'], color='red', linestyle='--', label=f'Mean AUC: {results["auc"]:.4f}')
plt.axvline(results['ci_lower'], color='green', linestyle=':', label=f'95% CI: [{results["ci_lower"]:.4f}, {results["ci_upper"]:.4f}]')
plt.axvline(results['ci_upper'], color='green', linestyle=':')
plt.xlabel('AUC Score')
plt.ylabel('Frequency')
plt.title('Bootstrap Distribution of AUC Scores')
plt.legend()
plt.grid(True)
plt.show()

from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier

def compare_models_auc(X, y, models, model_names, n_splits=5):
    """Compare AUC scores across multiple models using cross-validation"""
    from sklearn.model_selection import cross_val_score

    results = {}

    for model, name in zip(models, model_names):
        # Cross-validated AUC scores
        auc_scores = cross_val_score(
            model, X, y, cv=n_splits,
            scoring='roc_auc'
        )

        results[name] = {
            'mean_auc': np.mean(auc_scores),
            'std_auc': np.std(auc_scores),
            'all_scores': auc_scores
        }

        print(f"{name}:")
        print(f"  Mean AUC: {results[name]['mean_auc']:.4f}")
        print(f"  Std AUC: {results[name]['std_auc']:.4f}")
        print(f"  Individual AUCs: {results[name]['all_scores']}")
        print()

    # Plot comparison
    plt.figure(figsize=(10, 6))
    for name, result in results.items():
        plt.errorbar(name, result['mean_auc'], yerr=result['std_auc'],
                     fmt='o', capsize=5, label=name)

    plt.axhline(y=0.5, color='r', linestyle='--', label='Random Guessing')
    plt.ylabel('AUC Score')
    plt.title('Model Comparison by AUC Score')
    plt.legend()
    plt.grid(True)
    plt.show()

    return results

# Example comparison
models = [
    LogisticRegression(),
    RandomForestClassifier(n_estimators=100, random_state=42),
    SVC(probability=True, random_state=42),
    KNeighborsClassifier(n_neighbors=5)
]
model_names = ['Logistic Regression', 'Random Forest', 'SVM', 'k-NN']

auc_results = compare_models_auc(X, y, models, model_names)

from scipy.stats import wilcoxon, mannwhitneyu

def auc_significance_test(y_true, y_scores1, y_scores2, model_names):
    """Test if AUC difference between two models is statistically significant"""
    # Calculate AUC scores
    auc1 = roc_auc_score(y_true, y_scores1)
    auc2 = roc_auc_score(y_true, y_scores2)

    print(f"{model_names[0]} AUC: {auc1:.4f}")
    print(f"{model_names[1]} AUC: {auc2:.4f}")
    print(f"AUC Difference: {auc1 - auc2:.4f}")

    # Method 1: Wilcoxon signed-rank test on paired predictions
    # Create paired differences
    diff = y_scores1 - y_scores2

    # Wilcoxon test
    w_stat, w_p = wilcoxon(diff)
    print(f"\nWilcoxon signed-rank test:")
    print(f"  Statistic: {w_stat:.4f}")
    print(f"  p-value: {w_p:.4f}")

    # Method 2: Mann-Whitney U test on ranks
    # Create binary outcomes (1 if positive instance ranked higher, 0 otherwise)
    n = len(y_true)
    outcomes = []
    for i in range(n):
        for j in range(i+1, n):
            if y_true[i] != y_true[j]:
                if y_true[i] == 1:  # i is positive, j is negative
                    outcomes.append(1 if y_scores1[i] > y_scores1[j] else 0)
                else:  # i is negative, j is positive
                    outcomes.append(1 if y_scores1[j] > y_scores1[i] else 0)

    # Compare between models
    outcomes1 = []
    outcomes2 = []
    for i in range(n):
        for j in range(i+1, n):
            if y_true[i] != y_true[j]:
                if y_true[i] == 1:  # i is positive, j is negative
                    outcomes1.append(1 if y_scores1[i] > y_scores1[j] else 0)
                    outcomes2.append(1 if y_scores2[i] > y_scores2[j] else 0)
                else:  # i is negative, j is positive
                    outcomes1.append(1 if y_scores1[j] > y_scores1[i] else 0)
                    outcomes2.append(1 if y_scores2[j] > y_scores2[i] else 0)

    u_stat, u_p = mannwhitneyu(outcomes1, outcomes2)
    print(f"\nMann-Whitney U test:")
    print(f"  Statistic: {u_stat:.4f}")
    print(f"  p-value: {u_p:.4f}")

    # Interpretation
    alpha = 0.05
    if w_p < alpha:
        print(f"\nConclusion: The AUC difference is statistically significant (p < {alpha})")
        if auc1 > auc2:
            print(f"{model_names[0]} performs significantly better")
        else:
            print(f"{model_names[1]} performs significantly better")
    else:
        print(f"\nConclusion: The AUC difference is NOT statistically significant (p ≥ {alpha})")

# Example usage
# Train two models
model1 = LogisticRegression()
model2 = RandomForestClassifier(n_estimators=100, random_state=42)

model1.fit(X_train, y_train)
model2.fit(X_train, y_train)

y_scores1 = model1.predict_proba(X_test)[:, 1]
y_scores2 = model2.predict_proba(X_test)[:, 1]

auc_significance_test(y_test, y_scores1, y_scores2, ['Logistic Regression', 'Random Forest'])