Building a model to classify ECG data

The aim of this project is to use publicly available ECG data to build a model that classifies ECGs. In this notebook, we will prepare the data and train a model to predict the diagnosis of the ECG.

The PTB-XL ECG dataset is a large dataset of 21799 clinical 12-lead ECGs from 18869 patients of 10 second length. The raw waveform data was annotated by up to two cardiologists, who assigned potentially multiple ECG statements to each record. The data was downloaded from https://physionet.org/content/ptb-xl/1.0.3/.

Imports

import pandas as pd
import numpy as np
import wfdb
import ast
import matplotlib.pyplot as plt
import seaborn as sns
import gc

from sklearn.preprocessing import MultiLabelBinarizer
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc, accuracy_score, f1_score, roc_auc_score
from sklearn.utils.class_weight import compute_class_weight

import tensorflow as tf
from tensorflow.keras import layers, models
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Dropout, BatchNormalization
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.metrics import AUC 

Import data

def load_raw_data(df, sampling_rate, path):
    if sampling_rate == 100:
        data = [wfdb.rdsamp(path+f) for f in df.filename_lr]
    else:
        data = [wfdb.rdsamp(path+f) for f in df.filename_hr]
    data = np.array([signal for signal, meta in data])
    return data

path = '/Users/scc/projects/ecg_data/ptb-xl-a-large-publicly-available-electrocardiography-dataset-1.0.3/'
sampling_rate=100

Prepare data

Split data into train and test datasets

# load and convert annotation data
Y = pd.read_csv(path+'ptbxl_database.csv', index_col='ecg_id')
Y.scp_codes = Y.scp_codes.apply(lambda x: ast.literal_eval(x))

# Load raw signal data
X = load_raw_data(Y, sampling_rate, path)

# Load scp_statements.csv for diagnostic aggregation
agg_df = pd.read_csv(path+'scp_statements.csv', index_col=0)
agg_df = agg_df[agg_df.diagnostic == 1]

def aggregate_diagnostic(y_dic):
    tmp = []
    for key in y_dic.keys():
        if key in agg_df.index:
            tmp.append(agg_df.loc[key].diagnostic_class)
    return list(set(tmp))

# Apply diagnostic superclass
Y['diagnostic_superclass'] = Y.scp_codes.apply(aggregate_diagnostic)

# Split data into train and test - 10% test, 90% train
test_fold = 10

X_train = X[np.where(Y.strat_fold != test_fold)]
y_train = Y[(Y.strat_fold != test_fold)].diagnostic_superclass
X_test = X[np.where(Y.strat_fold == test_fold)]
y_test = Y[Y.strat_fold == test_fold].diagnostic_superclass

Convert diagnostic labels to binary format & normalize data

# Convert list labels to binary format
mlb = MultiLabelBinarizer()
y_train_binary = mlb.fit_transform(y_train)
y_test_binary = mlb.transform(y_test)

# Print number of classes
print(f"Number of diagnostic classes: {len(mlb.classes_)}")
print("Classes:", mlb.classes_)

# Normalize the ECG data
X_train_normalized = (X_train - X_train.mean()) / X_train.std()
X_test_normalized = (X_test - X_test.mean()) / X_test.std()

Number of diagnostic classes: 5
Classes: ['CD' 'HYP' 'MI' 'NORM' 'STTC']

Plot example file

sample_index = 50
sample_data = X_train[sample_index]

time = np.arange(sample_data.shape[0]) / 100  # divide by sampling rate to get seconds

# Create figure and axis with larger size
plt.figure(figsize=(15, 10))

# Plot all 12 leads
for i in range(12):
    plt.subplot(6, 2, i+1)  # 6 rows, 2 columns
    plt.plot(time, sample_data[:, i])
    plt.title(f'Lead {i+1}')
    plt.xlabel('Time (seconds)')
    plt.ylabel('Amplitude (mV)')
    plt.grid(True)

plt.tight_layout()
plt.show()

Plot distribution of diagnoses in training and test sets to make sure they're balanced

# Calculate diagnosis counts for training and test sets
train_counts = y_train_binary.sum(axis=0)
test_counts = y_test_binary.sum(axis=0)

# Set up the plot
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(15, 6))

# Plot training set distribution
x = np.arange(len(mlb.classes_))
ax1.bar(x, train_counts)
ax1.set_xticks(x)
ax1.set_xticklabels(mlb.classes_, rotation=45)
ax1.set_title('Training Set Distribution')
ax1.set_ylabel('Number of Cases')

# Plot test set distribution
ax2.bar(x, test_counts)
ax2.set_xticks(x)
ax2.set_xticklabels(mlb.classes_, rotation=45)
ax2.set_title('Test Set Distribution')
ax2.set_ylabel('Number of Cases')

# Add value labels on top of each bar
for i, v in enumerate(train_counts):
    ax1.text(i, v, str(int(v)), ha='center', va='bottom')
for i, v in enumerate(test_counts):
    ax2.text(i, v, str(int(v)), ha='center', va='bottom')

plt.tight_layout()
plt.show()

# Print percentages
print("\nDistribution in Training Set:")
for i, label in enumerate(mlb.classes_):
    percentage = (train_counts[i] / len(y_train_binary)) * 100
    print(f"{label}: {train_counts[i]:.0f} cases ({percentage:.1f}%)")

print("\nDistribution in Test Set:")
for i, label in enumerate(mlb.classes_):
    percentage = (test_counts[i] / len(y_test_binary)) * 100
    print(f"{label}: {test_counts[i]:.0f} cases ({percentage:.1f}%)")

Distribution in Training Set:
CD: 4402 cases (22.5%)
HYP: 2387 cases (12.2%)
MI: 4919 cases (25.1%)
NORM: 8551 cases (43.6%)
STTC: 4714 cases (24.0%)

Distribution in Test Set:
CD: 496 cases (22.6%)
HYP: 262 cases (11.9%)
MI: 550 cases (25.0%)
NORM: 963 cases (43.8%)
STTC: 521 cases (23.7%)

From this we can see that most samples fall into the NORMAL class, which is to be expected. The HYP class is the smallest, so we need to pay attention to the accuracy of predicting this diagnosis, as it will be the most difficult.

Define functions to create and evaluate models

In a first step, we tested models with different configurations. We decide to use a CNN model with 3 convolutional blocks with increasing number of filters. Early layers will detect simple patterns (for example peaks), while later layers will detect more complex patterns. The kernel size was set to 3, which means that each filter will look at 3 consecutive time steps - a relatively small window that should be a good compromise to detect both local features as well as broader patterns. By setting the max pooling size to 2, we halve the number of time steps after each convolutional block, reducing computation and making the model more robust to small shifts.

ReLU activation and Adam optimizer were chosen for this model because they are common choices for deep learning problems with good performance and minimal tuning required.

To start, we set epochs to 30 (30 iterations over the entire dataset). Batch size was set to 32, which is a compromise between training speed, memory usage and training accuracy. Validation split was set to 20%, which means that 20% of the training data is used for validation during training. Early stopping means the model will stop training if the validation loss does not improve for 5 consecutive epochs, which helps prevent overfitting.

From this base model, we further tested different configurations, including additional layers, different learning rates, class weights, data augmentation and optimizing the threshold. We found that a model with 3 Conv1D layers, 2 Dense layers, and a Dropout layer gave the best performance. We also found that threshold optimization and data augmentation improved the performance, while class weights did not. We performed further optimization based on this model.

models_results = []

Define Model Parameters and Functions

# enhanced model with 3 Conv1D layers, 2 Dense layers, and a Dropout layer
def create_enhanced_model():
    model = Sequential([
        Conv1D(64, kernel_size=3, activation='relu', padding='same', input_shape=(1000, 12)),
        BatchNormalization(),
        MaxPooling1D(pool_size=2),
        
        Conv1D(128, kernel_size=3, activation='relu', padding='same'),
        BatchNormalization(),
        MaxPooling1D(pool_size=2),
        
        Conv1D(256, kernel_size=3, activation='relu', padding='same'),
        BatchNormalization(),
        MaxPooling1D(pool_size=2),
        
        Flatten(),
        Dense(512, activation='relu'),  # Added layer
        Dropout(0.5),
        Dense(256, activation='relu'),
        Dense(5, activation='sigmoid')
    ])
    return model

# threshold optimization and data augmentation
def find_optimal_threshold(y_true, y_pred_proba):
    thresholds = np.arange(0.1, 0.9, 0.1)
    f1_scores = []
    
    for threshold in thresholds:
        y_pred = (y_pred_proba >= threshold).astype(int)
        f1 = f1_score(y_true, y_pred)
        f1_scores.append(f1)
    
    optimal_idx = np.argmax(f1_scores)
    return thresholds[optimal_idx], f1_scores[optimal_idx]

def evaluate_with_optimal_thresholds(y_true, y_pred_proba):
    optimal_thresholds = []
    best_f1_scores = []
    y_pred_optimal = np.zeros_like(y_pred_proba)
    
    for i in range(y_true.shape[1]):
        threshold, f1 = find_optimal_threshold(y_true[:, i], y_pred_proba[:, i])
        optimal_thresholds.append(threshold)
        best_f1_scores.append(f1)
        y_pred_optimal[:, i] = (y_pred_proba[:, i] >= threshold).astype(int)
    
    return y_pred_optimal, optimal_thresholds, best_f1_scores

def augment_hyp_samples(X, y, noise_level=0.001, shift_amount=50):
    hyp_indices = np.where(y[:, mlb.classes_.tolist().index('HYP')] == 1)[0]
    augmented_X = []
    augmented_y = []
    
    for idx in hyp_indices:
        # Original signal
        signal = X[idx]
        
        # Add noise
        noisy_signal = signal + np.random.normal(0, noise_level, signal.shape)
        augmented_X.append(noisy_signal)
        augmented_y.append(y[idx])
        
        # Time shifting
        shifted_signal = np.roll(signal, shift_amount, axis=0)
        augmented_X.append(shifted_signal)
        augmented_y.append(y[idx])
        
    return np.vstack([X, np.array(augmented_X)]), np.vstack([y, np.array(augmented_y)])

def train_and_evaluate_model(X_train, y_train, model, model_name, threshold=False, augment=False, noise_level=0.001, shift_amount=50, batch_size=32):
    # Compile model
    model.compile(optimizer='adam',
                 loss='binary_crossentropy',
                 metrics=['accuracy', AUC()])


    if augment:
        X_train_aug, y_train_binary = augment_hyp_samples(X_train, y_train, noise_level, shift_amount)

        X_train_normalized = (X_train_aug - np.mean(X_train_aug)) / np.std(X_train_aug)
        X_test_normalized = (X_test - np.mean(X_test)) / np.std(X_test)

    # Train model
    history = model.fit(X_train_normalized, y_train_binary,
                       validation_split=0.2,
                       epochs=30,
                       batch_size=batch_size,
                       callbacks=[EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)])
    
    # Get predictions
    y_pred_proba = model.predict(X_test_normalized)

    if threshold:
        # Use optimal thresholds
        y_pred_binary, thresholds, f1_scores = evaluate_with_optimal_thresholds(
            y_test_binary, y_pred_proba
        )
        extra_results = {'optimal_thresholds': thresholds}
        
    else:
        # Use default threshold (0.5)
        y_pred_binary = (y_pred_proba > 0.5).astype(int)
        f1_scores = [f1_score(y_test_binary[:, i], y_pred_binary[:, i]) 
                    for i in range(len(mlb.classes_))]
        extra_results = {}

    # Calculate common metrics
    results = {
        'model_name': model_name,
        'accuracy': accuracy_score(y_test_binary, y_pred_binary),
        'auc_scores': [roc_auc_score(y_test_binary[:, i], y_pred_proba[:, i]) 
                    for i in range(len(mlb.classes_))],
        'f1_scores': f1_scores,
        'history': history.history,
        **extra_results
    }
    
    return results

def plot_model_comparisons(models_results, figsize=(20, 8)):
    """
    Plot comparison of model performances.
    
    Args:
        models_results (list): List of dictionaries containing model results
        figsize (tuple): Figure size (width, height)
    """
    if len(models_results) == 1:
        # Keep existing single model layout
        plt.figure(figsize=(20, 8))
        result = models_results[0]
        
        # Plot 1: Overall Accuracy - with slimmer bar
        plt.subplot(1, 3, 1)
        bar_width = 0.3  # Reduced width for single bar
        plt.bar([0], [result['accuracy']], width=bar_width, color='skyblue', 
                edgecolor='black', linewidth=0.5)
        plt.text(0, result['accuracy'], f'{result["accuracy"]:.3f}', 
                ha='center', va='bottom')
        plt.title('Overall Accuracy')
        plt.ylabel('Accuracy')
        plt.xticks([0], [result['model_name']], rotation=45)
        plt.xlim(-0.5, 0.5)
        
        # Rest of single model plots...
        
    else:
        # New layout for multiple models - bigger figure
        plt.figure(figsize=(25, 12))
        
        # Plot 1: Overall Accuracy (top)
        plt.subplot(2, 1, 1)
        accuracies = [result['accuracy'] for result in models_results]
        model_names = [result['model_name'] for result in models_results]
        bars = plt.bar(model_names, accuracies, edgecolor='black', linewidth=0.5)
        
        # Add value labels on top of bars
        for bar in bars:
            height = bar.get_height()
            plt.text(bar.get_x() + bar.get_width()/2., height,
                    f'{height:.3f}', ha='center', va='bottom')
            
        plt.title('Overall Accuracy Comparison')
        plt.ylabel('Accuracy')
        plt.xticks(rotation=45)
        
        # Plot 2: AUC Scores by Class (bottom left)
        plt.subplot(2, 2, 3)
        x = np.arange(len(mlb.classes_))
        width = 0.8 / len(models_results)
        for i, result in enumerate(models_results):
            plt.bar(x + i*width, result['auc_scores'], width, 
                   label=result['model_name'], edgecolor='black', linewidth=0.5)
        plt.xlabel('Classes')
        plt.ylabel('AUC Score')
        plt.title('AUC Scores by Class')
        plt.xticks(x + width*len(models_results)/2, mlb.classes_, rotation=45)
        plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
        
        # Plot 3: F1 Scores by Class (bottom right)
        plt.subplot(2, 2, 4)
        for i, result in enumerate(models_results):
            plt.bar(x + i*width, result['f1_scores'], width, 
                   label=result['model_name'], edgecolor='black', linewidth=0.5)
        plt.xlabel('Classes')
        plt.ylabel('F1 Score')
        plt.title('F1 Scores by Class')
        plt.xticks(x + width*len(models_results)/2, mlb.classes_, rotation=45)
        plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left')
    
    # Common styling for all subplots
    for ax in plt.gcf().axes:
        ax.spines['top'].set_visible(False)
        ax.spines['right'].set_visible(False)
        
    plt.tight_layout()
    plt.show()

Run Model with base configuration

models_results.append(train_and_evaluate_model(X_train, y_train_binary, create_enhanced_model(), "Enhanced Model", threshold=True, augment=True))

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 72s 113ms/step - accuracy: 0.4670 - auc: 0.7426 - loss: 0.6999 - val_accuracy: 0.1346 - val_auc: 0.6422 - val_loss: 0.7512
Epoch 2/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 75s 122ms/step - accuracy: 0.5863 - auc: 0.8468 - loss: 0.4169 - val_accuracy: 0.1005 - val_auc: 0.6819 - val_loss: 0.7077
Epoch 3/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 81s 133ms/step - accuracy: 0.6306 - auc: 0.8837 - loss: 0.3606 - val_accuracy: 0.1680 - val_auc: 0.7496 - val_loss: 0.6134
Epoch 4/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 88s 144ms/step - accuracy: 0.6688 - auc: 0.9119 - loss: 0.3148 - val_accuracy: 0.2297 - val_auc: 0.8365 - val_loss: 0.4998
Epoch 5/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 88s 144ms/step - accuracy: 0.6925 - auc: 0.9280 - loss: 0.2859 - val_accuracy: 0.1481 - val_auc: 0.8073 - val_loss: 0.5750
Epoch 6/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 103s 169ms/step - accuracy: 0.7236 - auc: 0.9443 - loss: 0.2528 - val_accuracy: 0.2062 - val_auc: 0.8622 - val_loss: 0.5137
Epoch 7/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 101s 165ms/step - accuracy: 0.7551 - auc: 0.9612 - loss: 0.2111 - val_accuracy: 0.2279 - val_auc: 0.8546 - val_loss: 0.5735
Epoch 8/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 99s 162ms/step - accuracy: 0.7751 - auc: 0.9712 - loss: 0.1819 - val_accuracy: 0.3276 - val_auc: 0.8684 - val_loss: 0.5281
Epoch 9/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 94s 154ms/step - accuracy: 0.7945 - auc: 0.9782 - loss: 0.1574 - val_accuracy: 0.2490 - val_auc: 0.8866 - val_loss: 0.5234
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 28ms/step

Plot results from base model - the overall accuracy is 0.533 and AUCs between 80 and 90%. F1 scores are lowest for the HYP class at 0.505 und highest for NORM at 0.814.

# Plot comparisons
plot_model_comparisons(models_results)

gc.collect()
tf.keras.backend.clear_session()

From this base model, we will do some further hypertuning to determine whether we can improve the performance by changing the batch size or augmentation parameters. We know that we do not need to increase the epochs as we use early stopping and it has never reached 30.

batch_sizes = [16, 32, 64]

augment_configs = [
    {'noise_level': 0.005, 'shift_factor': 50},
    {'noise_level': 0.001, 'shift_factor': 20},
    {'noise_level': 0.01, 'shift_factor': 100}
]

Run Model with different configurations

for batch_size in batch_sizes:
    for aug_config in augment_configs:
        models_results.append(train_and_evaluate_model(
            X_train, 
            y_train_binary, 
            create_enhanced_model(), 
            f"model_b{batch_size}_n{aug_config['noise_level']}_s{aug_config['shift_factor']}", 
            threshold=True, 
            augment=True, 
            noise_level=aug_config['noise_level'], 
            shift_amount=aug_config['shift_factor'], 
            batch_size=batch_size))
        
        #clear memory between training runs
        gc.collect()
        tf.keras.backend.clear_session()

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 105s 83ms/step - accuracy: 0.4607 - auc: 0.7322 - loss: 0.7057 - val_accuracy: 0.1393 - val_auc: 0.6882 - val_loss: 0.6759
Epoch 2/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 93s 76ms/step - accuracy: 0.6051 - auc: 0.8668 - loss: 0.3837 - val_accuracy: 0.2172 - val_auc: 0.7387 - val_loss: 0.6775
Epoch 3/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 110s 90ms/step - accuracy: 0.6552 - auc: 0.8975 - loss: 0.3380 - val_accuracy: 0.2084 - val_auc: 0.8595 - val_loss: 0.5151
Epoch 4/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 105s 86ms/step - accuracy: 0.6810 - auc: 0.9173 - loss: 0.3062 - val_accuracy: 0.2090 - val_auc: 0.8497 - val_loss: 0.5095
Epoch 5/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 107s 88ms/step - accuracy: 0.7076 - auc: 0.9353 - loss: 0.2718 - val_accuracy: 0.2384 - val_auc: 0.8189 - val_loss: 0.6207
Epoch 6/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 110s 91ms/step - accuracy: 0.7462 - auc: 0.9545 - loss: 0.2283 - val_accuracy: 0.3370 - val_auc: 0.8912 - val_loss: 0.4861
Epoch 7/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 113s 93ms/step - accuracy: 0.7741 - auc: 0.9695 - loss: 0.1874 - val_accuracy: 0.3990 - val_auc: 0.9028 - val_loss: 0.4524
Epoch 8/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 116s 95ms/step - accuracy: 0.8038 - auc: 0.9807 - loss: 0.1473 - val_accuracy: 0.4987 - val_auc: 0.9264 - val_loss: 0.3872
Epoch 9/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 119s 97ms/step - accuracy: 0.8189 - auc: 0.9852 - loss: 0.1276 - val_accuracy: 0.4090 - val_auc: 0.8883 - val_loss: 0.5518
Epoch 10/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 122s 100ms/step - accuracy: 0.8344 - auc: 0.9890 - loss: 0.1084 - val_accuracy: 0.5192 - val_auc: 0.9330 - val_loss: 0.3962
Epoch 11/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 125s 102ms/step - accuracy: 0.8407 - auc: 0.9915 - loss: 0.0931 - val_accuracy: 0.4710 - val_auc: 0.9302 - val_loss: 0.4051
Epoch 12/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 126s 103ms/step - accuracy: 0.8476 - auc: 0.9926 - loss: 0.0854 - val_accuracy: 0.4412 - val_auc: 0.9092 - val_loss: 0.5838
Epoch 13/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 127s 104ms/step - accuracy: 0.8433 - auc: 0.9928 - loss: 0.0838 - val_accuracy: 0.4445 - val_auc: 0.9081 - val_loss: 0.5211
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 27ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 130s 105ms/step - accuracy: 0.4478 - auc: 0.7297 - loss: 0.7262 - val_accuracy: 0.0835 - val_auc: 0.6552 - val_loss: 0.7733
Epoch 2/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 130s 107ms/step - accuracy: 0.5871 - auc: 0.8488 - loss: 0.4082 - val_accuracy: 0.1077 - val_auc: 0.7388 - val_loss: 0.6950
Epoch 3/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 132s 108ms/step - accuracy: 0.6303 - auc: 0.8873 - loss: 0.3547 - val_accuracy: 0.1834 - val_auc: 0.7653 - val_loss: 0.6335
Epoch 4/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 131s 108ms/step - accuracy: 0.6649 - auc: 0.9071 - loss: 0.3231 - val_accuracy: 0.2929 - val_auc: 0.8477 - val_loss: 0.5051
Epoch 5/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 138s 114ms/step - accuracy: 0.6852 - auc: 0.9257 - loss: 0.2918 - val_accuracy: 0.2948 - val_auc: 0.8349 - val_loss: 0.5549
Epoch 6/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 136s 112ms/step - accuracy: 0.7113 - auc: 0.9428 - loss: 0.2566 - val_accuracy: 0.2412 - val_auc: 0.8585 - val_loss: 0.5178
Epoch 7/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 137s 112ms/step - accuracy: 0.7391 - auc: 0.9582 - loss: 0.2194 - val_accuracy: 0.2738 - val_auc: 0.8664 - val_loss: 0.5023
Epoch 8/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 139s 114ms/step - accuracy: 0.7811 - auc: 0.9724 - loss: 0.1766 - val_accuracy: 0.3182 - val_auc: 0.8986 - val_loss: 0.5030
Epoch 9/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 140s 115ms/step - accuracy: 0.7945 - auc: 0.9801 - loss: 0.1496 - val_accuracy: 0.4111 - val_auc: 0.9048 - val_loss: 0.4459
Epoch 10/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 141s 116ms/step - accuracy: 0.8095 - auc: 0.9859 - loss: 0.1250 - val_accuracy: 0.4154 - val_auc: 0.9154 - val_loss: 0.4628
Epoch 11/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 142s 117ms/step - accuracy: 0.8265 - auc: 0.9897 - loss: 0.1059 - val_accuracy: 0.3885 - val_auc: 0.9217 - val_loss: 0.4558
Epoch 12/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 144s 118ms/step - accuracy: 0.8257 - auc: 0.9908 - loss: 0.0969 - val_accuracy: 0.4162 - val_auc: 0.9273 - val_loss: 0.4853
Epoch 13/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 145s 119ms/step - accuracy: 0.8283 - auc: 0.9919 - loss: 0.0899 - val_accuracy: 0.4377 - val_auc: 0.9193 - val_loss: 0.5066
Epoch 14/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 150s 123ms/step - accuracy: 0.8404 - auc: 0.9931 - loss: 0.0809 - val_accuracy: 0.4658 - val_auc: 0.9255 - val_loss: 0.4271
Epoch 15/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 147s 121ms/step - accuracy: 0.8381 - auc: 0.9939 - loss: 0.0762 - val_accuracy: 0.5118 - val_auc: 0.9471 - val_loss: 0.3889
Epoch 16/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 149s 122ms/step - accuracy: 0.8368 - auc: 0.9939 - loss: 0.0722 - val_accuracy: 0.4191 - val_auc: 0.9318 - val_loss: 0.4579
Epoch 17/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 148s 122ms/step - accuracy: 0.8419 - auc: 0.9950 - loss: 0.0675 - val_accuracy: 0.4084 - val_auc: 0.9261 - val_loss: 0.4971
Epoch 18/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 150s 123ms/step - accuracy: 0.8458 - auc: 0.9955 - loss: 0.0602 - val_accuracy: 0.4814 - val_auc: 0.9276 - val_loss: 0.4872
Epoch 19/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 151s 124ms/step - accuracy: 0.8457 - auc: 0.9957 - loss: 0.0580 - val_accuracy: 0.3766 - val_auc: 0.9282 - val_loss: 0.4954
Epoch 20/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 150s 123ms/step - accuracy: 0.8461 - auc: 0.9966 - loss: 0.0519 - val_accuracy: 0.5579 - val_auc: 0.9437 - val_loss: 0.4725
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 29ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 149s 121ms/step - accuracy: 0.4570 - auc: 0.7301 - loss: 0.7145 - val_accuracy: 0.1253 - val_auc: 0.6600 - val_loss: 0.7096
Epoch 2/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 149s 122ms/step - accuracy: 0.5764 - auc: 0.8510 - loss: 0.4042 - val_accuracy: 0.1151 - val_auc: 0.7669 - val_loss: 0.6395
Epoch 3/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 148s 121ms/step - accuracy: 0.6252 - auc: 0.8841 - loss: 0.3593 - val_accuracy: 0.1760 - val_auc: 0.7602 - val_loss: 0.6257
Epoch 4/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 149s 122ms/step - accuracy: 0.6606 - auc: 0.9080 - loss: 0.3201 - val_accuracy: 0.1858 - val_auc: 0.7584 - val_loss: 0.7116
Epoch 5/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 150s 123ms/step - accuracy: 0.6948 - auc: 0.9282 - loss: 0.2846 - val_accuracy: 0.2392 - val_auc: 0.8293 - val_loss: 0.5536
Epoch 6/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 144s 118ms/step - accuracy: 0.7228 - auc: 0.9453 - loss: 0.2490 - val_accuracy: 0.3085 - val_auc: 0.8655 - val_loss: 0.5237
Epoch 7/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 139s 114ms/step - accuracy: 0.7542 - auc: 0.9618 - loss: 0.2088 - val_accuracy: 0.3655 - val_auc: 0.9041 - val_loss: 0.4088
Epoch 8/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 140s 115ms/step - accuracy: 0.7885 - auc: 0.9740 - loss: 0.1716 - val_accuracy: 0.4429 - val_auc: 0.9304 - val_loss: 0.3884
Epoch 9/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 143s 117ms/step - accuracy: 0.8046 - auc: 0.9831 - loss: 0.1382 - val_accuracy: 0.3471 - val_auc: 0.9186 - val_loss: 0.3874
Epoch 10/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 153s 125ms/step - accuracy: 0.8167 - auc: 0.9865 - loss: 0.1226 - val_accuracy: 0.4732 - val_auc: 0.9192 - val_loss: 0.4378
Epoch 11/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 152s 124ms/step - accuracy: 0.8312 - auc: 0.9905 - loss: 0.1008 - val_accuracy: 0.4289 - val_auc: 0.9223 - val_loss: 0.4638
Epoch 12/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 150s 123ms/step - accuracy: 0.8275 - auc: 0.9913 - loss: 0.0946 - val_accuracy: 0.3555 - val_auc: 0.9002 - val_loss: 0.6361
Epoch 13/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 154s 126ms/step - accuracy: 0.8299 - auc: 0.9919 - loss: 0.0913 - val_accuracy: 0.3042 - val_auc: 0.8988 - val_loss: 0.5908
Epoch 14/30
1219/1219 ━━━━━━━━━━━━━━━━━━━━ 146s 120ms/step - accuracy: 0.8370 - auc: 0.9937 - loss: 0.0775 - val_accuracy: 0.3534 - val_auc: 0.9293 - val_loss: 0.4796
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 28ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 98s 159ms/step - accuracy: 0.4644 - auc: 0.7396 - loss: 0.6711 - val_accuracy: 0.1067 - val_auc: 0.6759 - val_loss: 0.7047
Epoch 2/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 102s 167ms/step - accuracy: 0.5704 - auc: 0.8423 - loss: 0.4190 - val_accuracy: 0.2433 - val_auc: 0.6771 - val_loss: 0.7193
Epoch 3/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 108s 177ms/step - accuracy: 0.6328 - auc: 0.8847 - loss: 0.3605 - val_accuracy: 0.1623 - val_auc: 0.7612 - val_loss: 0.6393
Epoch 4/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 105s 172ms/step - accuracy: 0.6631 - auc: 0.9089 - loss: 0.3203 - val_accuracy: 0.2347 - val_auc: 0.8086 - val_loss: 0.5636
Epoch 5/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 103s 169ms/step - accuracy: 0.6966 - auc: 0.9294 - loss: 0.2829 - val_accuracy: 0.1887 - val_auc: 0.8429 - val_loss: 0.5269
Epoch 6/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 103s 168ms/step - accuracy: 0.7209 - auc: 0.9463 - loss: 0.2472 - val_accuracy: 0.2148 - val_auc: 0.8673 - val_loss: 0.5004
Epoch 7/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 103s 169ms/step - accuracy: 0.7652 - auc: 0.9621 - loss: 0.2089 - val_accuracy: 0.2408 - val_auc: 0.8758 - val_loss: 0.4796
Epoch 8/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 163ms/step - accuracy: 0.7852 - auc: 0.9716 - loss: 0.1802 - val_accuracy: 0.3370 - val_auc: 0.8801 - val_loss: 0.5384
Epoch 9/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 102s 168ms/step - accuracy: 0.8076 - auc: 0.9801 - loss: 0.1499 - val_accuracy: 0.3446 - val_auc: 0.9129 - val_loss: 0.4318
Epoch 10/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 101s 166ms/step - accuracy: 0.8090 - auc: 0.9855 - loss: 0.1273 - val_accuracy: 0.2946 - val_auc: 0.8942 - val_loss: 0.5017
Epoch 11/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 165ms/step - accuracy: 0.8268 - auc: 0.9889 - loss: 0.1093 - val_accuracy: 0.2751 - val_auc: 0.8839 - val_loss: 0.6026
Epoch 12/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 101s 165ms/step - accuracy: 0.8325 - auc: 0.9911 - loss: 0.0991 - val_accuracy: 0.4295 - val_auc: 0.9196 - val_loss: 0.4277
Epoch 13/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 102s 167ms/step - accuracy: 0.8407 - auc: 0.9914 - loss: 0.0920 - val_accuracy: 0.3945 - val_auc: 0.9357 - val_loss: 0.3980
Epoch 14/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 101s 165ms/step - accuracy: 0.8378 - auc: 0.9927 - loss: 0.0854 - val_accuracy: 0.3856 - val_auc: 0.9224 - val_loss: 0.4473
Epoch 15/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 164ms/step - accuracy: 0.8451 - auc: 0.9945 - loss: 0.0747 - val_accuracy: 0.3906 - val_auc: 0.9386 - val_loss: 0.3787
Epoch 16/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 164ms/step - accuracy: 0.8463 - auc: 0.9940 - loss: 0.0760 - val_accuracy: 0.3147 - val_auc: 0.9102 - val_loss: 0.5142
Epoch 17/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 164ms/step - accuracy: 0.8474 - auc: 0.9952 - loss: 0.0660 - val_accuracy: 0.2730 - val_auc: 0.9095 - val_loss: 0.5698
Epoch 18/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 163ms/step - accuracy: 0.8533 - auc: 0.9942 - loss: 0.0704 - val_accuracy: 0.3707 - val_auc: 0.9392 - val_loss: 0.4039
Epoch 19/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 102s 166ms/step - accuracy: 0.8480 - auc: 0.9953 - loss: 0.0628 - val_accuracy: 0.3538 - val_auc: 0.9441 - val_loss: 0.3797
Epoch 20/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 101s 165ms/step - accuracy: 0.8621 - auc: 0.9960 - loss: 0.0555 - val_accuracy: 0.4226 - val_auc: 0.9336 - val_loss: 0.4246
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 28ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 98s 159ms/step - accuracy: 0.4632 - auc: 0.7395 - loss: 0.6674 - val_accuracy: 0.3028 - val_auc: 0.6722 - val_loss: 0.6825
Epoch 2/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 100s 163ms/step - accuracy: 0.5921 - auc: 0.8524 - loss: 0.4060 - val_accuracy: 0.2033 - val_auc: 0.7416 - val_loss: 0.6260
Epoch 3/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 98s 161ms/step - accuracy: 0.6314 - auc: 0.8822 - loss: 0.3622 - val_accuracy: 0.1828 - val_auc: 0.7456 - val_loss: 0.6776
Epoch 4/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 97s 159ms/step - accuracy: 0.6618 - auc: 0.9093 - loss: 0.3208 - val_accuracy: 0.1368 - val_auc: 0.7923 - val_loss: 0.6119
Epoch 5/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 97s 159ms/step - accuracy: 0.6989 - auc: 0.9259 - loss: 0.2892 - val_accuracy: 0.2421 - val_auc: 0.8454 - val_loss: 0.5062
Epoch 6/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 96s 157ms/step - accuracy: 0.7194 - auc: 0.9424 - loss: 0.2564 - val_accuracy: 0.2882 - val_auc: 0.8532 - val_loss: 0.4922
Epoch 7/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 94s 155ms/step - accuracy: 0.7520 - auc: 0.9583 - loss: 0.2188 - val_accuracy: 0.2786 - val_auc: 0.8626 - val_loss: 0.5090
Epoch 8/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 95s 156ms/step - accuracy: 0.7800 - auc: 0.9707 - loss: 0.1835 - val_accuracy: 0.2835 - val_auc: 0.8933 - val_loss: 0.4725
Epoch 9/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 95s 155ms/step - accuracy: 0.7958 - auc: 0.9793 - loss: 0.1548 - val_accuracy: 0.3387 - val_auc: 0.9023 - val_loss: 0.4812
Epoch 10/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 95s 156ms/step - accuracy: 0.8201 - auc: 0.9846 - loss: 0.1304 - val_accuracy: 0.3446 - val_auc: 0.8985 - val_loss: 0.5176
Epoch 11/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 93s 152ms/step - accuracy: 0.8243 - auc: 0.9876 - loss: 0.1149 - val_accuracy: 0.3475 - val_auc: 0.9028 - val_loss: 0.4972
Epoch 12/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 103s 169ms/step - accuracy: 0.8327 - auc: 0.9899 - loss: 0.1034 - val_accuracy: 0.3376 - val_auc: 0.9090 - val_loss: 0.5518
Epoch 13/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 95s 155ms/step - accuracy: 0.8346 - auc: 0.9921 - loss: 0.0898 - val_accuracy: 0.3947 - val_auc: 0.9286 - val_loss: 0.4000
Epoch 14/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 89s 145ms/step - accuracy: 0.8471 - auc: 0.9930 - loss: 0.0838 - val_accuracy: 0.3965 - val_auc: 0.9303 - val_loss: 0.4388
Epoch 15/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 149ms/step - accuracy: 0.8491 - auc: 0.9934 - loss: 0.0775 - val_accuracy: 0.3532 - val_auc: 0.9082 - val_loss: 0.5488
Epoch 16/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 149ms/step - accuracy: 0.8543 - auc: 0.9946 - loss: 0.0705 - val_accuracy: 0.4080 - val_auc: 0.9367 - val_loss: 0.4306
Epoch 17/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 97s 159ms/step - accuracy: 0.8538 - auc: 0.9951 - loss: 0.0675 - val_accuracy: 0.3514 - val_auc: 0.9300 - val_loss: 0.4681
Epoch 18/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 94s 154ms/step - accuracy: 0.8525 - auc: 0.9959 - loss: 0.0589 - val_accuracy: 0.3186 - val_auc: 0.9172 - val_loss: 0.5317
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 27ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 94s 152ms/step - accuracy: 0.4683 - auc: 0.7423 - loss: 0.6813 - val_accuracy: 0.1159 - val_auc: 0.6298 - val_loss: 0.8712
Epoch 2/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 150ms/step - accuracy: 0.5738 - auc: 0.8461 - loss: 0.4139 - val_accuracy: 0.1629 - val_auc: 0.7072 - val_loss: 0.6946
Epoch 3/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 149ms/step - accuracy: 0.6382 - auc: 0.8867 - loss: 0.3561 - val_accuracy: 0.1216 - val_auc: 0.7221 - val_loss: 0.6724
Epoch 4/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 94s 153ms/step - accuracy: 0.6661 - auc: 0.9119 - loss: 0.3156 - val_accuracy: 0.1965 - val_auc: 0.7870 - val_loss: 0.5788
Epoch 5/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 150ms/step - accuracy: 0.6955 - auc: 0.9299 - loss: 0.2825 - val_accuracy: 0.1879 - val_auc: 0.8128 - val_loss: 0.5846
Epoch 6/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 148ms/step - accuracy: 0.7221 - auc: 0.9457 - loss: 0.2492 - val_accuracy: 0.2322 - val_auc: 0.8302 - val_loss: 0.5745
Epoch 7/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 89s 146ms/step - accuracy: 0.7524 - auc: 0.9585 - loss: 0.2182 - val_accuracy: 0.2798 - val_auc: 0.8724 - val_loss: 0.5034
Epoch 8/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 148ms/step - accuracy: 0.7842 - auc: 0.9724 - loss: 0.1778 - val_accuracy: 0.3163 - val_auc: 0.8939 - val_loss: 0.4542
Epoch 9/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 148ms/step - accuracy: 0.8019 - auc: 0.9811 - loss: 0.1471 - val_accuracy: 0.4445 - val_auc: 0.9200 - val_loss: 0.3930
Epoch 10/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 147ms/step - accuracy: 0.8266 - auc: 0.9866 - loss: 0.1226 - val_accuracy: 0.3522 - val_auc: 0.8924 - val_loss: 0.4985
Epoch 11/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 149ms/step - accuracy: 0.8212 - auc: 0.9884 - loss: 0.1135 - val_accuracy: 0.4907 - val_auc: 0.9273 - val_loss: 0.4025
Epoch 12/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 148ms/step - accuracy: 0.8294 - auc: 0.9902 - loss: 0.1018 - val_accuracy: 0.4562 - val_auc: 0.9396 - val_loss: 0.3693
Epoch 13/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 148ms/step - accuracy: 0.8339 - auc: 0.9920 - loss: 0.0914 - val_accuracy: 0.4361 - val_auc: 0.9168 - val_loss: 0.4692
Epoch 14/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 91s 149ms/step - accuracy: 0.8439 - auc: 0.9931 - loss: 0.0832 - val_accuracy: 0.4995 - val_auc: 0.9301 - val_loss: 0.3904
Epoch 15/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 89s 147ms/step - accuracy: 0.8497 - auc: 0.9939 - loss: 0.0741 - val_accuracy: 0.4552 - val_auc: 0.9153 - val_loss: 0.5176
Epoch 16/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 148ms/step - accuracy: 0.8483 - auc: 0.9947 - loss: 0.0685 - val_accuracy: 0.4076 - val_auc: 0.9151 - val_loss: 0.4999
Epoch 17/30
610/610 ━━━━━━━━━━━━━━━━━━━━ 90s 147ms/step - accuracy: 0.8403 - auc: 0.9952 - loss: 0.0669 - val_accuracy: 0.3840 - val_auc: 0.9300 - val_loss: 0.4385
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 26ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 232ms/step - accuracy: 0.4739 - auc: 0.7409 - loss: 0.6578 - val_accuracy: 0.1938 - val_auc: 0.6582 - val_loss: 0.7575
Epoch 2/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 237ms/step - accuracy: 0.5803 - auc: 0.8475 - loss: 0.4119 - val_accuracy: 0.1405 - val_auc: 0.7904 - val_loss: 0.5554
Epoch 3/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.6207 - auc: 0.8793 - loss: 0.3682 - val_accuracy: 0.2316 - val_auc: 0.7968 - val_loss: 0.5805
Epoch 4/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 239ms/step - accuracy: 0.6674 - auc: 0.9042 - loss: 0.3282 - val_accuracy: 0.1764 - val_auc: 0.8071 - val_loss: 0.5519
Epoch 5/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.6835 - auc: 0.9196 - loss: 0.3011 - val_accuracy: 0.2269 - val_auc: 0.7986 - val_loss: 0.6061
Epoch 6/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 237ms/step - accuracy: 0.7178 - auc: 0.9402 - loss: 0.2615 - val_accuracy: 0.2232 - val_auc: 0.8341 - val_loss: 0.5497
Epoch 7/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.7465 - auc: 0.9553 - loss: 0.2270 - val_accuracy: 0.2650 - val_auc: 0.8628 - val_loss: 0.5088
Epoch 8/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.7706 - auc: 0.9667 - loss: 0.1954 - val_accuracy: 0.3516 - val_auc: 0.9056 - val_loss: 0.4121
Epoch 9/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 233ms/step - accuracy: 0.7872 - auc: 0.9743 - loss: 0.1714 - val_accuracy: 0.3461 - val_auc: 0.8767 - val_loss: 0.5111
Epoch 10/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.8059 - auc: 0.9811 - loss: 0.1459 - val_accuracy: 0.4652 - val_auc: 0.9247 - val_loss: 0.3780
Epoch 11/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.8263 - auc: 0.9848 - loss: 0.1294 - val_accuracy: 0.3971 - val_auc: 0.8989 - val_loss: 0.4778
Epoch 12/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.8230 - auc: 0.9874 - loss: 0.1182 - val_accuracy: 0.2792 - val_auc: 0.8972 - val_loss: 0.5240
Epoch 13/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.8279 - auc: 0.9891 - loss: 0.1080 - val_accuracy: 0.3840 - val_auc: 0.9189 - val_loss: 0.4167
Epoch 14/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.8419 - auc: 0.9914 - loss: 0.0935 - val_accuracy: 0.3614 - val_auc: 0.9098 - val_loss: 0.4407
Epoch 15/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 238ms/step - accuracy: 0.8396 - auc: 0.9926 - loss: 0.0891 - val_accuracy: 0.3893 - val_auc: 0.9259 - val_loss: 0.4181
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 26ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 74s 237ms/step - accuracy: 0.4699 - auc: 0.7444 - loss: 0.6618 - val_accuracy: 0.2035 - val_auc: 0.5742 - val_loss: 0.8886
Epoch 2/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 237ms/step - accuracy: 0.5836 - auc: 0.8478 - loss: 0.4153 - val_accuracy: 0.1889 - val_auc: 0.7170 - val_loss: 0.6842
Epoch 3/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 240ms/step - accuracy: 0.6223 - auc: 0.8772 - loss: 0.3711 - val_accuracy: 0.1982 - val_auc: 0.7629 - val_loss: 0.6011
Epoch 4/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.6605 - auc: 0.9016 - loss: 0.3330 - val_accuracy: 0.1717 - val_auc: 0.7692 - val_loss: 0.6464
Epoch 5/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.6854 - auc: 0.9226 - loss: 0.2949 - val_accuracy: 0.2517 - val_auc: 0.8501 - val_loss: 0.4828
Epoch 6/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.7185 - auc: 0.9405 - loss: 0.2602 - val_accuracy: 0.3058 - val_auc: 0.8576 - val_loss: 0.4932
Epoch 7/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.7489 - auc: 0.9566 - loss: 0.2211 - val_accuracy: 0.2587 - val_auc: 0.8663 - val_loss: 0.4777
Epoch 8/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 231ms/step - accuracy: 0.7697 - auc: 0.9674 - loss: 0.1929 - val_accuracy: 0.3979 - val_auc: 0.9035 - val_loss: 0.4073
Epoch 9/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 235ms/step - accuracy: 0.7903 - auc: 0.9767 - loss: 0.1629 - val_accuracy: 0.2976 - val_auc: 0.8823 - val_loss: 0.4950
Epoch 10/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 239ms/step - accuracy: 0.8096 - auc: 0.9808 - loss: 0.1471 - val_accuracy: 0.3485 - val_auc: 0.8966 - val_loss: 0.4579
Epoch 11/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 235ms/step - accuracy: 0.8238 - auc: 0.9854 - loss: 0.1268 - val_accuracy: 0.3399 - val_auc: 0.9052 - val_loss: 0.4545
Epoch 12/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 239ms/step - accuracy: 0.8231 - auc: 0.9872 - loss: 0.1182 - val_accuracy: 0.4283 - val_auc: 0.9301 - val_loss: 0.3609
Epoch 13/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 235ms/step - accuracy: 0.8291 - auc: 0.9898 - loss: 0.1039 - val_accuracy: 0.3128 - val_auc: 0.9104 - val_loss: 0.4713
Epoch 14/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 237ms/step - accuracy: 0.8311 - auc: 0.9917 - loss: 0.0930 - val_accuracy: 0.3024 - val_auc: 0.8962 - val_loss: 0.5475
Epoch 15/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 233ms/step - accuracy: 0.8324 - auc: 0.9921 - loss: 0.0920 - val_accuracy: 0.3477 - val_auc: 0.9171 - val_loss: 0.4593
Epoch 16/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 234ms/step - accuracy: 0.8396 - auc: 0.9932 - loss: 0.0823 - val_accuracy: 0.3536 - val_auc: 0.9258 - val_loss: 0.4289
Epoch 17/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 73s 238ms/step - accuracy: 0.8407 - auc: 0.9939 - loss: 0.0765 - val_accuracy: 0.3811 - val_auc: 0.9255 - val_loss: 0.4459
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 26ms/step

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 230ms/step - accuracy: 0.4818 - auc: 0.7513 - loss: 0.6699 - val_accuracy: 0.1826 - val_auc: 0.6273 - val_loss: 0.8247
Epoch 2/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.5868 - auc: 0.8550 - loss: 0.4054 - val_accuracy: 0.1871 - val_auc: 0.7650 - val_loss: 0.6027
Epoch 3/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 234ms/step - accuracy: 0.6250 - auc: 0.8836 - loss: 0.3632 - val_accuracy: 0.1354 - val_auc: 0.7812 - val_loss: 0.5976
Epoch 4/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 233ms/step - accuracy: 0.6592 - auc: 0.9099 - loss: 0.3206 - val_accuracy: 0.1748 - val_auc: 0.8104 - val_loss: 0.5552
Epoch 5/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 72s 236ms/step - accuracy: 0.6965 - auc: 0.9272 - loss: 0.2884 - val_accuracy: 0.2238 - val_auc: 0.8340 - val_loss: 0.5519
Epoch 6/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.7193 - auc: 0.9439 - loss: 0.2538 - val_accuracy: 0.1951 - val_auc: 0.8561 - val_loss: 0.5214
Epoch 7/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.7440 - auc: 0.9577 - loss: 0.2206 - val_accuracy: 0.3147 - val_auc: 0.8577 - val_loss: 0.5261
Epoch 8/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 231ms/step - accuracy: 0.7678 - auc: 0.9685 - loss: 0.1904 - val_accuracy: 0.4152 - val_auc: 0.9044 - val_loss: 0.4280
Epoch 9/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.7903 - auc: 0.9765 - loss: 0.1652 - val_accuracy: 0.3102 - val_auc: 0.8964 - val_loss: 0.4414
Epoch 10/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.8169 - auc: 0.9835 - loss: 0.1360 - val_accuracy: 0.3850 - val_auc: 0.9065 - val_loss: 0.4655
Epoch 11/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 230ms/step - accuracy: 0.8169 - auc: 0.9869 - loss: 0.1204 - val_accuracy: 0.3914 - val_auc: 0.9160 - val_loss: 0.4364
Epoch 12/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 231ms/step - accuracy: 0.8325 - auc: 0.9891 - loss: 0.1081 - val_accuracy: 0.3553 - val_auc: 0.9198 - val_loss: 0.4487
Epoch 13/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.8387 - auc: 0.9905 - loss: 0.1005 - val_accuracy: 0.3432 - val_auc: 0.9138 - val_loss: 0.4541
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 25ms/step

Plot results of model comparison

plot_model_comparisons(models_results)

From these results we can select the model with batch size 64 and noise level 0.001 and shift factor 20. While the accuracy is not the highest, it performs best on the trickiest diagnosis (HYP) and has an overall balanced performance.

First, we need to run the model again as we cleared the memory after each training run.

# redefine function to return model and results
def train_and_save_model(X_train, y_train, model, model_name, threshold=False, augment=False, 
                           noise_level=0.01, shift_amount=100, batch_size=32):
    # Compile model
    model.compile(optimizer='adam',
                 loss='binary_crossentropy',
                 metrics=['accuracy', AUC()])


    if augment:
        X_train_aug, y_train_binary = augment_hyp_samples(X_train, y_train, noise_level, shift_amount)

        X_train_normalized = (X_train_aug - np.mean(X_train_aug)) / np.std(X_train_aug)
        X_test_normalized = (X_test - np.mean(X_test)) / np.std(X_test)

    # Train model
    history = model.fit(X_train_normalized, y_train_binary,
                       validation_split=0.2,
                       epochs=30,
                       batch_size=batch_size,
                       callbacks=[EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)])
    
    # Get predictions
    y_pred_proba = model.predict(X_test_normalized)

    if threshold:
        # Use optimal thresholds
        y_pred_binary, thresholds, f1_scores = evaluate_with_optimal_thresholds(
            y_test_binary, y_pred_proba
        )
        extra_results = {'optimal_thresholds': thresholds}
        
    else:
        # Use default threshold (0.5)
        y_pred_binary = (y_pred_proba > 0.5).astype(int)
        f1_scores = [f1_score(y_test_binary[:, i], y_pred_binary[:, i]) 
                    for i in range(len(mlb.classes_))]
        extra_results = {}

    # Calculate common metrics
    results = {
        'model_name': model_name,
        'accuracy': accuracy_score(y_test_binary, y_pred_binary),
        'auc_scores': [roc_auc_score(y_test_binary[:, i], y_pred_proba[:, i]) 
                    for i in range(len(mlb.classes_))],
        'f1_scores': f1_scores,
        'history': history.history,
        **extra_results
    }

    return model, results

Save model

best_model, results = train_and_save_model(
    X_train, 
    y_train_binary, 
    create_enhanced_model(), 
    "Optimized model", 
    threshold=True, 
    augment=True, 
    noise_level=0.001, 
    shift_amount=20, 
    batch_size=64
)

/Users/scc/miniconda3/envs/ml-med/lib/python3.11/site-packages/keras/src/layers/convolutional/base_conv.py:107: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)

Epoch 1/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 222ms/step - accuracy: 0.4687 - auc: 0.7436 - loss: 0.6428 - val_accuracy: 0.1370 - val_auc: 0.6335 - val_loss: 0.7825
Epoch 2/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 233ms/step - accuracy: 0.5898 - auc: 0.8514 - loss: 0.4098 - val_accuracy: 0.1778 - val_auc: 0.6770 - val_loss: 0.7126
Epoch 3/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.6174 - auc: 0.8769 - loss: 0.3727 - val_accuracy: 0.1977 - val_auc: 0.7343 - val_loss: 0.6409
Epoch 4/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 229ms/step - accuracy: 0.6626 - auc: 0.9014 - loss: 0.3325 - val_accuracy: 0.2039 - val_auc: 0.7688 - val_loss: 0.6143
Epoch 5/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 228ms/step - accuracy: 0.6783 - auc: 0.9194 - loss: 0.3016 - val_accuracy: 0.1873 - val_auc: 0.7412 - val_loss: 0.7343
Epoch 6/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 227ms/step - accuracy: 0.7101 - auc: 0.9372 - loss: 0.2677 - val_accuracy: 0.1852 - val_auc: 0.8276 - val_loss: 0.5622
Epoch 7/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 228ms/step - accuracy: 0.7354 - auc: 0.9520 - loss: 0.2351 - val_accuracy: 0.2978 - val_auc: 0.8635 - val_loss: 0.4960
Epoch 8/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 229ms/step - accuracy: 0.7612 - auc: 0.9624 - loss: 0.2077 - val_accuracy: 0.2966 - val_auc: 0.8431 - val_loss: 0.5565
Epoch 9/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.7881 - auc: 0.9732 - loss: 0.1756 - val_accuracy: 0.2706 - val_auc: 0.8459 - val_loss: 0.5744
Epoch 10/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 225ms/step - accuracy: 0.7950 - auc: 0.9784 - loss: 0.1562 - val_accuracy: 0.4416 - val_auc: 0.9016 - val_loss: 0.4463
Epoch 11/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.8172 - auc: 0.9836 - loss: 0.1368 - val_accuracy: 0.4031 - val_auc: 0.9034 - val_loss: 0.4552
Epoch 12/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.8168 - auc: 0.9854 - loss: 0.1285 - val_accuracy: 0.3143 - val_auc: 0.9085 - val_loss: 0.4528
Epoch 13/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.8340 - auc: 0.9887 - loss: 0.1107 - val_accuracy: 0.3397 - val_auc: 0.9057 - val_loss: 0.4654
Epoch 14/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 225ms/step - accuracy: 0.8338 - auc: 0.9906 - loss: 0.1011 - val_accuracy: 0.3514 - val_auc: 0.9067 - val_loss: 0.4867
Epoch 15/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 225ms/step - accuracy: 0.8383 - auc: 0.9911 - loss: 0.0973 - val_accuracy: 0.3485 - val_auc: 0.9265 - val_loss: 0.4284
Epoch 16/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 229ms/step - accuracy: 0.8390 - auc: 0.9927 - loss: 0.0873 - val_accuracy: 0.3852 - val_auc: 0.9252 - val_loss: 0.4490
Epoch 17/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 226ms/step - accuracy: 0.8448 - auc: 0.9936 - loss: 0.0801 - val_accuracy: 0.3573 - val_auc: 0.9047 - val_loss: 0.5397
Epoch 18/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 231ms/step - accuracy: 0.8488 - auc: 0.9935 - loss: 0.0793 - val_accuracy: 0.4092 - val_auc: 0.9395 - val_loss: 0.3873
Epoch 19/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 232ms/step - accuracy: 0.8433 - auc: 0.9947 - loss: 0.0714 - val_accuracy: 0.3891 - val_auc: 0.9232 - val_loss: 0.4859
Epoch 20/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 71s 233ms/step - accuracy: 0.8518 - auc: 0.9944 - loss: 0.0726 - val_accuracy: 0.3438 - val_auc: 0.9259 - val_loss: 0.4618
Epoch 21/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 230ms/step - accuracy: 0.8457 - auc: 0.9952 - loss: 0.0671 - val_accuracy: 0.3516 - val_auc: 0.9228 - val_loss: 0.5192
Epoch 22/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 69s 228ms/step - accuracy: 0.8566 - auc: 0.9960 - loss: 0.0602 - val_accuracy: 0.3368 - val_auc: 0.9302 - val_loss: 0.4405
Epoch 23/30
305/305 ━━━━━━━━━━━━━━━━━━━━ 70s 230ms/step - accuracy: 0.8460 - auc: 0.9961 - loss: 0.0596 - val_accuracy: 0.3813 - val_auc: 0.9334 - val_loss: 0.4530
69/69 ━━━━━━━━━━━━━━━━━━━━ 2s 24ms/step

best_model.save('ecg_classifier_v1.keras')

gc.collect()
tf.keras.backend.clear_session()