Why do my models keep getting exactly 0.5 AUC?

Question

I am currently doing a project in which I need to predict eye disease in a group of images. I am using the Keras built-in applications. I am getting good results on VGG16 and VGG19, but on the Xception architecture I keep getting AUC of exactly 0.5 every epoch.

I have tried different optimizers and learning rates, but nothing works. I solved the same problem with VGG19 by switching from RMSProp optimizer to Adam optimizer, but I can't get it to work for Xception.

   def buildModel():
    from keras.models import Model
    from keras.layers import Dense, Flatten
    from keras.optimizers import adam

    input_model = applications.xception.Xception(
        include_top=False,
        weights='imagenet',
        input_tensor=None,
        input_shape=input_sizes["xception"],
        pooling=None,
        classes=2)

    base_model = input_model

    x = base_model.output
    x = Flatten()(x)
    predictions = Dense(2, activation='softmax')(x)

    model = Model(inputs=base_model.input, outputs=predictions)
    for layer in base_model.layers:
        layer.trainable = False

    model.compile(optimizer=adam(lr=0.01), loss='binary_crossentropy', metrics=['accuracy'])

    return model


class Histories(keras.callbacks.Callback):

    def __init__(self, val_data):
        super(Histories, self).__init__()
        self.x_batch = []
        self.y_batch = []
        for i in range(len(val_data)):
            x, y = val_data.__getitem__(i)
            self.x_batch.extend(x)
            self.y_batch.extend(np.ndarray.astype(y, int))
        self.aucs = []
        self.specificity = []
        self.sensitivity = []
        self.losses = []
        return

    def on_train_begin(self, logs={}):
        initFile("results/xception_results_adam_3.txt")
        return

    def on_train_end(self, logs={}):
        return

    def on_epoch_begin(self, epoch, logs={}):
        return

    def on_epoch_end(self, epoch, logs={}):
        self.losses.append(logs.get('loss'))
        y_pred = self.model.predict(np.asarray(self.x_batch))
        con_mat = confusion_matrix(np.asarray(self.y_batch).argmax(axis=-1), y_pred.argmax(axis=-1))
        tn, fp, fn, tp = con_mat.ravel()
        sens = tp/(tp+fn)
        spec = tn/(tn+fp)
        auc_score = roc_auc_score(np.asarray(self.y_batch).argmax(axis=-1), y_pred.argmax(axis=-1))
        print("Specificity: %f Sensitivity: %f AUC: %f"%(spec, sens, auc_score))
        print(con_mat)
        self.sensitivity.append(sens)
        self.specificity.append(spec)
        self.aucs.append(auc_score)
        writeToFile("results/xception_results_adam_3.txt", epoch, auc_score, spec, sens, self.losses[epoch])
        return


# What follows is data from the Jupyter Notebook that I actually use to evaluate
#%% Initialize data
trainDirectory =      'RetinaMasks/train'
valDirectory =        'RetinaMasks/val'
testDirectory =       'RetinaMasks/test'

train_datagen = ImageDataGenerator(rescale=1. / 255)
test_datagen = ImageDataGenerator(rescale=1. / 255)

train_generator = train_datagen.flow_from_directory(
    trainDirectory,
    target_size=(299, 299),
    batch_size=16,
    class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(
    valDirectory,
    target_size=(299, 299),
    batch_size=16,
    class_mode='categorical')
    
test_generator = test_datagen.flow_from_directory(
    testDirectory,
    target_size=(299, 299),
    batch_size=16,
    class_mode='categorical')

#%% Create model
model = buildModel("xception")

#%% Initialize metrics
from keras.callbacks import EarlyStopping
from MetricsCallback import Histories
import keras

metrics = Histories(validation_generator)
es = EarlyStopping(monitor='val_loss', 
                   min_delta=0, 
                   patience=20,
                   verbose=0, 
                   mode='auto', 
                   baseline=None, 
                   restore_best_weights=False)

mcp = keras.callbacks.ModelCheckpoint("saved_models/xception.adam.lr0.1_{epoch:02d}.hdf5", 
                                      monitor='val_loss', 
                                      verbose=0, 
                                      save_best_only=False, 
                                      save_weights_only=False, 
                                      mode='auto',
                                      period=1)

#%% Train model
from StaticDataAugmenter import superDirectorySize

history = model.fit_generator(
    train_generator,
    steps_per_epoch=superDirectorySize(trainDirectory) // 16,
    epochs=100,
    validation_data=validation_generator,
    validation_steps=superDirectorySize(valDirectory) // 16,
    callbacks=[metrics, es, mcp],
    workers=8,
    shuffle=False
)

What causes this behavior, and how to prevent it?

Answer 1

Your learning rate is too high. Try lowering the learning rate.

I used to run into this when using transfer learning, I was fine-tuning at very high learning rates.

An extended AUC of 0.5 over multiple epochs in case of a binary classification means that your (convolutional) neural network is not able to distinguish between the classes at all. This is in turn because it's not able to learn anything.

Use learning_rates of 0.0001,0.00001,0.000001.

At the same time, you should try to unfreeze/make some layers trainable, due to the fact that you entire feature extractor is frozen; in fact this could be another reason why the network is incapable of learning anything.

I am quite confident that your problem will be solved if you lower your learning rate :).