Defining an AUC metric for Keras to support evaluation of validation dataset

Question

I am trying to implement an AUC metric for Keras so that I have AUC measurement after my validation set runs during a model.fit() run.

I define the metric as such:

def auc(y_true, y_pred):   
    keras.backend.get_session().run(tf.global_variables_initializer())
    keras.backend.get_session().run(tf.initialize_all_variables())
    keras.backend.get_session().run(tf.initialize_local_variables())

    #return K.variable(value=tf.contrib.metrics.streaming_auc(
    #    y_pred, y_true)[0], dtype='float32')
    return tf.contrib.metrics.streaming_auc(y_pred, y_true)[0]

This results in the following error which I don't know understand.

tensorflow.python.framework.errors_impl.FailedPreconditionError: 
Attempting to use uninitialized value auc/true_positives...

From online reading, it seems that the problem is 2-fold, a bug in tensorflow/keras and partially and issue with tensorflow being unable to initialize local variables from inference. Given these 2 issues, I do not see why I get this error or how to overcome it. Any suggestions?

I wrote two other metrics that work just fine:

# PFA, prob false alert for binary classifier
def binary_PFA(y_true, y_pred, threshold=K.variable(value=0.5)):
    y_pred = K.cast(y_pred >= threshold, 'float32')
    # N = total number of negative labels
    N = K.sum(1 - y_true)
    # FP = total number of false alerts, alerts from the negative class labels
    FP = K.sum(y_pred - y_pred * y_true)    
    return FP/N

# P_TA prob true alerts for binary classifier
def binary_PTA(y_true, y_pred, threshold=K.variable(value=0.5)):
    y_pred = K.cast(y_pred >= threshold, 'float32')
    # P = total number of positive labels
    P = K.sum(y_true)
    # TP = total number of correct alerts, alerts from the positive class labels
    TP = K.sum(y_pred * y_true)    
    return TP/P

Answer 1

Here are the tricks that I often use. Basically, this allows you to use whatever existing metrics in sklearn

from sklearn.metrics import roc_auc_score
import tensorflow as tf
def auc( y_true, y_pred ) :
    score = tf.py_func( lambda y_true, y_pred : roc_auc_score( y_true, y_pred, average='macro', sample_weight=None).astype('float32'),
                        [y_true, y_pred],
                        'float32',
                        stateful=False,
                        name='sklearnAUC' )
    return score

Now we can create a simple model to verify this metric.

from keras.layers import Input
from keras.models import Model

x = Input(shape=(100,))
y = Dense(10, activation='sigmoid')(x)
model = Model(inputs=x, outputs=y)
model.compile( 'sgd', loss='binary_crossentropy', metrics=[auc] )
print model.summary()


a = np.random.randn(1000,100)
b = np.random.randint(low=0,high=2,size=(1000,10))
model.fit( a, b )