TensorFlow: SparseSoftmaxCrossEntropyWithLogits Error?

Question

I'm right now trying to follow the getting started guide on tensorflow, and I'm coming up to a brick wall. I don't see any only support for the error I'm seeing, and I'm confident that my code does not steer too far off from what they have on their website.

Code:

import tensorflow as tf;
import tensorflow.contrib.eager as tfe;

tf.enable_eager_execution();

iris_dataset_url = 'http://download.tensorflow.org/data/iris_training.csv';
iris_csv_file = tf.keras.utils.get_file('iris_dataset.csv', iris_dataset_url);

iris_dataset_tests_url = 'http://download.tensorflow.org/data/iris_test.csv';
iris_tests_csv_file = tf.keras.utils.get_file('iris_tests_dataset.csv', iris_dataset_tests_url);

def iris_data_parse_line(line):
    default_feature = [[0.0], [0.0], [0.0], [0.0], [0.0]];
    parsed_line = tf.decode_csv(line, default_feature);

    features = tf.reshape(parsed_line[:-1], shape=(4,), name="features");
    label = tf.reshape(parsed_line[-1], shape=(), name="label");

    return features, label;

def prediction_loss_diff(features, label, model):
    predicted_label = model(features);
    return tf.losses.sparse_softmax_cross_entropy(label, predicted_label);

def gradient_tune(features, targets, model):
    with tf.GradientTape() as tape:
        prediction_loss = prediction_loss_diff(features, targets, model);
    return tape.gradient(prediction_loss, model.variables);

def train_model(training_dataset, model, optimizer):
    train_loss_results = []
    train_accuracy_results = []
    rounds = 201;


    for round_num in range(rounds):
        epoch_loss_avg = tfe.metrics.Mean();
        epoch_accuracy = tfe.metrics.Accuracy();

        for features, label in training_dataset:
            gradients = gradient_tune(features, label, model);
            optimizer.apply_gradients(
                    zip(gradients, model.variables),
                    global_step=tf.train.get_or_create_global_step());



def main():
    print("TensorFlow version: {}".format(tf.VERSION));
    print("Eager execution: {}".format(tf.executing_eagerly()));

    iris_dataset = (tf.data.TextLineDataset(iris_csv_file)
                           .skip(1)
                           .map(iris_data_parse_line)
                           .shuffle(1000)
                           .batch(32));

    model = tf.keras.Sequential([
        tf.keras.layers.Dense(10, activation="relu", input_shape=(4,)),
        tf.keras.layers.Dense(10, activation="relu"),
        tf.keras.layers.Dense(3)
    ]);

    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01);

    train_model(iris_dataset, model, optimizer);

if __name__ == "__main__":
    main();

The error seems to be happening in the prediction_loss_diff function. It is suppose to use the sparse_softmax_cross_entropy loss function. However I keep getting the error:

Traceback (most recent call last):
  File "main.py", line 69, in <module>
    main();
  File "main.py", line 66, in main
    train_model(iris_dataset, model, optimizer);
  File "main.py", line 41, in train_model
    gradients = gradient_tune(features, label, model);
  File "main.py", line 27, in gradient_tune
    prediction_loss = prediction_loss_diff(features, targets, model);
  File "main.py", line 23, in prediction_loss_diff
    return tf.losses.sparse_softmax_cross_entropy(label, predicted_label);
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/losses/losses_impl.py", line 853, in sparse_softmax_cross_entropy
    name="xentropy")
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 2050, in sparse_softmax_cross_entropy_with_logits
    precise_logits, labels, name=name)
  File "/usr/local/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 7504, in sparse_softmax_cross_entropy_with_logits
    _six.raise_from(_core._status_to_exception(e.code, message), None)
  File "<string>", line 2, in raise_from
tensorflow.python.framework.errors_impl.InternalError: Could not find valid device for node name: "SparseSoftmaxCrossEntropyWithLogits"
op: "SparseSoftmaxCrossEntropyWithLogits"
input: "dummy_input"
input: "dummy_input"
attr {
  key: "T"
  value {
    type: DT_FLOAT
  }
}
attr {
  key: "Tlabels"
  value {
    type: DT_FLOAT
  }
}

I'm not sure what it means "it is not finding the valid node device" but I assume it might have to do with something in the lower C wrappings? Are my inputs bad? Any help would be great, thank you.

Answer 1

It was an input issue. Oddly the error message was not fully insightful to what exactly was conflicting. However, reviewing the code diff from the getting started guide and mine, I noticed that my default_feature variable was initialized with all floats. I needed my label to be an integer to correctly label the data. Since the output of the sparse_softmax_cross_entropy function is a label, that is an int32/64, it cannot be compared to a float. This will cause an exception error with the C bindings, and lead to the stated exception. More info on the sparse_softmax_cross_entropy losses function.

So, instead of [[0.0], [0.0], [0.0], [0.0], [0.0]]; it should be [[0.0], [0.0], [0.0], [0.0], [0]];

Final code:

import tensorflow as tf;
import tensorflow.contrib.eager as tfe;

tf.enable_eager_execution();

iris_dataset_url = 'http://download.tensorflow.org/data/iris_training.csv';
iris_csv_file = tf.keras.utils.get_file('iris_dataset.csv', iris_dataset_url);

iris_dataset_tests_url = 'http://download.tensorflow.org/data/iris_test.csv';
iris_tests_csv_file = tf.keras.utils.get_file('iris_tests_dataset.csv', iris_dataset_tests_url);

def iris_data_parse_line(line):
    default_feature = [[0.0], [0.0], [0.0], [0.0], [0]]; #UPDATED SPOT!!!
    parsed_line = tf.decode_csv(line, default_feature);

    features = tf.reshape(parsed_line[:-1], shape=(4,), name="features");
    label = tf.reshape(parsed_line[-1], shape=(), name="label");

    return features, label;

def prediction_loss_diff(features, label, model):
    predicted_label = model(features);
    return tf.losses.sparse_softmax_cross_entropy(label, predicted_label);

def gradient_tune(features, targets, model):
    with tf.GradientTape() as tape:
        prediction_loss = prediction_loss_diff(features, targets, model);
    return tape.gradient(prediction_loss, model.variables);

def train_model(training_dataset, model, optimizer):
    train_loss_results = []
    train_accuracy_results = []
    rounds = 201;


    for round_num in range(rounds):
        epoch_loss_avg = tfe.metrics.Mean();
        epoch_accuracy = tfe.metrics.Accuracy();

        for features, label in training_dataset:
            gradients = gradient_tune(features, label, model);
            optimizer.apply_gradients(
                    zip(gradients, model.variables),
                    global_step=tf.train.get_or_create_global_step());



def main():
    print("TensorFlow version: {}".format(tf.VERSION));
    print("Eager execution: {}".format(tf.executing_eagerly()));

    iris_dataset = (tf.data.TextLineDataset(iris_csv_file)
                           .skip(1)
                           .map(iris_data_parse_line)
                           .shuffle(1000)
                           .batch(32));

    model = tf.keras.Sequential([
        tf.keras.layers.Dense(10, activation="relu", input_shape=(4,)),
        tf.keras.layers.Dense(10, activation="relu"),
        tf.keras.layers.Dense(3)
    ]);

    optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01);

    train_model(iris_dataset, model, optimizer);

if __name__ == "__main__":
    main();