Keras model.predict() slower on first iteration then gets faster

Question

I am trying to run model.predict() in a for loop multiple times and timing how long it takes on the same image. The data will be used to take an average of the time taken to run the prediction.

If I run the prediction in a separate script, it will run in around 300ms seconds on my MacBook. If I then run it iteratively in a for loop, the time taken will start at around 300ms on the first iteration, then drop to 80ms for the remaining iterations.

Is it because the first prediction is remaining in memory and Keras is doing something behind the scenes to shorten prediction time?

Any idea why this might be happening? Code is here:

#!/usr/bin/env python3

import argparse
import keras
from keras.applications.imagenet_utils import decode_predictions
from keras.applications.inception_v3 import preprocess_input
import os
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2' # Suppress CPU warnings
import time
from timeit import default_timer as timer
import datetime
import csv
import numpy as np

"""Define all model permutations for MobileNetsV1 and MobileNetsV2"""
# Define all V1 model permutations
# V1_MODELS = [(128,0.25)]
V1_MODELS = [(128, 0.25), (128, 0.5), (128, 0.75), (128, 1)]#,
#              (160, 0.25), (160, 0.5), (160, 0.75), (160, 1),
#              (192, 0.25), (192, 0.5), (192, 0.75), (192, 1),
#              (224, 0.25), (224, 0.5), (224, 0.75), (224, 1)]
# Define all V2 model permutations
V2_MODELS = [(96, 0.35), (96, 0.5), (96, 0.75), (96, 1), (96, 1.3), (96, 1.4),
             (128, 0.35), (128, 0.5), (128, 0.75), (128, 1), (128, 1.3), (128, 1.4),
             (160, 0.35), (160, 0.5), (160, 0.75), (160, 1), (160, 1.3), (160, 1.4),
             (192, 0.35), (192, 0.5), (192, 0.75), (192, 1), (192, 1.3), (192, 1.4),
             (224, 0.35), (224, 0.5), (224, 0.75), (224, 1), (224, 1.3), (224, 1.4)]


def save_result(model, time):
    with open(RESULTS_FILE_NAME, 'a', newline='') as csvfile:
        csv_writer = csv.writer(csvfile)
        csv_writer.writerow([model, time])

    # file = open(RESULTS_FILE_NAME, 'a')
    # file.write(text + '\n')
    # file.close()

if __name__ == "__main__":
    # Set up command line argument parser
    parser = argparse.ArgumentParser()
    parser.add_argument('--image', type=str, help='Path to the image to be tested', default='images/cheetah.jpg')
    parser.add_argument('--model', type=int, help='Specify model architecture as an integer V1: 1, V2: 2', default=1)
    parser.add_argument('--test', type=int, help='Specify the number of tests per model to perform', default=5)
    args = parser.parse_args()

    RESULTS_FILE_NAME = "results/MobileNetV{0}_result_{1}.csv".format(args.model, datetime.datetime.now().strftime("%Y%m%d%H%M%S"))

    # Holds total run time (each individual model time added to this variable)
    total_time = 0

    # Select model parameter list based on command line arguments (default = V1)
    if args.model == 1:
        MODEL_LIST = V1_MODELS
    elif args.model == 2:
        MODEL_LIST = V2_MODELS

    for model_params in MODEL_LIST:
        size = model_params[0]
        alpha = model_params[1]
        # Select MobileNet model based on command line arguments (default = V1)
        if args.model == 1:
            model = keras.applications.mobilenet.MobileNet(input_shape=(size, size, 3),
                                                           alpha=alpha,
                                                           depth_multiplier=1,
                                                           dropout=1e-3,
                                                           include_top=True,
                                                           weights='imagenet',
                                                           input_tensor=None,
                                                           pooling=None,
                                                           classes=1000)
        elif args.model == 2:
            model = keras.applications.mobilenet_v2.MobileNetV2(input_shape=(size, size, 3),
                                                               alpha=1.0,
                                                               depth_multiplier=1,
                                                               include_top=True,
                                                               weights='imagenet',
                                                               input_tensor=None,
                                                               pooling=None,
                                                               classes=1000)



        # model.summary()
        for num in range(args.test):

            # Start timing
            start_time = timer()

            # Preprocess the image TODO: should this be included in timing?
            img = keras.preprocessing.image.load_img(args.image, target_size=(size, size))
            x = keras.preprocessing.image.img_to_array(img)
            x = np.expand_dims(x, axis=0)
            x = preprocess_input(x)

            # Predict the category of the input image
            predictions = model.predict(x, verbose=1)

            # Print predictions
            #print('Predicted:', decode_predictions(predictions, top=3))

            # End timing
            end_time = timer()

            # Print total run time
            print("Size: {0}  Alpha: {1}".format(size, alpha))
            print("Time Taken: {} seconds".format(end_time-start_time))
            # save_result(str(model_params), str(end_time-start_time))
            total_time = total_time + (end_time-start_time)

    print("######################")
    print("Total Time: {} seconds".format(total_time))

Answer 1

The predict function is made during the first (and only the first) call to predict or predict_on_batch. That's one reason the first call takes more time.

For details, see the source code. Particularly, note when _make_predict_function is called and how it works.