how use grid search with fit generator in keras

Question

i want to grid search the parameter of the model with fit_generator as input in keras

i find below code in stack overflow and change it

1- but i don't understand how give the fit_generator or flow_from_directory to fit function(last line in the code)

2- how can add early stop?

thanks

from __future__ import print_function

import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D
from keras.wrappers.scikit_learn import KerasClassifier
from keras import backend as K
from sklearn.grid_search import GridSearchCV
from tqdm import tqdm      # a nice pretty percentage bar for tasks. Thanks to viewer Daniel Bühler for this suggestion
import os                  # dealing with directories
import numpy as np         # dealing with arrays
from random import shuffle # mixing up or currently ordered data that might lead our network astray in training.




num_classes = 10

# input image dimensions
img_rows, img_cols = 28, 28


input_shape = (img_rows, img_cols, 1)



def make_model(dense_layer_sizes, filters, kernel_size, pool_size):
    '''Creates model comprised of 2 convolutional layers followed by dense layers
    dense_layer_sizes: List of layer sizes.
        This list has one number for each layer
    filters: Number of convolutional filters in each convolutional layer
    kernel_size: Convolutional kernel size
    pool_size: Size of pooling area for max pooling
    '''

    model = Sequential()
    model.add(Conv2D(filters, kernel_size,
                     padding='valid',
                     input_shape=input_shape))
    model.add(Activation('relu'))
    model.add(Conv2D(filters, kernel_size))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=pool_size))
    model.add(Dropout(0.25))

    model.add(Flatten())
    for layer_size in dense_layer_sizes:
        model.add(Dense(layer_size))
    model.add(Activation('relu'))
    model.add(Dropout(0.5))
    model.add(Dense(num_classes))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy',
                  optimizer='adadelta',
                  metrics=['accuracy'])

    return model



class KerasClassifier(KerasClassifier):
     """ adds sparse matrix handling using batch generator
     """

     def fit(self, x, y, **kwargs):
         """ adds sparse matrix handling """
         if not issparse(x):
             return super().fit(x, y, **kwargs)

         ############ adapted from KerasClassifier.fit   ######################   
         if self.build_fn is None:
             self.model = self.__call__(**self.filter_sk_params(self.__call__))
         elif not isinstance(self.build_fn, types.FunctionType):
             self.model = self.build_fn(
                 **self.filter_sk_params(self.build_fn.__call__))
         else:
             self.model = self.build_fn(**self.filter_sk_params(self.build_fn))

         loss_name = self.model.loss
         if hasattr(loss_name, '__name__'):
             loss_name = loss_name.__name__
         if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
             y = to_categorical(y)
         ### fit => fit_generator
         fit_args = copy.deepcopy(self.filter_sk_params(Sequential.fit_generator))
         fit_args.update(kwargs)
         ############################################################
         self.model.fit_generator(
                     self.get_batch(x, y, self.sk_params["batch_size"]),
                                         samples_per_epoch=x.shape[0],
                                         **fit_args)                      
         return self                               

     def get_batch(self, x, y=None, batch_size=32):
         """ batch generator to enable sparse input """
         index = np.arange(x.shape[0])
         start = 0
         while True:
             if start == 0 and y is not None:
                 np.random.shuffle(index)
             batch = index[start:start+batch_size]
             if y is not None:
                 yield x[batch].toarray(), y[batch]
             else:
                 yield x[batch].toarray()
             start += batch_size
             if start >= x.shape[0]:
                 start = 0

     def predict_proba(self, x):
         """ adds sparse matrix handling """
         if not issparse(x):
             return super().predict_proba(x)

         preds = self.model.predict_generator(
                     self.get_batch(x, None, self.sk_params["batch_size"]), 
                                                val_samples=x.shape[0])
         return preds


dense_size_candidates = [[32], [64], [32, 32], [64, 64]]
my_classifier = KerasClassifier(make_model, batch_size=32)

validator = GridSearchCV(my_classifier,
                         param_grid={'dense_layer_sizes': dense_size_candidates,
                                     # epochs is avail for tuning even when not
                                     # an argument to model building function
                                     'epochs': [3, 6],
                                     'filters': [8],
                                     'kernel_size': [3],
                                     'pool_size': [2]},
                         scoring='neg_log_loss',
                         n_jobs=1)




batch_size = 20
validation_datagen = ImageDataGenerator(rescale=1./255)
train_datagen = ImageDataGenerator(rescale=1./255)
test_datagen = ImageDataGenerator(rescale=1./255)
train_generator = train_datagen.flow_from_directory(
        'd:/train',  # this is the target directory
        target_size=(width, height),  # all images will be resized to 150x150
        batch_size=batch_size,
        color_mode= "grayscale",
        class_mode='binary',
        shuffle=True
      #  ,save_to_dir='preview', save_prefix='cat', save_format='png'
        )  # since we use binary_crossentropy loss, we need binary labels

# this is a similar generator, for validation data
validation_generator = validation_datagen.flow_from_directory(
        'd:/validation',
        target_size=(width, height),
        batch_size=batch_size,
        color_mode= "grayscale",
        class_mode='binary')


test_generator = test_datagen.flow_from_directory(
        'd:/test',
        target_size=(width, height),
        batch_size=batch_size,
        color_mode= "grayscale",
        class_mode='binary')

validator.fit(??????

Answer 1

I 'm using this implementation, I hope it could help you.

from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping, ModelCheckpoint, CSVLogger
from keras.wrappers.scikit_learn import KerasClassifier

import types


class KerasBatchClassifier(KerasClassifier):

    def fit(self, X, y, **kwargs):

        # taken from keras.wrappers.scikit_learn.KerasClassifier.fit ###################################################
        if self.build_fn is None:
            self.model = self.__call__(**self.filter_sk_params(self.__call__))
        elif not isinstance(self.build_fn, types.FunctionType) and not isinstance(self.build_fn, types.MethodType):
            self.model = self.build_fn(**self.filter_sk_params(self.build_fn.__call__))
        else:
            self.model = self.build_fn(**self.filter_sk_params(self.build_fn))

        loss_name = self.model.loss
        if hasattr(loss_name, '__name__'):
            loss_name = loss_name.__name__

        if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
            y = to_categorical(y)

        ################################################################################################################


        datagen = ImageDataGenerator(
            rotation_range=45,
            width_shift_range=0.2,
            height_shift_range=0.2,
            shear_range=0.2,
            zoom_range=0.2,
            horizontal_flip=True,
            fill_mode='nearest'
        )

        if 'X_val' in kwargs and 'y_val' in kwargs:
            X_val = kwargs['X_val']
            y_val = kwargs['y_val']

            val_gen = ImageDataGenerator(
                horizontal_flip=True
            )
            val_flow = val_gen.flow(X_val, y_val, batch_size=32)
            val_steps = len(X_val) / 32

            early_stopping = EarlyStopping( patience=5, verbose=5, mode="auto")
            model_checkpoint = ModelCheckpoint("results/best_weights.{epoch:02d}-{loss:.5f}.hdf5", verbose=5, save_best_only=True, mode="auto")
        else:
            val_flow = None
            val_steps = None
            early_stopping = EarlyStopping(monitor="acc", patience=3, verbose=5, mode="auto")
            model_checkpoint = ModelCheckpoint("results/best_weights.{epoch:02d}-{loss:.5f}.hdf5", monitor="acc", verbose=5, save_best_only=True, mode="auto")

        callbacks = [early_stopping, model_checkpoint]

        epochs = self.sk_params['epochs'] if 'epochs' in self.sk_params else 100

        self.__history = self.model.fit_generator(
            datagen.flow(X, y, batch_size=32),  
            steps_per_epoch=len(X) / 32,
            validation_data=val_flow, 
            validation_steps=val_steps, 
            epochs=epochs,
            callbacks=callbacks
        )

        return self.__history

    def score(self, X, y, **kwargs):
        kwargs = self.filter_sk_params(Sequential.evaluate, kwargs)

        loss_name = self.model.loss
        if hasattr(loss_name, '__name__'):
            loss_name = loss_name.__name__
        if loss_name == 'categorical_crossentropy' and len(y.shape) != 2:
            y = to_categorical(y)
        outputs = self.model.evaluate(X, y, **kwargs)
        if type(outputs) is not list:
            outputs = [outputs]
        for name, output in zip(self.model.metrics_names, outputs):
            if name == 'acc':
                return output
        raise Exception('The model is not configured to compute accuracy. '
                        'You should pass `metrics=["accuracy"]` to '
                        'the `model.compile()` method.')

    @property
    def history(self):
        return self.__history

As you can see, it's specific to images, but you can adapt it to your specific needs.

I'm using it like this:

from sklearn.model_selection import GridSearchCV

model = KerasBatchClassifier(build_fn=create_model, epochs=epochs)

learn_rate   = [0.001, 0.01, 0.1]
epsilon      = [None, 1e-2, 1e-3]
dropout_rate = [0.25, 0.5]

param_grid   = dict(learn_rate=learn_rate, epsilon=epsilon, dropout_rate=dropout_rate)

grid = GridSearchCV(estimator=model, param_grid=param_grid)

grid_result = grid.fit(X_train, Y_train, X_val = X_test, y_val = Y_test)

Answer 2

There is a class called ParameterGrid, which in GridSearchCV() makes all combinations of the parameters given for the grid search. You can store them in a list. For example:

from sklearn.model_selection import ParameterGrid

parameters = {'epochs': [32, 64, 128],
              'batch_size':[24, 32, 48, 64],
}

list(ParameterGrid(parameters))

prints out

[{'batch_size': 24, 'epochs': 32},
 {'batch_size': 24, 'epochs': 64},
 {'batch_size': 24, 'epochs': 128},
 {'batch_size': 32, 'epochs': 32},
 {'batch_size': 32, 'epochs': 64},
 {'batch_size': 32, 'epochs': 128},
 {'batch_size': 48, 'epochs': 32},
 {'batch_size': 48, 'epochs': 64},
 {'batch_size': 48, 'epochs': 128},
 {'batch_size': 64, 'epochs': 32},
 {'batch_size': 64, 'epochs': 64},
 {'batch_size': 64, 'epochs': 128}]

In a loop for this list, you can train your model with these specific combinations. At the end of every loop you can check for the val_acc and val_loss with other functions.