How to freeze weights in certain layer with Keras?

Question

I am trying to freeze the weights of certain layer in a prediction model with Keras and mnist dataset, but it does not work. The code is like:

from keras.layers import Dense, Flatten
from keras.utils import to_categorical
from keras.models import Sequential, load_model
from keras.datasets import mnist
from keras.losses import categorical_crossentropy

import numpy as np

def load_data():
    (x_train, y_train), (x_test, y_test) = mnist.load_data()
    x_train = x_train.astype('float32')
    x_test = x_test.astype('float32')
    x_train /= 255
    x_test /= 255
    y_train = to_categorical(y_train, num_classes=10)
    y_test = to_categorical(y_test, num_classes=10)
    return x_train, y_train, x_test, y_test


def run():
    x_train, y_train, x_test, y_test = load_data()
    model = Sequential([Flatten(input_shape=(28, 28)),
                        Dense(300, name='dense1', activation='relu'),
                        Dense(100, name='dense2', activation='relu'),
                        Dense(10, name='dense3', activation='softmax')])
    model.trainable = True
    model.compile(optimizer='Adam',
                  metrics=['accuracy'],
                  loss=categorical_crossentropy)

    print(model.summary())
    model.fit(x_train, y_train, epochs=5, verbose=2)
    print(model.evaluate(x_test, y_test))
    return model

def freeze(model):
    x_train, y_train, x_test, y_test = load_data()

    name = 'dense1'

    weightsAndBias = model.get_layer(name=name).get_weights()

    # freeze the weights of this layer
    model.get_layer(name=name).trainable = False

    # record the weights before retrain
    weights_before = weightsAndBias[0]
    # retrain
    model.fit(x_train, y_train, verbose=2, epochs=1)
    weights_after = model.get_layer(name=name).get_weights()[0]

    if (weights_before == weights_after).all():
        print('the weights did not change!!!')
    else:
        print('the weights changed!!!!')

if __name__ == '__main__':
    model = run()
    freeze(model)

The program outputs 'the weights changed!!!!'. I do not understand why the weights of the layer named 'dense1' changes after setting model.get_layer(name=name).trainable = False.

Answer 1

You can do it by using:

model=Sequential()
layer=Dense(64,init='glorot_uniform',input_shape=(784,))
layer.trainable=False
model.add(layer)
layer2=Dense(784, activation='sigmoid',init='glorot_uniform')
layer2.trainable=True
model.add(layer2)
model.compile(loss='relu', optimizer=sgd,metrics = ['mae'])

Answer 2

You need to compile the graph after setting 'trainable'.
more info here

Answer 3

let me keep my layers freezed upto 5th layer, rest i will keep trainable
Here is more simple & more efficient code

for layer in model.layers[:5]:
    layer.trainable=False
for layer in model.layers[5:]:
    layer.trainable=True