Deeplearning4j xor example

Question

I'm trying to train an xor network with deeplearning4j, but i think i didn't really get how to use the dataset.

I wanted to create a NN with two inputs, two hidden neurons and one output neuron.

here is what i have:

package org.deeplearning4j.examples.xor;

import org.deeplearning4j.eval.Evaluation;
import org.deeplearning4j.nn.api.Layer;
import org.deeplearning4j.nn.api.OptimizationAlgorithm;
import org.deeplearning4j.nn.conf.MultiLayerConfiguration;
import org.deeplearning4j.nn.conf.NeuralNetConfiguration;
import org.deeplearning4j.nn.conf.Updater;
import org.deeplearning4j.nn.conf.distribution.UniformDistribution;
import org.deeplearning4j.nn.conf.layers.GravesLSTM;
import org.deeplearning4j.nn.conf.layers.RnnOutputLayer;
import org.deeplearning4j.nn.multilayer.MultiLayerNetwork;
import org.deeplearning4j.nn.weights.WeightInit;
import org.deeplearning4j.optimize.listeners.ScoreIterationListener;
import org.nd4j.linalg.api.ndarray.INDArray;
import org.nd4j.linalg.dataset.DataSet;
import org.nd4j.linalg.factory.Nd4j;
import org.nd4j.linalg.lossfunctions.LossFunctions.LossFunction;

public class XorExample {
    public static void main(String[] args) {

        INDArray input = Nd4j.zeros(4, 2);
        INDArray labels = Nd4j.zeros(4, 1);

        input.putScalar(new int[] { 0, 0 }, 0);
        input.putScalar(new int[] { 0, 1 }, 0);

        input.putScalar(new int[] { 1, 0 }, 1);
        input.putScalar(new int[] { 1, 1 }, 0);

        input.putScalar(new int[] { 2, 0 }, 0);
        input.putScalar(new int[] { 2, 1 }, 1);

        input.putScalar(new int[] { 3, 0 }, 1);
        input.putScalar(new int[] { 3, 1 }, 1);

        labels.putScalar(new int[] { 0, 0 }, 0);
        labels.putScalar(new int[] { 1, 0 }, 1);
        labels.putScalar(new int[] { 2, 0 }, 1);
        labels.putScalar(new int[] { 3, 0 }, 0);

        DataSet ds = new DataSet(input,labels);

        //Set up network configuration:
        MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).iterations(1)
            .learningRate(0.1)
            .list(2)
            .layer(0, new GravesLSTM.Builder().nIn(2).nOut(2)
                    .updater(Updater.RMSPROP)
                    .activation("tanh").weightInit(WeightInit.DISTRIBUTION)
                    .dist(new UniformDistribution(-0.08, 0.08)).build())
            .layer(1, new RnnOutputLayer.Builder(LossFunction.MCXENT).activation("softmax")        //MCXENT + softmax for classification
                    .updater(Updater.RMSPROP)
                    .nIn(2).nOut(1).weightInit(WeightInit.DISTRIBUTION)
                    .dist(new UniformDistribution(-0.08, 0.08)).build())
            .pretrain(false).backprop(true)
            .build();

            MultiLayerNetwork net = new MultiLayerNetwork(conf);
            net.init();
            net.setListeners(new ScoreIterationListener(1));

            //Print the  number of parameters in the network (and for each layer)
            Layer[] layers = net.getLayers();
            int totalNumParams = 0;
            for( int i=0; i<layers.length; i++ ){
                int nParams = layers[i].numParams();
                System.out.println("Number of parameters in layer " + i + ": " + nParams);
                totalNumParams += nParams;
            }
            System.out.println("Total number of network parameters: " + totalNumParams);

            net.fit(ds);


            Evaluation eval = new Evaluation(3);
            INDArray output = net.output(ds.getFeatureMatrix());
            eval.eval(ds.getLabels(), output);
            System.out.println(eval.stats());

    }
}

the output looks like that

Mär 20, 2016 7:03:06 PM com.github.fommil.jni.JniLoader liberalLoad
INFORMATION: successfully loaded C:\Users\LuckyPC\AppData\Local\Temp\jniloader5209513403648831212netlib-native_system-win-x86_64.dll
Number of parameters in layer 0: 46
Number of parameters in layer 1: 3
Total number of network parameters: 49
o.d.o.s.BaseOptimizer - Objective function automatically set to minimize. Set stepFunction in neural net configuration to change default settings.
o.d.o.l.ScoreIterationListener - Score at iteration 0 is 0.6931495070457458
Exception in thread "main" java.lang.IllegalArgumentException: Unable to getFloat row of non 2d matrix
    at org.nd4j.linalg.api.ndarray.BaseNDArray.getRow(BaseNDArray.java:3640)
    at org.deeplearning4j.eval.Evaluation.eval(Evaluation.java:107)
    at org.deeplearning4j.examples.xor.XorExample.main(XorExample.java:80)

Answer 1

Here is a solution I came up with.

public static void main(String[] args) throws IOException, InterruptedException {

    CSVDataSet dataSet = new CSVDataSet(new File("./train.csv"));
    CSVDataSetIterator trainingSetIterator = new CSVDataSetIterator(dataSet, dataSet.size());

    MultiLayerConfiguration configuration = new NeuralNetConfiguration.Builder()
            .weightInit(WeightInit.DISTRIBUTION).dist(new UniformDistribution(0, 1)).iterations(1150)
            .learningRate(1).seed(1)
            .optimizationAlgo(OptimizationAlgorithm.STOCHASTIC_GRADIENT_DESCENT).updater(Updater.SGD)
            .list(2)
            .backprop(true).pretrain(false)
            .layer(0, new DenseLayer.Builder().nIn(2).nOut(3).updater(Updater.SGD).build())
            .layer(1, new OutputLayer.Builder().nIn(3).nOut(1).build()).build();

    MultiLayerNetwork network = new MultiLayerNetwork(configuration);
    network.setListeners(new HistogramIterationListener(10), new ScoreIterationListener(100));
    network.init();

    long start = System.currentTimeMillis();
    network.fit(trainingSetIterator);
    System.out.println(System.currentTimeMillis() - start);

    try(DataOutputStream dos = new DataOutputStream(Files.newOutputStream(Paths.get("xor-coefficients.bin")))){
        Nd4j.write(network.params(), dos);
    }
    FileUtils.write(new File("xor-network-conf.json"), network.getLayerWiseConfigurations().toJson());
}

To test:

    MultiLayerConfiguration configuration = MultiLayerConfiguration.fromJson(FileUtils.readFileToString(new File("xor-network-conf.json")));

    try (DataInputStream dis = new DataInputStream(new FileInputStream("xor-coefficients.bin"))) {
        INDArray parameters = Nd4j.read(dis);

        MultiLayerNetwork network = new MultiLayerNetwork(configuration, parameters);
        network.init();

        List<INDArray> inputs = ImmutableList.of(Nd4j.create(new double[]{1, 0}),
                Nd4j.create(new double[]{0, 1}),
                Nd4j.create(new double[]{1, 1}),
                Nd4j.create(new double[]{0, 0}));

        List<INDArray> networkResults = inputs.stream().map(network::output).collect(toList());
        System.out.println(networkResults);
    }
}

with training data:

0,1,1

1,0,1

1,1,0

0,0,0

Answer 2

I believe there is a XOR example straight from their git repository!

The code is well documented and you can find it the repository here: https://github.com/deeplearning4j/dl4j-0.4-examples.git