Pybrain Neural Network failing to train correctly

Question

I've been working on creating a neural network using pybrain, and after training it with propagation for some reason it fails to train my network. Any data set I use with more than two classes in the out dimension will just pile all my observations into one category. Does anyone know why this is happening? Code and some output are below.

import scipy
import numpy
from pybrain.datasets            import ClassificationDataSet
from pybrain.utilities           import percentError
from pybrain.tools.shortcuts     import buildNetwork
from pybrain.supervised.trainers import BackpropTrainer
from pybrain.structure.modules   import SoftmaxLayer
from sklearn.metrics             import precision_score,recall_score,confusion_matrix
def makeDataset(CSVfile,ClassFile):
    #import the features to data, and their classes to dataClasses
    data=numpy.genfromtxt(CSVfile,delimiter=",")
    classes=numpy.genfromtxt(ClassFile,delimiter=",")
    print("Building the dataset from CSV files")
    #Initialize an empty Pybrain dataset, and populate it
    alldata=ClassificationDataSet(len(data[0]),1,nb_classes=3)
    for count in range(len((classes))):
        alldata.addSample(data[count],[classes[count]])
    return alldata



def makeNeuralNet(alldata,trainingPercent=.3,hiddenNeurons=5,trainingIterations=20):
    #Divide the data set into training and non-training data    
    testData, trainData = alldata.splitWithProportion(trainingPercent)
    testData._convertToOneOfMany( )
    trainData._convertToOneOfMany( )
    #Then build the network, and using backwards propogation to train it
    network = buildNetwork( trainData.indim, hiddenNeurons, trainData.outdim, outclass=SoftmaxLayer )
    trainer = BackpropTrainer( network, dataset=trainData, momentum=0.1, verbose=True, weightdecay=0.01)
    for i in range(trainingIterations):
        print("Training Epoch #"+str(i))
        trainer.trainEpochs( 1 )
    return [network,trainer]



def checkNeuralNet(trainer,alldata):
    predictedVals=trainer.testOnClassData(alldata)
    actualVals=list(alldata['target'])
##    for row in alldata['target']:
##        row=list(row)
##        index=row.index(1)
##        actualVals+=[index]
    print("-----------------------------")
    print("-----------------------------")
    print("The precision is "+str(precision_score(actualVals,predictedVals)))
    print("The recall is "+str(recall_score(actualVals,predictedVals)))
    print("The confusion matrix is as shown below:")
    print(confusion_matrix(actualVals,predictedVals))


CSVfile="/home/ubuntu/test.csv"
ClassFile="/home/ubuntu/test_Classes.csv"
#Build our dataset
alldata=makeDataset(CSVfile,ClassFile)
#Build and train the network
net=makeNeuralNet(alldata,trainingPercent=.7,hiddenNeurons=20,trainingIterations=20)
network=net[0]
trainer=net[1]
#Check it's strength
checkNeuralNet(trainer,alldata)

The last epoch of training has .09 error, as shown in the below output:

Training Epoch #19
Total error: 0.0968444196605

And yet when I go to print the confusion matrix, precision, and recall, I get the following as well as this weird error:

UserWarning: The sum of true positives and false positives are equal to zero for some labels. Precision is ill defined for those labels [1 2]. The precision and recall are equal to zero for some labels. fbeta_score is ill defined for those labels [1 2]. 
  average=average)
The precision is 0.316635552252
UserWarning: The sum of true positives and false positives are equal to zero for some labels. Precision is ill defined for those labels [1 2]. The precision and recall are equal to zero for some labels. fbeta_score is ill defined for those labels [1 2]. 
  average=average)
The recall is 0.562703787309
The confusion matrix is as shown below:
[[4487    0    0]
 [ 987    0    0]
 [2500    0    0]]

Answer 1

I had very similar problem, and I found SoftmaxLayer to be the cause. Try to replace it with something else, for example SigmoidLayer. If that is a problem in your case as well there is a good chance that this class is bugy.