How to ignore null fields in ExtJS data models?

Question

My question is very long... So, be patient :)

I'm working with models in ExtJS 4, but I'm having some problems with the associations, so I created a function to perform the automatic model creation for me. Lets suppose I need to parse the following JSON:

{ 
    "success": true, 
    "total": 28, 
    "itens": [{
        "id":1,
        "nome":"ACRE",
        "sigla":"AC",
        "pais":{
            "id":31,
            "nome":"BRASIL",
            "sigla":"BR"
        }
    },{
        "id":2,
        "nome":"ALAGOAS",
        "sigla":"AL",
        "pais":{
            "id":31,
            "nome":"BRASIL",
            "sigla":"BR"
        }
    }, ...]
}

The itens represent Provinces (Estados in Brazilian portuguese) which have a Country (País in brazilian portuguese). I tried to use ExtJS associations, but I thought that it works like Java relationships and I was wrong. Well, for this JSON I have these Java classes and these Ext Models (the models are created using the function provided too).

Pais.java

@Entity
// named queries here...
public class Pais implements Serializable {

    @Id
    @GeneratedValue
    private Long id;

    @NotNull
    @NotEmpty
    @Length( max = 100 )
    private String nome;

    @NotNull
    @NotEmpty
    @Column( unique = true )
    @Length( min = 2, max = 4 )
    private String sigla;

    // getters, setters, equals, hashCode and toString here

}

Estado.java

@Entity
// named queries here...
public class Estado implements Serializable {

    @Id
    @GeneratedValue
    private Long id;

    @NotNull
    @NotEmpty
    @Length( max = 100 )
    private String nome;

    @NotNull
    @NotEmpty
    @Column( unique = true )
    @Length( min = 2, max = 4 )
    private String sigla;

    @NotNull
    @ManyToOne
    private Pais pais;

    // getters, setters, equals, hashCode and toString here

}

Function to create the models

Ext.ns( "Uteis" );

// other utility functions here...

Uteis.createModel = function( modelData ) {

    var fields = modelData.fields;
    var processedFields = [];
    var normalFields = [];
    var relationFields = [];

    for ( var i in fields ) {

        if ( fields[i].type ) {

            switch ( fields[i].type ) {

                case "auto":
                case "string":
                case "int":
                case "float":
                case "boolean":
                case "date":
                    normalFields.push( fields[i] );
                    break;

                default:

                    var relationField = fields[i];

                    var prefix = relationField.name + ".";
                    var modelInstance = Ext.create( relationField.type );

                    modelInstance.fields.each( function( item, index, length ) {

                        var newField = {};

                        // I used this sintax to make possible create only some fields
                        // if I need in the future.
                        newField["name"] = prefix + item.name;
                        newField["type"] = item.type.type;

                        newField["convert"] = item.convert;
                        newField["dateFormat"] = item.dateFormat;
                        newField["defaultValue"] = item.defaultValue;
                        newField["mapping"] = item.mapping;
                        newField["persist"] = item.persist;
                        newField["sortDir"] = item.sortDir;
                        newField["sortType"] = item.sortType;
                        newField["useNull"] = item.useNull;

                        relationFields.push( newField );

                    });

                    break;

            }

        } else {
            normalFields.push( fields[i] );
        }

    }

    processedFields = normalFields.concat( relationFields );

    // debugging code
    /*console.log( "*** " + modelData.name );
    for ( var i in processedFields ) {
        console.log( processedFields[i] );
    }*/

    Ext.define( modelData.name, {
        extend: "Ext.data.Model",
        fields: processedFields
    });

};

Using function to create the models

Uteis.createModel({ 
    name: "Modelos.Pais",
    fields: [
        { name: "id",  type: "int" },
        { name: "nome",  type: "string" },
        { name: "sigla",  type: "string" }
    ]
});

Uteis.createModel({ 
    name: "Modelos.Estado",
    fields: [
        { name: "id",  type: "int" },
        { name: "nome",  type: "string" },
        { name: "sigla",  type: "string" },
        { name: "pais", type: "Modelos.Pais" } // <= references the model created above
    ]
});

The code above is correspondent to this. I created the function to automatize the creation of nested data fields (because the problem of the associations that I said).

Ext.define( "Modelos.Pais", {
    extend: "Ext.data.Model",
    fields: [
        { name: "id",  type: "int" },
        { name: "nome",  type: "string" },
        { name: "sigla",  type: "string" }
    ]
});

Ext.define( "Modelos.Estado", {
    extend: "Ext.data.Model",
    fields: [
        { name: "id",  type: "int" },
        { name: "nome",  type: "string" },
        { name: "sigla",  type: "string" },
        { name: "pais.id",  type: "int" },
        { name: "pais.nome",  type: "string" },
        { name: "pais.sigla",  type: "string" }
    ]
});

Ok, these models (created using my createModel function) work very well with my JsonStores. Until now, all of the mapping associations in the Java side were not null, so, my stores always have nested data to process. Now, I have to process some entities that can have null associations and my problems began. The store that needs to deal with this scenario not works (an exception is being thrown in the store operation, saying that the fields are null). I'm using Gson to create JSON from my entities. Its default behaviour is to not serialize null fields, them they will be undefined in the client side, so I though that if I serialize the null fields (sending a null) would make Ext to realize the null field and not try to process it. To do this I used this code to create the Gson:

Gson gson = new GsonBuilder().serializeNulls().create();

Ok, now the JSON with null associations is being generated, but the Ext continues to complain. I tried to use the field mapping and defaultValue configs with no success. To make the things simpler, lets use the example of Estados and Países (Privinces and Countries) where Pais is not @NotNull anymore. The JSON with a null pais would be like:

{ 
    "success": true, 
    "total": 28, 
    "itens": [{
        "id":1,
        "nome":"ACRE",
        "sigla":"AC",
        "pais":null   // <= here
    },{
        "id":2,
        "nome":"ALAGOAS",
        "sigla":"AL",
        "pais":{  // this is not null
            "id":31,
            "nome":"BRASIL",
            "sigla":"BR"
        }
    }, ...]
}

With this code, the pais.id, pais.nome and pais.sigla fields would not be available, since the pais property is null. So, my question is: How to make the store ignore some fields when they are null or undefined? I already tried to search for a solution with no success... Thank you very much!

Edit: After thinking the entire night in some possible solutions at server side, I implement a solution in the last 15 minutes, but I definatelly don't like it... Its a reflective method to traverse each object "object tree" before using Gson to set default values to fields that are null. It is working, but the JSON is becoming unnecessarily too big. The traversal method:

/**
 * A method to traverse the object tree and set "default" values to null fields.
 * 
 * @param target The object to be inspected.
 */
public static void traverseAndSetDefaultValue( Object target ) {

    try {

        for ( Field f : target.getClass().getDeclaredFields() ) {

            // ok to change...
            f.setAccessible( true );

            // is null? so create something
            if ( f.get( target ) == null ) {

                // new instance of the current field
                Object newInstance = null;

                // it needs to traverse to the next level?
                boolean okToTraverse = false;

                switch ( f.getType().getSimpleName() ) {

                    case "Byte":
                    case "Short":
                    case "Integer":
                        newInstance = 0;
                        break;

                    case "Long":
                        newInstance = 0L;
                        break;

                    case "Float":
                        newInstance = 0F;
                        break;

                    case "Double":
                        newInstance = 0D;
                        break;

                    case "Character":
                        newInstance = '\0';
                        break;

                    case "Boolean":
                        newInstance = Boolean.FALSE;
                        break;

                    case "String":
                        newInstance = "";
                        break;

                    case "List":
                        newInstance = new ArrayList();
                        break;

                    case "Set":
                        newInstance = new HashSet();
                        break;

                    default:
                        // calling the default constructor for no 
                        // "default" types
                        newInstance = f.getType().newInstance();
                        okToTraverse = true;
                        break;

                }

                f.set( target, newInstance );

                if ( okToTraverse ) {
                    traverseAndSetDefaultValue( newInstance );
                }

            }

        }

    } catch ( IllegalAccessException | InstantiationException exc ) {
        exc.printStackTrace();
    }

}

I would like what do you think about it... Thank you!

Edit 2: Hello again. I give up! :) I will use my solution that I posted above.I found some patches to improve the relationship with models and grids. I tested they, but the problem with null fields continues (at least the error disapeared). Ok, now is time to continue the development. When the application is finished I will go back to this issue to try to improve my solution. Thank you!

Answer 1

Not super sure if this is what the problem is, since you didn't give the exact error; but I've had problems with optional nested data before, and the solution was to create a mapping function in the model:

Ext.define( "Modelos.Estado", {
    extend: "Ext.data.Model",
    fields: [
        { name: "id",  type: "int" },
        { name: "nome",  type: "string" },
        { name: "sigla",  type: "string" },
        { name: "pais.id",  type: "int", mapping: function( o ) { return o.pais ? o.pais.id : null; } },
        { name: "pais.nome",  type: "string", mapping: function( o ) { return o.pais ? o.pais.nome : null; } },
        { name: "pais.sigla",  type: "string", mapping: function( o ) { return o.pais ? o.pais.sigla : null; } }
    ]
});

Answer 2

You can extend Ext.data.reader.Json as below:

Ext.define('Ext.data.reader.SafeJson', {
    extend: 'Ext.data.reader.Json',
    alias : 'reader.safejson',
    /**
     * @private
     * Returns an accessor function for the given property string. Gives support for properties such as the following:
     * 'someProperty'
     * 'some.property'
     * 'some["property"]'
     * This is used by buildExtractors to create optimized extractor functions when casting raw data into model instances.
     */
    createAccessor: (function() {
        var re = /[\[\.]/;

        return function(expr) {
            if (Ext.isEmpty(expr)) {
                return Ext.emptyFn;
            }
            if (Ext.isFunction(expr)) {
                return expr;
            }
            if (this.useSimpleAccessors !== true) {
                var i = String(expr).search(re);
                if (i >= 0) {
                    if (i > 0) {    // Check all property chain for existence. Return null if any level does not exist.
                        var a = [];
                        var l = expr.split('.');
                        var r = '';
                        for (var w in l) {
                            r = r + '.' + l[w];
                            a.push('obj' + r);
                        }
                        var v = "(" + a.join(" && ") + ") ? obj." + expr + " : null";
                        return Ext.functionFactory('obj', 'return (' + v + ')');
                    } else {
                        return Ext.functionFactory('obj', 'return obj' + (i > 0 ? '.' : '') + expr);
                    }
                }
            }
            return function(obj) {
                return obj[expr];
            };
        };
    }()),

        /**
     * @private
     * @method
     * Returns an accessor expression for the passed Field. Gives support for properties such as the following:
     *
     * - 'someProperty'
     * - 'some.property'
     * - 'some["property"]'
     *
     * This is used by buildExtractors to create optimized on extractor function which converts raw data into model instances.
     */
    createFieldAccessExpression: (function() {
        var re = /[\[\.]/;

        return function(field, fieldVarName, dataName) {
            var me     = this,
                hasMap = (field.mapping !== null),
                map    = hasMap ? field.mapping : field.name,
                result,
                operatorSearch;

            if (typeof map === 'function') {
                result = fieldVarName + '.mapping(' + dataName + ', this)';
            } else if (this.useSimpleAccessors === true || ((operatorSearch = String(map).search(re)) < 0)) {
                if (!hasMap || isNaN(map)) {
                    // If we don't provide a mapping, we may have a field name that is numeric
                    map = '"' + map + '"';
                }
                result = dataName + "[" + map + "]";
            } else {                
                if (operatorSearch > 0) {
                    var a = [];
                    var l = map.split('.');
                    var r = '';
                    for (var w in l) {
                        r = r + '.' + l[w];
                        a.push(dataName + r);
                    }
                    result = "("+a.join(" && ")+") ? "+dataName+"."+map+" : null";
                } else {
                    result = dataName + map;
                }
            }            
            return result;
        };
    }())
});

So you can successfully processing nested JSON-data with null nodes.

Example of JSON:

{
    root: [{
        id: 1,
        name: {
            name: "John",
            phone: "123"
        },          
    },
    {
        id: 4,
        name: null,         
    },
    ]
}

Working example with test data you can find here: http://jsfiddle.net/8Ftag/

ExtJS 4.1.1 tested