LibGDX mesh heightmap normals and lights

Question

I am trying to get mesh normals and lights working in LibGDX project.

I already have textured mesh generated from heightmap texture pixels.

The problem is I cannot get normals lighted up correctly. Also Im not 100% sure I have normal vertices correctly set up in TerrainChunk class.

Heres the main class code:

package com.me.terrain;

import com.badlogic.gdx.Game;
import com.badlogic.gdx.Gdx;
import com.badlogic.gdx.files.FileHandle;
import com.badlogic.gdx.graphics.Color;
import com.badlogic.gdx.graphics.GL20;
import com.badlogic.gdx.graphics.Mesh;
import com.badlogic.gdx.graphics.PerspectiveCamera;
import com.badlogic.gdx.graphics.Pixmap;
import com.badlogic.gdx.graphics.Texture;
import com.badlogic.gdx.graphics.VertexAttribute;
import com.badlogic.gdx.graphics.VertexAttributes.Usage;
import com.badlogic.gdx.graphics.g3d.utils.CameraInputController;
import com.badlogic.gdx.graphics.glutils.ShaderProgram;
import com.badlogic.gdx.math.Matrix3;
import com.badlogic.gdx.math.Matrix4;
import com.badlogic.gdx.math.Vector3;

public class Terra extends Game {

private PerspectiveCamera camera;
private CameraInputController camController;

private TerrainChunk chunk;
private Mesh mesh;

private ShaderProgram shader;
private Texture terrainTexture;

private final Matrix3 normalMatrix = new Matrix3();

private static final float[] lightPosition = { 5, 35, 5 };
private static final float[] ambientColor = { 0.2f, 0.2f, 0.2f, 1.0f };
private static final float[] diffuseColor = { 0.5f, 0.5f, 0.5f, 1.0f };
private static final float[] specularColor = { 0.7f, 0.7f, 0.7f, 1.0f };

private static final float[] fogColor = { 0.2f, 0.1f, 0.6f, 1.0f };

private Matrix4 model = new Matrix4();
private Matrix4 modelView = new Matrix4();

private final String vertexShader =
        "attribute vec4 a_position; \n" +
        "attribute vec3 a_normal; \n" +
        "attribute vec2 a_texCoord; \n" +
        "attribute vec4 a_color; \n" +

        "uniform mat4 u_MVPMatrix; \n" +
        "uniform mat3 u_normalMatrix; \n" +

        "uniform vec3 u_lightPosition; \n" +

        "varying float intensity; \n" +
        "varying vec2 texCoords; \n" +
        "varying vec4 v_color; \n" +

        "void main() { \n" +
        "    vec3 normal = normalize(u_normalMatrix * a_normal); \n" +
        "    vec3 light = normalize(u_lightPosition); \n" +
        "    intensity = max( dot(normal, light) , 0.0); \n" +

        "    v_color = a_color; \n" +
        "    texCoords = a_texCoord; \n" +

        "    gl_Position = u_MVPMatrix * a_position; \n" +
        "}";

private final String fragmentShader =
        "#ifdef GL_ES \n" +
        "precision mediump float; \n" +
        "#endif \n" +

        "uniform vec4 u_ambientColor; \n" +
        "uniform vec4 u_diffuseColor; \n" +
        "uniform vec4 u_specularColor; \n" +

        "uniform sampler2D u_texture; \n" +
        "varying vec2 texCoords; \n" +
        "varying vec4 v_color; \n" +

        "varying float intensity; \n" +

        "void main() { \n" +
        "    gl_FragColor = v_color * intensity * texture2D(u_texture, texCoords); \n" +
        "}";

@Override
public void create() {

    // Terrain texture size is 128x128
    terrainTexture = new Texture(Gdx.files.internal("data/concrete2.png"));

    // Height map (black/white) texture size is 32x32
    String heightMapFile = "data/heightmap.png";


    // position, normal, color, texture
    int vertexSize = 3 + 3 + 1 + 2;  

    chunk = new TerrainChunk(32, 32, vertexSize, heightMapFile);



    mesh = new Mesh(true, chunk.vertices.length / 3, chunk.indices.length,
            new VertexAttribute(Usage.Position, 3, ShaderProgram.POSITION_ATTRIBUTE),
            new VertexAttribute(Usage.Normal, 3, ShaderProgram.NORMAL_ATTRIBUTE),
            new VertexAttribute(Usage.ColorPacked, 4, ShaderProgram.COLOR_ATTRIBUTE),
            new VertexAttribute(Usage.TextureCoordinates, 2, ShaderProgram.TEXCOORD_ATTRIBUTE));

    mesh.setVertices(chunk.vertices);
    mesh.setIndices(chunk.indices);



    camera = new PerspectiveCamera(67, Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
    camera.position.set(5, 50, 5);
    camera.direction.set(3, 0, 0).sub(camera.position).nor();
    camera.near = 0.005f;
    camera.far = 300;
    camera.update();

    camController = new CameraInputController(camera);
    Gdx.input.setInputProcessor(camController);

    ShaderProgram.pedantic = false;

    shader = new ShaderProgram(vertexShader, fragmentShader);

}

@Override
public void render() {

    Gdx.gl.glViewport(0, 0, Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
    Gdx.gl.glEnable(GL20.GL_DEPTH_TEST);
    Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_DEPTH_BUFFER_BIT);

    camController.update();
    camera.update();


    // This is wrong?
    model.setToRotation(new Vector3(0, 1, 0), 45f);
    modelView.set(camera.view).mul(model);


    terrainTexture.bind();

    shader.begin();

    shader.setUniformMatrix("u_MVPMatrix", camera.combined);
    shader.setUniformMatrix("u_normalMatrix", normalMatrix.set(modelView).inv().transpose());

    shader.setUniform3fv("u_lightPosition", lightPosition, 0, 3);
    shader.setUniform4fv("u_ambientColor", ambientColor, 0, 4);
    shader.setUniform4fv("u_diffuseColor", diffuseColor, 0, 4);
    shader.setUniform4fv("u_specularColor", specularColor, 0, 4);

    shader.setUniformi("u_texture", 0);

    mesh.render(shader, GL20.GL_TRIANGLES);

    shader.end();

}
}

TerrainChunk class code:

final static class TerrainChunk {

    public final float[] heightMap;
    public final short width;
    public final short height;
    public final float[] vertices;
    public final short[] indices;

    public final int vertexSize;
    private final int positionSize = 3;

    public TerrainChunk(int width, int height, int vertexSize, String heightMapTexture) {

        if ((width + 1) * (height + 1) > Short.MAX_VALUE) {
            throw new IllegalArgumentException(            
                    "Chunk size too big, (width + 1)*(height+1) must be <= 32767");
        }

        this.heightMap = new float[(width + 1) * (height + 1)];
        this.width = (short) width;
        this.height = (short) height;
        this.vertices = new float[heightMap.length * vertexSize];
        this.indices = new short[width * height * 6];
        this.vertexSize = vertexSize;

        buildHeightmap(heightMapTexture);

        buildIndices();
        buildVertices();

        calcNormals(indices, vertices);

    }

    public void buildHeightmap(String pathToHeightMap) {

        FileHandle handle = Gdx.files.internal(pathToHeightMap);
        Pixmap heightmapImage = new Pixmap(handle);
        Color color = new Color();
        int idh = 0;

        for (int x = 0; x < this.width + 1; x++) {
            for (int y = 0; y < this.height + 1; y++) {
                Color.rgba8888ToColor(color, heightmapImage.getPixel(x, y));
                this.heightMap[idh++] = color.r;
            }
        }
    }

    public void buildVertices() {
        int heightPitch = height + 1;
        int widthPitch = width + 1;

        int idx = 0;
        int hIdx = 0;
        int strength = 10; // multiplier for height map

        float scale = 4f;

        for (int z = 0; z < heightPitch; z++) {
            for (int x = 0; x < widthPitch; x++) {

                // POSITION
                vertices[idx++] = scale * x;
                vertices[idx++] = heightMap[hIdx++] * strength;
                vertices[idx++] = scale * z;

                // NORMAL, skip these for now
                idx += 3;

                // COLOR
                vertices[idx++] = Color.WHITE.toFloatBits();

                // TEXTURE
                vertices[idx++] = (x / (float) width);
                vertices[idx++] = (z / (float) height);

            }
        }
    }

    private void buildIndices() {
        int idx = 0;
        short pitch = (short) (width + 1);
        short i1 = 0;
        short i2 = 1;
        short i3 = (short) (1 + pitch);
        short i4 = pitch;

        short row = 0;

        for (int z = 0; z < height; z++) {
            for (int x = 0; x < width; x++) {
                indices[idx++] = i1;
                indices[idx++] = i2;
                indices[idx++] = i3;

                indices[idx++] = i3;
                indices[idx++] = i4;
                indices[idx++] = i1;

                i1++;
                i2++;
                i3++;
                i4++;
            }

            row += pitch;
            i1 = row;
            i2 = (short) (row + 1);
            i3 = (short) (i2 + pitch);
            i4 = (short) (row + pitch);
        }
    }

    // Gets the index of the first float of a normal for a specific vertex
    private int getNormalStart(int vertIndex) {
        return vertIndex * vertexSize + positionSize;
    }

    // Gets the index of the first float of a specific vertex
    private int getPositionStart(int vertIndex) {
        return vertIndex * vertexSize;
    }

    // Adds the provided value to the normal
    private void addNormal(int vertIndex, float[] verts, float x, float y, float z) {

        int i = getNormalStart(vertIndex);

        verts[i] += x;
        verts[i + 1] += y;
        verts[i + 2] += z;
    }

    /*
     * Normalizes normals
     */
    private void normalizeNormal(int vertIndex, float[] verts) {

        int i = getNormalStart(vertIndex);

        float x = verts[i];
        float y = verts[i + 1];
        float z = verts[i + 2];

        float num2 = ((x * x) + (y * y)) + (z * z);
        float num = 1f / (float) Math.sqrt(num2);
        x *= num;
        y *= num;
        z *= num;

        verts[i] = x;
        verts[i + 1] = y;
        verts[i + 2] = z;
    }

    /*
     * Calculates the normals
     */
    private void calcNormals(short[] indices, float[] verts) {

        for (int i = 0; i < indices.length; i += 3) {
            int i1 = getPositionStart(indices[i]);
            int i2 = getPositionStart(indices[i + 1]);
            int i3 = getPositionStart(indices[i + 2]);

            // p1
            float x1 = verts[i1];
            float y1 = verts[i1 + 1];
            float z1 = verts[i1 + 2];

            // p2
            float x2 = verts[i2];
            float y2 = verts[i2 + 1];
            float z2 = verts[i2 + 2];

            // p3
            float x3 = verts[i3];
            float y3 = verts[i3 + 1];
            float z3 = verts[i3 + 2];

            // u = p3 - p1
            float ux = x3 - x1;
            float uy = y3 - y1;
            float uz = z3 - z1;

            // v = p2 - p1
            float vx = x2 - x1;
            float vy = y2 - y1;
            float vz = z2 - z1;

            // n = cross(v, u)
            float nx = (vy * uz) - (vz * uy);
            float ny = (vz * ux) - (vx * uz);
            float nz = (vx * uy) - (vy * ux);

            // normalize(n)
            float num2 = ((nx * nx) + (ny * ny)) + (nz * nz);
            float num = 1f / (float) Math.sqrt(num2);
            nx *= num;
            ny *= num;
            nz *= num;

            addNormal(indices[i], verts, nx, ny, nz);
            addNormal(indices[i + 1], verts, nx, ny, nz);
            addNormal(indices[i + 2], verts, nx, ny, nz);
        }

        for (int i = 0; i < (verts.length / vertexSize); i++) {
            normalizeNormal(i, verts);
        }
    }

}

What Im seeing is when I move camera the lights dont show correctly when I'm above terrain. They show more when Im under the terrain, though incorrectly even then I think.

pics:

1. below: http://i.imgur.com/TocCLfA.png

2. above: http://i.imgur.com/fwGhGDT.png

Answer 1

Solved the problem by debugging and drawing all normal positions with MeshPartBuilder / GL_LINES.

I found out that the normals were pointing inside the terrain. Changing normal direction was the solution.