How can I properly write this shader function in JS?

Question

What I want to happen:

For testing a game art style I thought of, I want to render a 3D world in pixel-art form. So for example, take a scene like this (but rendered with certain coloring / style so as to look good once pixelated):

And make it look something like this:

By playing with different ways of styling the 3D source I think the pixelated output could look nice. Of course to get this effect one just sizes the image down to ~80p and upscales it to 1080p with nearest neighbor resampling. But it's more efficient to render straight to an 80p canvas to begin with and just do the upscaling.

This is not typically how one would use a shader, to resize a bitmap in nearest neighbor format, but the performance on it is better than any other way I've found to make such a conversion in real time.

My code:

My buffer for the bitmap is stored in row major, as r1, g1, b1, a1, r2, g2, b2, a2... and I'm using gpu.js which essentially converts this JS func into a shader. My goal is to take one bitmap and return one at larger scale with nearest-neighbor scaling, so each pixel becomes a 2x2 square or 3x3 and so on. Assume inputBuffer is a scaled fraction of size of the output determined by the setOutput method.

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
  var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
  var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
  var remainder = this.thread.x % 4;
  return inputBuffer[(x * y) + remainder]; 
}).setOutput([width * height * 4]);

JSFiddle

Keep in mind it's iterating over a new buffer of the full size output, so I have to find the correct coordinates that will exist in the smaller sourceBuffer based on the current index in the outputBuffer (index is exposed by the lib as this.thread.x).

What's happening instead:

This, instead of making a nearest neighbor upscale, is making a nice little rainbow (above is the small normal render, below is the result of the shader, and to the right you can see some debug logging with stats about the input and output buffers):

What am I doing wrong?

Note: I asked a related question here, Is there a simpler (and still performant) way to upscale a canvas render with nearest neighbor resampling?

Answer 1

Update 1 - 25th May 2018

I was able to resolve most of the issues. There were quite a few

1. The logic of transformation was wrong, also the data was coming flipped for some reason, so I flipped cols and rows to start from bottom right

   var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
   var size = width[0] * height[0] * 4;
   var current_index = Math.floor((size - this.thread.x)/4); 
   var row = Math.floor(current_index / (width[0] * scale[0]) );
   var col = Math.floor((current_index % width[0])/scale[0]);
   var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col;
   var remainder = this.thread.x % 4;
   return inputBuffer[index_old * 4 + remainder];
   
   }).setOutput([width * height * 4]);
   

2. You were using width and height in floats, which I have changed to be calculated first and then scaled

   var smallWidth = Math.floor(window.innerWidth / scale);
   var smallHeight = Math.floor(window.innerHeight / scale);
   
   var width = smallWidth * scale;
   var height = smallHeight * scale;
   
   
   var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight);
   var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4);
   var outputBuffer = new Uint8ClampedArray(width * height * 4);
   

3. The canvas size was set to while inner width and height, you need to set it to just the image width and height

   context = canvas.getContext('2d');
   canvas.width = width;
   canvas.height = height;
   

Below is the final JSFiddle for the same

https://jsfiddle.net/are5Lbw8/6/

Results:

Final Code for reference

var container;
var camera, scene, renderer;
var mouseX = 0;
var mouseY = 0;
var scale = 4;
var windowHalfX = window.innerWidth / 2;
var windowHalfY = window.innerHeight / 2;
var smallWidth = Math.floor(window.innerWidth / scale);
var smallHeight = Math.floor(window.innerHeight / scale);

var width = smallWidth * scale;
var height = smallHeight * scale;


var rt = new THREE.WebGLRenderTarget(smallWidth, smallHeight);
var frameBuffer = new Uint8Array(smallHeight * smallHeight * 4);
var outputBuffer = new Uint8ClampedArray(width * height * 4);
var output;
var divisor = 2;
var divisorHalf = divisor / 2;
var negativeDivisorHalf = -1 * divisorHalf;
var canvas;
var context;
var gpu = new GPU();

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
/*   var y = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
  var x = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
  var remainder = this.thread.x % 4;
  return inputBuffer[(x * y) + remainder];
   */
   var size = width[0] * height[0] * 4;
   var current_index = Math.floor((size - this.thread.x)/4); 
   var row = Math.floor(current_index / (width[0] * scale[0]) );
   var col = Math.floor((current_index % width[0])/scale[0]);
   var index_old = Math.floor(row * (width[0] / scale[0])) + width[0] - col;
   var remainder = this.thread.x % 4;
   return inputBuffer[index_old * 4 + remainder];

}).setOutput([width * height * 4]);
console.log(window.innerWidth);
console.log(window.innerHeight);
init();
animate();

function init() {
  container = document.createElement('div');
  document.body.appendChild(container);
  canvas = document.createElement('canvas');
  document.body.appendChild(canvas);
  context = canvas.getContext('2d');
  canvas.width = width;
  canvas.height = height;
  camera = new THREE.PerspectiveCamera(45, window.innerWidth / window.innerHeight, 1, 2000);
  camera.position.z = 100;

  // scene
  scene = new THREE.Scene();
  var ambient = new THREE.AmbientLight(0xbbbbbb);
  scene.add(ambient);
  var directionalLight = new THREE.DirectionalLight(0xdddddd);
  directionalLight.position.set(0, 0, 1);
  scene.add(directionalLight);

  // texture
  var manager = new THREE.LoadingManager();
  manager.onProgress = function(item, loaded, total) {
    console.log(item, loaded, total);
  };
  var texture = new THREE.Texture();
  var onProgress = function(xhr) {
    if (xhr.lengthComputable) {
      var percentComplete = xhr.loaded / xhr.total * 100;
      console.log(Math.round(percentComplete, 2) + '% downloaded');
    }
  };
  var onError = function(xhr) {};
  var imgLoader = new THREE.ImageLoader(manager);
  imgLoader.load('https://i.imgur.com/P6158Su.jpg', function(image) {
    texture.image = image;
    texture.needsUpdate = true;
  });

  // model
  var objLoader = new THREE.OBJLoader(manager);
  objLoader.load('https://s3-us-west-2.amazonaws.com/s.cdpn.io/286022/Bulbasaur.obj', function(object) {
    object.traverse(function(child) {
      if (child instanceof THREE.Mesh) {
        child.material.map = texture;
      }
    });
    object.scale.x = 45;
    object.scale.y = 45;
    object.scale.z = 45;
    object.rotation.y = 3;
    object.position.y = -10.5;
    scene.add(object);
  }, onProgress, onError);
  renderer = new THREE.WebGLRenderer({
    alpha: true,
    antialias: false
  });
  renderer.setPixelRatio(window.devicePixelRatio);
  renderer.setSize(smallWidth, smallHeight);
  container.appendChild(renderer.domElement);
  renderer.context.webkitImageSmoothingEnabled = false;
  renderer.context.mozImageSmoothingEnabled = false;
  renderer.context.imageSmoothingEnabled = false;
  document.addEventListener('mousemove', onDocumentMouseMove, false);
  window.addEventListener('resize', onWindowResize, false);
}

function onWindowResize() {
  windowHalfX = (window.innerWidth / 2) / scale;
  windowHalfY = (window.innerHeight / 2) / scale;
  camera.aspect = (window.innerWidth / window.innerHeight) / scale;
  camera.updateProjectionMatrix();
  renderer.setSize(smallWidth, smallHeight);
}

function onDocumentMouseMove(event) {

  mouseX = (event.clientX - windowHalfX) / scale;
  mouseY = (event.clientY - windowHalfY) / scale;

}

function animate() {
  requestAnimationFrame(animate);
  render();
}

var flag = 0;

function render() {
  camera.position.x += (mouseX - camera.position.x) * .05;
  camera.position.y += (-mouseY - camera.position.y) * .05;
  camera.lookAt(scene.position);
  renderer.render(scene, camera);
  renderer.render(scene, camera, rt);
  renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer);
  //console.time('gpu');
  console.log(frameBuffer, [width], [height], [scale]);
  var outputBufferRaw = pixelateMatrix(frameBuffer, [width], [height], [scale]);
  //console.timeEnd('gpu');
  if (flag < 15) {
    console.log('source', frameBuffer);
    console.log('output', outputBufferRaw);

    var count = 0;
    for (let i = 0; i < frameBuffer.length; i++) {
      if (frameBuffer[i] != 0) {
        count++;
      }
    }
    console.log('source buffer length', frameBuffer.length)
    console.log('source non zero', count);

    var count = 0;
    for (let i = 0; i < outputBufferRaw.length; i++) {
      if (outputBufferRaw[i] != 0) {
        count++;
      }
    }
    console.log('output buffer length', outputBufferRaw.length)
    console.log('output non zero', count);
  }
  outputBuffer = new Uint8ClampedArray(outputBufferRaw);
  output = new ImageData(outputBuffer, width, height);
  context.putImageData(output, 0, 0);
  flag++;
}

Original Answer

I have gotten close but two issues are left

1. The image is getting inverted
2. Sometimes your inputBuffer size is not a multiple of 4, which causes it to misbehave.

Below is code I used

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
   var current_index = Math.floor(this.thread.x/4); 
   var row = Math.floor(current_index / (width[0] * scale[0]) );
   var col = Math.floor((current_index % width[0])/scale[0]);
   var index_old = Math.floor(row * (width[0] / scale[0])) + col;
   var remainder = this.thread.x % 4;
   return inputBuffer[index_old * 4 + remainder];

}).setOutput([width * height * 4]);

Below is the JSFiddle

https://jsfiddle.net/are5Lbw8/

And below is the current output

Answer 2

I think the function should look like:

var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, height, scale) {
    var x = Math.floor((this.thread.x / (width[0] * 4)) / scale[0]);
    var y = Math.floor((this.thread.x % (width[0] * 4)) / scale[0]);
    var finalval = y * (Math.floor(width[0]/scale[0]) * 4) + (x * 4);
    var remainder = this.thread.x % 4;
    return inputBuffer[finalval + remainder];
}).setOutput([width * height * 4]);

Basically, get x and y in similar manner as you, scale x and y, then converting back from (x,y) you multiple the y value by the new scaled width and add the x value. Not sure how you got x*y for that part of it.

Answer 3

Final Answer

Tarun's answer helped me get to my final solution, so his bounty was well deserved, but I actually learned about a feature (graphical output paired with context sharing for direct buffer output to the render target) of gpu.js that allows for roughly 30x faster rendering, bringing the total time for shading and rendering the output from 30ms+ to ~1ms, and this is without an additional optimization I now know to be possible to send the array buffer to the GPU even faster, but I just didnt have any motivation to get the shading / rendering time down lower than 1ms.

var canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
document.body.appendChild(canvas);
var gl = canvas.getContext('webgl');
var gpu = new GPU({
  canvas,
  gl
});
var pixelateMatrix = gpu.createKernel(function(inputBuffer, width, scale, size, purity, div) {
    var subX = Math.floor(this.thread.x / scale[0]);
    var subY = Math.floor(this.thread.y / scale[0]);
    var subIndex = ((subX * 4) + (subY * width[0] * 4));
    var rIndex = subIndex;
    var gIndex = subIndex + 1;
    var bIndex = subIndex + 2;
    var r = ((inputBuffer[rIndex] * purity[0]) + inputBuffer[rIndex - 4] + inputBuffer[rIndex + 4]) / (div[0]);
    var g = ((inputBuffer[gIndex] * purity[0]) + inputBuffer[gIndex - 4] + inputBuffer[gIndex + 4]) / (div[0]);
    var b = ((inputBuffer[bIndex] * purity[0]) + inputBuffer[bIndex - 4] + inputBuffer[bIndex + 4]) / (div[0]);
    this.color(r / 255, g / 255, b / 255);
  }).setOutput([width, height]).setGraphical(true);

inputBuffer is simply the buffer retrieved via three.js's readRenderTargetPixels method.

renderer.render(scene, camera, rt);
renderer.readRenderTargetPixels(rt, 0, 0, smallWidth, smallHeight, frameBuffer);
pixelateMatrix(frameBuffer, [smallWidth], [scale], [size], [purity], [div]);

Side Note

Can we just marvel for a moment about how much power WebGL brings to the browser? That's 8.2944 million multi-operation tasks carried out in just ~1ms. ~64 billion total maximum math ops per second for my shader by my count. That's insanity. Can that even be right? Is my math wrong on that? I see nvidia's self driving AI is performing 24 trillion ops/s, so I guess these numbers on my 1060 are within the realm of possibility. It's just incredible though.

GPU.js does a just fantastic job of optimizing matrix operations to run on the GPU without the need for learning shader code, and the creator is extremely active on the project, responding to issues usually in a matter of hours. Highly recommend you guys give the lib a try. Especially awesome for machine learning throughput.