Mixing two audio buffers, put one on background of another by using web Audio Api

Question

I want to mix two audio sources by put one song as background of another into single source.

for example, i have input :

<input id="files" type="file" name="files[]" multiple onchange="handleFilesSelect(event)"/>

And script to decode this files:

window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new window.AudioContext();
var sources = [];
var files = [];
var mixed = {};

function handleFilesSelect(event){
    if(event.target.files.length <= 1)
          return false;

     files = event.target.files;
     readFiles(mixAudioSources);
}

function readFiles(index, callback){
    var freader = new FileReader();
    var i = index ? index : 0;

    freader.onload = function (e) {     
        context.decodeAudioData(e.target.result, function (buf) {

            sources[i] = context.createBufferSource();
            sources[i].connect(context.destination);
            sources[i].buffer = buf;

            if(files.length > i+1){
                readFiles(i + 1, callback);
            } else {
                if(callback){
                    callback();
                }
            }
        });
    };

    freader.readAsArrayBuffer(files[i]);
}

function mixAudioSources(){
    //So on our scenario we have here two decoded audio sources in "sources" array.
    //How we can mix that "sources" into "mixed" variable by putting "sources[0]" as background of "sources[1]"
}

So how i can mix this sources into one source? For example i have two files, how i can put one source as background of another and put this mix into single source?

Another scenario: if i read input stream from microphone for example and i want to put this input on background song (some kind of karaoke) it is possible to do this work on client with html5 support? What about performance? Maybe better way to mix this audio sources on server side?

If it possible, so what the possible implementation of mixAudioSources function?

Thanks.

Answer 1

Two approach originally posted at Is it possible to mix multiple audio files on top of each other preferably with javascript, adjusted to process File objects at change event of <input type="file"> element.

The first approach utilizes OfflineAudioContext(), AudioContext.createBufferSource(), AudioContext.createMediaStreamDestination(), Promise constructor, Promise.all(), MediaRecorder() to mix audio tracks, then offer mixed audio file for download.

var div = document.querySelector("div");

function handleFilesSelect(input) {
  div.innerHTML = "loading audio tracks.. please wait";
  var files = Array.from(input.files);
  var duration = 60000;
  var chunks = [];
  var audio = new AudioContext();
  var mixedAudio = audio.createMediaStreamDestination();
  var player = new Audio();
  var context;
  var recorder;
  var description = "";
  
  player.controls = "controls";
  
  function get(file) {
    description += file.name.replace(/\..*|\s+/g, "");
    return new Promise(function(resolve, reject) {
      var reader = new FileReader;
      reader.readAsArrayBuffer(file);
      reader.onload = function() {
        resolve(reader.result)
      }
    })
  }

  function stopMix(duration, ...media) {
    setTimeout(function(media) {
      media.forEach(function(node) {
        node.stop()
      })
    }, duration, media)
  }

  Promise.all(files.map(get)).then(function(data) {
      var len = Math.max.apply(Math, data.map(function(buffer) {
        return buffer.byteLength
      }));
      context = new OfflineAudioContext(2, len, 44100);
      return Promise.all(data.map(function(buffer) {
          return audio.decodeAudioData(buffer)
            .then(function(bufferSource) {
              var source = context.createBufferSource();
              source.buffer = bufferSource;
              source.connect(context.destination);
              return source.start()
            })
        }))
        .then(function() {
          return context.startRendering()
        })
        .then(function(renderedBuffer) {
          return new Promise(function(resolve) {
            var mix = audio.createBufferSource();
            mix.buffer = renderedBuffer;
            mix.connect(audio.destination);
            mix.connect(mixedAudio);
            recorder = new MediaRecorder(mixedAudio.stream);
            recorder.start(0);
            mix.start(0);
            div.innerHTML = "playing and recording tracks..";
            // stop playback and recorder in 60 seconds
            stopMix(duration, mix, recorder)

            recorder.ondataavailable = function(event) {
              chunks.push(event.data);
            };

            recorder.onstop = function(event) {
              var blob = new Blob(chunks, {
                "type": "audio/ogg; codecs=opus"
              });
              console.log("recording complete");
              resolve(blob)
            };
          })
        })
        .then(function(blob) {
          console.log(blob);
          div.innerHTML = "mixed audio tracks ready for download..";
          var audioDownload = URL.createObjectURL(blob);
          var a = document.createElement("a");
          a.download = description + "." + blob.type.replace(/.+\/|;.+/g, "");
          a.href = audioDownload;
          a.innerHTML = a.download;
          document.body.appendChild(a);
          a.insertAdjacentHTML("afterend", "<br>");
          player.src = audioDownload;
          document.body.appendChild(player);
        })
    })
    .catch(function(e) {
      console.log(e)
    });

}

<!DOCTYPE html>
<html>

<head>
</head>

<body>
  <input id="files" 
         type="file" 
         name="files[]" 
         accept="audio/*" 
         multiple 
         onchange="handleFilesSelect(this)" />
  <div></div>
</body>

</html>

The second approach uses AudioContext.createChannelMerger(), AudioContext.createChannelSplitter()

var div = document.querySelector("div");

function handleFilesSelect(input) {

  div.innerHTML = "loading audio tracks.. please wait";
  var files = Array.from(input.files);
  var chunks = [];
  var channels = [
    [0, 1],
    [1, 0]
  ];
  var audio = new AudioContext();
  var player = new Audio();
  var merger = audio.createChannelMerger(2);
  var splitter = audio.createChannelSplitter(2);
  var mixedAudio = audio.createMediaStreamDestination();
  var duration = 60000;
  var context;
  var recorder;
  var audioDownload;
  var description = "";

  player.controls = "controls";

  function get(file) {
    description += file.name.replace(/\..*|\s+/g, "");
    console.log(description);
    return new Promise(function(resolve, reject) {
      var reader = new FileReader;
      reader.readAsArrayBuffer(file);
      reader.onload = function() {
        resolve(reader.result)
      }
    })
  }

  function stopMix(duration, ...media) {
    setTimeout(function(media) {
      media.forEach(function(node) {
        node.stop()
      })
    }, duration, media)
  }

  Promise.all(files.map(get)).then(function(data) {
      return Promise.all(data.map(function(buffer, index) {
          return audio.decodeAudioData(buffer)
            .then(function(bufferSource) {
              var channel = channels[index];
              var source = audio.createBufferSource();
              source.buffer = bufferSource;
              source.connect(splitter);
              splitter.connect(merger, channel[0], channel[1]);          
              return source
            })
        }))
        .then(function(audionodes) {
          merger.connect(mixedAudio);
          merger.connect(audio.destination);
          recorder = new MediaRecorder(mixedAudio.stream);
          recorder.start(0);
          audionodes.forEach(function(node, index) {
            node.start(0)
          });
          
          div.innerHTML = "playing and recording tracks..";
          
          stopMix(duration, ...audionodes, recorder);

          recorder.ondataavailable = function(event) {
            chunks.push(event.data);
          };

          recorder.onstop = function(event) {
            var blob = new Blob(chunks, {
              "type": "audio/ogg; codecs=opus"
            });
            audioDownload = URL.createObjectURL(blob);
            var a = document.createElement("a");
            a.download = description + "." + blob.type.replace(/.+\/|;.+/g, "");
            a.href = audioDownload;
            a.innerHTML = a.download;
            player.src = audioDownload;
            document.body.appendChild(a);
            document.body.appendChild(player);
          };
        })
    })
    .catch(function(e) {
      console.log(e)
    });
}

<!DOCTYPE html>
<html>

<head>
</head>

<body>
  <input id="files" 
         type="file" 
         name="files[]" 
         accept="audio/*" 
         multiple onchange="handleFilesSelect(this)" />
  <div></div>
</body>

</html>

Answer 2

I just want to complement the excellent answer of guest271314 and post here the solution based on answer of guest271314 for second scenario (the second source is microphone input). Actually is client karaoke. Script:

window.AudioContext = window.AudioContext || window.webkitAudioContext;
var context = new window.AudioContext();
var playbackTrack = null;

function handleFileSelect(event){

     var file = event.files[0];
     var freader = new FileReader();

    freader.onload = function (e) {     
        context.decodeAudioData(e.target.result, function (buf) {

            playbackTrack = context.createBufferSource();
            playbackTrack.buffer = buf;

            var karaokeButton = document.getElementById("karaoke_start");
            karaokeButton.style.display = "inline-block";
            karaokeButton.addEventListener("click", function(){
                startKaraoke();
            });
        });
    };

    freader.readAsArrayBuffer(file);
}

function stopMix(duration, mediaRecorder) {
    setTimeout(function(mediaRecorder) {
      mediaRecorder.stop();
      context.close();
    }, duration, mediaRecorder)
 }

function startKaraoke(){

    navigator.mediaDevices.getUserMedia({audio: true,video: false})
        .then(function(stream) {
            var mixedAudio = context.createMediaStreamDestination();
            var merger = context.createChannelMerger(2);
            var splitter = context.createChannelSplitter(2);
            var duration = 5000;

            var chunks = [];
            var channel1 = [0,1];
            var channel2 = [1, 0];

            var gainNode = context.createGain();
            playbackTrack.connect(gainNode);
            gainNode.connect(splitter);
            gainNode.gain.value = 0.5; // From 0 to 1
            splitter.connect(merger, channel1[0], channel1[1]);

            var microphone = context.createMediaStreamSource(stream);
            microphone.connect(splitter);
            splitter.connect(merger, channel2[0], channel2[1]);

            merger.connect(mixedAudio);
            merger.connect(context.destination);

            playbackTrack.start(0);
            var mediaRecorder = new MediaRecorder(mixedAudio.stream);
            mediaRecorder.start(1);

            mediaRecorder.ondataavailable = function (event) {
                chunks.push(event.data);
            }

            mediaRecorder.onstop = function(event) {
              var player = new Audio();
              player.controls = "controls";

              var blob = new Blob(chunks, {
                "type": "audio/mp3"
              });

              audioDownload = URL.createObjectURL(blob);
              var a = document.createElement("a");
              a.download = "karaokefile." + blob.type.replace(/.+\/|;.+/g, "");
              a.href = audioDownload;
              a.innerHTML = a.download;
              player.src = audioDownload;
              document.body.appendChild(a);
              document.body.appendChild(player);
            };

            stopMix(duration, mediaRecorder);
        })
        .catch(function(error) {
          console.log('error: ' + error);
        });

}

And Html:

<input id="file" 
         type="file" 
         name="file" 
         accept="audio/*" 
         onchange="handleFileSelect(this)" />
  <span id="karaoke_start" style="display:none;background-color:yellow;cursor:pointer;">start karaoke</span>

Here the working plnkr example: plnkr