Kurento endpoints order in order to record screencast

Question

I'm trying to record a webcam with kurento media server, here is the function I'm using at the backend:

var startScreen = (sessionId, ws, sdpOffer, callback) => {
    console.log("Start screen")
  getKurentoClient((error, kurentoClient) => {
    if (error) {
      return callback(error)
    }

    kurentoClient.create('MediaPipeline', (error, pipeline) => {
      if (error) {
        return callback(error)
      }


     var recordParams = {
        stopOnEndOfStream: true,
        mediaProfile: 'WEBM_VIDEO_ONLY',
        uri: 'file:///PATH/TO/VIDEO/screen.webm'
      }
      pipeline.create('RecorderEndpoint', recordParams, (error, recorder) => {
        if (error) return callback(error)

        screenRecorder = recorder
        recorder.record(() => console.log("recording"))
        pipeline.create('WebRtcEndpoint', (error, webRtcEndpoint) => {
          if (error) {
            pipeline.release()
            return callback(error)
          }

        screenRtcEndpoint = webRtcEndpoint

          if (candidatesQueue[sessionId]) {
            while(candidatesQueue[sessionId].length) {
              var candidate = candidatesQueue[sessionId].shift()
              webRtcEndpoint.addIceCandidate(candidate)
            }
          }
          webRtcEndpoint.on('OnIceCandidate', (event) => {
            var candidate = kurento.register.complexTypes.IceCandidate(event.candidate)
            ws.send(JSON.stringify({
              id: 'iceCandidateScreen',
              candidate: candidate
            }))
          })

          webRtcEndpoint.processOffer(sdpOffer, (error, sdpAnswer) => {
            if (error) {
              pipeline.release()
              return callback(error)
            }

            sessions[sessionId] = {
              'pipeline': pipeline,
              'webRtcEndpoint': webRtcEndpoint
            }

          webRtcEndpoint.connect(webRtcEndpoint, error => {
            if (error) {
              return callback(error)
            }
            console.log("Connect webrtcEndpoint")
            webRtcEndpoint.connect(screenRecorder, error => {
              if (error) {
                return callback(error)
              }
              console.log("connect to the screen recorder")
            })

            callback(null, sdpAnswer)
            })

          })

          webRtcEndpoint.gatherCandidates((error) => {
            if (error) {
              return callback(error)
            }
          })
        })
      })
    })
  })
}

the pipeline looks something like this:

mediaPipeline -> recorderEndpoint and recorder.record -> WebRtcEndpoint connect webrtcendpoint -> connect recorder endpoint

at the front end i do this:

mediaConstrains = {
  audio: false,
  video: {
    mandatory: {
      maxWidth: 640,
      maxHeight: 480,
      maxFrameRate: 15,
      minFrameRate: 1
    }
  }
}

var getMediaConstrains = () => mediaConstrains

var setMediaConstrains = (config) => {
  mediaConstrains = config
}

var startScreen = () => {
  var options = {
    mediaConstrains: mediaConstrains,
    onicecandidate: onIceCandidate,
    configuration: { iceServers: [
        {'url': 'turn:numb.viagenie.ca:3478',
          'username': '[email protected]',
        'credential': 'passwordrandom'}
    ] }
  }

  webRtcPeerScreen = kurentoUtils.WebRtcPeer.WebRtcPeerSendonly(options, function (error) {
    if (error) {
      return onError(error)
    }

    this.generateOffer(onOfferScreen)
  })
}

This is for only screencast which is not working, I'm using almost the exact same code to record a webcam which its fully working, This is the stop function:

var stop = () => {
  if (webRtcPeerWebcam && webRtcPeerScreen) {
    webRtcPeerWebcam.dispose()
    webRtcPeerWebcam = null
    webRtcPeerScreen.dispose()
    webRtcPeerScreen = null

    var message = {
      id: 'stop'
    }
    sendMessage(message)
  }
}

Where I dispose the pipeline at the front end but, I send this message to the backend and then this wss message calls this function:

var stop = (sessionId) => {
  if (sessions[sessionId]) {
    var pipeline = sessions[sessionId].pipeline
    pipeline.release()
    delete sessions[sessionId]
    delete candidatesQueue[sessionId]
  }
}

I think this maybe the problem of why this is not working for recording the screencast or maybe I'm connecting the pipelines improperly

Anyway thanks!

PD: i found this on KMS logs:

KurentoMediaElementImpl MediaElementImpl.cpp:434 mediaFlowOutStateChange() <kmswebrtcendpoint373> Media N OT Flowing OUT in pad default with type video 

Answer 1

The first and most important issue, is that you are processing candidates, and gathering those candidates, before the SDP negotiation. This won't work, so I think that your webrtc is not working at all, regardless how you are connecting the endpoints.

Second, you are creating the recorder before the WebRtcEndpoint, and after that invoking record. The recorder does not have anything connected, and no media is flowing IN. I'd suggest you to invoke the record method after you connect the WebRtcEndpoint to the recorder and after media is flowing. For that purpose, you can add a listener to the event MediaStateChanged like so

webRtcEndpoint.on('MediaStateChanged', function (event) {
  if ((event.oldState !== event.newState) && (event.newState === 'CONNECTED')) {
    // Start recording
    recorderEndpoint.record();
  }
});

Connect the WebRtcEndpoint to the recorder right after you create it.

Also, just as a side note, this line does not make sense as the endpoint is sendonly

webRtcEndpoint.connect(webRtcEndpoint, error => {

Just to summarize, this is what I'm suggesting. Don't forget to fill in whichever gaps you might find, as without filling the callbacks from OnIceCandidate and so on, it won't work.

var startScreen = (sessionId, ws, sdpOffer, callback) => {
  console.log("Start screen")
  getKurentoClient((error, kurentoClient) => {
    if (error) {
      return callback(error)
    }

    kurentoClient.create('MediaPipeline', (error, pipeline) => {
      pipeline.create('RecorderEndpoint', recordParams, (error, recorder) => {
        pipeline.create('WebRtcEndpoint', (error, webRtcEndpoint) => {
          webRtcEndpoint.connect(screenRecorder, error => {
            webRtcEndpoint.processOffer(sdpOffer, (error, sdpAnswer) => {
              // The SDP negotiation must be completed before processing candidates
              callback(null, sdpAnswer)
              if (candidatesQueue[sessionId]) {
                while (candidatesQueue[sessionId].length) {
                  var candidate = candidatesQueue[sessionId].shift()
                  webRtcEndpoint.addIceCandidate(candidate)
                }
              }

              webRtcEndpoint.on('MediaStateChanged', function(event) {
                // This will start recording right after media starts flowing
                if ((event.oldState !== event.newState) && (event.newState === 'CONNECTED')) {
                  // Start recording
                  recorderEndpoint.record();
                }
              })
              
              webRtcEndpoint.on('OnIceCandidate', (event) => { /* your code here */ })
                // Candidates must be gathered after the negotiation is completed, and the
                // event handler is bound
              webRtcEndpoint.gatherCandidates((error) => { /* your code here */ })
            })
          })
        })
      })
    })
  })
}