Audio mixing with canvas streaming using Flashphoner

[KaranBhansali91]

Hi,

Currently we are using canvas streaming in our platform. Is there Flashphoner api to support multiple audio streams such as microphone + mp3 with canvas streaming?

 

[Max]

Good day.
There are two options:
1. Publish two separate audio only streams from microphone and from mp3 (playing in canvas), then play them on client side separately, or mix them on server side
2. Publish two audio tracks in one WebRTC connection using SFU functions. In this case, you can choose audio track to play on players side.

 

[KaranBhansali91]

Hi,
Using mix them on server side method, from same client can I publish audio stream and video stream as canvas stream simultaneously , then mix them on server side ? Following is the code for video streaming from canvas

room
    .publish({
      display: document.getElementById("localDisplay"),
      constraints: constraints,
      cacheLocalResources: true,
      receiveVideo: false,
      receiveAudio: false,
      disableConstraintsNormalization: true,
      useCanvasMediaStream: true,
    })

can I add the below code to publish audio stream using canvas , localAudioDisplay is a canvas element

room
    .publish({
      display: document.getElementById("localAudioDisplay"),
      constraints: constraints,     
      useCanvasMediaStream: true,
    })

 

[Max]

can I add the below code to publish audio stream using canvas , localAudioDisplay is a canvas element

To capture audio from canvas, use AudioContext. See Canvas Streaming example on GitHub:

function createCanvasStream() {
    var canvasContext = canvas.getContext("2d");
    var canvasStream = canvas.captureStream(30);
    mockVideoElement = document.createElement("video");
    mockVideoElement.setAttribute("playsinline", "");
    mockVideoElement.setAttribute("webkit-playsinline", "");
    mockVideoElement.src = '../../dependencies/media/test_movie.mp4';
    mockVideoElement.loop = true;
    mockVideoElement.muted = true;
    var useRequestAnimationFrame = $("#usedAnimFrame").is(':checked');
    mockVideoElement.addEventListener("play", function () {
        var $this = this;
        (function loop() {
            if (!$this.paused && !$this.ended) {
                canvasContext.drawImage($this, 0, 0);
                if (useRequestAnimationFrame) {
                    requestAnimationFrame(loop);
                } else {
                    setTimeout(loop, 1000 / 30); // drawing at 30fps
                }
            }
        })();
    }, 0);
    if (!$("#sendVideo").is(':checked')) {
        canvasStream.removeTrack(canvasStream.getVideoTracks()[0]);
    }
    mockVideoElement.play();
    if ($("#sendAudio").is(':checked')) {
        mockVideoElement.muted = false;
        try {
            var audioContext = new (window.AudioContext || window.webkitAudioContext)();
        } catch (e) {
            console.warn("Failed to create audio context");
        }
        var source = audioContext.createMediaElementSource(mockVideoElement);
        var destination = audioContext.createMediaStreamDestination();
        source.connect(destination);
        canvasStream.addTrack(destination.stream.getAudioTracks()[0]);
    }
    return canvasStream;
}