Using Self-Segmentation javascript of Mediapipe to pass user selfie as a texture on Networked-Aframe for multiplaying experiences

Well my target is to make a multiplaying 3D environment where the persons are represented by cubes that have as textures their selfie without the background. It is a kind of cheap virtual production without chroma key background. The background is removed with MediaPipe Selfie-Segmentation. The issue is that instead of having the other player texture on the cube (P1 should see P2, and P2 should see P1, each one sees his selfie. This means that P1 sees P1 and P2 sees P2 which is bad.

Live demo: https://vrodos-multiplaying.iti.gr/plain_aframe_mediapipe_testbed.html

Instructions: You should use two machines to test it Desktops, Laptops or Mobiles. Use only Chrome as Mediapipe is not working at other browsers. In case webpage jams, reload the webpage (Mediapipe is sometimes sticky). At least two machines should load the webpage in order to start the multiplaying environment.

Code:

<html>
<head>
    <script src="https://aframe.io/releases/1.2.0/aframe.min.js"></script>

    <!-- Selfie Segmentation of Mediapipe -->
    <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/control_utils.css" crossorigin="anonymous">

    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/camera_utils.js" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/control_utils.js" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/drawing_utils.js" crossorigin="anonymous"></script>
    <script src="https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/selfie_segmentation.js" crossorigin="anonymous"></script>

    <!-- Networked A-frame -->
    <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.3.0/socket.io.slim.js"></script>
    <script src="/easyrtc/easyrtc.js"></script>
    <script src="https://unpkg.com/networked-aframe/dist/networked-aframe.min.js"></script>
</head>
<body>


<a-scene networked-scene="
         adapter: easyrtc;
         video: true;
         debug: true;
         connectOnLoad: true;">

    <a-assets>
        <template id="avatar-template">
            <a-box material="alphaTest: 0.5; transparent: true; side: both;"
                   width="1"
                   height="1"
                   position="0 0 0" rotation="0 0 0"
                   networked-video-source-mediapiped></a-box>
        </template>
    </a-assets>


    <a-entity id="player"
              networked="template:#avatar-template;attachTemplateToLocal:false;"
              camera wasd-controls look-controls>
    </a-entity>

    <a-plane position="0 -2 -4" rotation="-90 0 0" width="4" height="4" color="#7BC8A4" ></a-plane>
    <a-sky color="#777"></a-sky>

</a-scene>


</body>



<script>

// Networked Aframe : Register new component for streaming the Selfie-Segmentation video stream
AFRAME.registerComponent('networked-video-source-mediapiped', {

    schema: {
    },

    dependencies: ['material'],

    init: function () {

        this.videoTexture = null;
        this.video = null;
        this.stream = null;


        this._setMediaStream = this._setMediaStream.bind(this);

        NAF.utils.getNetworkedEntity(this.el).then((networkedEl) => {

            const ownerId = networkedEl.components.networked.data.owner;

            if (ownerId) {

                NAF.connection.adapter.getMediaStream(ownerId, "video")
                    .then(this._setMediaStream)
                    .catch((e) => NAF.log.error(`Error getting media stream for ${ownerId}`, e));
            } else {
                // Correctly configured local entity, perhaps do something here for enabling debug audio loopback
            }
        });
    },


    _setMediaStream(newStream) {

        if(!this.video) {
            this.setupVideo();
        }

        if(newStream != this.stream) {

            if (this.stream) {
                this._clearMediaStream();
            }

            if (newStream) {
                this.video.srcObject = canvasElement.captureStream(30);

                this.videoTexture = new THREE.VideoTexture(this.video);
                this.videoTexture.format = THREE.RGBAFormat;

                // Mesh to send
                const mesh = this.el.getObject3D('mesh');
                mesh.material.map = this.videoTexture;
                mesh.material.needsUpdate = true;
            }

            this.stream = newStream;
        }
    },

    _clearMediaStream() {

        this.stream = null;

        if (this.videoTexture) {

            if (this.videoTexture.image instanceof HTMLVideoElement) {
                // Note: this.videoTexture.image === this.video
                const video = this.videoTexture.image;
                video.pause();
                video.srcObject = null;
                video.load();
            }

            this.videoTexture.dispose();
            this.videoTexture = null;
        }
    },

    remove: function() {
        this._clearMediaStream();
    },

    setupVideo: function() {
        if (!this.video) {
            const video = document.createElement('video');
            video.setAttribute('autoplay', true);
            video.setAttribute('playsinline', true);
            video.setAttribute('muted', true);
            this.video = video;
        }
    }
});


// -----  Mediapipe ------
const controls = window;
//const mpSelfieSegmentation = window;
const examples = {
    images: [],
    // {name: 'name', src: 'https://url.com'},
    videos: [],
};
const fpsControl = new controls.FPS();
let activeEffect = 'background';
const controlsElement = document.createElement('control-panel');


var canvasElement = document.createElement('canvas');
canvasElement.height= 1000;
canvasElement.width = 1000;
var canvasCtx = canvasElement.getContext('2d');

// --------
function drawResults(results) {
    canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
    canvasCtx.drawImage(results.segmentationMask, 0, 0, canvasElement.width, canvasElement.height);
    canvasCtx.globalCompositeOperation = 'source-in';
    canvasCtx.drawImage(results.image, 0, 0, canvasElement.width, canvasElement.height);
}

const selfieSegmentation = new SelfieSegmentation({
    locateFile: (file) => {
        console.log(file);
        return `https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/${file}`;
    }
});

selfieSegmentation.onResults(drawResults);
// -------------

new controls
    .ControlPanel(controlsElement, {
        selfieMode: true,
        modelSelection: 1,
        effect: 'background',
    })
    .add([
        new controls.StaticText({title: 'MediaPipe Selfie Segmentation'}),
        fpsControl,
        new controls.Toggle({title: 'Selfie Mode', field: 'selfieMode'}),
        new controls.SourcePicker({
            onSourceChanged: () => {
                selfieSegmentation.reset();
            },
            onFrame: async (input, size) => {
                const aspect = size.height / size.width;
                let width, height;
                if (window.innerWidth > window.innerHeight) {
                    height = window.innerHeight;
                    width = height / aspect;
                } else {
                    width = window.innerWidth;
                    height = width * aspect;
                }
                canvasElement.width = width;
                canvasElement.height = height;
                await selfieSegmentation.send({image: input});
            },
            examples: examples
        }),
        new controls.Slider({
            title: 'Model Selection',
            field: 'modelSelection',
            discrete: ['General', 'Landscape'],
        }),
        new controls.Slider({
            title: 'Effect',
            field: 'effect',
            discrete: {'background': 'Background', 'mask': 'Foreground'},
        }),
    ])
    .on(x => {
        const options = x;
        //videoElement.classList.toggle('selfie', options.selfieMode);
        activeEffect = x['effect'];
        selfieSegmentation.setOptions(options);
    });
</script>

</html>

Screenshot:

Here Player 1 sees Player 1 stream instead of Player 2 stream (Grrrrrr):

Solution

Well, I found it. The problem was that a MediaStream can have many video tracks. See my answer here:

https://github.com/networked-aframe/networked-aframe/issues/269

Unfortunately networked-aframe EasyRtcAdapter does not support many MediaStreams, but it is easy to add another video track and then get videotrack[1] instead of videotrack[0]. I should make a special EasyRtcAdapter to avoid having two video tracks and avoid overstressing bandwidth.