How can I get the current frame of an HTML video as a JavaScript array?

Question

On the press of a button, I would like to convert the current video frame from the webcam to an array or tfjs tensor.

I am unable to find a way to:

• Get the array
• Crop and resize the array to the largest possible 64x64x3 image

I would like to use/implement some functions (getFrameFromVideo and crop_to_size) that can do something along the lines of the following:

const video = document.getElementById("cam");
var arr_frame;
var cropped_frame;
var tensor_frame;

function crop_to_size(img, xsize, ysize) {
  // crop the image
  return cropped_img;
}

function predict_img() {
  arr_frame = getFrameFromVideo(video);
  cropped_frame = crop_to_size(arr_frame, 64, 64);
  tensor_frame = tf.tensor(cropped_frame);
  // do stuff with tensor
}

That can be called from a button such as below

<video id="cam" autoplay muted></video>
<button onclick="predict_img()">
  predict current frame
</button>

Answer 1

Yes. You can do it. Play the webcam on a <video> then put image on <canvas> then get its data. In between resize, and remove the alpha-channel.

const video = document.querySelector('video');
const canvas = document.querySelector('canvas');
const ctx = canvas.getContext('2d');

var constraints = {
  audio: false
};

navigator.mediaDevices.getUserMedia(constraints).then(function(stream) {
  video.srcObject = stream;
}).catch((err) => {
  console.log("playing video instead of webcam")
  video.crossOrigin = "anonymous"
  video.src = "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/BigBuckBunny.mp4";

});

function snapshot() {

  // resize and crop and center
  const aspectRatio = video.videoWidth / video.videoHeight;
  const targetWidth = 64
  const targetHeight = 64
  let newWidth, newHeight, startX, startY;
  if (aspectRatio > 1) {
    newWidth = targetWidth;
    newHeight = targetHeight / aspectRatio;
    startX = 0;
    startY = (targetHeight - newHeight) / 2; 
  } else {
    newWidth = targetWidth * aspectRatio;
    newHeight = targetHeight;
    startX = (targetWidth - newWidth) / 2; 
    startY = 0;
  }

  // clear canvas 
  ctx.fillStyle = '#00ff00';
  ctx.fillRect(0, 0, canvas.width, canvas.height);

  ctx.drawImage(video, startX, startY, newWidth, newHeight);

  // imageData is 64 x 64 x 4 = (rgba)
  const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);

  // convert to 64x64x3
  const pixels = [];
  for (let i = 0; i < imageData.data.length; i += 4) {
    pixels.push(imageData.data[i]);
    pixels.push(imageData.data[i + 1]);
    pixels.push(imageData.data[i + 2]);
  }

  console.log("" + pixels)
}

<video autoplay muted style="width: 300px; height: 150px"></video>
<canvas width="64" height="64" style="border: 1px solid red"></canvas>
<button onclick="snapshot()">Take a snapshot</button>