let video;

let predictions = [];

​

function setup() {

  createCanvas(640, 480);

  video = createCapture(VIDEO);

  video.size(width, height);

​

  facemesh = ml5.facemesh(video, modelReady);

​

  // This sets up an event that fills the global variable "predictions"

  // with an array every time new predictions are made

  facemesh.on("predict", (results) => {

    predictions = results;

  });

​

  // Hide the video element, and just show the canvas

  video.hide();

  setInterval(() => {

    sendKeypointsToArduino()

  }, 1000 / 4)

}

​

function modelReady() {

  console.log("Model ready!");

}

​

function draw() {

  image(video, 0, 0, width, height);

  // background(0)

  drawKeypoints();

​

  // send to your arduino

  // do a moving window average (because the values vary so much)

  // send at the right interval (instead of 30fps)

​

  if (

    predictions.length > 0 &&

    predictions[0].scaledMesh &&

    predictions[0].scaledMesh.length > 0

  ) {

    // We can call both functions to draw all keypoints

    // drawKeypoints();

    debugger;

    document.getElementById("debug").innerHTML =

      "z" + predictions[0].scaledMesh[296, 66, 199, 215, 435, 1, 50, 280][2];

  }

}

​

let collectionOfKeypoints = []

​

// A function to draw ellipses over the detected keypoints

function drawKeypoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

​

    // TODO: keep collecting / adding keypoints here

    // TODO: figure out the data structure to add them in

    collectionOfKeypoints.push({x: 1, y: 2, z: 3})

    // Draw facial keypoints.

//     for (let j = 0; j < keypoints.length; j += 1) {

//       const [x, y] = keypoints[j];

​

//       fill(0, 255, 0);

//       ellipse(x, y, 5, 5);

//     }

    // Draw facial keypoints.

    for (let j = 0; j < keypoints.length; j += 1) {

      if (j == 296) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 66) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 199) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 215) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 435) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 1) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 50) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      } else if (j == 280) {

        const [x, y, z] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, z, z);

      }

    }

  }

}

​

​

// A function to draw ellipses over the detected keypoints

function sendKeypointsToArduino() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

​

    // Draw facial keypoints.

//     for (let j = 0; j < keypoints.length; j += 1) {

//       const [x, y] = keypoints[j];

​

//       fill(0, 255, 0);

//       ellipse(x, y, 5, 5);

//     }

    // Draw facial keypoints.

    for (let j = 0; j < keypoints.length; j += 1) {

      if (j == 296) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 66) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 199) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 215) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 435) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 1) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 50) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      } else if (j == 280) {

        const [x, y] = keypoints[j];

        fill(0, 255, 0);

        ellipse(x, y, 5, 5);

      }

    }

  }

  // Average using the collection

  let sumX = 0

  let sumY = 0

  let counter = 0

  for (let kp of collectionOfKeypoints) {

    sumX = sumX + kp.x

    sumY = sumY + kp.Y

    counter +1

  }

  let avgX = sumX / counter

  let avgY = sumY / counter

  // TODO: Send the data to arduino

  // Clear the collection

  collectionOfKeypoints = []

}

​