// IMA Low Res Summer 2022, Make Art with AI

// Project 1: Interactive Experience with Body Movement

​

// test that ml5 is loaded

console.log("ml5 version:", ml5.version);

​

// facemesh variables

let facemesh;

let video;

let predictions = [];

let person;

​

// image assets

let imgRightEye;

let flash;

let mute;

let speaker;

let lens;

​

// keypoint variables

let lipTX = 0;

let lipTY = 0;

let lipBX = 0;

let lipBY = 0;

let lipsDist = 0;

let eyeLXTop = 0;

let eyeLYTop = 0;

let eyeLXBot = 0;

let eyeLYBot = 0;

let eyeLDist = 0;

let eyeRXTop = 0;

let eyeRYTop = 0;

let eyeRXBot = 0;

let eyeRYBot = 0;

let eyeRDist = 0;

​

// preload images

function preload() {

  imgRightEye = loadImage("rightEyeLens.png");

  flash = loadImage("flash.png");

  mute = loadImage("muteIcon.png");

  speaker = loadImage("speaker.png");

  lens = loadImage("lens.png");

}

​

function setup() {

  createCanvas(640, 480);

​

  // webcam

  video = createCapture(VIDEO);

  video.size(width, height);

  // video.hide();

​

  // load facemesh

  facemesh = ml5.facemesh(video, modelReady);

​

  // set up event to fill "predictions" with an array

  // every time new predictions are made

  facemesh.on("face", (results) => {

    predictions = results;

    person = predictions[0];

  });

}

​

// confirm that facemesh model is ready

function modelReady() {

  console.log("Model ready!");

}

​

function draw() {

  // put camera on the canvas

  push();

  translate(width, 0);

  scale(-1, 1);

  image(video, 0, 0, width, height);

  pop();

​

  // call function to draw keypoints

  updateKeypoints();

  // show certain images based on facial movements

//   // track top & bottom lip

//   fill(255, 0, 0);

//   circle(lipTX, lipTY, 30);

//   fill(255, 255, 0);

//   circle(lipBX, lipBY, 30);

//   // track left brow & bottom lid

//   fill(255, 0, 0);

//   circle(eyeLXTop, eyeLYTop, 10);

//   fill(255, 255, 0);

//   circle(eyeLXBot, eyeLYBot, 10);

//   // track right brow & bottom lid

//   fill(255, 0, 0);

//   circle(eyeRXTop, eyeRYTop, 10);

//   fill(255, 255, 0);

//   circle(eyeRXBot, eyeRYBot, 10);

  push();

  imageMode(CENTER);

  // LIPS

  // show speaker when talking

  if (lipsDist > 30) { 

  image(speaker, lipTX, lipTY, 100, 75);

  } else { // show mute icon when not talking

    image(mute, lipTX+2, lipTY+10, 30, 30);  

  }

  // EYES

  // show camera lens on eyes at all times  

  image(lens, eyeLXTop, ((eyeLYTop + eyeLYBot)/2), 100, 100);

  image(lens, eyeRXTop, ((eyeRYTop + eyeRYBot)/2), 100, 100);

  // show camera flash when blinking

  if (eyeLDist < 45.5) { // left eye distance

  image(flash, eyeLXTop, ((eyeLYTop + eyeLYBot)/2), 150, 150);

  }

  if (eyeRDist < 43) { // right eye distance

  image(flash, eyeRXTop, ((eyeRYTop + eyeRYBot)/2), 150, 150);

  }

  pop();

}

​

// play camera flash animation

function mouseClicked() {

  // testing

  console.log("Mouse clicked");

}

​

function updateKeypoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypointsTopLip = predictions[i].annotations.lipsUpperOuter;

    const keypointsBottomLip = predictions[i].annotations.lipsLowerOuter;

    const keypointsRightEyeTop = predictions[i].annotations.rightEyebrowUpper;

    const keypointsRightEyeBottom = predictions[i].annotations.rightEyeLower0;

    const keypointsLeftEyeTop = predictions[i].annotations.leftEyebrowUpper;

    const keypointsLeftEyeBottom = predictions[i].annotations.leftEyeLower0;    

// show single keypoints for top & bottom lip    

/*    for (let j = 0; j < keypointsTopLip.length; j += 1) {

      const [x, y] = keypointsTopLip[j];

      fill(0, 255, 0);

      text(j, x, y-20); // 6

    }

    for (let k = 0; k < keypointsBottomLip.length; k += 1) {

      const [x, y] = keypointsBottomLip[k];

      fill(0, 255, 0);

      text(k, x, y-20); //5 

    }

*/   

    // isolate middle keypoint for lips

    lipTX = width - keypointsTopLip[5][0]; // width - x

    lipTY = keypointsTopLip[5][1];

    lipBX = width - keypointsBottomLip[4][0];

    lipBY = keypointsBottomLip[4][1];

    lipsDist = dist(lipTX, lipTY, lipBX, lipBY);

    text(lipsDist.toFixed(2), lipTX - 50, lipTY); // show distance between top & bottom lip

​

    // isolate middle keypoint for eyes

    eyeRXTop = width - keypointsRightEyeTop[3][0];

    eyeRYTop = keypointsRightEyeTop[3][1];

    eyeRXBot = width - keypointsRightEyeBottom[5][0];

    eyeRYBot = keypointsRightEyeBottom[5][1];

    eyeRDist = dist(eyeRXTop, eyeRYTop, eyeRXBot, eyeRYBot);

    eyeLXTop = width - keypointsLeftEyeTop[3][0];

    eyeLYTop = keypointsLeftEyeTop[3][1];

    eyeLXBot = width - keypointsLeftEyeBottom[4][0];

    eyeLYBot = keypointsLeftEyeBottom[4][1];

    eyeLDist = dist(eyeLXTop, eyeLYTop, eyeLXBot, eyeLYBot);

    text(eyeLDist.toFixed(2), eyeLXTop - 70, eyeLYTop); // show distance between keypoints for left eye

    text(eyeRDist.toFixed(2), eyeRXTop + 75, eyeRYTop); // show distance between keypoints for right eye

    // place camera lens over right eye

    for (let l = 0; l < keypointsRightEyeTop.length; l += 1) {

      const [x, y] = keypointsRightEyeTop[l];

    }

    for (let m = 0; m < keypointsRightEyeBottom.length; m += 1) {

      const [x, y] = keypointsRightEyeBottom[m];

    }

​

    // draw facial keypoints for left eye

    for (let n = 0; n < keypointsLeftEyeTop.length; n += 1) {

      const [x, y] = keypointsLeftEyeTop[n];

    }

    for (let o = 0; o < keypointsLeftEyeBottom.length; o += 1) {

      const [x, y] = keypointsLeftEyeBottom[o];

    }

  }

}

​