//This sketch tracks your facial features and depending on your mood, it will affect the canvas environment.

//Happy: If you smile, then the sun will come up and blush marks will appear on your cheeks.

//Sad: If you raise your eyebrows and frown, then you will begin to cry and it will darken.

//Angry: If your eyebrows are crossed, then the user's face will become red with smoke coming out of their ears.

​

//Global Variables --------------------------------------------------------------

let facemesh;

let myVideo;

let predictions = [];

const cryingParticles = [];

let faceTooFar = false;

​

let happySoundGate = true;

let sadSoundGate = true;

let angrySoundGate = true;

​

//Facemesh Setup --------------------------------------------------------------

function preload() {

  soundFormats('mp3');

  Happy = loadSound("Audio/Happy.mp3");

  Sad = loadSound("Audio/Sad.mp3");

  Angry = loadSound("Audio/Angry.mp3");

  Flowers = loadImage("Images/Happy_Flowers.png");

  Clouds = loadImage("Images/Dark_Clouds.png");

  Fire = loadImage("Images/Fire.gif");

}

​

function setup() {

  createCanvas(640, 480);

  myVideo = createCapture(VIDEO);

  textAlign(CENTER);

  myVideo.size(width, height);

  myVideo.hide();

  ml5.flipImage(myVideo);

  //Initiate the audio tracks

  Happy.play();

  Sad.play();

  Angry.play();

  Happy.setVolume(0);

  Sad.setVolume(0);

  Angry.setVolume(0);

  facemesh = ml5.facemesh(myVideo, modelReady);

  // Fills 'predictions' with an array every time new predictions of the facepoints are made

  facemesh.on("predict", results => {predictions = results; });

}

​

function modelReady() {

  console.log("Model ready!");

}

​

//Facemesh Testing Functions  --------------------------------------------------------------

//A function to draw ellipses over the detected keypoints

function drawKeyPoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

​

    // Draw facial keypoints.

    for (let j = 0; j < keypoints.length; j += 1) {

      const [x, y] = keypoints[j];

​

      fill(0, 255, 0);

      ellipse(x, y, 5, 5);

    }

  }

}

​

//Testing function to see which points are where for which part of the face

function drawTestPoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].annotations.rightEyebrowUpper;

​

    // Draw facial keypoints.

    for (let j = 0; j < keypoints.length; j += 1) {

      const [x, y] = keypoints[j];

​

      fill(0, 255, 0);

      textSize(10);

      text(j, x, y);

    }

  }

}

​

//Emotion Checker Function ----------------------------------------------------------------

function EmotionCheck() {

  //Condition if there is a happy face: 

  for (let i = 0; i < predictions.length; i += 1) {

    //Getting the upperLip component to test for happiness or sadness 

    const upperLip = predictions[i].annotations.lipsUpperOuter;

    //Breaking upperLip components into their x and y coordinates and c

    const [upperLipLeft_x, upperLipLeft_y] = upperLip[0];

    const [upperLipRight_x, upperLipRight_y] = upperLip[10];

    let lipDistance = dist(upperLipLeft_x, upperLipLeft_y, upperLipRight_x, upperLipRight_y);

    //Happy Emotion: Uses lips to check if happy and if so, activates the blushes on the cheeks

    const rightCheek = predictions[i].annotations.rightCheek;

    const leftCheek = predictions[i].annotations.leftCheek;

    //Sad Emotion: Uses lips and brows to check if sad and if so, activates crying below the eyes

    const rightEye = predictions[i].annotations.rightEyeLower0; 

    const leftEye = predictions[i].annotations.leftEyeLower0; 

    //Angry Emotion: Uses lips and eyebrows to check if angry. 

    const rightEyebrow = predictions[i].annotations.rightEyebrowUpper;

    const leftEyebrow = predictions[i].annotations.leftEyebrowUpper;

    const [rightEyebrow5_x, rightEyebrow5_y] = rightEyebrow[5];

    const [rightEyebrow7_x, rightEyebrow7_y] = rightEyebrow[7];

    let browDistance = dist(rightEyebrow5_x, rightEyebrow5_y, rightEyebrow7_x, rightEyebrow7_y);

​

    //Checking Conditions: ----------------------------------------------------------------

    //If the person is smiling, then it will call the happy function

    if(lipDistance > 100 && !faceTooFar) {

      HappyEmotion(rightCheek, leftCheek);

    }

    //If the person is frowning and raises their eyebrows, then it will call the sad function

    else if(lipDistance < 80 && browDistance > 33 && !faceTooFar) {

      SadEmotion(rightEye, leftEye);

    }

    //Luckily if you make an angry face, it keeps the lip level at 90, thus it doesn't call function. 

    //This checks whether user makes an angry face and if so, call angry function

    else if(lipDistance > 80 && lipDistance < 100 && browDistance < 29 && !faceTooFar) {

      AngryEmotion();

    }

    else {

      Happy.stop();

      Sad.stop();

      Angry.stop();

      happySoundGate = true;

      sadSoundGate = true;

      angrySoundGate = true;

    }

    //This checks if the user's face is too far from the webcam 

    if(lipDistance < 65) {

      faceTooFar = true; 

      happySoundGate = true;

      sadSoundGate = true;

      angrySoundGate = true;

      Happy.stop();

      Sad.stop();

      Angry.stop();

    }

    else {faceTooFar = false;}

    //Testing parameters for console

    // console.log('Lip Distance: ' + lipDistance);

    // console.log('Brow Distance: ' + browDistance);

    // fill('black');

    // text(lipDistance, width/2, height/4);

    // text(browDistance, width/2, 3*height/4);

    if(lipDistance >= 80 && lipDistance <= 85) {

      fill('lightgreen');

      noStroke();

      rect(0, 0, 20, 20);

    }

  }

}

​

​

//Emotions: --------------------------------------------------------------------------------

//Happy Emotion Execution: 

function HappyEmotion(rightCheek, leftCheek) {

  const [rightCheek_x, rightCheek_y] = rightCheek[0];

  const [leftCheek_x, leftCheek_y] = leftCheek[0];

  fill(255, 227, 277, 150);

  noStroke();

  ellipse(rightCheek_x, rightCheek_y, 50, 50);

  ellipse(leftCheek_x, leftCheek_y, 50, 50);

  //Happy Decorations

  image(Flowers, 0, 250, Flowers.width/0.9);

  fill('yellow');

  ellipse(0, 0, 300, 300);

  fill('#FFEB3B');

  ellipse(0, 0, 200, 200);

  background(255,254,224, 50);

  if(happySoundGate) {

    Happy.play();

    Happy.loop();

    Happy.setVolume(0.2);

​

    Sad.stop();

    Angry.stop(); 

    happySoundGate = false;

    sadSoundGate = true;

    angrySoundGate = true;

  }

}

​

//Sad Emotion Execution:

function SadEmotion(rightEye, leftEye) {

  const [rightEye_x, rightEye_y] = rightEye[4];

  const [leftEye_x, leftEye_y] = leftEye[4];

  cryingParticles.push(new Tear(rightEye_x, rightEye_y+5));

  cryingParticles.push(new Tear(leftEye_x, leftEye_y+5));

​

  for (let i = cryingParticles.length-1; i >= 0; i--) {

    cryingParticles[i].update();

    cryingParticles[i].show();

    if (cryingParticles[i].finished()) {

      cryingParticles.splice(i, 1);

    }

  }

  //Sad Decorations

  background(9,0,136, 50);

  image(Clouds, 0, -100, Clouds.width/2, Clouds.height/4);

  if(sadSoundGate) {

    Sad.play();

    Sad.loop();

    Sad.setVolume(0.2);

    Happy.stop();

    Angry.stop();

    happySoundGate = true;

    sadSoundGate = false;

    angrySoundGate = true; 

  }

}

​

//Angry Emotion Execution:

function AngryEmotion() {  

  //Angry Decorations

  background(255,82,82, 100);

  image(Fire, 0, height/2+50, Fire.width/0.75, Fire.height/1.25);

  if(angrySoundGate) {

    Angry.play();

    Angry.loop();

    Angry.setVolume(0.2);

    Happy.stop();

    Sad.stop();

    happySoundGate = true;

    sadSoundGate = true;

    angrySoundGate = false; 

  }

}

​

​

//Drawing Function --------------------------------------------------------------

function draw() {

  image(myVideo, 0, 0, width, height);

​

  //If the user's face is too far, it will print a helper box to make them move closer to it. 

  if(faceTooFar) {

    noFill();

    stroke(0);

    strokeWeight(5);

    rect(210, 100, 230, 310);

    strokeWeight(1);

    textSize(50);

    fill('black');

    text('MOVE CLOSER!', width/2, 75);

  }

  //These function calls are to test where points are on a specified landmark.

  // drawTestPoints();

  // drawKeyPoints();

  EmotionCheck();

  // console.log(predictions);

}

​

​

class Tear {

  //Constructor to initialize the particle based on initial variables

  constructor(handX, handY){

    this.x = handX;

    this.y = handY;

    this.alpha = 255;

    this.size = random(5,10);

    this.assignVelocity();

  }

  //This assigns the velocity on the particle depending on what it is

  assignVelocity() {

        this.vx = random(-2, 2);

        this.vy = random(1, 6);

  }

  //Then the alpha value is below 0, then we return a true (to remove particle from array)

   finished() {

    return this.alpha < 0;

  }

​

  //This function updates the velocities and alpha value of the particles

  update() {

    this.alpha -= 5; 

    this.x += this.vx;

    this.y += this.vy;

  }

​

  //This function draws the particles

  show() {

    noStroke();

    fill(0, random(120, 180), 255, this.alpha); 

    ellipse(this.x, this.y, this.size, 20);

  }

}