​

let faceapi;

let detections = [];

let expressionValues=[];

let video;

let canvas;

let happyEmojis=[];

let vx;

let vy;

let songs = [];

let currentSongIndex = -1; 

​

// loading grpahics and music

​

function preload() {

    happyEmojis[0]= loadImage('1.png'); 

    happyEmojis[1] = loadImage('2.png'); 

    happyEmojis[2] = loadImage('3.png'); 

    happyEmojis[3] = loadImage('4.png'); 

    happyEmojis[4]= loadImage('5.png'); 

    happyEmojis[5] = loadImage('6.png'); 

    happyEmojis[6] = loadImage('7.png'); 

    happyEmojis[7] = loadImage('8.png'); 

    happyEmojis[8]= loadImage('9.png'); 

    happyEmojis[9] = loadImage('10.png'); 

    happyEmojis[10] = loadImage('11.png'); 

    happyEmojis[11] = loadImage('12.png'); 

    happyEmojis[12]= loadImage('13.png'); 

    happyEmojis[13] = loadImage('14.png'); 

    happyEmojis[14] = loadImage('15.png'); 

    happyEmojis[15] = loadImage('16.png'); 

    happyEmojis[16]= loadImage('17.png'); 

    happyEmojis[17] = loadImage('18.png'); 

    happyEmojis[18] = loadImage('19.png'); 

    happyEmojis[19] = loadImage('20.png'); 

    happyEmojis[20] = loadImage('21.png'); 

    happyEmojis[21] = loadImage('22.png'); 

    happyEmojis[22] = loadImage('23.png'); 

    songs[0] = loadSound('song1.mp3');

    songs[1] = loadSound('song2.mp3');

    songs[2] = loadSound('song3.mp3');

    songs[3] = loadSound('song4.mp3');

    songs[4] = loadSound('song5.mp3');

}

​

// Setting up the canvas and video settings.

function setup() {

    canvas = createCanvas(windowWidth, windowHeight);

    canvas.id('canvas');

    video = createCapture(VIDEO);

    video.size(windowWidth, windowHeight);

    video.id('video');

  const faceOptions = {

    withLandmarks: true,

    withExpressions: true,

    withDescriptors: true,

    minConfidence: 0.5

  };

​

  //Initialize the model: 

  faceapi = ml5.faceApi(video, faceOptions, faceReady);

  image1 = new Emoji(happyEmojis[0],random(0,width-250),0,1,1);

  image2 = new Emoji(happyEmojis[1],random(0,width-250),0,0.5,1);

  image3 = new Emoji(happyEmojis[2],random(0,width-250),0,0.5,1);

  image4 = new Emoji(happyEmojis[3],random(0,width-250),0,1,1.5);

  image5 = new Emoji(happyEmojis[4],random(0,width-250),0,1,0.5);

  image6 = new Emoji(happyEmojis[5],random(0,width-250),0,1,1);

  image7 = new Emoji(happyEmojis[6],random(0,width-250),0,1,1.5);

  image8 = new Emoji(happyEmojis[7],random(0,width-250),0,1,0.5);

  image9 = new Emoji(happyEmojis[8],random(0,width-250),0,2,1);

  image10 = new Emoji(happyEmojis[9],random(0,width-250),0,1,1.5);

  image11 = new Emoji(happyEmojis[10],random(0,width-250),0,1,0.5);

  image12 = new Emoji(happyEmojis[11],random(0,width-250),0,1,1.5);

  image13 = new Emoji(happyEmojis[12],random(0,width-250),0,2,1);

  image14= new Emoji(happyEmojis[13],random(0,width-250),0,1,2);

  image15= new Emoji(happyEmojis[14],random(0,width-250),0,1,1.5);

  image16= new Emoji(happyEmojis[15],random(0,width-250),0,1,1.5);

  image17 = new Emoji(happyEmojis[16],random(0,width-250),0,1,1);

  image18 = new Emoji(happyEmojis[17],random(0,width-250),0,1,1);

  image19 = new Emoji(happyEmojis[18],random(0,width-250),0,1,1.5);

  image20 = new Emoji(happyEmojis[19],random(0,width-250),0,1,0.5);

  image21 = new Emoji(happyEmojis[20],random(0,width-250),0,1,1.5);

  image22 = new Emoji(happyEmojis[21],random(0,width-250),0,1,0.5);

  image23 = new Emoji(happyEmojis[22],random(0,width-250),0,1,0.5);

}

​

function windowResized() {

    // Adjust canvas and video size when window is resized

    resizeCanvas(windowWidth, windowHeight);

    video.size(windowWidth, windowHeight);

}

​

​

function draw(){

  // for (let i=0; i < happyEmojis.length ;i++ ){

  //   image(happyEmojis[i],random(0,480),random(0,360),30,30);

  // }

  // image(happyEmojis[0],0,0,30,30);

  // image(happyEmojis[1],40,10,30,30);

  // image(happyEmojis[2],50,20,30,30);

  // image(happyEmojis[3],70,20,30,30);

  clear();

  drawExpressions(detections, 20, 20, 14);

  if (expressionValues.length > 1 && expressionValues[1] === 1) { // Assuming index 1 is for 'happy'

    image1.display();

    image1.update();

    image2.display();

    image2.update();

    image3.display();

    image3.update();

    image4.display();

    image4.update();

    image5.display();

    image5.update();

    // playSong(4);

  }

  if (expressionValues.length > 1 && expressionValues[4] === 1) { // Assuming index 1 is for 'happy'

    image11.display();

    image11.update();

    image12.display();

    image12.update();

    image13.display();

    image13.update();

    image14.display();

    image14.update();

    // playSong(1);

  }

  if (expressionValues.length > 1 && expressionValues[3] === 1) { // Assuming index 1 is for 'happy'

    image15.display();

    image15.update();

    image16.display();

    image16.update();

    image17.display();

    image17.update();

    image18.display();

    image18.update();

    image23.display();

    image23.update();

    // playSong(2);

  }

  if (expressionValues.length > 1 && expressionValues[2] === 1) { // Assuming index 1 is for 'happy'

    image7.display();

    image7.update();

    image8.display();

    image8.update();

    image9.display();

    image9.update();

    image10.display();

    image10.update();

    // playSong(3);

  }

  if (expressionValues.length > 1 && expressionValues[0] === 1) { // Assuming index 1 is for 'happy'

    image6.display();

    image6.update();

    // image16.display();

    // image16.update();

    image19.display();

    image19.update();

    image20.display();

    image20.update();

    image21.display();

    image21.update();

    image22.display();

    image22.update();

    // playSong(0);

  }

​

  if (expressionValues.length > 1 && expressionValues[1] === 1) {

    playSong(3);

  } else if (expressionValues.length > 1 && expressionValues[4] === 1) {

    // play song 1

    playSong(0);

  } else if (expressionValues.length > 1 && expressionValues[3] === 1) {

    // play song 2

    playSong(1);

  } else if (expressionValues.length > 1 && expressionValues[2] === 1) {

    // play song 3

    playSong(2);

  } else if (expressionValues.length > 1 && expressionValues[0] === 1) {

    // play song 5

    playSong(4);

  }

}

​

function playSong(index) {

  //stop any currently playing song

  for (let i = 0; i < songs.length; i++) {

    if (i !== index && songs[i].isPlaying()) {

      songs[i].stop();

    }

  }

​

  // play the selected song

  if (!songs[index].isPlaying()) {

    songs[index].play();

  }

}

​

​

class Emoji {

    constructor(img,x,y,vx, vy) {

        this.img = img;

        this.x = x;

      this.y = y;

        this.vx = vx;

        this.vy = vy;

    }

    update() {

        this.x += this.vx;

        this.y += this.vy;

        // Check for canvas boundaries

        if (this.x < -130 || this.x > width -200) this.vx *= -1;

        if (this.y < -110 || this.y > height -150) this.vy *= -1;

    }

​

    display() {

        image(this.img, this.x, this.y, 500, 500);

    }

}

​

function keyTyped() {

  // $$$ For some reason on Chrome/Mac you may have to press f twice to toggle. Works correctly on Firefox/Mac

  if (key === 'f') {

    toggleFullscreen();

  }

  // uncomment to prevent any default behavior

  // return false;

}

​

// Toggle fullscreen state. Must be called in response

// to a user event (i.e. keyboard, mouse click)

function toggleFullscreen() {

  let fs = fullscreen(); // Get the current state

  fullscreen(!fs); // Flip it!

}

​

function faceReady() {

  faceapi.detect(gotFaces);// Start detecting faces:

}

​

// Got faces:

function gotFaces(error, result) {

  if (error) {

    console.log(error);

    return;

  }

​

  detections = result;　//Now all the data in this detections: 

  // console.log(detections);

​

  clear();//Make back ground transparent:

  // drawBoxs(detections);//Draw detection box:

  // drawLandmarks(detections);

​

  storeExpressions(detections); 

  faceapi.detect(gotFaces);

}

​

// function drawBoxs(detections){

//   if (detections.length > 0) {

//     for (f=0; f < detections.length; f++){

//       let {_x, _y, _width, _height} = detections[0].alignedRect._box;

//       stroke(44, 169, 225);

//       strokeWeight(1);

//       noFill();

//       rect(_x, _y, _width, _height);

//     }

//   }

// }

​

// function drawLandmarks(detections){

//   if (detections.length > 0) {

//     for (f=0; f < detections.length; f++){

//       let points = detections[f].landmarks.positions;

//       for (let i = 0; i < points.length; i++) {

//         stroke(44, 169, 225);

//         strokeWeight(3);

//         point(points[i]._x, points[i]._y);

//       }

//     }

//   }

// }

​

function mapExpressions(expressions) {

    const expressionOrder = ['neutral', 'happy', 'sad', 'disgusted', 'surprised'];

    let expressionValues = [];

​

    expressionOrder.forEach(expression => {

        let value = expressions[expression] > 0.5 ? 1 : 0; // Threshold can be adjusted.

        expressionValues.push(value);

    });

​

    return expressionValues;

}

​

function storeExpressions(detections) {

    if (detections.length > 0) {

        let expressions = detections[0].expressions;

        expressionValues = mapExpressions(expressions);

        console.log(expressionValues);

​

    }

}

​

function drawExpressions(detections, x, y, textYSpace){

  if(detections.length > 0){

    let {neutral, happy, angry, sad, disgusted, surprised, fearful} = detections[0].expressions;

    textFont('Helvetica Neue');•

    textSize(14);

    noStroke();

    fill(44, 169, 225);

​

    text("Neutral:       " + nf(neutral*100, 2, 2)+"%", x, y);

    text("Happiness: " + nf(happy*100, 2, 2)+"%", x, y+textYSpace);

    text("Sad:            "+ nf(sad*100, 2, 2)+"%", x, y+textYSpace*2);

    text("Disgusted: " + nf(disgusted*100, 2, 2)+"%", x, y+textYSpace*3);

    text("Surprised:  " + nf(surprised*100, 2, 2)+"%", x, y+textYSpace*4);

  }else{

    text("Neutral: ", x, y);

    text("Happiness: ", x, y + textYSpace);

    text("Sad: ", x, y + textYSpace*2);

    text("Disgusted: ", x, y + textYSpace*3);

    text("Surprised: ", x, y + textYSpace*4);

  }

}

​

function keyPressed() {

  if (key == " ") {

    // important to have in order to start the serial connection!!

    setUpSerial();

  }

}

​

// This function will be called by the web-serial library

// with each new *line* of data. The serial library reads

// the data until the newline and then gives it to us through

// this callback function

function readSerial(data) {

  ////////////////////////////////////

  //READ FROM ARDUINO HERE

  ////////////////////////////////////

  console.log(expressionValues);

  if (data != null) {

    // make sure there is actually a message

    // split the message

    let fromArduino = split(trim(data), ",");

    // if the right length, then proceed

    if (fromArduino.length == 3) {

      // only store values here

      // do everything with those values in the main draw loop

      // We take the string we get from Arduino and explicitly

      // convert it to a number by using int()

      // e.g. "103" becomes 103

      rVal = int(fromArduino[0]);

      alpha = int(fromArduino[1]);

      alph = int(fromArduino[2]);

      // alp = int(fromArduino[3]);

      // al = int(fromArduino[4]);

      console.log("1",rVal);

      console.log("2",alpha);

      console.log("3", alph);

      // console.log('4',alp);

      // console.log('5', al);

    }

​

    //////////////////////////////////

    //SEND TO ARDUINO HERE (handshake)

    //////////////////////////////////

    let sendToArduino = expressionValues[0] + "," + expressionValues[1] + "," + expressionValues[2] + ","  + expressionValues[3] + ","  + expressionValues[4] + "\n";

    writeSerial(sendToArduino);

  }

}