let detections = [];

let expressionValues=[];

let video;

let canvas;

let happyEmojis=[];

let vx;

let vy;

// let emojis=[];

// let img;

​

function preload() {

    happyEmojis[0]= loadImage('1.png'); 

    happyEmojis[1] = loadImage('2.png'); 

    happyEmojis[2] = loadImage('3.png'); 

    happyEmojis[3] = loadImage('4.png'); 

    happyEmojis[4]= loadImage('5.png'); 

    happyEmojis[5] = loadImage('6.png'); 

    happyEmojis[6] = loadImage('7.png'); 

    happyEmojis[7] = loadImage('8.png'); 

    happyEmojis[8]= loadImage('9.png'); 

    happyEmojis[9] = loadImage('10.png'); 

    happyEmojis[10] = loadImage('11.png'); 

    happyEmojis[11] = loadImage('12.png'); 

    happyEmojis[12]= loadImage('13.png'); 

    happyEmojis[13] = loadImage('14.png'); 

    happyEmojis[14] = loadImage('15.png'); 

    happyEmojis[15] = loadImage('16.png'); 

    happyEmojis[16]= loadImage('17.png'); 

    happyEmojis[17] = loadImage('18.png'); 

    // happyEmojis[18] = loadImage('19.png'); 

    // happyEmojis[19] = loadImage('20.png'); 

}

​

function setup() {

  canvas = createCanvas(600, 500);

  canvas.id('canvas');

​

  // Creat the video: 

  video = createCapture(VIDEO);

  video.size(width, height);

  video.id('video');

  const faceOptions = {

    withLandmarks: true,

    withExpressions: true,

    withDescriptors: true,

    minConfidence: 0.5

  };

​

  //Initialize the model: 

  faceapi = ml5.faceApi(video, faceOptions, faceReady);

  image1 = new Emoji(happyEmojis[0],random(0,width-250),0,2,1);

  image2 = new Emoji(happyEmojis[1],random(0,width-250),0,1,2);

  image3 = new Emoji(happyEmojis[2],random(0,width-250),0,2,3);

  image4 = new Emoji(happyEmojis[3],random(0,width-250),0,3,2);

  image5 = new Emoji(happyEmojis[4],random(0,width-250),0,2,1);

  image6 = new Emoji(happyEmojis[5],random(0,width-250),0,1,2);

  image7 = new Emoji(happyEmojis[6],random(0,width-250),0,2,3);

  image8 = new Emoji(happyEmojis[7],random(0,width-250),0,3,2);

  image9 = new Emoji(happyEmojis[8],random(0,width-250),0,2,1);

  image10 = new Emoji(happyEmojis[9],random(0,width-250),0,1,2);

  image11 = new Emoji(happyEmojis[10],random(0,width-250),0,2,3);

  image12 = new Emoji(happyEmojis[11],random(0,width-250),0,3,2);

  image13 = new Emoji(happyEmojis[12],random(0,width-250),0,2,1);

  image14= new Emoji(happyEmojis[13],random(0,width-250),0,1,2);

  image15= new Emoji(happyEmojis[14],random(0,width-250),0,2,3);

  image16= new Emoji(happyEmojis[15],random(0,width-250),0,3,2);

  image17 = new Emoji(happyEmojis[16],random(0,width-250),0,2,1);

  image18 = new Emoji(happyEmojis[17],random(0,width-250),0,1,2);

  // image19 = new Emoji(happyEmojis[2],random(0,600),0,2,3);

  // image20 = new Emoji(happyEmojis[3],random(0,600),0,3,2);

}

function draw(){

  // for (let i=0; i < happyEmojis.length ;i++ ){

  //   image(happyEmojis[i],random(0,480),random(0,360),30,30);

  // }

  // image(happyEmojis[0],0,0,30,30);

  // image(happyEmojis[1],40,10,30,30);

  // image(happyEmojis[2],50,20,30,30);

  // image(happyEmojis[3],70,20,30,30);

  clear();

  if (expressionValues.length > 1 && expressionValues[1] === 1) { // Assuming index 1 is for 'happy'

    image1.display();

    image1.update();

    image2.display();

    image2.update();

    image3.display();

    image3.update();

    image4.display();

    image4.update();

    image5.display();

    image5.update();

  }

  if (expressionValues.length > 1 && expressionValues[4] === 1) { // Assuming index 1 is for 'happy'

    image11.display();

    image11.update();

    image12.display();

    image12.update();

    image13.display();

    image13.update();

    image14.display();

    image14.update();

  }

  if (expressionValues.length > 1 && expressionValues[3] === 1) { // Assuming index 1 is for 'happy'

    image15.display();

    image15.update();

    image16.display();

    image16.update();

    image17.display();

    image17.update();

    image18.display();

    image18.update();

  }

  if (expressionValues.length > 1 && expressionValues[2] === 1) { // Assuming index 1 is for 'happy'

    image7.display();

    image7.update();

    image8.display();

    image8.update();

    image9.display();

    image9.update();

    image10.display();

    image10.update();

  }

  if (expressionValues.length > 1 && expressionValues[0] === 1) { // Assuming index 1 is for 'happy'

    image6.display();

    image6.update();

    // image16.display();

    // image16.update();

    // image17.display();

    // image17.update();

    // image18.display();

    // image18.update();

  }

}

​

class Emoji {

    constructor(img,x,y,vx, vy) {

        this.img = img;

        this.x = x;

      this.y = y;

        this.vx = vx;

        this.vy = vy;

    }

    update() {

        this.x += this.vx;

        this.y += this.vy;

        // Check for canvas boundaries

        if (this.x < -130 || this.x > width -200) this.vx *= -1;

        if (this.y < -110 || this.y > height -150) this.vy *= -1;

    }

​

    display() {

        image(this.img, this.x, this.y, 300, 300);

    }

}

​

function faceReady() {

  faceapi.detect(gotFaces);// Start detecting faces:

}

​

// Got faces:

function gotFaces(error, result) {

  if (error) {

    console.log(error);

    return;

  }

​

  detections = result;　//Now all the data in this detections: 

  // console.log(detections);

​

  clear();//Make back ground transparent:

  drawBoxs(detections);//Draw detection box:

  drawLandmarks(detections);

  // drawExpressions(detections, 20, 250, 14);

  storeExpressions(detections); 

  faceapi.detect(gotFaces);

}

​

function drawBoxs(detections){

  if (detections.length > 0) {

    for (f=0; f < detections.length; f++){

      let {_x, _y, _width, _height} = detections[0].alignedRect._box;

      stroke(44, 169, 225);

      strokeWeight(1);

      noFill();

      rect(_x, _y, _width, _height);

    }

  }

}

​

function drawLandmarks(detections){

  if (detections.length > 0) {

    for (f=0; f < detections.length; f++){

      let points = detections[f].landmarks.positions;

      for (let i = 0; i < points.length; i++) {

        stroke(44, 169, 225);

        strokeWeight(3);

        point(points[i]._x, points[i]._y);

      }

    }

  }

}

​

function mapExpressions(expressions) {

    const expressionOrder = ['neutral', 'happy', 'sad', 'disgusted', 'surprised'];

    let expressionValues = [];

​

    expressionOrder.forEach(expression => {

        let value = expressions[expression] > 0.5 ? 1 : 0; // Threshold can be adjusted.

        expressionValues.push(value);

    });

​

    return expressionValues;

}

​

function storeExpressions(detections) {

    if (detections.length > 0) {

        let expressions = detections[0].expressions;

        expressionValues = mapExpressions(expressions);

        console.log(expressionValues);

​

    }

}

​

function keyPressed() {

  if (key == " ") {

    // important to have in order to start the serial connection!!

    setUpSerial();

  }

}

​

// This function will be called by the web-serial library

// with each new *line* of data. The serial library reads

// the data until the newline and then gives it to us through

// this callback function

function readSerial(data) {

  ////////////////////////////////////

  //READ FROM ARDUINO HERE

  ////////////////////////////////////

  console.log(expressionValues);

  if (data != null) {

    // make sure there is actually a message

    // split the message

    let fromArduino = split(trim(data), ",");

    // if the right length, then proceed

    if (fromArduino.length == 5) {

      // only store values here

      // do everything with those values in the main draw loop

      // We take the string we get from Arduino and explicitly

      // convert it to a number by using int()

      // e.g. "103" becomes 103

      rVal = int(fromArduino[0]);

      alpha = int(fromArduino[1]);

      alph = int(fromArduino[2]);

      alp = int(fromArduino[3]);

      al = int(fromArduino[4]);

      console.log(rVal);

      console.log(alpha);

      console.log("3", alph);

      console.log('4',alp);

      console.log('5', al);

    }

​

    //////////////////////////////////

    //SEND TO ARDUINO HERE (handshake)

    //////////////////////////////////

    let sendToArduino = expressionValues[0] + "," + expressionValues[1] + "," + expressionValues[2] + ","  + expressionValues[3] + ","  + expressionValues[4] + "\n";

    writeSerial(sendToArduino);

  }

}