let video;

let predictions = [];

let timestamp = 0;

var mouth_openness = [];

var dt_mouth_openness = [];

var flg_speaking = false;

var timestamp_speaking;

​

var faces = [];

​

function setup() {

  createCanvas(1920 / 4, 1080 / 4);

  video = createCapture(VIDEO);

  video.size(width, height);

​

  facemesh = ml5.facemesh(video, modelReady);

​

  // This sets up an event that fills the global variable "predictions"

  // with an array every time new predictions are made

  facemesh.on("predict", (results) => {

    console.log(millis()-timestamp);

    predictions = results;

​

    for( let i = 0; i < predictions.length; i++ ){

    const keypoints = predictions[i].scaledMesh;

    const [x1, y1, z1] = keypoints[13];

    const [x2, y2, z2] = keypoints[14];

    const [x3, y3, z3] = keypoints[263];

    const [x4, y4, z4] = keypoints[33];

​

    mouth_openness[mouth_openness.length] = [];

    dt_mouth_openness[dt_mouth_openness.length] = [];

​

    mouth_openness[i].push(

      dist(x1, y1, z1, x2, y2, z2) / dist(x3, y3, z3, x4, y4, z4)

    );

    }

    timestamp = millis();

  });

​

  // Hide the video element, and just show the canvas

  video.hide();

}

​

function modelReady() {

  console.log("Model ready!");

}

​

function draw() {

  // image(video, 0, 0, width, height);

  background(0);

  image(video, 0, 0, width, height);

  // We can call both functions to draw all keypoints

  drawKeypoints();

​

  stroke(0);

  fill(255);

  rect(0, height / 2 - 50, 100, 100);

​

  for (let j = 0; j < mouth_openness.length; j++) {

    noFill();

    beginShape();

    for (let i = 0; i < mouth_openness[j].length; i++) {

      vertex(i, height / 2 + 100 * mouth_openness[j][i]);

    }

    endShape();

​

    if (dt_mouth_openness.length > 0) {

      stroke(255, 0, 0);

      beginShape();

      for (let i = 0; i < dt_mouth_openness[j].length; i++) {

        vertex(i, height / 2 + 100 * dt_mouth_openness[j][i]);

      }

      endShape();

    }

  }

  // 口に動きがあればspeakingと表示する

  for( let i = 0; i < dt_mouth_openness.length; i++ ){

//    console.log(dt_mouth_openness[i][dt_mouth_openness[i].length-1]);

    if( dt_mouth_openness[i][dt_mouth_openness[i].length-1] > 0.01 ){

      flg_speaking = true;

      timestamp_speaking = millis();

    }

  }

  if( flg_speaking)

}

​

// A function to draw ellipses over the detected keypoints

function drawKeypoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

    //    console.log(keypoints);

    // Draw facial keypoints.

    for (let j = 0; j < keypoints.length; j += 1) {

      const [x, y] = keypoints[j];

      let c = [[255,0,0],[0,255,0],[0,0,255]];

      stroke(c[i][0],c[i][1],c[i][2]);

      ellipse(x / 4, y / 4, 1, 1);

    }

    const [x1, y1, z1] = keypoints[13];

    const [x2, y2, z2] = keypoints[14];

    const [x3, y3, z3] = keypoints[263];

    const [x4, y4, z4] = keypoints[33];

​

    // ellipse(x1 / 4, y1 / 4, 10, 10);

    // ellipse(x2 / 4, y2 / 4, 10, 10);

    // circle(x3 / 4, y3 / 4, 10);

    // circle(x4 / 4, y4 / 4, 10);

​

    mouth_openness[mouth_openness.length] = [];

    dt_mouth_openness[dt_mouth_openness.length] = [];

​

    mouth_openness[i].push(

      dist(x1, y1, z1, x2, y2, z2) / dist(x3, y3, z3, x4, y4, z4)

    );

​

    if (mouth_openness[i].length >= 2) {

      dt_mouth_openness[i].push(

        mouth_openness[i][mouth_openness[i].length - 1] -

          mouth_openness[i][mouth_openness[i].length - 2]

      );

    }

    if (mouth_openness[i].length > 100) {

      mouth_openness[i].shift();

    }

    if (dt_mouth_openness.length > 0) {

      if (dt_mouth_openness[i].length > 100) {

        dt_mouth_openness[i].shift();

      }

    }

  }

}

​