// Aug 3, 2023

// Felix Z.Y. Yuan

​

// This mirror is designed to reflect two side of your emotion

​

// The first side is the desire to gain attention, where you are trapped in a digital calculator-like world with a spotlight on you. You can move your mouse to control the spotlight, try and see what will happen to you mask persona. Also, you can move your face mask with WASD, and when you shout, you mask persona gets closer to the spotlight.

​

// The second side is the desire to just dance slowly by yourself, where you are really "blue" and time seems slow. But if you try slowly dancing, putting your hand higher/lower or left/right, you'll get something for yourself.

​

// Hope you have fun! ^ ^

​

let xball = [];

let yball = [];

let poses=[];

let capture;

let cap2;

let facemesh;

let poseNet;

let predictions = [];

let mic;

let margin = 50;

let dX = 0,

  dY = 0; //move mask by mouse

let rSpotlight = 0;

let tX = 0,

  tY = 0; //move mask by wasd

let vX = 0,

  vY = 0; //move mouse by sound

let WASDspeed = 3;

let soundMultiple = 1000; //level of sound sensitivity

let soundLowest = 0.05; //filter too small sound\

​

function setup() {

  createCanvas(640, 480);

  font = loadFont("PressStart2P-Regular.ttf");

​

  //facemesh for attention

  capture = createCapture(VIDEO);

  capture.size(640, 480);

  capture.hide();

  facemesh = ml5.facemesh(capture, modelReady);

  facemesh.on("predict", (results) => {

    predictions = results;

  });

​

  theSound=loadSound("123.wav");

  //posnet for dancing

  cap2 = createCapture(VIDEO);

  cap2.size(640, 480);

  cap2.hide();

  poseNet = ml5.poseNet(cap2, modelReady);

  poseNet.on("pose", function (results) {

    poses = results;

  });

  body_keypoints_layer = createGraphics(width, height);

  skeleton_layer = createGraphics(width, height);

​

  //mic

  mic = new p5.AudioIn();

  mic.start();

​

  for (let i = 0; i < 10; i++) {

    xball.push(new ball1(random(width), random(height)));

    yball.push(new ball2(random(width), random(height)));

  }

}

​

function modelReady() {

  console.log("Model ready!");

}

​

function draw() {

  if (key == "1" || key == "w" || key == "a" || key == "s" || key == "d") {

    drawFame();

  } else if (key == "2") {

    drawDance();

  } else {

    drawOpening();

  }

}

​

function drawOpening() {

  background("rgba(0,0,0,0.78)");

​

  push();

​

  textFont(font);

  textAlign(CENTER);

  fill("rgb(220,220,220)");

  textSize(30);

  text("what do you want?", width / 2, height / 3);

​

  textAlign(LEFT);

  text("1.to get fame", width / 9, (4 * height) / 6);

  text("2.to solo dance", width / 9, (5 * height) / 6);

​

  pop();

}

​

function drawFame() {

  push();

  textFont(font);

  textSize(10);

  background("rgb(203,4,4)");

  capture.loadPixels();

  let gap = 10;

  for (let i = 0; i < capture.width; i += gap) {

    for (let j = 0; j < capture.height; j += gap) {

      let index = (j * capture.width + i) * 4;

      let red = capture.pixels[index];

      let green = capture.pixels[index + 1];

      let blue = capture.pixels[index + 2];

      let alpha = capture.pixels[index + 3];

      noStroke();

      fill("rgb(220,220,220)");

      let val = map(green, 0, 255, 0, 6);

      let char = "";

      if (val > 5) {

        char = "8";

      } else if (val > 4) {

        char = "5";

      } else if (val > 3) {

        char = "4";

      } else if (val > 2) {

        char = "7";

      } else if (val > 1) {

        char = "7";

      } else if (val > 0) {

        char = "1";

      }

      textAlign(CENTER, CENTER);

      text(char, i, j);

    }

  }

​

  drawFamePoints();

​

  pop();

}

​

function drawFamePoints() {

  let micLevel = mic.getLevel();

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

    // Draw facial keypoints.

​

    if (key == "w") {

      tY = tY - WASDspeed;

      if (xX < mouseX) {

        if (micLevel > soundLowest) {

          vX = micLevel * soundMultiple;

          vY = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      } else {

        if (micLevel > soundLowest) {

          vX = -micLevel * soundMultiple;

          vY = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      }

    } else if (key == "a") {

      tX = tX - WASDspeed;

      if (yY < mouseY) {

        if (micLevel > soundLowest) {

          vY = micLevel * soundMultiple;

          vX = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      } else {

        if (micLevel > soundLowest) {

          vY = -micLevel * soundMultiple;

          vX = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      }

    } else if (key == "s") {

      tY = tY + WASDspeed;

      if (xX < mouseX) {

        if (micLevel > soundLowest) {

          vX = micLevel * soundMultiple;

          vY = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      } else {

        if (micLevel > soundLowest) {

          vX = -micLevel * soundMultiple;

          vY = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      }

    } else if (key == "d") {

      tX = tX + WASDspeed;

      if (yY < mouseY) {

        if (micLevel > soundLowest) {

          vY = micLevel * soundMultiple;

          vX = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      } else {

        if (micLevel > soundLowest) {

          vY = -micLevel * soundMultiple;

          vX = 0;

        } else {

          vX = 0;

          vY = 0;

        }

      }

    } else {

      tX = 0;

      tY = 0;

      vX = 0;

      vY = 0;

    }

​

    for (let j = 0; j < keypoints.length; j += 1) {

      const [x, y] = keypoints[j];

​

      if (mouseX < x) {

        dX = map(mouseX, 0, x, -75, 0);

      } else {

        dX = map(mouseX, x, width, 0, 75);

      }

      if (mouseY < y) {

        dY = map(mouseY, 0, y, -75, 0);

      } else {

        dY = map(mouseY, y, height, 0, 75);

      }

​

      rSpotlight = map((abs(dX) + abs(dY)) / 2, 0, 75, 800, 0);

​

      xX = x + tX + vX;

      yY = y + tY + vY;

​

      if (dist(xX, yY, mouseX, mouseY) > rSpotlight / 4) {

        faceText = random(["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]);

        fill("rgb(0,0,129)");

        text(faceText, xX, yY);

      } else {

        push();

        textSize(18);

        fill("rgba(0,0,0,0.78)");

        text("E", xX, yY);

        theSound.play(0,1);

        pop();

      }

    }

  }

​

  rSpotlight = map((abs(dX) + abs(dY)) / 2, 0, 75, 400, 0);

  cursor(CROSS);

  fill(245, 222, 179, 130);

  noStroke();

  circle(mouseX, mouseY, rSpotlight);

}

​

class ball1 {

  constructor(x, y) {

    this.x = x;

    this.y = y;

    this.dir = true;

  }

​

  move() {

    if (this.dir == true) {

      this.x = this.x - 5;

    } else {

      this.x = this.x + 5;

    }

  }

​

  reDir(tf) {

    this.dir = tf;

  }

​

  update() {

    if (this.x < 0 || this.x > width) {

      this.x = random(width);

      this.y = random(height);

    }

  }

​

  drawBall() {

    this.move();

    this.update();

    push();

    fill("rgb(203,4,4)");

    noStroke();

    circle(this.x, this.y, 10);

    pop();

  }

}

​

class ball2 {

  constructor(x, y) {

    this.x = x;

    this.y = y;

    this.dir = true;

  }

  move() {

    if (this.dir == true) {

      this.y = this.y - 5;

    } else {

      this.y = this.y + 5;

    }

  }

​

  reDir(tf) {

    this.dir = tf;

  }

​

  update() {

    if (this.y < 0 || this.y>height) {

      this.x = random(width);

      this.y = random(height);

    }

  }

​

  drawBall() {

    this.move();

    this.update();

    push();

    fill("rgb(203,4,4)");

    noStroke();

    circle(this.x, this.y, 10);

    pop();

  }

}

​

function drawDance() {

  cap2.loadPixels();

  for (let i = 0; i < cap2.pixels.length; i += 4) {

    cap2.pixels[i] = 0; //red

    cap2.pixels[i + 1] = 0; //green

    cap2.pixels[i + 2] += random(100, 130); //blue

    cap2.pixels[i + 3] = 10; //alpha

  }

  cap2.updatePixels();

  image(cap2, 0, 0, width, height);

​

  // Loop through all the poses detected

  for (let i = 0; i < poses.length; i++) {

    // For each pose detected, loop through all the keypoints

    let pose = poses[i].pose;

    let lHand = pose.keypoints[10];

    let lShoulder = pose.keypoints[6];

    let rHand = pose.keypoints[9];

    let rShoulder = pose.keypoints[5];

​

    if (lHand.position.x < lShoulder.position.x) {

      for (let i = 0; i < 10; i++) {

        xball[i].reDir(true);

      }

    } else {

      for (let i = 0; i < 10; i++) {

        xball[i].reDir(false);

      }

    }

​

    if (rHand.position.y < rShoulder.position.y) {

      for (let i = 0; i < 10; i++) {

        yball[i].reDir(true);

      }

    } else {

      for (let i = 0; i < 10; i++) {

        yball[i].reDir(false);

      }

    }

  }

​

  for (let i = 0; i < 10; i++) {

    xball[i].drawBall();

    yball[i].drawBall();

  }

}