//work in progress, pixel array interaction integration forthcoming, "reflection" on glass  

//mouth open - fog in window

//right hand - hearts on window and secret mermaid

​

// reference from collaboration w/ Yiting Liu

// reference from Themis Garcia https://medium.com/computational-media/icm-media-week-2-e5a2455ef45d

​

var faceapi;

let video;

let detections;

let img;

let fogged;

let dolphins;

let hearts;

let circleMask;

​

let poseNet;

let poses = [];

​

const detection_options = {

  withLandmarks: true,

  withDescriptors: false,

}

​

​

​

function preload() {

  img = loadImage('SubWindow.png');

  dolphins = loadImage('DolphinsWindow.png');

  fogged = loadImage('FoggyWindow.png');

  hearts = loadImage('MermaidWindow.png');

​

}

​

function setup() {

  createCanvas(589, 592);

  // load up your video

  video = createCapture(VIDEO);

  video.size(320, 240);

  video.mask(img);

​

  image(img, 0, 0, width, height);

  //video.hide(); // Hide the video element, and just show the canvas

  faceapi = ml5.faceApi(video, detection_options, modelReady);

  textAlign(RIGHT);

​

  // Create a new poseNet method with a single detection

  poseNet = ml5.poseNet(video, modelReady);

  // This sets up an event that fills the global variable "poses"

  // with an array every time new poses are detected

  poseNet.on('pose', function(results) {

    poses = results;

  });

​

​

  function modelReady() {

    console.log('ready!');

    console.log(faceapi);

    faceapi.detect(gotResults);

  }

​

  function gotResults(err, result) {

    if (err) {

      console.log(err);

      return

    }

    // console.log(result)

    detections = result;

​

    image(img, 0, 0, width, height);

    if (detections) {

      if (detections.length > 0) {

        drawLandmarks(detections);

      }

​

    }

    faceapi.detect(gotResults);

​

  }

}

function draw() {

  image(img, 0, 0);

  fogDraw();

}

​

​

​

function drawLandmarks(detections) {

​

  for (let i = 0; i < detections.length; i++) {

​

    const mouth = detections[i].parts.mouth;

​

    const mouth_x = int(mouth[3]._x);

    const mouth_y = int(mouth[3]._y);

​

    const mouth2_x = int(mouth[9]._x);

    const mouth2_y = int(mouth[9]._y);

​

    let d = int(dist(mouth_x, mouth_y, mouth2_x, mouth2_y));

    // if mouth is open

    if (d > 20) {

​

      image(img, 0, 0, width, height);

      image(fogged, 0, 0);

​

    } else {

      image(img, 0, 0, width, height);

      image(dolphins, 0, 0);

​

​

    }

​

  }

}

    function fogDraw() {

    for (let i = 0; i < poses.length; i++) {

      // For each pose detected, loop through all the keypoints

      let pose = poses[i].pose;

      // A keypoint is an object describing a body part (like   rightArm or leftShoulder)

      let rightWrist = pose.keypoints[10];

      if (rightWrist.score > 0.5) {

        image(hearts, 0, 0);

        console.log('here');

      }

​

    }

​

  }

​

​

​

​

​