Concept: Gestures to Communicate

Model: Posenet for Pose Classification

Project Collaborators: Priya Bandodkar, Manisha Laroia, Sananda Dutta

Class : Intro to AI | Final Group Project

Date: 10 April 2020

Code Reference Credit: The Coding Train | Daniel Shiffman

*/

​

let video;

let poseNet;

let pose;

let skeleton;

​

let brain;

let poseLabel = "";

​

let state = 'done';

let targetLabel;

​

function keyPressed() {

  if (key == 't') {

    brain.normalizeData();

    brain.train({epochs: 50}, finished); 

  } else if (key == 's') {

    brain.saveData();

  } else {

    targetLabel = key;

    console.log(targetLabel);

    setTimeout(function() {

      console.log('started');

      state = 'started';

      setTimeout(function() {

        console.log('stopped');

        state = 'done';

      }, 10000);

    }, 5000);

  }

}

​

function setup() {

  createCanvas(640, 480);

  video = createCapture(VIDEO);

  video.hide();

  poseNet = ml5.poseNet(video, modelLoaded);

  poseNet.on('pose', gotPoses);

​

  let options = {

    inputs: 34,

    outputs: 6,

    task: 'classification',

    debug: true

  }

  brain = ml5.neuralNetwork(options);

  const modelInfo = {

    model: 'modelTrained-new/model.json',

    metadata: 'modelTrained-new/model_meta.json',

    weights: 'modelTrained-new/model.weights.bin',

  };

  brain.load(modelInfo, brainLoaded);

​

  // brain.loadData('gestures-new.json', dataReady); //Call trained data for gestures 

}

​

function brainLoaded() {

  console.log('pose classification is set!');

  classifyPose();

}

​

function classifyPose() {

  if (pose) {

    let inputs = [];

    for (let i = 0; i < pose.keypoints.length; i++) {

      let x = pose.keypoints[i].position.x;

      let y = pose.keypoints[i].position.y;

      inputs.push(x);

      inputs.push(y);

    }

    brain.classify(inputs, myResult);

  } else {

    setTimeout(classifyPose, 100);

  }

}

​

function myResult(error, results) {  

  console.log(results);

  console.log(results[0].label);

  if (results[0].confidence > 0.75) {

    poseLabel = results[0].label.toUpperCase();

  }

  classifyPose();

}

​

function dataReady() {

  brain.normalizeData();

  brain.train({

    epochs: 50

  }, finished);

}

​

function finished() {

  console.log('model trained');

  brain.save();

  classifyPose();

}

​

function gotPoses(poses) {

  if (poses.length > 0) {

    pose = poses[0].pose;

    skeleton = poses[0].skeleton;

    if (state == 'collecting') {

      let inputs = [];

      for (let i = 0; i < pose.keypoints.length; i++) {

        let x = pose.keypoints[i].position.x;

        let y = pose.keypoints[i].position.y;

        inputs.push(x);

        inputs.push(y);

      }

      let target = [targetLabel];

      brain.addData(inputs, target);

    }

  }

}

​

function modelLoaded() {

  console.log('poseNet is ready!');

}

​

function draw() {

  push();

  translate(video.width, 0);

  scale(-1, 1);

  image(video, 0, 0, video.width, video.height);

​

  if (pose) {

    for (let i = 0; i < skeleton.length; i++) {

      let a = skeleton[i][0];

      let b = skeleton[i][1];

      strokeWeight(2);

      stroke(0);

​

      line(a.position.x, a.position.y, b.position.x, b.position.y);

    }

    for (let i = 0; i < pose.keypoints.length; i++) {

      let x = pose.keypoints[i].position.x;

      let y = pose.keypoints[i].position.y;

      fill(255, 0, 0);

      stroke(0);

      ellipse(x, y, 20, 20);

    }

  }

  pop();

  fill(0);

  rect (0, 0, 300, 60);

  fill(255);

  noStroke(255);

  textSize(40);

  textAlign(LEFT, TOP);

  text(poseLabel, 10, 10);

}