Concept: Gestures to Communicate

Model: Posenet for Pose Classification

Project Collaborators: Priya Bandodkar, Manisha Laroia, Sananda Dutta

Class : Intro to AI | Final Group Project

Date: 10 April 2020

Code Reference Credit: The Coding Train | Daniel Shiffman

*/

​

let video;

let poseNet;

let pose;

let skeleton;

let storeData;

​

let state = 'done';

let targetLabel;

let poseName = "";

​

function keyPressed() {

  if (key == 's') {

    storeData.saveData();

 } else {

    targetLabel = key;

    console.log(targetLabel);

    setTimeout(function() {

      console.log('started');

      state = 'started';

      setTimeout(function() {

        console.log('stopped');

        state = 'done';

      }, 10000);

    }, 5000);

  }

​

}

​

function setup() {

  createCanvas(640, 480);

  video = createCapture(VIDEO);

  video.hide();

  poseNet = ml5.poseNet (video, modelLoaded);

  poseNet.on('pose', gotPoses);

  let options = {

    inputs: 34,

    outputs: 6,

    task: 'classification',

    debug: true

  }

  storeData = ml5.neuralNetwork(options);

​

  const modelInfo = {

    model: 'Modeltrained/model.json',

    metadata: 'Modeltrained/model_meta.json',

    weights: 'Modeltrained/model.weights.bin',

  };

  storeData.load(modelInfo, storeDataLoaded);

​

 // storeData.loadData('gestures.json, dataReady); //Call trained data for gestures 

}

​

function storeDataLoaded() {

  console.log('pose classified!');

  //classifyPose();

}

/*

function classifyPose() {

  if (pose) {

    let inputs = [];

    for (let i = 0; i < pose.keypoints.length; i++) {

      let x = pose.keypoints[i].position.x;

      let y = pose.keypoints[i].position.y;

      inputs.push(x);

      inputs.push(y);

    }

    storeData.classify(inputs, gotResult);

    } 

    else {

      setTimeout(classifyPose, 100);

    }

}

​

function gotResult(error, results) {  

  if (results[0].confidence > 0.75) {

    poseLabel = results[0].label.toUpperCase();

  }

  classifyPose();

}

      */           

function dataReady() {

    storeData.normalizeData();

    storeData.train({epochs: 50

    }, finished);

  }

function finished () {

  console.log('model is trained!');

  storeData.save();

  //classifyPose();

}

​

function gotPoses(poses) {

  //console.log(poses);

  if(poses.length > 0) {

   pose = poses[0].pose;

    skeleton = poses [0].skeleton;

    if (state == 'started') { 

      let inputs = [];

​

      for (let i = 0; i < pose.keypoints.length; i++) {

        let x = pose.keypoints[i].position.x;

        let y = pose.keypoints[i].position.y;

        inputs.push(x);

        inputs.push(y);

      }

​

      let target = [targetLabel];

      storeData.addData(inputs, target);

    }

  }

}

function modelLoaded() {

  console.log('poseNet loaded!');

}

​

function draw() {

  push();

  translate(video.width, 0);

  scale(-1, 1);

  image(video, 0 , 0, video.width, video.height);

  if (pose) {

    for (let i = 0; i < skeleton.length; i++) {

      let a = skeleton[i][0];

      let b = skeleton[i][1];

      strokeWeight(2);

      stroke(0);

​

      line(a.position.x, a.position.y, b.position.x, b.position.y);

    }

    for (let i = 0; i < pose.keypoints.length; i++) {

      let x = pose.keypoints[i].position.x;

      let y = pose.keypoints[i].position.y;

      fill(255, 0, 0);

      stroke(0);

      ellipse(x, y, 20, 20);

    }

 }

​

  pop();

​

  fill(0, 255, 0);

  noStroke(255);

  textSize(60);

  textAlign(CENTER, CENTER);

  text(poseName, width / 2, height / 2);

​

}