// This code is based on

// ml5.js: Pose Classification

// The Coding Train / Daniel Shiffman

// https://thecodingtrain.com/Courses/ml5-beginners-guide/7.2-pose-classification.html

// https://youtu.be/FYgYyq-xqAw

​

// Separated into three sketches

// 1: Data Collection: https://editor.p5js.org/yining/sketches/dCoPm-Opb

// 2: Model Training: https://editor.p5js.org/yining/sketches/IrBFfXbSF

// 3: Model Deployment: https://editor.p5js.org/yining/sketches/6cFF9-L-Z

​

let video;

let model;

let predictions = [];

​

let brain;

let poseLabel = "";

​

function preload() {

  video = createCapture(VIDEO, () => {

    loadHandTrackingModel();

  });

  video.hide();

}

​

async function loadHandTrackingModel() {

  // Load the handpose model.

  model = await handpose.load();

  predictHand();

  let options = {

    inputs: 63,

    outputs: 2,

    task: 'classification',

    debug: true

  }

  // brain = ml5.neuralNetwork(options);

  const modelInfo = {

    model: 'model/model.json',

    metadata: 'model/model_meta.json',

    weights: 'model/model.weights.bin',

  };

  // brain.load(modelInfo, brainLoaded);

  let modelJson = await fetch(modelInfo.model);

  modelJson = await modelJson.text();

  const modelJsonFile = new File([modelJson], 'model.json', { type: 'application/json' });

​

  let weightsBlob = await fetch(modelInfo.weights);

  weightsBlob = await weightsBlob.blob();

  const weightsBlobFile = new File([weightsBlob], 'model.weights.bin', {

    type: 'application/macbinary',

  });

​

  brain = await tf.loadLayersModel(tf.io.browserFiles([modelJsonFile, weightsBlobFile]));

  // console.log('brain', brain)

  brainLoaded();

}

​

async function predictHand() {

  predictions = await model.estimateHands(video.elt);

  setTimeout(() => predictHand(), 100);

}

​

function setup() {

  createCanvas(640, 480);

}

​

function brainLoaded() {

  console.log('Hand pose classification ready!');

  classifyPose();

}

​

async function classifyPose() {

  if (predictions.length > 0) {

    const landmarks = predictions[0].landmarks;

    let inputs = [];

    for (let i = 0; i < landmarks.length; i++) {

      inputs.push(landmarks[i][0] / 640);

      inputs.push(landmarks[i][1] / 480);

      inputs.push((landmarks[i][2] + 80) / 80);

    }

    // brain.classify(inputs, gotResult);

    const output = tf.tidy(() => {

      return brain.predict(tf.tensor(inputs, [1, 63]));

    });

    const result = await output.array();

    gotResult(result);

  } else {

    setTimeout(classifyPose, 100);

  }

}

​

function gotResult(results) {

  if (results[0] && results[0][0]) {

    if (results[0][0] > 0.5) poseLabel = '1';

    if (results[0][1] > 0.5) poseLabel = '2';

  }

  //console.log(results[0].confidence);

  classifyPose();

}

​

function modelLoaded() {

  console.log('Hand pose model ready');

}

​

function draw() {

  if (model) image(video, 0, 0);

  if (predictions.length > 0) {

    // We can call both functions to draw all keypoints and the skeletons

    drawKeypoints();

    drawSkeleton();

  }

​

  fill(255, 0, 255);

  noStroke();

  textSize(100);

  textAlign(CENTER, CENTER);

  text(poseLabel, width / 2, height / 2);

}

​

// A function to draw ellipses over the detected keypoints

function drawKeypoints()  {

    let prediction = predictions[0];

    for (let j = 0; j < prediction.landmarks.length; j++) {

      let keypoint = prediction.landmarks[j];

      fill(255, 0, 0);

      noStroke();

      ellipse(keypoint[0], keypoint[1], 10, 10);

    }

}

​

// A function to draw the skeletons

function drawSkeleton() {

    let annotations = predictions[0].annotations;

    stroke(255, 0, 0);

    for (let j = 0; j < annotations.thumb.length - 1; j++) {

      line(annotations.thumb[j][0], annotations.thumb[j][1], annotations.thumb[j + 1][0], annotations.thumb[j + 1][1]);

    }

    for (let j = 0; j < annotations.indexFinger.length - 1; j++) {

      line(annotations.indexFinger[j][0], annotations.indexFinger[j][1], annotations.indexFinger[j + 1][0], annotations.indexFinger[j + 1][1]);

    }

    for (let j = 0; j < annotations.middleFinger.length - 1; j++) {

      line(annotations.middleFinger[j][0], annotations.middleFinger[j][1], annotations.middleFinger[j + 1][0], annotations.middleFinger[j + 1][1]);

    }

    for (let j = 0; j < annotations.ringFinger.length - 1; j++) {

      line(annotations.ringFinger[j][0], annotations.ringFinger[j][1], annotations.ringFinger[j + 1][0], annotations.ringFinger[j + 1][1]);

    }

    for (let j = 0; j < annotations.pinky.length - 1; j++) {

      line(annotations.pinky[j][0], annotations.pinky[j][1], annotations.pinky[j + 1][0], annotations.pinky[j + 1][1]);

    }

​

    line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.thumb[0][0], annotations.thumb[0][1]);

    line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.indexFinger[0][0], annotations.indexFinger[0][1]);

    line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.middleFinger[0][0], annotations.middleFinger[0][1]);

    line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.ringFinger[0][0], annotations.ringFinger[0][1]);

    line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.pinky[0][0], annotations.pinky[0][1]);

​

}