// This code is based on

// ml5.js: Pose Classification

// The Coding Train / Daniel Shiffman

// https://thecodingtrain.com/Courses/ml5-beginners-guide/7.2-pose-classification.html

// https://youtu.be/FYgYyq-xqAw

​

// Separated into three sketches

// 1: Data Collection: https://editor.p5js.org/yining/sketches/dCoPm-Opb

// 2: Model Training: https://editor.p5js.org/yining/sketches/IrBFfXbSF

// 3: Model Deployment: https://editor.p5js.org/yining/sketches/6cFF9-L-Z

​

let video;

let model;

let predictions = [];

​

let brain;

let poseLabel = "";

​

function preload() {

  video = createCapture(VIDEO, () => {

    loadHandTrackingModel();

  });

  video.hide();

  font = loadFont('assets/ImperatorBronze.ttf');

}

​

async function loadHandTrackingModel() {

  // Load the handpose model.

  model = await handpose.load();

  predictHand();

  let options = {

    inputs: 63,

    outputs: 2,

    task: 'classification',

    debug: true

  }

  // brain = ml5.neuralNetwork(options);

  const modelInfo = {

    model: 'model/model.json',

    metadata: 'model/model_meta.json',

    weights: 'model/model.weights.bin',

  };

  // brain.load(modelInfo, brainLoaded);

  let modelJson = await fetch(modelInfo.model);

  modelJson = await modelJson.text();

  const modelJsonFile = new File([modelJson], 'model.json', {

    type: 'application/json'

  });

​

  let weightsBlob = await fetch(modelInfo.weights);

  weightsBlob = await weightsBlob.blob();

  const weightsBlobFile = new File([weightsBlob], 'model.weights.bin', {

    type: 'application/macbinary',

  });

​

  brain = await tf.loadLayersModel(tf.io.browserFiles([modelJsonFile, weightsBlobFile]));

  // console.log('brain', brain)

  brainLoaded();

}

​

async function predictHand() {

  predictions = await model.estimateHands(video.elt);

  setTimeout(() => predictHand(), 100);

}

​

function setup() {

  createCanvas(640, 480, WEBGL);

}

​

function brainLoaded() {

  console.log('Hand pose classification ready!');

  classifyPose();

}

​

async function classifyPose() {

  if (predictions.length > 0) {

    const landmarks = predictions[0].landmarks;

    let inputs = [];

    for (let i = 0; i < landmarks.length; i++) {

      inputs.push(landmarks[i][0] / 640);

      inputs.push(landmarks[i][1] / 480);

      inputs.push((landmarks[i][2] + 80) / 80);

    }

    // brain.classify(inputs, gotResult);

    const output = tf.tidy(() => {

      return brain.predict(tf.tensor(inputs, [1, 63]));

    });

    const result = await output.array();

    gotResult(result);

  } else {

    setTimeout(classifyPose, 100);

  }

}

​

function gotResult(results) {

  if (results[0] && results[0][0]) {

    if (results[0][0] > 0.5) poseLabel = 'idle';

    if (results[0][1] > 0.5) poseLabel = 'magic'

    if (results[0][2] > 0.5) poseLabel = 'close';

  }

  //console.log(results[0].confidence);

  classifyPose();

}

​

function modelLoaded() {

  console.log('Hand pose model ready');

}

​

function draw() {

  if (model) image(video, -(width / 2), -(height / 2));

  if (predictions.length > 0) {

    // We can call both functions to draw all keypoints and the skeletons

    // drawKeypoints();

    // drawSkeleton();

    drawSpells();

  }

​

  fill(255, 0, 255);

  noStroke();

  textFont(font);

  textSize(100);

  textAlign(CENTER, CENTER);

  // text(poseLabel, 0, 0);

}

​

// A function to draw ellipses over the detected keypoints

function drawKeypoints() {

  let prediction = predictions[0];

  for (let j = 0; j < prediction.landmarks.length; j++) {

    let keypoint = prediction.landmarks[j];

    fill(255, 0, 0);

    noStroke();

    ellipse(keypoint[0], keypoint[1], 10, 10);

  }

}

​

// A function to draw the skeletons

function drawSkeleton() {

  let annotations = predictions[0].annotations;

  stroke(255, 0, 0);

  for (let j = 0; j < annotations.thumb.length - 1; j++) {

    line(annotations.thumb[j][0], annotations.thumb[j][1], annotations.thumb[j + 1][0], annotations.thumb[j + 1][1]);

  }

  for (let j = 0; j < annotations.indexFinger.length - 1; j++) {

    line(annotations.indexFinger[j][0], annotations.indexFinger[j][1], annotations.indexFinger[j + 1][0], annotations.indexFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.middleFinger.length - 1; j++) {

    line(annotations.middleFinger[j][0], annotations.middleFinger[j][1], annotations.middleFinger[j + 1][0], annotations.middleFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.ringFinger.length - 1; j++) {

    line(annotations.ringFinger[j][0], annotations.ringFinger[j][1], annotations.ringFinger[j + 1][0], annotations.ringFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.pinky.length - 1; j++) {

    line(annotations.pinky[j][0], annotations.pinky[j][1], annotations.pinky[j + 1][0], annotations.pinky[j + 1][1]);

  }

​

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.thumb[0][0], annotations.thumb[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.indexFinger[0][0], annotations.indexFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.middleFinger[0][0], annotations.middleFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.ringFinger[0][0], annotations.ringFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.pinky[0][0], annotations.pinky[0][1]);

​

}

​

function drawSpells(){

  let annotations = predictions[0].annotations;

    push();

    translate(-(width / 2), -(height / 2));

    translate(annotations.middleFinger[0][0], annotations.middleFinger[0][1], 7);

    let d = dist(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.indexFinger[3][0], annotations.indexFinger[3][1]);

    let f = d / 110;

​

    let cGreen = color(57, 255, 20);

    let cOrange = color(255, 131, 0, 180);

    let cRed = color(255, 80, 0, 200);

    let cWhite = color(255, 255, 255, 130);

​

    if (poseLabel === "close") {

      // draw two outer rings

      push();

      rotateZ(frameCount * 0.01);

      noFill();

      strokeWeight(0.8 * f);

      stroke(cGreen);

      torus(70 * f, 4 * f, 24, 3);

      pop();

​

      push();

      rotateZ(frameCount * -0.01);

      noFill();

      strokeWeight(0.8 * f);

      stroke(cGreen);

      torus(56 * f, 3 * f, 24, 3);

      pop();

    } else if (poseLabel === "magic") {

      // draw 2 outer rings

      push();

      rotateZ(frameCount * 0.01);

      translate(0, 0, 8);

      noFill();

      strokeWeight(0.8 * f);

      stroke(cOrange);

      torus(70 * f, 4 * f, 24, 3);

      pop();

​

      push();

      rotateZ(frameCount * -0.01);

      noFill();

      strokeWeight(0.8 * f);

      stroke(cRed);

      torus(56 * f, 3 * f, 24, 3);

      pop();

      // draw 2 squares

      push();

      rotateZ(frameCount * 0.01);

      translate(0, 0, 10);

      noFill();

      stroke(cOrange);

      strokeWeight(0.5 * f);

      box(70 * f, 70 * f, 1, 1, 1);

      strokeWeight(0.7 * f);

      box(68 * f, 68 * f, 1, 1, 1);

      pop();

​

      push();

      rotateZ(PI * 90 / 360 + frameCount * 0.01);

      noFill();

      stroke(cOrange)

      strokeWeight(0.5 * f);

      box(66 * f, 66 * f, 1, 1, 1);

      strokeWeight(0.7 * f);

      box(64 * f, 64 * f, 1, 1, 1);

      pop();

      // draw 3 spheres

      push();

      rotateX(frameCount * 0.02);

      noFill();

      stroke(cRed);

      sphere(26 * f, 16, 6);

      pop();

​

      push();

      rotateY(frameCount * 0.02);

      noFill();

      stroke(cRed);

      sphere(26 * f, 16, 6);

      pop();

​

      push();

      rotateZ(frameCount * 0.02);

      noFill();

      stroke(cRed);

      sphere(26 * f, 16, 6);

      pop();

    }

    pop();

}