let video;

let hands = [];

let thumb;

let indexFinger;

let palmBase;

let nn;

let label = "";

​

function preload() {

  // For this example to work across all browsers

  // "webgl" or "cpu" needs to be set as the backend

  ml5.setBackend("webgl");

​

  handPose = ml5.handPose();

}

​

function setup() {

  createCanvas(640, 480);

​

  video = createCapture(VIDEO);

  video.size(width, height);

  video.hide();

​

  // create a neural network for classifying

  // between rock, paper and scissors

  let options = {

    task: "classification",

    debug: true,

  };

  nn = ml5.neuralNetwork(options);

​

  handPose.detectStart(video, gotHands);

}

​

function gotHands(results) {

  //console.log(results);

  hands = results;

}

​

function draw() {

  image(video, 0, 0, width, height);

​

  // check if we have a prediction

  if (hands.length > 0) {

    // get the index finger

    indexFinger = hands[0].index_finger_tip;

    ellipse(indexFinger.x, indexFinger.y, 10, 10);

​

    // get the middle finger

    middleFinger = hands[0].middle_finger_tip;

    ellipse(middleFinger.x, middleFinger.y, 10, 10);

​

    // get the wrist

    wrist = hands[0].wrist;

    ellipse(wrist.x, wrist.y, 10, 10);

  }

​

  textSize(48);

  fill(255);

  textAlign(CENTER);

  text(label, width / 2, height / 2);

}

​

function keyPressed() {

  // create four inputs for the neural network

  // from the relative coordinates between index finger and wrist

  // and middle finger and palm base

  let inputs = [

    indexFinger.x - wrist.x,

    indexFinger.y - wrist.y,

    middleFinger.x - wrist.x,

    middleFinger.y - wrist.y,

  ];

  //console.log(inputs);

​

  if (key == "1") {

    nn.addData(inputs, ["rock"]);

    console.log("Added rock");

  } else if (key == "2") {

    nn.addData(inputs, ["paper"]);

    console.log("Added paper");

  } else if (key == "3") {

    nn.addData(inputs, ["scissors"]);

    console.log("Added scissors");

  } else if (key == "t") {

    nn.normalizeData();

    let options = {

      epochs: 64,

      batchSize: 12,

    };

    nn.train(options, doneTraining);

  } else if (key == "c") {

    nn.classify(inputs, doneClassifying);

  }

}

​

function doneTraining() {

  console.log("Done training!");

  // we could immediately start classifying here if we wanted

}

​

function doneClassifying(results) {

  console.log(results[0].label);

  label = results[0].label;

​

  // automatically classify again

  let inputs = [

    indexFinger.x - wrist.x,

    indexFinger.y - wrist.y,

    middleFinger.x - wrist.x,

    middleFinger.y - wrist.y,

  ];

  nn.classify(inputs, doneClassifying);

}

​