* 👋 Hello! This is an ml5.js example made and shared with ❤️.

 * Learn more about the ml5.js project: https://ml5js.org/

 * ml5.js license and Code of Conduct: https://github.com/ml5js/ml5-next-gen/blob/main/LICENSE.md

 *

 * This example demonstrates hand tracking on live video through ml5.handPose.

 */

​

let handPose;

let video;

let hands = [];

​

​

// function preload() {

//   // Load the handPose model

//   handPose = ml5.handPose();

// }

​

function setup() {

  createCanvas(windowWidth, windowHeight);

  // Create the webcam video and hide it

  video = createCapture(VIDEO);

  video.size(width, height);

  video.hide();

  // // start detecting hands from the webcam video

  // handPose.detectStart(video, gotHands);

   // Load the Handpose model

  handPose = ml5.handPose(video, modelReady);

  handPose.on('predict', gotHands);

}

​

function draw() {

  // Draw the webcam video

  image(video, 0, 0, width, height);

  // Check for two hands

  if (hands.length === 2) {

    let hand1Fingers = detectFingers(hands[0]);

    let hand2Fingers = detectFingers(hands[1]);

​

    // Both hands are open palms

    if (hand1Fingers === 5 && hand2Fingers === 5) {

      sendCommand('X'); // Stop

      return; // Skip further processing

    }

  }

​

  // Process single hand if only one is detected

  if (hands.length > 0) {

    let hand = hands[0]; // Only process the first detected hand

    let extendedFingers = detectFingers(hand);

​

    // Send command based on the number of fingers extended

    if (extendedFingers === 1) {

      sendCommand('L'); // Move Left

    } else if (extendedFingers === 2) {

      sendCommand('R'); // Move Right

    } else if (extendedFingers === 3) {

      sendCommand('B'); // Move Backward

    } else if (extendedFingers === 4) {

      sendCommand('F'); // Move Forward

    } else if (extendedFingers === 5) {

      sendCommand('S'); // Spin

    }

​

    // Draw keypoints

    drawKeypoints(hand);

  }

​

  // Draw all the tracked hand points

  for (let i = 0; i < hands.length; i++) {

    let hand = hands[i];

    for (let j = 0; j < hand.keypoints.length; j++) {

      let keypoint = hand.keypoints[j];

      fill(0, 255, 0);

      noStroke();

      circle(keypoint.x, keypoint.y, 10);

    }

  }

}

​

// Callback function for when handPose outputs data

function gotHands(results) {

  // save the output to the hands variable

  hands = results;

}

​

// Detect the number of fingers extended

function detectFingers(hand) {

  let extended = 0;

  const fingerTips = [4, 8, 12, 16, 20]; // Tip points for thumb, index, middle, ring, pinky

​

  for (let i = 0; i < fingerTips.length; i++) {

    let tip = hand.keypoints[fingerTips[i]];

    let dip = hand.keypoints[fingerTips[i] - 2]; // Point just below the fingertip

​

    // Check if the fingertip is higher than the dip point

    if (tip.y < dip.y) {

      extended++;

    }

  }

  return extended;

}

​

// Draw all the tracked hand points

function drawKeypoints(hand) {

  for (let i = 0; i < hand.keypoints.length; i++) {

    let keypoint = hand.keypoints[i];

    fill(0, 255, 0);

    noStroke();

    circle(keypoint.x, keypoint.y, 10);

  }

}

​

// Send command to Arduino

function sendCommand(command) {

  if (serial && serial.isOpen()) {

    serial.write(command + '\n'); // Send command with a newline

  }

}

​

function keyPressed() {

  if (key === ' ') { // Space key

    setUpSerial(); // Call the serial setup function

  }

}