let handPose;

let video;

let hands = [];

let actionText = "";

let prevActionText = ""; // To keep track of the previous action

let gameImage;

let GameInstructions = true;

​

function preload() {

  // Load the handPose model

  handPose = ml5.handPose();

​

  gameImage = loadImage("GameInstructions.png");

}

​

function setup() {

  createCanvas(windowWidth, windowHeight);

  // Create the webcam video and hide it

  video = createCapture(VIDEO);

  video.size(width, height);

  video.hide();

  // Start detecting hands from the webcam video

  handPose.detectStart(video, gotHands);

}

​

function draw() {

  if (GameInstructions) {

    image(gameImage, 0, 0, width, height); // Display the instructions image

  } else {

    // Draw the webcam video

    image(video, 0, 0, width, height);

​

    // Reset actionText each frame

    actionText = "";

​

    if (hands.length > 0) {

      // Handle gestures for one or two hands

      if (hands.length === 2) {

        // Two hands detected: "stops"

        actionText = "stops";

      } else {

        // One hand detected

        let hand = hands[0];

​

        // Check for one raised finger

        if (isOneFingerRaised(hand)) {

          actionText = "spins";

        } else {

          // Get the average position of the hand keypoints

          let handX = 0;

          let handY = 0;

          for (let i = 0; i < hand.keypoints.length; i++) {

            handX += hand.keypoints[i].x;

            handY += hand.keypoints[i].y;

          }

          handX /= hand.keypoints.length;

          handY /= hand.keypoints.length;

​

          // Check which region the hand is in

          if (handX < width / 2 && handY < height / 2) {

            actionText = "moves forward"; // Top Left

          } else if (handX >= width / 2 && handY < height / 2) {

            actionText = "moves back"; // Top Right

          } else if (handX < width / 2 && handY >= height / 2) {

            actionText = "turns left"; // Bottom Left

          } else if (handX >= width / 2 && handY >= height / 2) {

            actionText = "turns right"; // Bottom Right

          }

        }

      }

    } else {

      // No hands detected: Robot stops

      actionText = "stops";

    }

​

    // Send command to Arduino if actionText has changed and serial is active

    if (actionText !== prevActionText && serialActive) {

      writeSerial(actionText + "\r\n");

      console.log("Sent to Arduino:", actionText);

      prevActionText = actionText;

    }

​

    // Draw all the tracked hand points

    for (let i = 0; i < hands.length; i++) {

      let hand = hands[i];

​

      // Draw keypoints

      for (let j = 0; j < hand.keypoints.length; j++) {

        let keypoint = hand.keypoints[j];

        fill(0, 255, 0);

        noStroke();

        circle(keypoint.x, keypoint.y, 10);

      }

    }

​

    // Display the action text

    fill(255, 0, 0);

    textSize(32);

    text(actionText, 10, 30);

​

    // Regions for debugging and easy visibility

    noFill();

    stroke(255, 0, 0);

    strokeWeight(2);

​

    // Top Left

    rect(0, 0, width / 2, height / 2);

​

    // Top Right

    rect(width / 2, 0, width / 2, height / 2);

​

    // Bottom Left

    rect(0, height / 2, width / 2, height / 2);

​

    // Bottom Right

    rect(width / 2, height / 2, width / 2, height / 2);

  }

}

​

// Callback function for when handPose outputs data

function gotHands(results) {

  hands = results;

}

​

// Function to check if one finger is raised

function isOneFingerRaised(hand) {

  let wrist = hand.keypoints[0]; // Wrist keypoint

​

  // Indices for finger tips

  let fingerTipIndices = [4, 8, 12, 16, 20];

​

  let raisedFingers = 0;

​

  for (let i = 0; i < fingerTipIndices.length; i++) {

    let fingertip = hand.keypoints[fingerTipIndices[i]];

    let distance = dist(wrist.x, wrist.y, fingertip.x, fingertip.y);

    if (distance > 100) {

      // Threshold to consider a finger as raised

      raisedFingers++;

    }

  }

​

  return raisedFingers === 1; // Return true if exactly one finger is raised

}

​

function keyPressed() {

  if (key === " ") {

    setUpSerial(); // Call the serial setup function

    GameInstructions = false;

  }

}

​

function windowResized() {

  // Resize the canvas and adjust video size when the window is resized

  resizeCanvas(windowWidth, windowHeight);

  video.size(windowWidth, windowHeight);

}

​

//Debugging

function readSerial(data) {

  console.log("Received from Arduino:", data);

}

​