let poseNet; // PoseNet object

let poses = []; // Detected poses

let video; // Video capture object

​

function setup() {

  createCanvas(640, 480);

​

  // Set up serial connection

  serial = new p5.SerialPort();

  serial.open('/dev/tty.usbmodem11301'); // Replace with the appropriate port name

  serial.on('open', openSerial); // Call openSerial when the serial port is opened

​

  // Set up PoseNet

  video = createCapture(VIDEO);

  video.size(width, height);

  video.hide(); // Hide the video element

  poseNet = ml5.poseNet(video, modelReady);

  poseNet.on('pose', gotPoses);

}

​

function openSerial() {

  serial.write('s'); // Start signal to Arduino

}

​

function modelReady() {

  console.log('Model loaded');

}

​

function gotPoses(results) {

  poses = results;

}

​

function draw() {

  // Display video capture

  image(video, 0, 0, width, height);

​

  // We can call both functions to draw all keypoints and the skeletons

  drawKeypoints();

  drawSkeleton();

​

  // Process detected poses and control car wheels

  if (poses.length > 0) {

    // Analyze poses and control wheels

    analyzePoses();

  }

}

​

function analyzePoses() {

  // Assuming single person pose estimation

  let pose = poses[0].pose;

​

  // Extract hand keypoints

  let handKeypoints = pose.keypoints.filter((keypoint) =>

    keypoint.part.startsWith('left') || keypoint.part.startsWith('right')

  );

​

  // Check if both hands are detected

  if (handKeypoints.length >= 2) {

    // Get positions of left and right hands

    let leftHand = handKeypoints.find((keypoint) => keypoint.part.startsWith('left')).position;

    let rightHand = handKeypoints.find((keypoint) => keypoint.part.startsWith('right')).position;

​

    // Map hand positions to control signals

    let signal1, signal2;

​

    // Control signals for left hand gesture

    if (leftHand.y < height / 2) {

      signal1 = 255; // Move forward

    } else {

      signal1 = 0; // Stop

    }

​

    // Control signals for right hand gesture

    if (rightHand.y < height / 2) {

      signal2 = 255; // Move forward

    } else {

      signal2 = 0; // Stop

    }

​

    // Send control signals to Arduino

    sendControlSignals(signal1, signal2);

  }

}

​

// Send control signals to Arduino

function sendControlSignals(signal1, signal2) {

  const data = signal1 + ',' + signal2;

  serial.write(data); // Send data to Arduino

}

​

function drawKeypoints() {

  // Loop through all the poses detected

  for (let i = 0; i < poses.length; i++) {

    // For

    let pose = poses[i].pose;

    for (let j = 0; j < pose.keypoints.length; j++) {

      // A keypoint is an object describing a body part (like rightArm or leftShoulder)

      let keypoint = pose.keypoints[j];

      // Only draw an ellipse if the pose probability is bigger than 0.2

      if (keypoint.score > 0.2) {

        fill(255, 0, 0);

        noStroke();

        ellipse(keypoint.position.x, keypoint.position.y, 10, 10);

      }

    }

  }

}

​

// A function to draw the skeletons

function drawSkeleton() {

  // Loop through all the skeletons detected

  for (let i = 0; i < poses.length; i++) {

    let skeleton = poses[i].skeleton;

    // For every skeleton, loop through all body connections

    for (let j = 0; j < skeleton.length; j++) {

      let partA = skeleton[j][0];

      let partB = skeleton[j][1];

      stroke(255, 0, 0);

      line(partA.position.x, partA.position.y, partB.position.x, partB.position.y);

    }

  }

}

​