let video;

let hands = [];

​

let regions = [];

let isPaired = false;

​

let bodySegmentation;

let segmentation;

​

let options = {

  maskType: "person", // This will generate the body silhouette

};

​

let soundFile;

​

function preload() {

  // Load the handPose model

  handPose = ml5.handPose();

  // Load the sound file

  soundFile = loadSound('succession.mp3');

  bodySegmentation = ml5.bodySegmentation("SelfieSegmentation", options);

}

​

function setup() {

  createCanvas(640, 480);

  // Create the webcam video and hide it

  video = createCapture(VIDEO);

  video.size(640, 480);

  video.hide();

  // start detecting hands from the webcam video

  handPose.detectStart(video, gotHands);

  // Start body segmentation

  bodySegmentation.detectStart(video, gotResults);

​

  // Define 10 regions across the width of the canvas

  let regionWidth = width / 10;

  for (let i = 0; i < 10; i++) {

    regions.push({

      x: i * regionWidth,

      y: 0,

      w: regionWidth,

      h: height,

      handsInRegion: [],

    });

  }

  // Set the sound to loop

  soundFile.setLoop(true);

}

​

function draw() {

  background(255);

​

  // Check if segmentation is available

  if (segmentation && segmentation.mask) {

    // Draw the mirrored segmentation mask

    push();

    translate(width, 0); // Move to the right edge

    scale(-1, 1); // Mirror horizontally

    image(segmentation.mask, 0, 0);

    pop();

  }

​

  // Check which regions have hand points

  for (let region of regions) {

    region.handsInRegion = [];

  }

​

  for (let i = 0; i < hands.length; i++) {

    let hand = hands[i];

    for (let keypoint of hand.keypoints) {

      let mirroredX = width - keypoint.x;

​

      for (let region of regions) {

        if (

          mirroredX > region.x &&

          mirroredX < region.x + region.w &&

          keypoint.y > region.y &&

          keypoint.y < region.h

        ) {

          if (!region.handsInRegion.includes(i)) {

            region.handsInRegion.push(i);

          }

        }

      }

    }

  }

​

  // Check paired regions

  let pairedActive = false;

  for (let i = 0; i < 5; i++) {

    let firstRegion = regions[i];

    let correspondingRegion = regions[i + 5];

    if (

      firstRegion.handsInRegion.length > 0 &&

      correspondingRegion.handsInRegion.length > 0

    ) {

      pairedActive = true;

      fill(255, 0, 0, 100);

    } else {

      fill(255, 255, 255, 50);

    }

    noStroke();

    rect(firstRegion.x, firstRegion.y, firstRegion.w, firstRegion.h);

    rect(correspondingRegion.x, correspondingRegion.y, correspondingRegion.w, correspondingRegion.h);

  }

​

  // Play/pause sound based on paired region detection

  if (pairedActive && !isPaired) {

    soundFile.play();

    isPaired = true;

  } else if (!pairedActive && isPaired) {

    soundFile.pause();

    isPaired = false;

  }

​

  // Debug - draw the regions and their labels

  // noStroke();

  // fill(255, 255, 0);

  // textSize(16);

  // for (let i = 0; i < 10; i++) {

  //   let region = regions[i];

  //   text(i + 1, region.x + region.w / 2, height / 2);

  // }

​

  //Draw all detected hand keypoints

  // for (let hand of hands) {

  //   for (let keypoint of hand.keypoints) {

  //     let mirroredX = width - keypoint.x;

  //     fill(0, 255, 0);

  //     noStroke();

  //     circle(mirroredX, keypoint.y, 10);

  //   }

  // }

}

​

function gotHands(results) {

  hands = results;

}

​

function gotResults(result) {

  segmentation = result;

}

​