let video;

let hands = [];

​

let regions = [];

let isPaired = false;

let buttonR = [];

​

let soundFile;

let start = false;

​

function preload() {

  // Load the handPose model

  handPose = ml5.handPose();

  // Load the sound file

  soundFile = loadSound('succession.mp3');

  button = loadImage("WiiStart.png");

}

​

function setup() {

  createCanvas(640, 480);

  // Create the webcam video and hide it

  video = createCapture(VIDEO);

  video.size(640, 480);

  video.hide();

  // start detecting hands from the webcam video

  handPose.detectStart(video, gotHands);

​

  // Define 10 regions across the width of the canvas

  let regionWidth = width / 10;

  for (let i = 0; i < 10; i++) {

    regions.push({

      x: i * regionWidth,

      y: 0,

      w: regionWidth,

      h: height,

      handsInRegion: [],

    });

  }

  for (let i = 0; i < 1; i++) {

    buttonR.push({

      x: 180,

      y: 360,

      w: 280,

      h: 81,

      handsInBRegion: [],

    });

  }

  // Set the sound to loop

  soundFile.setLoop(true);

}

​

function draw() {

  background(0);

​

  // Mirror the video feed

  push();

  translate(width, 0);

  scale(-1, 1);

  image(video, 0, 0, width, height);

  pop();

​

  // Check which regions have hand points

  for (let region of regions) {

    region.handsInRegion = [];

  }

​

  for (let i = 0; i < hands.length; i++) {

    let hand = hands[i];

    for (let keypoint of hand.keypoints) {

      let mirroredX = width - keypoint.x;

​

      for (let region of regions) {

        if (

          mirroredX > region.x &&

          mirroredX < region.x + region.w &&

          keypoint.y > region.y &&

          keypoint.y < region.h

        ) {

          if (!region.handsInRegion.includes(i)) {

            region.handsInRegion.push(i);

          }

        }

      }

    }

  }

​

  // Check paired regions

  let pairedActive = false;

  for (let i = 0; i < 5; i++) {

    let firstRegion = regions[i];

    let correspondingRegion = regions[i + 5];

    if (

      firstRegion.handsInRegion.length > 0 &&

      correspondingRegion.handsInRegion.length > 0

    ) {

      pairedActive = true;

      fill(255, 0, 0, 100);

    } else {

      fill(255, 255, 255, 50);

    }

    noStroke();

    rect(firstRegion.x, firstRegion.y, firstRegion.w, firstRegion.h);

    rect(correspondingRegion.x, correspondingRegion.y, correspondingRegion.w, correspondingRegion.h);

  }

​

  // Play/pause sound based on paired region detection

  if (pairedActive && !isPaired && start == true) {

    soundFile.play();

    isPaired = true;

  } else if (!pairedActive && isPaired) {

    soundFile.pause();

    isPaired = false;

  }

​

  // Debug - draw the regions and their labels

  noStroke();

  fill(255, 255, 0);

  textSize(16);

  for (let i = 0; i < 10; i++) {

    let region = regions[i];

    text(i + 1, region.x + region.w / 2, height / 2);

  }

  if (start == false){

    fill(150, 210, 210);

    rect(0, 0, 640, 480);

    fill(255);

    textSize(50);

    stroke(0);

    strokeWeight(4);

    text("Placeholder", 200, 70);

    textSize(25);

    let desc = 'Welcome. This is an experience for two.\nPlease stand in front of the camera, side by side.\nImagine this screen is divided down the middle in half.\nEach person stands in one half.\nNow raise one hand. See your hand.\nTry to synchronize your hand movement.\n\nTogether, touch the Start button to begin.';

    // Left.

    text(desc, 30, 120);

    image(button, 180, 360, 280, 81);

    let handsInButtonRegion = new Set();

    for (let i = 0; i < hands.length; i++) {

      let hand = hands[i];

      for (let keypoint of hand.keypoints) {

        let mirroredX = width - keypoint.x;

        if (

          mirroredX > buttonR[0].x &&

          mirroredX < buttonR[0].x + buttonR[0].w &&

          keypoint.y > buttonR[0].y &&

          keypoint.y < buttonR[0].y + buttonR[0].h

        ) {

          handsInButtonRegion.add(i);

          break; // Only need one keypoint per hand in the region

        }

      }

    }

​

    // If two or more hands are pressing the button, start the experience

    if (handsInButtonRegion.size >= 2) {

      start = true;

    }

  }

​

  // Draw all detected hand keypoints

  for (let hand of hands) {

    for (let keypoint of hand.keypoints) {

      let mirroredX = width - keypoint.x;

      fill(0, 255, 0);

      noStroke();

      circle(mirroredX, keypoint.y, 10);

    }

  }

}

​

​

function gotHands(results) {

  hands = results;

}