let handPose;

let hands = [];

let synth;

let notes = ['C5', 'Bb4', 'G4', 'F4', 'Db4', 'C4'];

let numRows = 16;

let grid = Array(numRows).fill().map(() => Array(6).fill(false));

let playheadY = 0;

let isPlaying = true;

let lastBeatTime = 0;

let tempo = 420;

let currentBeat = 0;

let circles = [];

let horizontalStrings = [];

let verticalStrings = [];

let squares = [];

let divisions = [];

let verticalSquares = [];

const PADDING = 100;

let lastClickPos = { x: null, y: null };

​

function preload() {

  handPose = ml5.handPose({ flipped: true });

}

​

function setup() {

  createCanvas(640, 480);

  video = createCapture(VIDEO, { flipped: true });

  video.hide();

  handPose.detectStart(video, gotHands);

  synth = new p5.PolySynth();

}

​

function gotHands(results) {

  hands = results;

}

​

function draw() {

  image(video, 0, 0);

  // Visual effects

  updateAndDrawEffects(divisions);

  updateAndDrawEffects(squares);

  updateAndDrawEffects(horizontalStrings);

  updateAndDrawEffects(verticalStrings);

  updateAndDrawEffects(verticalSquares);

  // Grid with padding

  let gridWidth = width - (PADDING * 2);

  let cellWidth = gridWidth / 6;

  let cellHeight = height / numRows;

  for (let row = 0; row < numRows; row++) {

    for (let col = 0; col < 6; col++) {

      if (grid[row][col]) fill(255, 255, 0, 100);

      else noFill();

      stroke(255, 50);

      rect(PADDING + (col * cellWidth), row * cellHeight, cellWidth, cellHeight);

    }

  }

  // Playhead

  if (isPlaying) {

    let beatInterval = (60 / tempo) * 1000;

    let progress = (millis() - lastBeatTime) / beatInterval;

    playheadY = (currentBeat * cellHeight + progress * cellHeight) % height;

    stroke(255, 255, 0);

    strokeWeight(2);

    line(PADDING, playheadY, width - PADDING, playheadY);

    if (millis() - lastBeatTime >= beatInterval) {

      playBeat();

      currentBeat = (currentBeat + 1) % numRows;

      lastBeatTime = millis();

    }

  }

  // Hand interaction

  if (hands.length > 0) {

    let hand = hands[0];

    let index = hand.index_finger_tip;

    let thumb = hand.thumb_tip;

    let x = (index.x + thumb.x) * 0.5;

    let y = (index.y + thumb.y) * 0.5;

    let d = dist(index.x, index.y, thumb.x, thumb.y);

    if (d < 20) {

      circles.push(new FadingCircle(x, y, 40));

    }

    handleHandGesture(x, y, d);

  }

  circles = circles.filter(circle => {

    circle.draw();

    return circle.update();

  });

}

​

function updateAndDrawEffects(effectsArray) {

  for (let i = effectsArray.length - 1; i >= 0; i--) {

    effectsArray[i].update();

    effectsArray[i].draw();

    if (!effectsArray[i].isAlive) {

      effectsArray.splice(i, 1);

    }

  }

}

​

function handleHandGesture(x, y, d) {

  let gridWidth = width - (PADDING * 2);

  let cellWidth = gridWidth / 6;

  let cellHeight = height / numRows;

  if (d < 20 && lastClickPos.x === null) {

    lastClickPos.x = x;

    lastClickPos.y = y;

  } else if (d >= 20 && lastClickPos.x !== null) {

    let col = floor((lastClickPos.x - PADDING) / cellWidth);

    let row = floor(lastClickPos.y / cellHeight);

    if (col >= 0 && col < 6 && row >= 0 && row < numRows) {

      grid[row][col] = !grid[row][col];

    }

    lastClickPos = { x: null, y: null };

  }

}

​

function playBeat() {

  for (let i = 0; i < notes.length; i++) {

    if (grid[currentBeat][i]) {

      synth.play(notes[i], 0.3, 0, 0.2);

      switch(i) {

        case 0: // C5

          let y = random(height);

          horizontalStrings.push(new HorizontalString(y, 1000));

          break;

        case 1: // Bb4

          let x = random(width);

          verticalStrings.push(new VerticalString(x, 1000));

          break;

        case 2: // G4

          circles.push(new Circle(random(width), random(height), random(40, 80)));

          break;

        case 3: // F4

          let div = new DivisionEffect(1000);

          div.divisions = floor(random(2, 6));

          divisions.push(div);

          break;

        case 4: // Db4

          squares.push(new MovingSquare(1500));

          break;

        case 5: // C4

          verticalSquares.push(new VerticalSquare(1500));

          break;

      }

    }

  }

}