p5.disableFriendlyErrors = true;

​

// Constants

const VIDEO_WIDTH = 640;

const VIDEO_HEIGHT = 480;

​

// p5.sound/Audio Globals

let osc;

let playing;

let avg = [0, 0];

​

// ML5/Video Globals

let handpose;

let video;

let predictions = [];

​

function setup() {

  let cnv = createCanvas(VIDEO_WIDTH, VIDEO_HEIGHT);

  // ML5/Video Initialization

  text("Loading video...", 10, 10);

  video = createCapture(VIDEO);

  video.size(VIDEO_WIDTH, VIDEO_HEIGHT);

  text("Loading model...", 10, 25);

  handpose = ml5.handpose(video, function() { 

    console.log("Model ready!");

  });

  handpose.on("predict", results => {

    predictions = results;

  });

  text("Model ready!", 10, 40);

  video.hide();

  // p5.sound/Audio Initialization

  osc = new p5.Oscillator("sine");

  lp = new p5.LowPass();

  osc.disconnect();

  osc.connect(lp);

  cnv.mousePressed(playOscillator);

  cnv.mouseReleased(stopOscillator);

}

​

function draw() {

  background(220);

  // Video display / drawing

  push();

  translate(width, 0);

  scale(-1, 1);

  //image(video, 0, 0, VIDEO_WIDTH, VIDEO_HEIGHT);

  pop();

  drawKeypoints();

  if(mouseIsPressed) {

    drawCursor();

  }

  // See framerate

  text("fps: " + frameRate().toFixed(1), 5, 15);

  // Calculate center of points from handpose, update avg, draw avg point

  avgKeypoints();

  // Audio

  let osc_freq = constrain(map(avg[0], 0, VIDEO_WIDTH, 100, 1000), 100, 1000);

  let amp = constrain(map(mouseY, VIDEO_HEIGHT, 0, 0, 0.5), 0, 0.5);

  if(playing) {

    osc.amp(amp, 0.1);

    osc.freq(osc_freq, 0.05);

  } else {

    osc.amp(0, 0.1);

  }

}

​

function avgKeypoints() {

  if (predictions.length == 1) {

    let x = predictions[0]["landmarks"].map((v, i) => { return v[0] });

    let y = predictions[0]["landmarks"].map((v, i) => { return v[1] });

    x = x.reduce((s, v) => { return s + v; }) / x.length;

    y = y.reduce((s, v) => { return s + v; }) / y.length;

    avg = [x, y];

    push();

    translate(width, 0);

    scale(-1, 1);

    fill(0, 0, 255);

    noStroke();

    ellipse(x, y, 10, 10);

    pop();

  }

}

​

function drawKeypoints() {

  for (let i = 0; i < predictions.length; i++) {

    const prediction = predictions[i];

    push();

    translate(width, 0);

    scale(-1, 1);

    for (let j = 0; j < prediction.landmarks.length; j ++) {

      const keypoint = prediction.landmarks[j];

      fill(0, 255, 0);

      noStroke();

      ellipse(keypoint[0], keypoint[1], 10, 10);

    }

    pop();

  }

}

​

function drawCursor() {

  push();

  fill(255, 0, 0);

  noStroke();

  ellipse(mouseX, mouseY, 10, 10);

  pop();

}

​

function playOscillator() {

  osc.start();

  playing = true;

}

​

function stopOscillator() {

  osc.amp(0, 0.1);

  playing = false;

}