let video;

let poseNet;

let noseX = 0; //nose

let noseY = 0;

let hand1X = 0; //left hand

let hand1Y = 0;

let hand2X = 0; //right hand

let hand2Y = 0;

let soundFile = []; //music file

let ball = {}; //panning channels

// let Raleway;

​

​

let index = 0;

​

​

//load music

function preload() {

  soundFormats('mp3', 'ogg');

  soundFile[0] = loadSound('06_AcousticGtrDI.mp3');

  soundFile[1] = loadSound('02_SaxophoneCloseMic2.mp3');

  soundFile[2] = loadSound('04_AcousticGtrCloseMics.mp3');

}

​

​

function setup() {

  createCanvas(680,500);

  video = createCapture(VIDEO);

  video.hide();

  poseNet = ml5.poseNet(video, modelReady);

  poseNet.on('pose', gotPoses);

  fill(39,40,34);

  image(video, 0, 0);

  fft = new p5.FFT(0.8,64);//music visualization with smooth factor

  soundFile[0].amp();//visualization

   soundFile[1].amp();

   soundFile[2].amp();

​

  soundFile[2].play();//music start

  soundFile[0].play();//music start

  soundFile[1].play();//music start

}

​

function gotPoses(poses) {

  //console.log(poses);

  if (poses.length > 0) {

    let nX = poses[0].pose.keypoints[0].position.x;

    let nY = poses[0].pose.keypoints[0].position.y;

    let h1X = poses[0].pose.keypoints[9].position.x; //left hand

    let h1Y = poses[0].pose.keypoints[9].position.y;

    let h2X = poses[0].pose.keypoints[10].position.x; //right hand

    let h2Y = poses[0].pose.keypoints[10].position.y;

​

    noseX = lerp(noseX, nX, 0.5); //nose

    noseY = lerp(noseY, nY, 0.5);

    hand1X = lerp(hand1X, h1X, 0.5); //left hand

    hand1Y = lerp(hand1Y, h1Y, 0.5);

    hand2X = lerp(hand2X, h2X, 0.5); //right hand

    hand2Y = lerp(hand2Y, h2Y, 0.5);

​

  }

}

​

function modelReady() {

  //console.log('model ready');

}

​

function draw() {

​

​

  var COLOURS = ['#F92672', '#66D9EF', '#A6E22E', '#FD971F', '#272822', '#F92672', '#66D9EF', '#69D2E7', '#A7DBD8', '#E0E4CC', '#F38630', '#FA6900', '#FF4E50', '#F9D423'];

​

  //image(video, 0, 0); //real 

  noStroke();

  fill(39,40,34,30); //background for colors

  rect(0, 0,680,500);

​

​

  fill(random(COLOURS));

  ellipse(noseX, noseY, 50); //face main channel

  ellipse(hand1X, hand1Y, 30); //left hand

  ellipse(hand2X, hand2Y, 30); //right hand

​

  //music

  musicStart();

  //panning channels

  ball.x = constrain(noseX,0,width);

  channelsMove();

  //text

  fill(205);

  text(16)

  text("- - - - - - - - - - - - - - - -", 240,300)

​

}

​

​

​

function musicStart(){ //start music 

  if(noseY > 300){     //nose 

    soundFile[0].setVolume(0.0001);

  }else{

    soundFile[0].setVolume(1.0);

    fill(205);

    textSize(16)

    text("NO MUSIC", 290,350)

  }

​

  if(hand2X < 200 && hand2Y < 480){  //right hand

    soundFile[1].setVolume(1.0);  

    fill(167, 219, 216,60); //background for colors

    rect(10,50,200,400,80);

    musicVisualliation();

  }else{

    soundFile[1].setVolume(0.0001);//volume disappear

    fill(205);

    textSize(16)

    text('RAISE RIGHT HAND', 24,height/2)//text

}

  if(hand1X > 450 && hand1Y < 480){ //left hand

    soundFile[2].setVolume(1.0);

    fill(249, 212, 35,60); //background for colors

    rect(470,50,200,400,80);

    musicVisualliation();

    }else{

    soundFile[2].setVolume(0.001);//volume disappear

    fill(205);

    textSize(16)

    text('RAISE LEFT HAND', 490,height/2)//text

    }

}

​

​

function channelsMove(){ //left & right ear channels

  let panning = map(ball.x, 240, 400, -1.0, 1.0);

  soundFile[0].pan(panning);

  soundFile[1].pan(panning);

  soundFile[2].pan(panning);  

}

​

​

function musicVisualliation(){ //圈圈 visualization amp

  var spectrum = fft.analyze();

​

  for(var i = 0; i < spectrum.length; i ++){ //left

    var amp = spectrum[i];

​

    noFill();

    strokeWeight(1);

    stroke(39,40,34,40);

    ellipse(110, height/2, amp);

    ellipse(570, height/2, amp);

  }

}

​

​

​

​

​

​

​