The example lets you experiment with k-means clustering. You can add data-points by showing hand gestures and hitting the space bar to capture them. If you want to see the output, simply click on Train & Display. If you want to know more how this clustering algorithm is working, check out the video by Computerphile (https://www.youtube.com/watch?v=yR7k19YBqiw) on this topic. 

​

Built with Kmeans Clustering and Handpose from ml5.js and p5.js 

Created by Tom-Lucas Säger 2021

 */

​

// We start by declaring and defining a few variables that will come in handy later.

let video;

let predictions = [];

let data = [];

let data_collection = [];

let pixelArray = [];

let textVar =

  "Model loading please wait…";

let redrawing = false;

​

// We create our video, a slider the button to start training and initialise HandPose 

function setup() {

  pixelDensity(1);

  createCanvas(640, 480);

  video = createCapture(VIDEO);

  video.hide();

​

  slider = createSlider(1, 10, 2, 1);

  slider.input(sliderAdjusted);

  sliderLabel = createP("Number of Clusters: " + slider.value());

​

  textSize(16);

  let trainButton = createButton("Train & Display");

  trainButton.position(width - 115, height + 115);

  trainButton.mousePressed(trainModel);

​

  const handpose = ml5.handpose(video, modelReady);

  handpose.on("predict", gotResults);

​

  textAlign(CENTER, TOP);

  fill(0, 255, 0);

  textSize(48);

  noStroke();

}

//The video is drawn, as well as the keypoints from Handpose, we also show the user instructions on what to do. The function redrawExamples() is only called when we have called trainModel()

function draw() {

  background(255);

  image(video, 0, 0, width, height);

  drawKeypoints();

  text(textVar, width / 2, height / 2);

  if (redrawing) {

    redrawExamples();

  }

}

//Updates the value for our slider, when it is changed 

function sliderAdjusted() {

  sliderLabel.html("Number of Cluster: " + slider.value());

}

//Gets called once HandPose has loaded 

function modelReady() {

  console.log("Model ready!");

  textVar =

  "Bring you hand in the view \n and press space-bar \n to add an example ";

}

//Gets called once HandPose has results

function gotResults(results) {

  predictions = results;

}

//Handpose keypoints are drawn

function drawKeypoints() {

  data = [];

  for (let i = 0; i < predictions.length; i += 1) {

    const prediction = predictions[i];

    for (let j = 0; j < prediction.landmarks.length; j += 1) {

      const keypoint = prediction.landmarks[j];

      ellipse(keypoint[0], keypoint[1], 10, 10);

      fill(0, 255, 0);

      data.push(keypoint[0], keypoint[1]);

    }

  }

}

//This functions gets called when a key is pressed, we then define what happens

function keyPressed() {

  //If the user presses the space bar (keycode 32) and their hand is present,

  //we push the data from HandPose in one array and every pixel from the canvas in another, so we can redraw it later  

  if (keyCode === 32 && data.length > 0) {

    video.loadPixels();

    //We check if the pixel Array is not empty, this happens if the user scans to quickly. 

    if(video.pixels[4] == 0){

      textVar = "Not so fast, please";

      video.updatePixels();

    }

    else if (video.pixels[4] > 0){

    pixelArray.push(video.pixels);

    data_collection.push(data);

    textVar = "";

    }

    video = createCapture(VIDEO);

    video.hide();

  //If the user presses space bar and their hand is not present, we prompt them to bring their hand in the view 

  } else if (keyCode === 32 && data.length == 0) {

    textVar = "Bring you hand in the view \n and press space-bar";

  }

}

//This function is called once the user presses the trainButton it will start the kMeans-training

function trainModel() {

  let k;

  if (slider.value() > data_collection.length) {

    k = data_collection.length;

  } else {

    k = slider.value();

  }

  const options = {

    k: k,

    maxIter: 400,

    threshold: 0.5,

  };

  kmeans = ml5.kmeans(data_collection, options, clustersCalculated);

}

//Once the training is done, this function is called. It sets redrawing to true, so redrawExamples is called in the draw loop

function clustersCalculated() {

  redrawing = true;

}

//This function redraws our captured examples this video by the Coding Train explains very well how this is done. 

//Then we assign them the values from kMeans

function redrawExamples() {

  let scale = Math.ceil(Math.sqrt(pixelArray.length));

  let position;

  background(0);

  for (let j = 0; j < scale; j++) {

    for (let i = 0; i < scale; i++) {

      if (i + j * scale >= data_collection.length) {

        break;

      }

      for (let y = 0; y < height; y += scale) {

        for (x = 0; x < width; x += scale) {

          position = i + j * scale;

          let index = (x + y * width) * 4;

          let r = pixelArray[position][index];

          let g = pixelArray[position][index + 1];

          let b = pixelArray[position][index + 2];

          fill(r, g, b);

          rect(

            x / scale + (i * width) / scale,

            y / scale + (j * height) / scale,

            1

          );

        }

      }

      //The number of the cluster gets drawn

      fill(0, 255, 0);

      textAlign(CENTER, BOTTOM);

      textSize(200 / scale);

      text(

        kmeans.dataset[position].centroid,

        width / scale + (i * width) / scale - width / scale / 2,

        height / scale + (j * height) / scale

      );

    }

  }

}

​