​

​

let serial;// variable to hold an instance of the serialport library

// Create a KNN classifier

let portName = '/dev/cu.usbmodem146201';// fill in your serial port name here

let outByte = 0;// for outgoing data

​

let lightswitch1 = false;

let lightswitch2 = false;

let lightswitch3 = false;

​

const classifier = knnClassifier.create();

​

const LABELS_MAP = {

  'LowLight': 0,

  'MedLight': 1,

  'HighLight': 2

}

​

function preload() {

  video = createCapture(VIDEO, () => {

    loadHandTrackingModel();

  });

  video.hide();

  // Create the UI buttons

  createButtons();

}

​

function setup() {

  const canvas = createCanvas(600, 400);

  canvas.parent('canvasContainer');

  //  serial = new p5.SerialPort();    // make a new instance of the serialport library

  //  serial.on('error', serialError); // callback for errors

  // serial.open(portName);           // open a serial port

}

​

function serialError(err) {

  console.log('Something went wrong with the serial port. ' + err);

}

​

async function loadHandTrackingModel() {

  // Load the MediaPipe handpose model.

  model = await handpose.load();

  select('#status').html('Hand Tracking Model Loaded')

  predictHand();

}

​

function draw() {

  background(255);

  if (model) image(video, 0, 0);

  if (predictions.length > 0) {

    // We can call both functions to draw all keypoints and the skeletons

    drawKeypoints();

    drawSkeleton();

  }

  if(lightswitch1 == true){

    light1();

  }

  if(lightswitch2 == true){

    light2();

  }

  if(lightswitch3 == true){

    light3();

  }

}

​

function light1(){

  fill(240,230,140);

  translate(100, 200);

  noStroke();

  for (let i = 0; i < 10; i ++) {

    ellipse(0, 10, 20, 50);

    rotate(PI/5);

  }

}

​

​

function light2(){

  fill(240,230,140);

  translate(100, 200);

  noStroke();

  for (let i = 0; i < 10; i ++) {

    ellipse(0, 10, 20, 80);

    rotate(PI/5);

  }

}

​

​

function light3(){

  fill(240,230,140);

  translate(100, 200);

  noStroke();

  for (let i = 0; i < 10; i ++) {

    ellipse(0, 10, 20, 120);

    rotate(PI/5);

  }

}

​

async function predictHand() {

  // Pass in a video stream (or an image, canvas, or 3D tensor) to obtain a

  // hand prediction from the MediaPipe graph.

  predictions = await model.estimateHands(video.elt);

​

  setTimeout(() => predictHand(), 200);

}

​

// Add the current hand tracking data to the classifier

function addExample(label) {

  if (predictions.length > 0) {

    const features = predictions[0].landmarks;

    const tensors = tf.tensor(features)

    // Add an example with a label to the classifier

    classifier.addExample(tensors, label);

    updateCounts();

  } else {

    console.log('No gesture is detected')

  }

}

​

// Predict the current frame.

async function classify() {

  // Get the total number of labels from classifier

  const numLabels = classifier.getNumClasses();

  if (numLabels <= 0) {

    console.error('There is no examples in any label');

    return;

  }

  if (predictions.length > 0) {

    const results = await classifier.predictClass(tf.tensor(predictions[0].landmarks));

    if (results.confidences) {

      const confidences = results.confidences;

      // result.label is the label that has the highest confidence

      if (results.label) {

        select('#result').html(results.label);

        select('#confidence').html(`${confidences[results.label] * 100} %`);

      }

      console.log(results.label);

      select('#confidenceRock').html(`${confidences['LowLight'] ? confidences['LowLight'] * 100 : 0} %`);

      if(results.label == "LowLight"){

        lightswitch1 = true;

      } else {

        lightswitch1 = false;

      }

      select('#confidencePaper').html(`${confidences['MedLight'] ? confidences['MedLight'] * 100 : 0} %`);

    if(results.label == "MedLight"){

        lightswitch2 = true;

      } else {

        lightswitch2 = false;

      }

      select('#confidenceScissor').html(`${confidences['HighLight'] ? confidences['HighLight'] * 100 : 0} %`);

       if(results.label == "HighLight"){

        lightswitch3 = true;

      } else {

        lightswitch3 = false;

      }

    }

    classify();

  } else {

    setTimeout(() => classify(), 1000);

  }

}

​

// Update the example count for each label  

function updateCounts() {

  const counts = classifier.getClassExampleCount();

​

  select('#exampleRock').html(counts['LowLight'] || 0);

  select('#examplePaper').html(counts['MedLight'] || 0);

  select('#exampleScissor').html(counts['HighLight'] || 0);

}

​

// Clear the examples in one label

function clearLabel(label) {

  classifier.clearClass(label);

  updateCounts();

}

​

// Clear all the examples in all labels

function clearAllLabels() {

  classifier.clearAllClasses();

  updateCounts();

}

​

// A util function to create UI buttons

function createButtons() {

  // When the A button is pressed, add the current frame

  // from the video with a label of "LowLight" to the classifier

  buttonA = select('#addClassRock');

  buttonA.mousePressed(function() {

    addExample('LowLight');

  });

​

  // When the B button is pressed, add the current frame

  // from the video with a label of "MedLight" to the classifier

  buttonB = select('#addClassPaper');

  buttonB.mousePressed(function() {

    addExample('MedLight');

  });

​

  // When the C button is pressed, add the current frame

  // from the video with a label of "HighLight" to the classifier

  buttonC = select('#addClassScissor');

  buttonC.mousePressed(function() {

    addExample('HighLight');

  });

​

  // Reset buttons

  resetBtnA = select('#resetRock');

  resetBtnA.mousePressed(function() {

    clearLabel('LowLight');

  });

  resetBtnB = select('#resetPaper');

  resetBtnB.mousePressed(function() {

    clearLabel('MedLight');

  });

  resetBtnC = select('#resetScissor');

  resetBtnC.mousePressed(function() {

    clearLabel('HighLight');

  });

​

  // Predict button

  buttonPredict = select('#buttonPredict');

  buttonPredict.mousePressed(classify);

​

  // Clear all classes button

  buttonClearAll = select('#clearAll');

  buttonClearAll.mousePressed(clearAllLabels);

}

​

// A function to draw ellipses over the detected keypoints

function drawKeypoints()  {

  let prediction = predictions[0];

  for (let j = 0; j < prediction.landmarks.length; j++) {

    let keypoint = prediction.landmarks[j];

    fill(255, 0, 0);

    noStroke();

    ellipse(keypoint[0], keypoint[1], 10, 10);

  }

}

​

// A function to draw the skeletons

function drawSkeleton() {

  let annotations = predictions[0].annotations;

  stroke(255, 0, 0);

  for (let j = 0; j < annotations.thumb.length - 1; j++) {

    line(annotations.thumb[j][0], annotations.thumb[j][1], annotations.thumb[j + 1][0], annotations.thumb[j + 1][1]);

  }

  for (let j = 0; j < annotations.indexFinger.length - 1; j++) {

    line(annotations.indexFinger[j][0], annotations.indexFinger[j][1], annotations.indexFinger[j + 1][0], annotations.indexFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.middleFinger.length - 1; j++) {

    line(annotations.middleFinger[j][0], annotations.middleFinger[j][1], annotations.middleFinger[j + 1][0], annotations.middleFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.ringFinger.length - 1; j++) {

    line(annotations.ringFinger[j][0], annotations.ringFinger[j][1], annotations.ringFinger[j + 1][0], annotations.ringFinger[j + 1][1]);

  }

  for (let j = 0; j < annotations.pinky.length - 1; j++) {

    line(annotations.pinky[j][0], annotations.pinky[j][1], annotations.pinky[j + 1][0], annotations.pinky[j + 1][1]);

  }

​

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.thumb[0][0], annotations.thumb[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.indexFinger[0][0], annotations.indexFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.middleFinger[0][0], annotations.middleFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.ringFinger[0][0], annotations.ringFinger[0][1]);

  line(annotations.palmBase[0][0], annotations.palmBase[0][1], annotations.pinky[0][0], annotations.pinky[0][1]);

}

​