Classification to sounds 

​

Train a Neural Network to distinguish between class A and class B and play back sounds for each class

​

Built with featureExtractor model from ml5js and p5js

​

Created by Andreas Refsgaard 2020

*/

​

let featureExtractor;

let classifier;

let video;

let loss;

let imagesOfA = 0; 

let imagesOfB = 0;

let classificationResult;

let confidence = 0;

let lastClass = '';

​

let soundA;

let soundB;

​

function preload() {

  //if loading your own sound files rememeber to re-name them

  soundA = loadSound('A.mp3');

  soundB = loadSound('B.mp3');

}

​

function setup() {

  createCanvas(640, 480);

  // Create a video element

  video = createCapture(VIDEO);

  video.size(640, 480);

  video.hide();

  // Extract the already learned features from MobileNet

  featureExtractor = ml5.featureExtractor('MobileNet', modelReady);

  // Create a new classifier using those features and give the video we want to use

  const options = { numLabels: 2 }; //Specify the number of classes/labels

  classifier = featureExtractor.classification(video, options);  

  // Set up the UI buttons

  setupButtons();

}

​

/**************  YOU CAN MAKE CHANGES HERE *************/

function draw() {

  background(122);

  image(video, 0, 0, width, height);

  textSize(64);

  //NOTE: how the code looks for if the lastClass variable has different from last time. This stops the sound file repeatedly triggering. This method is very useful for lots of actions like this. 

  if (classificationResult == 'A' && lastClass !== 'A') {

    soundA.play();

    soundB.stop();

  } else if (classificationResult == 'B' && lastClass !== 'B') {

    soundB.play();

    soundA.stop();

  }

  lastClass = classificationResult;

}

​

/**************************************/

​

​

// A function to be called when the model has been loaded

function modelReady() {

  select('#modelStatus').html('Base Model (MobileNet) loaded!');

}

​

​

// Classify the current frame.

function classify() {

  classifier.classify(gotResults);

}

​

// A util function to create UI buttons

function setupButtons() {

  // When the A button is pressed, add the current frame

  // from the video with a label of "A" to the classifier

  buttonA = select('#ButtonA');

  buttonA.mousePressed(function() {

    classifier.addImage('A');

    select('#amountOfAImages').html(imagesOfA++);

  });

​

  // When the B button is pressed, add the current frame

  // from the video with a label of "B" to the classifier

  buttonB = select('#ButtonB');

  buttonB.mousePressed(function() {

    classifier.addImage('B');

    select('#amountOfBImages').html(imagesOfB++);

  });

​

  // Train Button

  train = select('#train');

  train.mousePressed(function() {

    classifier.train(function(lossValue) {

      if (lossValue) {

        loss = lossValue;

        select('#loss').html('Loss: ' + loss);

      } else {

        select('#loss').html('Done Training! Final Loss: ' + loss);

      }

    });

  });

​

  // Predict Button

  buttonPredict = select('#buttonPredict');

  buttonPredict.mousePressed(classify);

  // Save model

  saveBtn = select('#save');

  saveBtn.mousePressed(function() {

    classifier.save();

  });

​

  // Load model

  loadBtn = select('#load');

  loadBtn.changed(function() {

    classifier.load(loadBtn.elt.files, function(){

      select('#modelStatus').html('Custom Model Loaded!');

    });

  });

}

​

// Show the results

function gotResults(err, result) {

  // Display any error

  if (err) {

    console.error(err);

  }  

  select('#result').html(result[0].label);

  select('#confidence').html(nf(result[0].confidence,0,2));

​

  classificationResult = result[0].label;

  confidence = result[0].confidence;

  classify();

}