xxxxxxxxxx
291
// ROCK PAPER SCISSORS DETECTOR
//forked from Golan Levin example: https://editor.p5js.org/golan/sketches/4UggchtU-
// Hand Pose Detector using p5.js + handsfree.js + ml5.js. Based on:
// https://editor.p5js.org/ml5/sketches/NeuralNetwork_pose_classifier
// https://learn.ml5js.org/#/reference/neural-network?id=examples#/
//
// See: https://unpkg.com/handsfree@8.5.1/build/lib/handsfree.js
// See: https://unpkg.com/ml5@latest/dist/ml5.min.js
// Note: this downloads large models the first time it's run.
// Your mileage may vary. Use good lighting.
let handsfree; // The handsfree.js tracker
let webcam; // A webcam video (for display only)
// Interface
let dataButton;
let dataLabel;
let trainButton;
const N_LANDMARKS = 21;
const CATEGORIES = ["A", "B", "C"];
let sampleCounts = [0, 0, 0];
let bTrainingCompleted = false;
let theResults;
//------------------------------------------
function setup() {
createCanvas(640, 480);
// Create a webcam object. It's just for show.
webcam = createCapture(VIDEO);
webcam.size(640, 480);
webcam.hide();
// Configure handsfree.js to track hands
handsfree = new Handsfree({
showDebug: false /* shows or hides the camera */,
hands: true /* acquire hand data? */,
});
handsfree.start();
// ML5:
console.log("ml5 version:", ml5.version);
dataLabel = createSelect();
for (var i = 0; i < CATEGORIES.length; i++) {
dataLabel.option(CATEGORIES[i]);
}
dataButton = createButton("add example");
dataButton.mousePressed(addExample);
trainButton = createButton("train model");
trainButton.mousePressed(trainModel);
// Save and download the model
let saveBtn = createButton("save model");
saveBtn.mousePressed(function () {
brain.save();
});
// Create the model.
let options = {
inputs: N_LANDMARKS * 2,
outputs: CATEGORIES.length,
task: "classification",
debug: true,
};
brain = ml5.neuralNetwork(options);
}
//------------------------------------------
function draw() {
background("white");
drawVideoBackground();
drawHandPoints();
if (bTrainingCompleted) {
classify();
if (theResults) {
console.log(theResults[0].label);
//important line!!!!
if (theResults[0].label == "A") {
//do something
console.log("gesture 1");
//add a shape/image etc
}
else if (theResults[0].label == "B") {
}
else if (theResults[0].label == "C") {
}
}
}
drawResults();
}
//------------------------------------------
// Add a training example
function addExample() {
let inputs = getLandmarks();
if (inputs && inputs.length > 0) {
let target = dataLabel.value();
brain.addData(inputs, [target]);
for (var i = 0; i < CATEGORIES.length; i++) {
if (target === CATEGORIES[i]) {
sampleCounts[i]++;
}
}
}
}
//------------------------------------------
// Train the model
function trainModel() {
brain.normalizeData();
let options = {
epochs: 30,
};
brain.train(options, finishedTrainingCallback);
bTrainingCompleted = true;
}
//------------------------------------------
// Begin prediction
function finishedTrainingCallback() {
print("Finished Training");
}
//------------------------------------------
// Classify
function classify() {
let inputs = getLandmarks();
if (inputs && inputs.length > 0) {
brain.classify(inputs, gotResultsCallback);
}
}
//------------------------------------------
function gotResultsCallback(error, results) {
if (results) {
theResults = results;
}
}
//------------------------------------------
function drawResults() {
noStroke();
fill("lightgray");
rect(0, 0, width, 110);
if (bTrainingCompleted) {
if (theResults && theResults.length > 0) {
for (var j = 0; j < CATEGORIES.length; j++) {
var jthCategory = CATEGORIES[j];
for (var i = 0; i < theResults.length; i++) {
var ithLabel = theResults[i].label;
if (ithLabel === jthCategory) {
var ithConfidence = theResults[i].confidence;
var str = ithLabel + ": ";
resultLabel = str;
resultConfidence = ithConfidence;
str += nf(ithConfidence, 1, 2);
fill("black");
noStroke();
text(str, 120, 25 + j * 30);
stroke("black");
fill("white");
rect(10, 10 + j * 30, 100, 20);
fill("darkgray");
rect(10, 10 + j * 30, 100 * ithConfidence, 20);
}
}
}
}
} else {
for (var j = 0; j < CATEGORIES.length; j++) {
var str = CATEGORIES[j] + " samples: " + sampleCounts[j];
fill("black");
noStroke();
text(str, 10, 25 + j * 30);
}
}
}
function drawHandPoints() {
if (handsfree.data.hands) {
if (handsfree.data.hands.multiHandLandmarks) {
var handLandmarks = handsfree.data.hands.multiHandLandmarks;
var nHands = handLandmarks.length;
var handVertexIndices = [
[17, 0, 1, 5, 9, 13, 17] /* palm */,
[1, 2, 3, 4] /* thumb */,
[5, 6, 7, 8] /* index */,
[9, 10, 11, 12] /* middle */,
[13, 14, 15, 16] /* ring */,
[17, 18, 19, 20] /* pinky */,
];
// Draw lines connecting the parts of the fingers
noFill();
stroke("lime");
strokeWeight(3);
for (var h = 0; h < nHands; h++) {
for (var f = 0; f < handVertexIndices.length; f++) {
// finger
beginShape();
for (var j = 0; j < handVertexIndices[f].length; j++) {
var ji = handVertexIndices[f][j];
var jx = handLandmarks[h][ji].x;
var jy = handLandmarks[h][ji].y;
jx = map(jx, 0, 1, width, 0);
jy = map(jy, 0, 1, 0, height);
vertex(jx, jy);
}
endShape();
}
}
// Draw just the points of the hands
stroke("black");
fill("red");
strokeWeight(1);
for (var h = 0; h < nHands; h++) {
for (var i = 0; i <= 20; i++) {
var px = handLandmarks[h][i].x;
var py = handLandmarks[h][i].y;
px = map(px, 0, 1, width, 0);
py = map(py, 0, 1, 0, height);
circle(px, py, 9);
}
}
}
}
}
//------------------------------------------
function getLandmarks() {
if (handsfree.data.hands) {
if (handsfree.data.hands.multiHandLandmarks) {
var handLandmarks = handsfree.data.hands.multiHandLandmarks;
var nHands = handLandmarks.length;
if (nHands > 0) {
const landmarkData = [];
var whichHand = 0;
var ax = 0;
var ay = 0;
for (var i = 0; i <= 20; i++) {
ax += handLandmarks[whichHand][i].x;
ay += handLandmarks[whichHand][i].y;
}
ax /= 21;
ay /= 21;
for (var i = 0; i <= 20; i++) {
var px = handLandmarks[whichHand][i].x;
var py = handLandmarks[whichHand][i].y;
landmarkData.push(px - ax);
landmarkData.push(py - ay);
}
return landmarkData;
}
}
}
return null;
}
//------------------------------------------
function drawVideoBackground() {
push();
translate(width, 0);
scale(-1, 1);
tint(255, 255, 255, 160);
image(webcam, 0, 0, width, height);
tint(255);
pop();
}