xxxxxxxxxx
283
// ROCK PAPER SCISSORS DETECTOR
//forked from Golan Levin example: https://editor.p5js.org/golan/sketches/4UggchtU-
// Hand Pose Detector using p5.js + handsfree.js + ml5.js. Based on:
// https://editor.p5js.org/ml5/sketches/NeuralNetwork_pose_classifier
// https://learn.ml5js.org/#/reference/neural-network?id=examples#/
//
// See: https://unpkg.com/handsfree@8.5.1/build/lib/handsfree.js
// See: https://unpkg.com/ml5@latest/dist/ml5.min.js
// Note: this downloads large models the first time it's run.
// Your mileage may vary. Use good lighting.
let handsfree; // The handsfree.js tracker
let webcam; // A webcam video (for display only)
// Interface
let dataButton;
let dataLabel;
let trainButton;
const N_LANDMARKS = 21;
const CATEGORIES = ["A", "B", "C"];
let sampleCounts = [0, 0, 0];
let bTrainingCompleted = false;
let theResults;
//------------------------------------------
function setup() {
createCanvas(640, 480);
// Create a webcam object. It's just for show.
webcam = createCapture(VIDEO);
webcam.size(640, 480);
webcam.hide();
// Configure handsfree.js to track hands
handsfree = new Handsfree({
showDebug: false, /* shows or hides the camera */
hands: true /* acquire hand data? */
});
handsfree.start();
// ML5:
console.log('ml5 version:', ml5.version);
// dataLabel = createSelect();
// for (var i=0; i<CATEGORIES.length; i++){
// dataLabel.option(CATEGORIES[i]);
// }
// dataButton = createButton('add example');
// dataButton.mousePressed(addExample);
// trainButton = createButton('train model');
// trainButton.mousePressed(trainModel);
// // Save and download the model
// let saveBtn = createButton('save model');
// saveBtn.mousePressed(function () {
// brain.save();
// });
// Create the model.
let options = {
inputs: (N_LANDMARKS*2),
outputs: CATEGORIES.length,
task: 'classification',
debug: true
}
brain = ml5.neuralNetwork(options);
// Load model explictly pointing to each file
const modelDetails = {
model: 'model/model.json',
metadata: 'model/model_meta.json',
weights: 'model/model.weights.bin'
}
brain.load(modelDetails, modelReady);
}
function modelReady() {
console.log('model loaded!')
console.log('ready to classify --> press connect')
bTrainingCompleted = true;
classify();
}
//------------------------------------------
function draw() {
background('white');
drawVideoBackground();
drawHandPoints();
if (bTrainingCompleted) classify();
drawResults();
}
//------------------------------------------
// Add a training example
function addExample() {
let inputs = getLandmarks();
if (inputs && inputs.length > 0) {
let target = dataLabel.value();
brain.addData(inputs, [target]);
for (var i=0; i<CATEGORIES.length; i++){
if (target === CATEGORIES[i]){
sampleCounts[i]++;
}
}
}
}
//------------------------------------------
// Train the model
function trainModel() {
brain.normalizeData();
let options = {
epochs: 30
}
brain.train(options, finishedTrainingCallback);
bTrainingCompleted = true;
}
//------------------------------------------
// Begin prediction
function finishedTrainingCallback() {
print("Finished Training");
}
//------------------------------------------
// Classify
function classify() {
let inputs = getLandmarks();
if (inputs && inputs.length > 0) {
brain.classify(inputs, gotResultsCallback);
}
}
//------------------------------------------
function gotResultsCallback(error, results) {
if (results){
theResults = results;
}
}
//------------------------------------------
function drawResults(){
noStroke();
fill('lightgray');
rect(0,0,width,110);
if (bTrainingCompleted){
if (theResults && (theResults.length > 0)){
for (var j=0; j<CATEGORIES.length; j++){
var jthCategory = CATEGORIES[j];
for (var i=0; i<theResults.length; i++){
var ithLabel = theResults[i].label;
if (ithLabel === jthCategory){
var ithConfidence = theResults[i].confidence;
var str = ithLabel + ": ";
str += nf(ithConfidence,1,2);
fill('black');
noStroke();
text(str, 120, 25+j*30);
stroke('black');
fill('white');
rect(10,10+j*30, 100,20);
fill('darkgray');
rect(10,10+j*30, 100*ithConfidence,20);
}
}
}
}
} else {
for (var j=0; j<CATEGORIES.length; j++){
var str = CATEGORIES[j] + " samples: " + sampleCounts[j];
fill('black');
noStroke();
text(str, 10, 25+j*30);
}
}
}
function drawHandPoints() {
if (handsfree.data.hands) {
if (handsfree.data.hands.multiHandLandmarks) {
var handLandmarks = handsfree.data.hands.multiHandLandmarks;
var nHands = handLandmarks.length;
var handVertexIndices = [
[17,0,1,5,9,13,17], /* palm */
[1,2,3,4], /* thumb */
[5,6,7,8], /* index */
[9,10,11,12], /* middle */
[13,14,15,16], /* ring */
[17,18,19,20], /* pinky */
];
// Draw lines connecting the parts of the fingers
noFill();
stroke('lime');
strokeWeight(3);
for (var h = 0; h < nHands; h++) {
for (var f=0; f<handVertexIndices.length; f++){ // finger
beginShape();
for (var j=0; j<handVertexIndices[f].length; j++){
var ji = handVertexIndices[f][j];
var jx = handLandmarks[h][ji].x;
var jy = handLandmarks[h][ji].y;
jx = map(jx, 0, 1, width, 0);
jy = map(jy, 0, 1, 0, height);
vertex(jx, jy);
}
endShape();
}
}
// Draw just the points of the hands
stroke('black');
fill('red');
strokeWeight(1);
for (var h = 0; h < nHands; h++) {
for (var i = 0; i <= 20; i++) {
var px = handLandmarks[h][i].x;
var py = handLandmarks[h][i].y;
px = map(px, 0, 1, width, 0);
py = map(py, 0, 1, 0, height);
circle(px, py, 9);
}
}
}
}
}
//------------------------------------------
function getLandmarks(){
if (handsfree.data.hands) {
if (handsfree.data.hands.multiHandLandmarks) {
var handLandmarks = handsfree.data.hands.multiHandLandmarks;
var nHands = handLandmarks.length;
if (nHands > 0){
const landmarkData = [];
var whichHand = 0;
var ax = 0;
var ay = 0;
for (var i = 0; i <= 20; i++) {
ax += handLandmarks[whichHand][i].x;
ay += handLandmarks[whichHand][i].y;
}
ax /= 21;
ay /= 21;
for (var i = 0; i <= 20; i++) {
var px = handLandmarks[whichHand][i].x;
var py = handLandmarks[whichHand][i].y;
landmarkData.push( px - ax );
landmarkData.push( py - ay );
}
return landmarkData;
}
}
}
return null;
}
//------------------------------------------
function drawVideoBackground(){
push();
translate(width, 0);
scale(-1,1);
tint(255,255,255, 160);
image(webcam, 0, 0, width, height);
tint(255);
pop();
}