xxxxxxxxxx
448
/*
DGIF-6037-301 Experiment 3
Olivia Pasian, Paul Van Rijn, Kasper Zhang - group 3
Blue Gaze
Observe the world via an USB camera
Use ml5.ps to detect people, poses and body points
assess the body counts to determine the effect on the bird's mood
dance based on the mood
*/
//-----------------------------------------------------------
// Declare variables for video, pose detection, and data storage
let video;
let bodyPose;
let poses = [];
let boundingBoxes = [];
let flipVideo = true;
let showVideo = true;
let distance;
let birdBehavior;
let timer = 10;
let specialTimer = 10;
let statisticsDict = { happy: 0, neutral: 0, sad: 0 };
let numbersOfPeople;
let peopleFacingCamera = false;
let peopleTooClose = false;
let tooManyPeople = false;
let bodyPointLeftEar = 3;
let bodyPointRightEar = 4;
let bodyPointLeftEye = 1;
let bodyPointRightEye = 2;
let port;
let connectBtn;
// Dance routines by Olivia and Kasper
// each row is a step
// columns represent the position for each servo as a percent angle
// head, left wing, right wing, body tilt
let sadDance = [
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
[60, 70, 30, 100],
[30, 80, 20, 100],
[30, 70, 30, 100],
[60, 80, 20, 100],
];
let happyDance = [
[15, 0, 100, 30],
[70, 0, 100, 50],
[75, 30, 70, 50],
[75, 0, 100, 30],
[75, 10, 90, 30],
[70, 30, 70, 30],
[15, 10, 90, 30],
[10, 30, 70, 50],
[10, 50, 50, 50],
[15, 50, 50, 30],
[15, 0, 100, 30],
[70, 0, 100, 50],
[75, 30, 70, 50],
[75, 0, 100, 30],
[75, 10, 90, 30],
[70, 30, 70, 30],
[15, 10, 90, 30],
[10, 30, 70, 50],
[10, 50, 50, 50],
[15, 50, 50, 30],
[15, 0, 100, 30],
[70, 0, 100, 50],
[75, 30, 70, 50],
[75, 0, 100, 30],
[75, 10, 90, 30],
[70, 30, 70, 30],
[15, 10, 90, 30],
[10, 30, 70, 50],
[10, 50, 50, 50],
[15, 50, 50, 30],
[15, 0, 100, 30],
[70, 0, 100, 50],
[75, 30, 70, 50],
[75, 0, 100, 30],
[75, 10, 90, 30],
[70, 30, 70, 30],
[15, 10, 90, 30],
[10, 30, 70, 50],
[10, 50, 50, 50],
[15, 50, 50, 30],
[15, 0, 100, 30],
[70, 0, 100, 50],
[75, 30, 70, 50],
[75, 0, 100, 30],
[75, 10, 90, 30],
[70, 30, 70, 30],
[15, 10, 90, 30],
[10, 30, 70, 50],
[10, 50, 50, 50],
[15, 50, 50, 30],
];
// head a b body
let neutralDance = [
[10, 50, 50, 30],
[10, 55, 45, 30],
[15, 60, 40, 30],
[15, 60, 40, 30],
[25, 70, 30, 30],
[40, 60, 40, 30],
[40, 60, 40, 30],
[70, 60, 60, 30],
[70, 50, 50, 30],
[70, 50, 50, 30],
[70, 70, 30, 30],
[50, 70, 30, 30],
[50, 55, 45, 30],
[20, 50, 50, 30],
];
let danceStep = 0;
let danceStepTime = 200; //milliseconds between steps
let nextStep = 0;
let danceTime = 0;
// States
let state = "Neutral";
// Initialize servos
class bodyPart {
constructor(servoMin, servoMax, servoNum, bodyIndex) {
this.bodyIndex = bodyIndex;
this.servoValue = -1;
this.servoMin = servoMin >= 0 && servoMin <= 255 ? servoMin : 0;
this.servoMax = servoMax >= 0 && servoMax <= 255 ? servoMax : 0;
this.servoMax =
this.servoMax < this.servoMin ? this.servoMin : this.servoMax;
this.servoNum = servoNum;
}
setServoValue(rawServoVal, rangeStart, rangeEnd) {
this.servoValue = round(
map(rawServoVal, rangeStart, rangeEnd, this.servoMin, this.servoMax, true)
);
}
}
// relate the bird body parts to the servos
let bodyParts = [
new bodyPart(40, 140, 0, 0), //head
new bodyPart(0, 120, 1, 9), //l wing
new bodyPart(80, 150, 2, 10), //r wing
new bodyPart(70, 90, 3, 0), // body
];
//-----------------------------------------------------------
// Preload function to load the ML5 body pose model
function preload() {
bodyPose = ml5.bodyPose("MoveNet", { flipped: flipVideo });
}
//-----------------------------------------------------------
// State machine function to control dance behavior
function performDance(dance) {
danceTime = millis();
if (danceTime > nextStep) {
nextStep = danceTime + danceStepTime;
for (let i = 0; i < bodyParts.length; i++) {
bodyParts[i].setServoValue(dance[danceStep][i], 0, 100);
}
danceStep = (danceStep + 1) % dance.length;
sendDataToArduino();
}
}
function specialPerformDance(dance) {
if (specialTimer === 0) {
specialTimer = 10;
}
while (specialTimer > 0) {
if (frameCount % 60 === 0 && timer > 0) {
timer--;
}
danceTime = millis();
if (danceTime > nextStep) {
nextStep = danceTime + danceStepTime;
for (let i = 0; i < bodyParts.length; i++) {
bodyParts[i].setServoValue(dance[danceStep][i], 0, 100);
}
danceStep = (danceStep + 1) % dance.length;
sendDataToArduino();
}
}
}
// console print status once per second
function countForTenAndResult() {
if (timer === 0) {
timer = 3;
statisticsDict = { happy: 0, neutral: 0, sad: 0 };
}
if (frameCount % 60 === 0 && timer > 0) {
timer--;
}
if (timer === 0) {
console.log("-------------------------------------");
let finalEmotion = getFinalEmotion();
if (finalEmotion === "happy") {
state = "Happy";
} else if (finalEmotion === "sad") {
state = "Sad";
} else {
state = "Neutral";
}
timer = 10; // reset the timer for the next neutral check cycle
statisticsDict = { happy: 0, neutral: 0, sad: 0 };
}
}
function simpleCountForTen() {
if (specialTimer === 0) {
specialTimer = 10;
}
if (frameCount % 60 === 0 && timer > 0) {
timer--;
}
}
// update the mood statistics
function getBirdBehavior() {
peopleFacingCamera = poses.filter((pose) => pose.facingCamera).length;
peopleTooClose = poses.filter((pose) => pose.personTooClose).length;
if (poses.length > 4 || peopleTooClose > 0) {
birdBehavior = "sad";
} else if (poses.length <= 4 && peopleFacingCamera < poses.length) {
birdBehavior = "happy";
} else {
birdBehavior = "neutral";
}
statisticsDict[birdBehavior]++;
}
// determine the emotional state based on the statistics so far
function getFinalEmotion() {
if (statisticsDict["happy"] >= 50) return "happy";
if (
statisticsDict["sad"] >= statisticsDict["happy"] &&
statisticsDict["sad"] >= statisticsDict["neutral"]
)
return "sad";
return "neutral";
}
//-----------------------------------------------------------
// Setup function to initialize the canvas, video, and start pose detection
function setup() {
createCanvas(640, 480);
video = createCapture(VIDEO, { flipped: flipVideo });
video.size(640, 480);
video.hide();
bodyPose.detectStart(video, gotPoses);
initializeSerial();
createConnectButton();
}
//-----------------------------------------------------------
// Draw function to manage states and handle video feed
function draw() {
background(255);
if (showVideo) {
image(video, 0, 0, width, height);
}
console.log("This is the current state:", state);
console.log("This is numbers of people:", poses.length);
console.log("Is Someone get too close:", peopleTooClose);
console.log("Numbers of people face to the bird", peopleFacingCamera);
switch (state) {
case "Neutral":
performDance(neutralDance);
countForTenAndResult();
getBirdBehavior();
console.log(statisticsDict);
break;
case "Happy":
performDance(happyDance);
if (danceStep === 0) {
state = "Neutral"; // return to Neutral after the dance
}
break;
case "Sad":
performDance(sadDance);
if (danceStep === 0) state = "Neutral"; // return to Neutral after the dance
break;
}
updateConnectButton();
}
//-----------------------------------------------------------
// Callback function when poses are detected
// assess poses for significant factors
function gotPoses(results) {
poses = results || [];
boundingBoxes = poses.map((pose) => pose.box).filter((box) => box != null);
// check the ears for distance from camera
poses.forEach((pose, personIndex) => {
const bodyPoint1 = getKeypoint(bodyPointLeftEar, personIndex);
const bodyPoint2 = getKeypoint(bodyPointRightEar, personIndex);
if (
bodyPoint1 &&
bodyPoint2 &&
bodyPoint1.confidence > 0.1 &&
bodyPoint2.confidence > 0.1
) {
distance = measureDistance(bodyPoint1, bodyPoint2);
pose.personTooClose = distance > 140;
}
// check eye confidence for facing camera
const bodyPoint3 = getKeypoint(bodyPointLeftEye, personIndex);
const bodyPoint4 = getKeypoint(bodyPointRightEye, personIndex);
pose.facingCamera =
bodyPoint3 &&
bodyPoint4 &&
bodyPoint3.confidence > 0.4 &&
bodyPoint4.confidence > 0.4;
});
}
//-----------------------------------------------------------
// Helper function to safely get keypoint data
function getKeypoint(pointIndex, personIndex = 0) {
if (!poses || poses.length === 0) return null;
if (!poses[personIndex]) return null;
const keypoints = poses[personIndex].keypoints;
if (!keypoints) return null;
return keypoints[pointIndex] || null;
}
//-----------------------------------------------------------
// Function to measure distance between two points
function measureDistance(point1, point2) {
if (!point1 || !point2) return null;
const dx = point2.x - point1.x;
const dy = point2.y - point1.y;
const distance = Math.sqrt(dx * dx + dy * dy);
stroke(255, 165, 0);
line(point1.x, point1.y, point2.x, point2.y);
const midX = (point1.x + point2.x) / 2;
const midY = (point1.y + point2.y) / 2;
noStroke();
fill(255, 165, 0);
textSize(12);
text(`${Math.round(distance)}px`, midX, midY);
return distance;
}
//-----------------------------------------------------------
// Serial connection helper functions for Arduino
function sendDataToArduino() {
if (port.opened()) {
let outString = "";
for (let j = 0; j < 4; j++) {
if (bodyParts[j].servoValue >= 0) {
outString = outString + str(bodyParts[j].servoValue);
if (j < 3) outString = outString + ",";
}
}
outString = outString + "\n";
port.write(outString);
}
}
function initializeSerial() {
port = createSerial();
let usedPorts = usedSerialPorts();
if (usedPorts.length > 0) {
port.open(usedPorts[0], 57600);
}
}
function createConnectButton() {
connectBtn = createButton("Connect to Arduino");
connectBtn.position(80, 120);
connectBtn.mousePressed(connectBtnClick);
}
function updateConnectButton() {
if (!port.opened()) {
connectBtn.html("Connect to Arduino");
} else {
connectBtn.html("Disconnect");
}
}
function connectBtnClick() {
if (!port.opened()) {
port.open("Arduino", 57600);
} else {
port.close();
}
}