let handpose;

let video;

let predictions = [];

let handPredictions = [];

​

function setup() {

  createCanvas(640, 480);

  video = createCapture(VIDEO);

  video.size(width, height);

​

  // Initialize FaceMesh model

  facemesh = ml5.facemesh(video, modelReady);

  facemesh.on('predict', results => {

    predictions = results;

  });

​

  // Initialize HandPose model

  handpose = ml5.handpose(video, () => {

    console.log('HandPose model ready');

  });

  handpose.on('predict', results => {

    handPredictions = results;

  });

​

  video.hide();

}

​

function modelReady() {

  console.log('FaceMesh model ready');

}

​

function draw() {

  image(video, 0, 0, width, height);

​

  // Check if mouth is open and get mouth position

  let mouthOpen = false;

  let mouthPosition;

  if (predictions.length > 0) {

    const keypoints = predictions[0].scaledMesh;

    // Mouth keypoints (bottom lip: 17, upper lip: 13)

    let upperLip = keypoints[13];

    let lowerLip = keypoints[17];

    mouthPosition = createVector((upperLip[0] + lowerLip[0]) / 2, (upperLip[1] + lowerLip[1]) / 2);

    const mouthDist = dist(upperLip[0], upperLip[1], lowerLip[0], lowerLip[1]);

    if (mouthDist > 15) { // Threshold for mouth open

      mouthOpen = true;

    }

  }

​

  // Get index finger position

  let indexFingerPosition;

  if (handPredictions.length > 0) {

    const keypoints = handPredictions[0].landmarks;

    // Index finger tip is keypoint 8

    let indexFingerTip = keypoints[8];

    indexFingerPosition = createVector(indexFingerTip[0], indexFingerTip[1]);

  }

​

  // Pigeon behavior

  if (mouthOpen && indexFingerPosition) {

    let pigeon = '🕊️';

    let pigeonPos = p5.Vector.lerp(mouthPosition, indexFingerPosition, 0.5);

    textSize(32);

    text(pigeon, pigeonPos.x, pigeonPos.y);

  }

}

​

function windowResized() {

  resizeCanvas(windowWidth, windowHeight);

}

​