let faceMesh;

let handPose;

let faces = [];

let hands = [];

let capturedFace = null;

let isPinched = false;

let gravity = 0.5;

let pinchOffsetX = 0;

let pinchOffsetY = 0;

let triangles;

​

function preload() {

  faceMesh = ml5.faceMesh({ maxFaces: 1, flipped: true });

  handPose = ml5.handPose({ flipped: true });

  triangles = faceMesh.getTriangles();

}

​

function gotFaces(results) {

  faces = results;

}

​

function gotHands(results) {

  hands = results;

}

​

function setup() {

  createCanvas(1280, 960);  // Doubled canvas size

  video = createCapture(VIDEO, { flipped: true });

  video.size(640, 480);     // Keep original capture size

  video.hide();

  faceMesh.detectStart(video, gotFaces);

  handPose.detectStart(video, gotHands);

}

​

function getPinchPosition() {

  if (hands.length > 0) {

    let hand = hands[0];

    let index = hand.index_finger_tip;

    let thumb = hand.thumb_tip;

    return {

      x: (index.x + thumb.x) / 2,

      y: (index.y + thumb.y) / 2,

      isPinched: dist(index.x, index.y, thumb.x, thumb.y) < 20

    };

  }

  return null;

}

​

function updateFallingFace() {

  if (capturedFace && !isPinched) {

    capturedFace.y += capturedFace.vy;

    capturedFace.vy += gravity;

  }

}

​

function captureFace(face, pinchPos) {

  // Store the pinch position relative to the face

  capturedFace = {

    originalFace: face,

    x: 0,

    y: 0,

    vy: 0,

    pinchX: pinchPos.x,  // Store initial pinch position

    pinchY: pinchPos.y

  };

}

​

function updatePinchedFace(pinchPos) {

  if (capturedFace && isPinched) {

    // Calculate the difference between current and initial pinch position

    capturedFace.x = pinchPos.x - capturedFace.pinchX;

    capturedFace.y = pinchPos.y - capturedFace.pinchY;

  }

}

​

function draw() {

  background(0);

  // Scale everything 2x

  scale(2);

  // Draw video at 50% opacity for reference

  tint(255);

  image(video, 0, 0);

  noTint();

​

  // Get pinch state and position

  let pinchPos = getPinchPosition();

  if (pinchPos) {

    let wasPinched = isPinched;

    isPinched = pinchPos.isPinched;

    // Capture face when pinch starts

    if (isPinched && !wasPinched && faces.length > 0) {

      captureFace(faces[0], pinchPos);

    }

    // Update face position when pinched

    if (isPinched) {

      updatePinchedFace(pinchPos);

    }

  } else {

    isPinched = false;

  }

  // Update falling face

  updateFallingFace();

  // Draw the captured face if it exists

  if (capturedFace) {

    push();

    translate(capturedFace.x, capturedFace.y);

    video.loadPixels();

    beginShape(TRIANGLES);

    for (let i = 0; i < triangles.length; i++) {

      let indices = triangles[i];

      let pointA = capturedFace.originalFace.keypoints[indices[0]];

      let pointB = capturedFace.originalFace.keypoints[indices[1]];

      let pointC = capturedFace.originalFace.keypoints[indices[2]];

      // Calculate center of triangle for color sampling

      let cx = (pointA.x + pointB.x + pointC.x) / 3;

      let cy = (pointA.y + pointB.y + pointC.y) / 3;

      // Sample color from original video position

      let index = (floor(cx) + floor(cy) * video.width) * 4;

      let r = video.pixels[index];

      let g = video.pixels[index + 1];

      let b = video.pixels[index + 2];

      noStroke();

      fill(r, g, b);

      vertex(pointA.x, pointA.y);

      vertex(pointB.x, pointB.y);

      vertex(pointC.x, pointC.y);

    }

    endShape();

    pop();

  }

  // Reset captured face if it falls off screen

  if (capturedFace && !isPinched && capturedFace.y > height/2) {  // Adjusted for scale

    capturedFace = null;

  }

}