//ml5js code taken from https://github.com/ml5js/ml5-examples/tree/release/p5js/ObjectDetector

let video;

let pg;

let changeMode = false;

​

let objectDetector;

​

let invertImg;

let imageMask;

let objects = [];

let status;

// let listOfColors = [color('#FF0000'), color('#FF0000'), color('#FF0000')];

// var listOfColors = [color('#aabf12'), color('#33ab12'), color('#165512'), color('#fe3fa2'), color('#a345cd')];

var colorArray = ['#FF0000', '#FF0000', '#FF0000', '#FF6633', '#FFB399', '#FF33FF', '#FFFF99', '#00B3E6', '#E6B333', '#3366E6', '#999966', '#99FF99', '#B34D4D',

  '#80B300', '#809900', '#E6B3B3', '#6680B3', '#66991A',

  '#FF99E6', '#CCFF1A', '#FF1A66', '#E6331A', '#33FFCC',

  '#66994D', '#B366CC', '#4D8000', '#B33300', '#CC80CC',

  '#E666B3', '#33991A', '#CC9999', '#B3B31A', '#00E680'

];

// let questions = {

//   //People

//   person: ['How do you know this person?', 'Where are they from?', 'What challenges have they faced?', 'What do you value about them?', 'What is the power equation between you and them?'],

​

//   // Animal

//   bird: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   cat: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   dog: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   horse: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   sheep: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   cow: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   elephant: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   bear: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   zebra: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

//   giraffe: ['Who are they?', 'How did they get here?', 'What impact do they have on the environment?', 'What challenges do they face?', 'How do you relate to them?'],

​

//   //Objects

//   backpack: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   bed: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   toothbrush: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   umbrella: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   handbag: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   tie: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   suitcase: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   frisbee: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   skis: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   snowboard: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   sports_ball: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   kite: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   baseball_bat: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   baseball_glove: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   tennis_racket: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   surfboard: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   skateboard: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   bottle: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   wine_glass: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   cup: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   fork: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   knife: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   spoon: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   bowl: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   chair: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   couch: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   dining_table: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   toilet: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   book: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   vase: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   clock: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   scissors: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   teddy_bear: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   hair_drier: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

//   sink: ['Who made this?', 'What is it made of?', 'What happens it to when it becomes waste?', 'Does the design of this disadvantage any user?', 'How did this end up here?'],

​

//   //Food

//   banana: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   apple: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   sandwich: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   orange: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   brocolli: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   carrot: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   hot_dog: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   pizza: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   donut: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

//   cake: ['Where did this come from?', 'Who was behind the creation of this?', 'What impact does this have on the environment?', 'What happens to this after you have finished with it?', 'What value does this bring to you?'],

​

//   //Road infra

//   traffic_light: ['Why was this created?', 'Whom does this benefit?', 'Is this designed to harm or impend anyone?', 'Who created this?', 'How does this affect the environment?'],

//   fire_hydrant: ['Why was this created?', 'Whom does this benefit?', 'Is this designed to harm or impend anyone?', 'Who created this?', 'How does this affect the environment?'],

//   stop_sign: ['Why was this created?', 'Whom does this benefit?', 'Is this designed to harm or impend anyone?', 'Who created this?', 'How does this affect the environment?'],

//   parking_meter: ['Why was this created?', 'Whom does this benefit?', 'Is this designed to harm or impend anyone?', 'Who created this?', 'How does this affect the environment?'],

//   bench: ['Why was this created?', 'Whom does this benefit?', 'Is this designed to harm or impend anyone?', 'Who created this?', 'How does this affect the environment?'],

​

//   //Electronics

//   tv: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   remote: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   cell_phone: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   laptop: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   mouse: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   keyboard: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   microwave: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   toaster: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   oven: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

//   refrigerator: ['Who made this?', 'What purpose does this serve?', 'What imapct does it have on the environment?', 'What does it require from you?', 'What happens to it after it becomes waste?'],

​

//   //Transportation

//   car: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   bicycle: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   bus: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   airplane: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   train: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   truck: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

//   boat: ['Why do you use this?', 'What is the environmental footprint?', 'Who created this?', 'Whom does this primarily benefit?', 'What are the unconventional ways in which this is used?'],

​

//   //Plant

//   potted_plant: ['Who planted this?', 'How does it affect the environment?', 'Who benefits from this plant?', 'How did this come to be here?', 'What is your relationship with it?']

// };

​

function setup() {

  createCanvas(640, 600);

  pixelDensity(1);

​

  objectDetector = ml5.objectDetector('cocossd', modelReady);

  // objectDetector = ml5.objectDetector('yolo', modelReady);

​

  video = createCapture(VIDEO);

  video.size(640, 480);

  video.hide();

​

  pg = createGraphics(width, height);

  invertImg = createGraphics(width, height);

}

​

function shutter() {

  pg.image(video, 0, 0);

  invertImg.image(video, 0, 0);

  invertImg.filter(INVERT);

  changeMode = true;

  objectDetector.detect(pg, gotResult);

​

}

​

function draw() {

  background(220);

  buttonSection();

​

​

  if (changeMode) {

    // image(pg, 0, 0);

    if (status != undefined) {

      image(pg, 0, 0)

​

      for (let i = 0; i < objects.length; i++) {

        textAlign(LEFT);

        noStroke();

        fill(colorArray[i]);

        text(objects[i].label + " " + nfc(objects[i].confidence * 100.0, 2) + "%", objects[i].x + 10, objects[i].y + 15);

        push()

        blendMode(DIFFERENCE);

        noFill();

        // fill(255, 20);

        strokeWeight(2);

        stroke(255, 0, 0);

        stroke(colorArray[i]);

        // imageMask = createGraphics(objects[i].width, objects[i].height);

        // imageMask.rect(objects[i].x, objects[i].y, objects[i].width, objects[i].height, 10);

        // imageMask.fill('rgba(0, 0, 0, 1)');

        // ( masked = invertImg.get()).mask(imageMask.get());

        // image(masked, 0, 0);

        // invertImg.mask(imageMask);

        // image(invertImg, objects[i].x, objects[i].y);

        rect(objects[i].x, objects[i].y, objects[i].width, objects[i].height, 10);

​

        let labelTag = objects[i].label;

        labelTag = labelTag.replace(" ", "_");

        // console.log(typeof questions[person]);

        // console.log(typeof questions[toothbrush]);

        push()

        blendMode(SCREEN);

        for (var j = 0; j < questions[labelTag].length; j++) {

          fill(255);

          noStroke();

          text(questions[labelTag][j], objects[i].x + 10, objects[i].y + 35 + (15 * j));

​

          // console.log(questions[labelTag][j]);

        }

        pop()

        pop()

      }

    }

  } else {

    image(video, 0, 0);

  }

}

​

function modelReady() {

  console.log("model Ready!")

  status = true;

}

​

// A function to run when we get any errors and the results

function gotResult(err, results) {

  if (err) {

    console.log(err);

  }

  // console.log(results)

  objects = results;

}

​

function resetScreen() {

  changeMode = false;

}

​

function buttonSection() {

​

  //Shutter Button

  noFill();

  stroke(40);

  strokeWeight(5);

  ellipse(width / 2, 530, 50);

  noStroke();

  fill(240);

  ellipse(width / 2, 530, 50);

  textAlign(CENTER);

  fill(0);

  text('Shutter', width/2, 575);

  if (dist(mouseX, mouseY, width / 2, 530) < 20) {

    fill(200);

    ellipse(width / 2, 530, 50);

  }

​

  //Clear Button

  noFill();

  stroke(40);

  strokeWeight(5);

  ellipse(width / 2 - 200, 530, 30);

  noStroke();

  fill(40);

  ellipse(width / 2 - 200, 530, 30);

  fill(0);

  text('Clear', width/2-200, 565);

  if (dist(mouseX, mouseY, width / 2 - 200, 530) < 15) {

    fill(200);

    ellipse(width / 2 - 200, 530, 30);

  }

​

  //Save Button

  noFill();

  stroke(40);

  strokeWeight(5);

  ellipse(width / 2 + 200, 530, 30);

  noStroke();

  fill(140);

  ellipse(width / 2 + 200, 530, 30);

    fill(0);

  text('Save', width/2+200, 565);

  if (dist(mouseX, mouseY, width / 2 + 200, 530) < 15) {

    fill(200);

    ellipse(width / 2 + 200, 530, 30);

  }

}

​

function mousePressed() {

  if (dist(mouseX, mouseY, width / 2, 530) < 25) {

    fill(140);

    shutter();

  }

  if (dist(mouseX, mouseY, width / 2 - 200, 530) < 15) {

    fill(140);

    resetScreen();

  }

  if (dist(mouseX, mouseY, width / 2 + 200, 530) < 15) {

    fill(140);

    save("Questioning-Camera.png");

  }

}

​