Code references

The random notes code is derived from https://happycoding.io/examples/p5js/ and thereafter altered to fit the assignment and my vision. the parallax code by dakerlogend is used to retrieve the facemesh. Facemesh is then used to move the man.

​

Frequencies

The random notes are generated through an array that holds frequencies of two minutes of Dmitri Shostakovich symphony No. 4 in C minor, Op. 43 Presto (rapid) where a clear build-up and climax is visible as the symbolism for insanity, estrangement, and eventually downfall (https://www.youtube.com/watch?v=TwNgpRdiu9k; 1:09-3:09). The frequencies are derived with a pitch analysis with the program 'Praat' by Paul Boersma and David Weenink. Each frequency represents (on average) 0.5/120 seconds. Furthermore, more information from this sound fragment is used, such as BPM (average of 'presto') and intensity (dB).

​

Song choice

Shostakovich's fourth sympony is composed 1935 and 1936 which is in the same time as the production of Not to be Reproduced of Rene Magritte (1937). Shostakovich's work represents the chaotism of Stalin's Great Terror and reveals the insanity and alienation of the time, and at the end, downfall, by a great deal of 'musical' surrealism (Freya Par. (2019). An introduction to Shostakovich's Symphony No. 4. Retrieved from: https://www.classical-music.com/features/articles/introduction-shostakovichs-symphony-no-4/.

​

Image selection

Not to be Reproduced from Rene Magritte (40_2022) is used as the canvas for this representation of 'developing insanity and alienation' and is photoshopped to create the outline of chaos of the man and book in the mirror which creates the alternate reality in this picture. 

*/

​

//facemesh

let facemesh;

let video;

​

//save predictions into an array

let predictions = [];

​

//middle piont between the eyes

var eyepoint = 168;

​

var pointX;

var pointY;

var centervaluex = 208;

var centervaluey = 381;

var xDiff;

var yDiff;

​

//decibel ranges from audiofile

const minIntensity = 45;

const maxIntensity = 85;

​

//presto means 150-170 BPM

// ranges are divided by 2 so the sketch will play

const minSpeed = 150 / 5;

const maxSpeed = 170 / 5;

​

//margin to the sides

const margin = 200;

​

//endpoint xDiff

let end = 127;

let finaltouch = 100;

​

let note;

​

// list of frequencies (on average 2 per second)

let notearray = [

  112,

  697,

  89,

  87,

  88,

  104,

  110,

  104,

  111,

  93,

  210,

  95,

  199,

  100,

  1801,

  149,

  88,

  88,

  89,

  148,

  98,

  92,

  185,

  166,

  156,

  171,

  211,

  103,

  202,

  149,

  146,

  122,

  98,

  216,

  98,

  132,

  196,

  254,

  206,

  207,

  104,

  104,

  87,

  179,

  168,

  149,

  130,

  116,

  103,

  97,

  277,

  312,

  353,

  351,

  369,

  435,

  418,

  435,

  418,

  499,

  403,

  351,

  424,

  836,

  175,

  2372,

  582,

  555,

  547,

  582,

  109,

  352,

  88,

  122,

  1377,

  1331,

  2823,

  1656,

  1395,

  1120,

  1792,

  1927,

  1956,

  2022,

  2281,

  4437,

  2060,

  1484,

  4423,

  3652,

  2910,

  2232,

  4760,

  1948,

  4302,

  1928,

  4036,

  124,

  123,

  131,

  294,

  147,

  78,

  88,

  92,

  1046,

  1328,

  4080,

  148,

  147,

  83,

  87,

  98,

  104,

  1470,

  1586,

  443,

  443,

  444,

  173,

  522,

  589,

  512,

  167,

  518,

  469,

  936,

  751,

  441,

  1363,

  82,

  83,

  726,

  1133,

  1137,

  1155,

  1400,

  296,

  294,

  92,

  99,

  88,

  84,

  84,

  86,

  90,

  1254,

  100,

  99,

  2923,

  107,

  110,

  124,

  124,

  248,

  250,

  165,

  98,

  268,

  252,

  268,

  373,

  371,

  148,

  98,

  98,

  1268,

  104,

  3768,

  3580,

  221,

  769,

  763,

  82,

  83,

];

​

let speed;

let prevNote;

let prevSpeed;

​

//Images derived from Not to be Reproduced

let img;

let img2;

let img3;

let img4;

​

//Load images

function preload() {

  img = loadImage("cutout_final4.png");

  img2 = loadImage("empty withouth background.png");

  img3 = loadImage("man_3.png");

  img4 = loadImage("empty.png");

}

​

function setup() {

  //create canvas

  createCanvas(img.width, img.height);

​

  //setup video frame

  video = createCapture(VIDEO);

  video.size = (width, height);

​

  facemesh = ml5.facemesh(video, modelReady);

​

  //Filling global variable "predictions with an array"

  facemesh.on("predict", (results) => {

    predictions = results;

  });

​

  //hide video

  video.hide();

  //create background image

​

  //randomized speed of drawing the lines; to create chaos

  speed = random(minSpeed, maxSpeed);

  prevNote = note;

  prevSpeed = speed;

}

​

function modelReady() {

  console.log("Model ready!");

}

​

function draw() {

  //creating the speed

  const frameDivider = floor(60 / speed);

  if (frameCount % frameDivider == 0) {

    playSynth();

  }

​

  image(img4, 0, 0);

  //call function of lines

  lines();

​

  //front images

  image(img, 0, 0);

​

  img3.resize(635, 797);

  image(img3, xDiff, -35);

​

  //draw keypoints

  drawKeypoints();

​

  // Read values to callibrate man's position to sittingposition of the viewer

  // console.log(xDiff, pointX);

}

​

function playSynth() {

  for (let i = 0; i < notearray.length; i++) {

    note = notearray[i];

​

    //create visual movement

    note += random(-notearray[i], notearray[i]);

    note = constrain(note, minIntensity, maxIntensity);

​

    speed += random([-5, 0, 5]);

    speed = constrain(speed, minSpeed, maxSpeed);

  }

}

​

function drawKeypoints() {

  for (let i = 0; i < predictions.length; i += 1) {

    const keypoints = predictions[i].scaledMesh;

​

    //set X and Y values of eyepoint to a parameter

    [pointX, pointY] = keypoints[eyepoint];

​

    //calibrating images to eyepoint

    xDiff = pointX - centervaluex;

    yDiff = pointY - centervaluey;

​

    //keep in frame

    //make sure that the image does not go past the abstract reflection

    if (xDiff > end) {

      xDiff = end;

    }

    if (xDiff < 0) {

      xDiff = 0;

    }

    if (yDiff > height) {

      yDiff = height;

    }

    if (yDiff < 0) {

      yDiff = 0;

    }

  }

}

​

function lines() {

  //stay in canvas transformed to fill the silhouet of the man and book in the mirror

  const prevSpeedX = map(prevSpeed, minSpeed, maxSpeed, margin, width - margin);

  const prevNoteY = map(

    prevNote,

    minIntensity,

    maxIntensity,

    height - margin,

    margin

  );

  const speedX = map(speed, minSpeed, maxSpeed, margin, width - margin);

  const noteY = map(note, minIntensity, maxIntensity, height - margin, margin);

​

  prevNote = note;

  prevSpeed = speed;

​

  // get color of the pixels of the image

  img.loadPixels();

  const pixelX = random(img.width);

  const pixelY = random(img.height);

  const pixelColor = img.get(pixelX, pixelY);

​

  //draw strokecolors

  stroke(pixelColor);

  strokeWeight(random(150, 170));

  //draw the lines generated by the frequencies

  //xDiff is used to hide also the little end by the book

  line(

    prevSpeedX + finaltouch - xDiff,

    prevNoteY,

    speedX + finaltouch - xDiff,

    noteY

  );

}

​