Computational Media: Pixels & Video Webcam Manipulation with ml5 and createGraphics

Final sketch

p5.js Web Editor

with Resources and References used at the end of the code

Green & Pink Filter only

// 🔛 Make a project that manipulates images or video on the pixel level.
// 🔜 Turn video feed into scribbles.
// 🤔 Make vertical stripes everywhere except on my face.
// 😊 Turn path from my hand into a liquified version of the pixels.
// 🙋🏻‍♀️ with noise?

let videoFeed;
let videoWidth = 640;
let videoHeight = 480;
let canvas;

function setup() {
  canvas = createCanvas(videoWidth, videoHeight);  
  videoFeed = createCapture(VIDEO, { flipped: true });
  videoFeed.size(videoWidth, videoHeight);
  // videoFeed.hide();
  
  pixelDensity(1); // Dealing with retina screens where 1 pixel corresponds to 4 on the screen, though not needed anymore.
}

function draw() {
  background(220);
  videoFeed.loadPixels();
  loadPixels();
  
  // This is what the background function does essentially. Fill every pixel with color.
  for (let y = 0; y < height; y++){
    for (let x = 0; x < width; x++) {
      // Image processing 101
      let index = (x + y * width)*4;
        // Index is start of the pixel, signaling how many pixels are there.
      
      // Fav tones of magenta // R, G, B:
        // 115, 0, 58
        // 180, 79, 111
      
      let r = videoFeed.pixels[index+0];
      let g = videoFeed.pixels[index+1];
      let b = videoFeed.pixels[index+2];
      
      // At least one of the values has to come from the videoFeed, otherwise we're just paiting the pixels without any shape:
      pixels[index+0] = random(115,180);
      pixels[index+1] = g;
      pixels[index+2] = random(58, 111);
      pixels[index+3] = 255;
    }
  }
  updatePixels();
}

// Resources:
// Coding Train The Pixel Array p5 https://www.youtube.com/watch?v=nMUMZ5YRxHI&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=3
// Coding Train Brightness Mirror https://www.youtube.com/watch?v=rNqaw8LT2ZU&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=4

Detecting index and middle fingers on the hand

//// ML5 Handpose Variables /////
let handPose;
let hands = [];

function preload() {
  handPose = ml5.handPose({ flipped: true });
}

function mousePressed() {
  console.log(hands);
}

function gotHands(results) {
  hands = results;
}

function setup() {
  // [ rest of filter code ]  
  
  handPose.detectStart(videoFeed, gotHands);
}

function draw() {
  // [ rest of filter code ]
  
  ////// ML5 HAND DETECTION //////
  if (hands.length > 0) {
    let hand = hands[0]; // Using the first hand in the array
    let index = hand.index_finger_tip;
    let middle = hand.middle_finger_tip;
    
    if (hand.confidence > 0.1) {
      noStroke();
      fill(255, 0, 255);
      circle(index.x, index.y, 16);
      circle(middle.x, middle.y, 30);
      
      
      ///// TO SEE ALL THE KEYPOINTS IN A HAND ////
      // for (let i=0; i < hand.keypoints.length; i++) {
        //let keypoint = hand.keypoints[i];

        // Visual representation of hands' keypoints 
        //fill(255, 0, 255);
        //noStroke();
        //circle(keypoint.x, keypoint.y, 16);
      //}
    }
  }
}

Using createGraphics() to leave a trail

// 🔛 Make a project that manipulates images or video on the pixel level.
// 🔜 Turn video feed into scribbles.
// 🤔 Make vertical stripes everywhere except on my face.
// 😊 Turn path from my hand into a liquified version of the pixels.
// 🙋🏻‍♀️ with noise?
// OR if dist() = 0, then filter can change colors entirely or we can create a column overlay

/// VideoFeed variables ///
let videoFeed;
let videoWidth = 640;
let videoHeight = 480;
let canvas;

//// ML5 Handpose /////
let handPose;
let hands = [];

/// PAINTING ////
let paintingCanvas;

function preload() {
  handPose = ml5.handPose({ flipped: true });
}

function mousePressed() {
  console.log(hands);
}

function gotHands(results) {
  hands = results;
}

function setup() {
  canvas = createCanvas(videoWidth, videoHeight);
  paintingCanvas = createGraphics(videoWidth, videoHeight);
  paintingCanvas.clear();
  
  videoFeed = createCapture(VIDEO, { flipped: true });
  videoFeed.size(videoWidth, videoHeight);
  videoFeed.hide();
  pixelDensity(1); // Dealing with retina screens where 1 pixel corresponds to 4 on the screen, though not needed anymore.
  
  handPose.detectStart(videoFeed, gotHands);
}

function draw() {
  videoFeed.loadPixels();
  loadPixels();
  
  // This is what the background function does essentially. Fill every pixel with color.
  for (let y = 0; y < height; y++){
    for (let x = 0; x < width; x++) {
      // Image processing 101
      let index = (x + y * width)*4;
        // Index is start of the pixel, signaling how many pixels are there.
      
      // Fav tones of magenta // R, G, B:
        // 115, 0, 58
        // 180, 79, 111
      
      let r = videoFeed.pixels[index+0];
      let g = videoFeed.pixels[index+1];
      let b = videoFeed.pixels[index+2];
      
      // At least one of the values has to come from the videoFeed, otherwise we're just painting the pixels without any shape:
      pixels[index+0] = random(115,180);
      pixels[index+1] = g;
      pixels[index+2] = random(58, 111);
      pixels[index+3] = 255;
    }
  }
  
  // For video effect
  updatePixels();
  
  // Drawing createGraphics() on top of video effect
  image(paintingCanvas, 0, 0);

  
  ////// ML5 HAND DETECTION //////
  if (hands.length > 0) {
    let hand = hands[0]; // Using the first hand in the array
    let index = hand.index_finger_tip;
    let middle = hand.middle_finger_tip;
    
    if (hand.confidence > 0.1) {      
      paintingCanvas.noStroke();
      paintingCanvas.fill(255, 0, 255);
      paintingCanvas.circle(index.x, index.y, 16);
      paintingCanvas.circle(middle.x, middle.y, 30);
      
      
      ///// TO SEE ALL THE KEYPOINTS IN A HAND ////
      // for (let i=0; i < hand.keypoints.length; i++) {
        //let keypoint = hand.keypoints[i];

        // Visual representation of hands' keypoints 
        //fill(255, 0, 255);
        //noStroke();
        //circle(keypoint.x, keypoint.y, 16);
      //}
      
      // Can use dist function to check if distance between two points is 0. Then if yes, execute a function, to change filter for example.
    }
  }
}

// In order to make it look like you're painting with your fingers, we can either store all the coordinates through which my hand.keypoint has passed or we can use CreateGraphics();

///// RESOURCES /////

// Coding Train The Pixel Array p5 https://www.youtube.com/watch?v=nMUMZ5YRxHI&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=3
// Coding Train Brightness Mirror https://www.youtube.com/watch?v=rNqaw8LT2ZU&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=4
// Hand Pose ML5 documentation https://editor.p5js.org/ml5/sketches/QGH3dwJ1A
// Coding Train Hand Pose Detection with ML5 https://www.youtube.com/watch?v=vfNHdVbE-l4&t=43s
// Noise p5 reference https://p5js.org/examples/repetition-noise/
// Coding Train createGraphics(); https://youtu.be/TaluaAD9MKA?si=ZEHIeC40MQbNlryN
// Collage (image Manipulation) https://www.youtube.com/watch?v=Ra1IHHeWBPs&t=78s

After creating the rectangles in the same spot as the where my index.x and index.y is, I still have to resolve the fact that the color values for RGB are not being read as I intended.

Possible Solutions to ml5 making my editor slow

Decrease resolution from video feed
Use Google Chrome to view the sketch
Increase frameRate();
Look into HandPose model options

Adding elements based on index and middle finger positions

// 🔛 Make a project that manipulates images or video on the pixel level.
// 🔜 Turn video feed into scribbles.
// 🤔 Make vertical stripes everywhere except on my face.
// 😊 Turn path from my hand into a liquified version of the pixels.
// 🙋🏻‍♀️ with noise?


// Future iterations of this sketch from a technical standpoint for the existing goal:
  // - Fix slowness problem - blog lists possible solutions
  // - Evaluate whether I would like to use the sizes array for anything
  // - If dist(thumb and index) = 0, then filter can change colors entirely or we can create a column overlay
  // - Add possibility to save an image
  // - Add offset and more points per finger touch
  // - Figure out how to alpha value for the drawn circles and rectangles

/// VideoFeed variables ///
let videoFeed;
let videoWidth = 640;
let videoHeight = 480;
let canvas;

//// ML5 Handpose /////
let handPose;
let hands = [];

/// PAINTING ////
let paintingCanvas;
let cols;
let rows;
let size = 15; // Size of each pixelated re-rendering
let sizes = []; // 2D array to store sizes of each square based on brightness 🌸


function preload() {
  handPose = ml5.handPose({ flipped: true });
}

function mousePressed() {
  console.log(hands);
}

function gotHands(results) {
  hands = results;
}

function setup() {
  canvas = createCanvas(videoWidth, videoHeight);
  paintingCanvas = createGraphics(videoWidth, videoHeight);
  paintingCanvas.clear();
  
  /// FOR PAINTING ///
  cols = width/size; // Number of columns calculation
  rows = height/size; // Number of rows calculation 
  
  // Initializing the `sizes` array as a 2D array filled with zeros
      // It's possible that sizes is no longer needed 👾
  for (let i = 0; i < cols; i++){
    sizes[i] = [];
    for (let j = 0; j < rows; j++){
      sizes[i][j] = 0; // Setting initial size of each square to 0
    }
  }
  
  rectMode(CENTER); // Drawing rectangles from their center
  
  videoFeed = createCapture(VIDEO, { flipped: true });
  videoFeed.size(videoWidth, videoHeight);
  videoFeed.hide();
  pixelDensity(1); // Dealing with retina screens where 1 pixel corresponds to 4 on the screen
  
  handPose.detectStart(videoFeed, gotHands);
}

function draw() {
  frameRate(500); // Nothing seems to be changing though
  videoFeed.loadPixels();
  loadPixels();
  
  // This is what the background function does essentially. Fill every pixel with color.
  for (let y = 0; y < height; y++){
    for (let x = 0; x < width; x++) {
      // Image processing 101
      let index = (x + y * width)*4;
        // Index is start of the pixel, signaling how many pixels are there.
      
      // Fav tones of magenta // R, G, B:
        // 115, 0, 58
        // 180, 79, 111
      
      let r = videoFeed.pixels[index+0];
      let g = videoFeed.pixels[index+1];
      let b = videoFeed.pixels[index+2];
      
      // At least one of the values has to come from the videoFeed, otherwise we're just painting the pixels without any shape:
      pixels[index+0] = random(115,180);
      pixels[index+1] = g;
      pixels[index+2] = random(58, 111);
      pixels[index+3] = 255;
    }
  }
  
  // For video effect
  updatePixels();
  
  // Drawing createGraphics() on top of video effect
  image(paintingCanvas, 0, 0);

  
  ////// ML5 HAND DETECTION //////
  if (hands.length > 0) {
    let hand = hands[0]; // Using the first hand in the array
    let index = hand.index_finger_tip;
    let middle = hand.middle_finger_tip;
    
    if (hand.confidence > 0.1) {      
      // paintingCanvas.noStroke();
      // paintingCanvas.fill(255, 0, 255);
      // paintingCanvas.circle(index.x, index.y, 16);
      // paintingCanvas.circle(middle.x, middle.y, 30);
        // circle(x, y, d)
      
      // Looping through each cell in the grid
      for (let i=0; i < cols; i++ ) {
        for (let j=0; j < rows; j++) {
          
          // 🌎  Get positions at the cell's center in the video feed
          let x = i * size;
          let y = j * size;
          // Calculate pixel index in video
          let index2 = (x + y * width ) * 4; // 🌸 Check if issue
          
          // Extract RGB values to create the color
          let r2 = random(115,180); // videoFeed.pixels[index2];
          let g2 = videoFeed.pixels[index2 + 1];
          let b2 = videoFeed.pixels[index2 + 2];
            // Missing alpha value
          let c = color(r2, g2, b2);
          
          // Map the brightness to a size range, making brighter areas smaller 🌸
          sizes[i][j] = map(brightness(c), 0, 100, size * 2, 0);
          
          paintingCanvas.fill(c); // Fill with cell color
          paintingCanvas.noStroke(); // No outline for rectangles
          //           paintingCanvas.stroke(c);
          //           paintingCanvas.point(i, j);
          
          // Draw the square at the index handPose position and mapped size
          // rect(x, y, w, h)
          paintingCanvas.rect(
            index.x, 
            index.y, 
            sizes[i][j], 
            sizes[i][j]);
          
          paintingCanvas.circle(
            middle.x + 20, 
            middle.y, 
            20);
        }
      }
      
      
      ///// TO SEE ALL THE KEYPOINTS IN A HAND ////
      // for (let i=0; i < hand.keypoints.length; i++) {
        //let keypoint = hand.keypoints[i];

        // Visual representation of hands' keypoints 
        //fill(255, 0, 255);
        //noStroke();
        //circle(keypoint.x, keypoint.y, 16);
      //}
      
      // Can use dist function to check if distance between two points is 0. Then if yes, execute a function, to change filter for example.
    }
  }
}

// In order to make it look like you're painting with your fingers, we can either store all the coordinates through which my hand.keypoint has passed or we can use CreateGraphics();

///// RESOURCES /////

// HandPose documentation https://docs.ml5js.org/#/reference/handpose
// Coding Train The Pixel Array p5 https://www.youtube.com/watch?v=nMUMZ5YRxHI&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=3
// Coding Train Brightness Mirror https://www.youtube.com/watch?v=rNqaw8LT2ZU&list=PLRqwX-V7Uu6aKKsDHZdDvN6oCJ2hRY_Ig&index=4
// Hand Pose ML5 documentation https://editor.p5js.org/ml5/sketches/QGH3dwJ1A
// Hand Pose Painting by Coding Train https://editor.p5js.org/codingtrain/sketches/LCEHJm6PA + https://thecodingtrain.com/tracks/ml5js-beginners-guide/ml5/hand-pose
// Coding Train Hand Pose Detection with ML5 https://www.youtube.com/watch?v=vfNHdVbE-l4&t=43s
// Noise p5 reference https://p5js.org/examples/repetition-noise/
// Coding Train createGraphics(); https://youtu.be/TaluaAD9MKA?si=ZEHIeC40MQbNlryN
// Collage (image Manipulation) https://www.youtube.com/watch?v=Ra1IHHeWBPs&t=78s
// Create Raster Graphics from Images by Patt Vira https://youtu.be/ruth6FZ2RK0?si=-NPkZQUZAUBmjP4Z
// HandPose Particle Emitters https://editor.p5js.org/codingtrain/sketches/t7l5pYDDI
// Pixel Painting https://happycoding.io/tutorials/p5js/images/pixel-painter
// Happy Coding Tutorials https://happycoding.io/tutorials/p5js/
// FrameRate reference https://p5js.org/reference/p5/frameRate/

//// QUESTIONS ////
// Liquify effect?https://stackoverflow.com/questions/74420562/can-i-let-shape-with-liquify-effect-like-photoshop-by-p5-js


//// NOTES /////
// get() can only be used for images