notebook-examples/webcam_eye_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*this notebook works in any Sandbox environments* "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Webcam (audio/video) processing\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Usage:\n",
    "\n",
    "Using `ipywebrtc` you can create a `MediaStream` out of:\n",
    "- Any ipywidget using `WidgetStream`\n",
    "- A video file using `VideoStream`\n",
    "- An image file using `ImageStream`\n",
    "- An audio file using `AudioStream`\n",
    "- Your webcam/camera using `CameraStream`\n",
    "\n",
    "From this `MediaStream` you can:\n",
    "- Record a movie using `VideoRecorder`\n",
    "- Record an image snapshot using `ImageRecorder`\n",
    "- Record an audio fragment using `AudioRecorder`\n",
    "- Stream it to peers using the simple `chat` function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywebrtc import CameraStream, ImageRecorder, VideoRecorder, AudioRecorder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "camera = CameraStream(constraints=\n",
    "                      {'facing_mode': 'user',\n",
    "                       'audio': False,\n",
    "                       'video': { 'width': 320, 'height': 240 }\n",
    "                       })\n",
    "#camera"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import Image, HBox\n",
    "import PIL.Image\n",
    "import io\n",
    "import numpy as np\n",
    "from skimage.filters import sobel\n",
    "from skimage.color.adapt_rgb import adapt_rgb, each_channel\n",
    "from skimage import filters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_recorder = ImageRecorder(stream=camera)\n",
    "image_recorder.recording = True\n",
    "#image_recorder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "face_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_frontalface_default.xml')\n",
    "eye_cascade = cv2.CascadeClassifier(cv2.data.haarcascades + 'haarcascade_eye.xml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_detected_eye(face):  \n",
    "    img = face.copy()  \n",
    "    fr = face_cascade.detectMultiScale(img)  \n",
    "    for (x,y,w,h) in fr:\n",
    "        cv2.rectangle(img,(x,y),(x+w,y+h),(255,0,0),2)\n",
    "        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)\n",
    "        roi_gray = gray[y:y+h, x:x+w]\n",
    "        roi_color = img[y:y+h, x:x+w]\n",
    "        eyes = eye_cascade.detectMultiScale(roi_color, scaleFactor=1.5, minNeighbors=5)\n",
    "        print(\"Found {0} eyes!\".format(len(eyes)))\n",
    "        for (ex,ey,ew,eh) in eyes:\n",
    "            cv2.rectangle(roi_color,(ex,ey),(ex+ew,ey+eh),(0,255,0),2)\n",
    "    return img"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Press the Camera Button to start"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "editable": true,
    "slideshow": {
     "slide_type": ""
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(Image(value=b''), ImageRecorder(image=Image(value=b\"\\x89PNG\\r\\n\\x1a\\n\\x00\\x00\\x00\\rIHDR\\x00\\x00…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Press the Camera Button to start\n",
    "out = Image()\n",
    "stop_process = False\n",
    "\n",
    "def process_image(_):\n",
    "    if stop_process:\n",
    "        return\n",
    "    im_in = PIL.Image.open(io.BytesIO(image_recorder.image.value))\n",
    "   # result = get_detected_face(np.array(im_in)[...,:3])\n",
    "    result = get_detected_eye(np.array(im_in)[...,:3])\n",
    "    \n",
    "    im_out = PIL.Image.fromarray(result)\n",
    "    f = io.BytesIO()\n",
    "    im_out.save(f, format='jpeg')\n",
    "    out.value = f.getvalue()\n",
    "    image_recorder.recording = True\n",
    "\n",
    "image_recorder.image.observe(process_image, names=['value'])\n",
    "image_recorder.recording = True\n",
    "HBox([out, image_recorder])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "stop_process = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ipywidgets import Widget\n",
    "Widget.close_all()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}