notebooks/emotion_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "678ccd40-80f1-4bcf-8052-aa81f36b4958",
   "metadata": {},
   "source": [
    "# Emotion Detection Proof of Concept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f0c28324-8a8f-43c2-b86a-96108e61aba6",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-08 22:14:24.632383: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-02-08 22:14:24.632429: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-02-08 22:14:24.633424: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-02-08 22:14:24.639512: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-02-08 22:14:25.300591: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "import dataclasses\n",
    "\n",
    "from fer import Video\n",
    "from fer import FER\n",
    "import scenedetect\n",
    "import cv2\n",
    "from pytube import YouTube\n",
    "\n",
    "from numpy import ndarray\n",
    "import pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5b2791f-e768-4b29-9b16-815da0478fbf",
   "metadata": {},
   "source": [
    "#### Download video"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "230b5c2a-cb2c-4b7a-8602-886b6001c12d",
   "metadata": {},
   "outputs": [],
   "source": [
    "yt_id = \"aQoFrRq6Ds8\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a965ed61-12aa-4b54-8449-4f1802bdfe17",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = f\"https://www.youtube.com/watch?v={yt_id}\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76885b24-135f-462e-a82c-b1f1710fb629",
   "metadata": {},
   "source": [
    "###### Get the first 720p stream url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "80c913ea-df9f-4b3f-83ad-d7ed6c1c507b",
   "metadata": {},
   "outputs": [],
   "source": [
    "yt = YouTube(url)\n",
    "video_stream = yt.streams\n",
    "stream_url = video_stream.filter(res=\"720p\").first().url"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "029a8981-3cf4-4d66-8c76-a33cd9f19b52",
   "metadata": {},
   "source": [
    "##### stream the video for cv2 and scenedetect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "038f8a03-2bf4-4856-b08d-b0da37c89579",
   "metadata": {},
   "outputs": [],
   "source": [
    "capture = cv2.VideoCapture(stream_url)\n",
    "video = scenedetect.VideoCaptureAdapter(capture)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4588c68-73bb-469d-8967-5c0e0fb216a4",
   "metadata": {},
   "source": [
    "##### detect the scenes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "84805256-c60b-4992-a19f-6957a420b5ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:pyscenedetect:Downscale factor set to 5, effective resolution: 256 x 137\n",
      "INFO:pyscenedetect:Detecting scenes...\n"
     ]
    }
   ],
   "source": [
    "scene_manager = scenedetect.SceneManager()\n",
    "scene_manager.add_detector(scenedetect.ContentDetector())\n",
    "scene_manager.detect_scenes(video=video)\n",
    "scene_list = scene_manager.get_scene_list()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e481a13-674a-4684-90fc-4b9663dfec08",
   "metadata": {},
   "source": [
    "Restart the video so that we can run emotion detection on it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "ac1ee6d0-82ae-4d79-a94f-37e90f6a5eae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "capture.set(cv2.CAP_PROP_POS_FRAMES, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c4ef2e9-cf25-48b1-92b8-8e4ddb282c1e",
   "metadata": {},
   "source": [
    "#### Emotion detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a74fa0a0-9d8c-4a00-a80e-dda0e9685fdd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-08 22:14:37.158122: E external/local_xla/xla/stream_executor/cuda/cuda_driver.cc:274] failed call to cuInit: CUDA_ERROR_UNKNOWN: unknown error\n",
      "2024-02-08 22:14:37.158148: I external/local_xla/xla/stream_executor/cuda/cuda_diagnostics.cc:129] retrieving CUDA diagnostic information for host: max\n",
      "2024-02-08 22:14:37.158156: I external/local_xla/xla/stream_executor/cuda/cuda_diagnostics.cc:136] hostname: max\n",
      "2024-02-08 22:14:37.158244: I external/local_xla/xla/stream_executor/cuda/cuda_diagnostics.cc:159] libcuda reported version is: 545.29.6\n",
      "2024-02-08 22:14:37.158267: I external/local_xla/xla/stream_executor/cuda/cuda_diagnostics.cc:163] kernel reported version is: 545.29.6\n",
      "2024-02-08 22:14:37.158273: I external/local_xla/xla/stream_executor/cuda/cuda_diagnostics.cc:241] kernel version seems to match DSO: 545.29.6\n"
     ]
    }
   ],
   "source": [
    "detector = FER(mtcnn=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c5718232-50ef-4ded-8d01-d8196a5bb225",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pandas.DataFrame(columns=[\"frame\", \"timestamp\", \"emotion\", \"score\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "6448a155-82bd-475d-b5ab-a6394f1f4625",
   "metadata": {},
   "outputs": [],
   "source": [
    "fps = capture.get(cv2.CAP_PROP_FPS)\n",
    "success, image = capture.read()\n",
    "count = 0\n",
    "length = int(capture.get(cv2.CAP_PROP_FRAME_COUNT))\n",
    "\n",
    "while success:\n",
    "    # to save time, we only run the emotion detection on 1 out of every 3 rows\n",
    "    if count % 3 ==0:\n",
    "        emotion, score = detector.top_emotion(image) # 'happy', 0.99\n",
    "        predictions.loc[len(predictions)] = {\n",
    "            \"frame\": count,\n",
    "            \"timestamp\": count/fps,\n",
    "            \"emotion\": emotion,\n",
    "            \"score\": score,\n",
    "        }\n",
    "    success, image = capture.read()\n",
    "    count += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0664dfbb-0f4e-49d7-bd4b-3d8694069aa1",
   "metadata": {},
   "source": [
    "#### Collect tags into groups of \"scenes\"\n",
    "Why are we using scene detection? We split the video into many images and those images are assigned emotions by the emotion detection algorithm. If every image in a single scene was assigned \"happy\" then we want only one \"happy\" tag to be created at the beginning of the scene. However, if the next scene was assigned \"happy\" we want to create an additional tag on the video to indicate that it is a separate \"happy\" scene."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "19b5dce0-bd8d-48b9-be57-280dff4bd9f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dataclasses.dataclass\n",
    "class Scene:\n",
    "    scene_number: int\n",
    "    start_frame: int\n",
    "    end_frame: int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1b5f17bd-bf82-4aa7-a68f-df67c25d2567",
   "metadata": {},
   "outputs": [],
   "source": [
    "scene_boundaries = []\n",
    "for i, scene in enumerate(scene_list):\n",
    "    scene_boundaries.append(\n",
    "        Scene(\n",
    "            i,\n",
    "            scene[0].get_frames(),\n",
    "            scene[1].get_frames()-1,\n",
    "        )\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5c8d5800-d1d9-43ac-b217-8aa249d55ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_scene(frame):\n",
    "    for scene in scene_boundaries:\n",
    "        if frame < scene.end_frame:\n",
    "            return scene.scene_number\n",
    "    return scene_boundaries[-1].scene_number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "957af1ac-7284-4d69-b2f6-ec713dd4a140",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions[\"scene\"] = predictions[\"frame\"].map(get_scene)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ef4617ee-42bc-4e4d-95d7-7680388cb62d",
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions.to_csv(f\"initial_predictions - {yt_id}.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9893f40f-2d5b-4097-81ac-c8703f1498a6",
   "metadata": {},
   "source": [
    "#### If you don't want to rerun the predictions, start here\n",
    "### cleaning up the output predictions\n",
    "The emotion detection alogorithm will produce some very low score predictions and sometimes it will produce no prediction. We assign low score and missing predictions a value of \"no_emotion\" based on the parameters we set in the cell below. The \"neutral\" prediction is extremely common, so to avoid filling the data with an unnnecessary number of \"neutral\" tags, we set a minimum threshold to only include \"neutral\" when it is a very strong prediction."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "14fc0cb4-f10b-49a8-8d64-828ddb547d8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "minimum_score = .8\n",
    "exclude_neutral_min_score = .96"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "72452556-d224-45dd-901b-b298f2b6ce9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "output = pandas.read_csv(f\"initial_predictions - {yt_id}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1dcbd2be-db4f-44c4-94c8-742e1b03a898",
   "metadata": {},
   "outputs": [],
   "source": [
    "output[\"old_emotion\"] = output[\"emotion\"] # this is for debugging purposes\n",
    "output[\"score\"] = output[\"score\"].fillna(0)\n",
    "\n",
    "output[\"emotion\"] = output.apply(lambda x: \"no_emotion\" if x[\"score\"] < minimum_score else x[\"emotion\"], axis=1)\n",
    "output[\"emotion\"] = output.apply(\n",
    "    lambda x: \"no_emotion\" if x[\"emotion\"] == \"neutral\" and x[\"score\"] < exclude_neutral_min_score else x[\"emotion\"], \n",
    "    axis=1\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79a9d6eb-6c8c-4993-aee5-5ff7eedf03e5",
   "metadata": {},
   "source": [
    "#### detect shifts in emotions\n",
    "A shift in emotion is when the emotion changes within a given scene. If the first 10 seconds of a scene showed a happy person and the next 10 seconds showed the person being sad, then we would want 2 tags: one \"happy\" tag at the beginning of the scene and then a \"sad\" tag at the 10 second mark when the emotion shifts. In order to determine an emotion \"shift\" we want to look at a window of predictions and store the mode of that"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ec035b87-d5ff-43fe-928f-389c272949d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# how many images to consider at a time when deciding if an emotion shift\n",
    "steps = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e82f5d06-d7f9-40c7-b082-08d398b69964",
   "metadata": {},
   "outputs": [],
   "source": [
    "def most_common_tag_and_frame(list_of_rows):\n",
    "    \"\"\"\n",
    "    Detect the most common emotion that was detected in a collection of images \n",
    "    and the frame that this emotion occurs on first\n",
    "    Returns None if there is a tie or if all of the rows are labeled \"no_emotion\"\n",
    "    \"\"\"\n",
    "    tags = []\n",
    "    for r in list_of_rows:\n",
    "        if r[\"emotion\"] != \"no_emotion\":\n",
    "            tags.append(r[\"emotion\"])\n",
    "    if not tags:\n",
    "        return None\n",
    "    counts = {tag: tags.count(tag) for tag in tags}\n",
    "    if len(set(tags)) > 1 and len(set(counts.values())) == 1:\n",
    "        return None\n",
    "    most_common = max(set(tags), key=counts.get)\n",
    "    for row in list_of_rows:\n",
    "        if row[\"emotion\"] == most_common:\n",
    "            prevalence = counts[most_common]/len(tags)\n",
    "            return most_common, row[\"frame\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "f9773219-b363-40dc-bb8c-38756559f712",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_emotion_shifts(dataframe_of_scenes):\n",
    "    \"\"\"\n",
    "    returns a list of frames that indicate when the emotion shifted\n",
    "    \"\"\"\n",
    "    rows = []\n",
    "    for i, r in dataframe_of_scenes.iterrows():\n",
    "        rows.append(r)\n",
    "    if len(dataframe_of_scenes) < steps:\n",
    "        frame = most_common_tag_and_frame(rows)[1]\n",
    "        return [frame, ]\n",
    "\n",
    "    output_list = []\n",
    "    i = 0\n",
    "    last_emotion = None\n",
    "    while i + steps <= len(rows):\n",
    "        most_common = most_common_tag_and_frame(rows[i:i+steps])\n",
    "        if most_common is not None:\n",
    "            current_emotion = most_common[0]\n",
    "            frame = most_common[1]\n",
    "            if last_emotion != current_emotion:\n",
    "                last_emotion = current_emotion\n",
    "                output_list.append(frame)\n",
    "        i += 1\n",
    "    return output_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "36cc2296-c500-4412-9353-614230869c4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "emotion_shifts = []\n",
    "for scene in output[\"scene\"].unique():\n",
    "    scene_df = output[output[\"scene\"] == scene]\n",
    "    if len(scene_df):\n",
    "        emotion_shifts.extend(\n",
    "            get_emotion_shifts(scene_df),\n",
    "        )\n",
    "\n",
    "output[\"emotion_shift\"] = output[\"frame\"].map(lambda x: x in emotion_shifts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "76fd153f-bdc0-4ed3-9314-6af54e40f3f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "output[\"emotion_shift\"] = output[\"frame\"].map(lambda x: x in emotion_shifts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "97d96238-58d2-4253-a8dd-d16db8d56d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "output.to_csv(f\"emotion_detection - {yt_id}.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c343d123-9652-4e05-b101-5a43c299ea76",
   "metadata": {},
   "source": [
    "#### Add the tags to the web app database"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "25fc1f39-4341-4d82-a2d1-2150b054e924",
   "metadata": {},
   "outputs": [],
   "source": [
    "from videobookmarks.tag import get_video_details\n",
    "from videobookmarks.datamodel.datamodel import PostgresDataModel\n",
    "import os\n",
    "from datetime import date"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "885835a1-b83a-4132-8a1a-aaf4d4bb15b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "DB_URL = os.getenv(\"DB_URL\")\n",
    "datamodel = PostgresDataModel(DB_URL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "901108f8-7760-4a03-b56a-8dd97a464dbd",
   "metadata": {},
   "outputs": [],
   "source": [
    "video_id = datamodel.load_video_id(yt_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "b6d1d4c8-26eb-4070-bf34-79df4f55557d",
   "metadata": {},
   "outputs": [],
   "source": [
    "if video_id is None:\n",
    "    video_details = get_video_details(yt_id)\n",
    "    video_id = datamodel.create_video_id(\n",
    "        yt_id,\n",
    "        video_details[\"thumbnail_url\"],\n",
    "        video_details[\"title\"],\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "72633238-9f81-4a57-8e43-d39ead2f0e52",
   "metadata": {},
   "outputs": [],
   "source": [
    "video_id = datamodel.load_video_id(yt_id)\n",
    "user_id = datamodel.get_user_with_name(\"mesterhammerfic\").id\n",
    "description = f\"\"\"\n",
    "minimum score: {minimum_score} | \n",
    "exclude neutral: {exclude_neutral_min_score} | \n",
    "step size: {steps}\n",
    "\"\"\"\n",
    "tag_list_id = datamodel.create_tag_list(f\"emotion detection test {date.today()}\", description, user_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "d3294792-cee3-4352-b29b-96a8b2a2265e",
   "metadata": {},
   "outputs": [],
   "source": [
    "for index, row in output[(output[\"emotion_shift\"] == True) & (output.emotion != \"no_emotion\")].iterrows():\n",
    "    datamodel.add_tag(\n",
    "        row[\"emotion\"],\n",
    "        row[\"timestamp\"],\n",
    "        tag_list_id,\n",
    "        video_id,\n",
    "        user_id,\n",
    "    )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}