one_shot_talking_face2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/brahianrosswill/StudioRvC/blob/main/one_shot_talking_face2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "KKpvfa8a6SSn",
        "outputId": "631af66f-55e4-4878-9ea7-a9c8130881a3",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Updated git hooks.\n",
            "Git LFS initialized.\n",
            "/content\n",
            "fatal: destination path 'pocketsphinx' already exists and is not an empty directory.\n",
            "/content/pocketsphinx\n",
            "[ 88%] Built target pocketsphinx\n",
            "[ 88%] Built target pocketsphinx_main\n",
            "[ 88%] Built target pocketsphinx_batch\n",
            "[ 90%] Built target pocketsphinx_mdef_convert\n",
            "[ 93%] Built target pocketsphinx_jsgf2fsg\n",
            "[ 95%] Built target pocketsphinx_lm_convert\n",
            "[ 97%] Built target pocketsphinx_lm_eval\n",
            "[100%] Built target pocketsphinx_pitch\n",
            "\u001b[36mInstall the project...\u001b[0m\n",
            "-- Install configuration: \"\"\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us.lm.bin\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/cmudict-en-us.dict\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/README\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/variances\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/means\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/mdef\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/transition_matrices\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/feat.params\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/noisedict\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us/sendump\n",
            "-- Up-to-date: /usr/local/share/pocketsphinx/model/en-us/en-us-phone.lm.bin\n",
            "-- Up-to-date: /usr/local/share/man/man1/pocketsphinx.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/pocketsphinx_batch.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/pocketsphinx_mdef_convert.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/sphinx_lm_convert.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/sphinx_lm_eval.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/sphinx_lm_sort.1\n",
            "-- Up-to-date: /usr/local/share/man/man1/sphinx_pitch.1\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_batch\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_mdef_convert\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_jsgf2fsg\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_lm_convert\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_lm_eval\n",
            "-- Up-to-date: /usr/local/bin/pocketsphinx_pitch\n",
            "-- Up-to-date: /usr/local/lib/libpocketsphinx.a\n",
            "-- Up-to-date: /usr/local/include/include\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/vad.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/alignment.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/mllr.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/export.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/logmath.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/lattice.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/err.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/model.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/search.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/endpointer.h\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/prim_type.h\n",
            "-- Up-to-date: /usr/local/include/include\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx\n",
            "-- Up-to-date: /usr/local/include/include/pocketsphinx/sphinx_config.h\n",
            "-- Up-to-date: /usr/local/lib/pkgconfig/pocketsphinx.pc\n",
            "/content\n",
            "fatal: destination path 'one-shot-talking-face' already exists and is not an empty directory.\n",
            "/content/one-shot-talking-face\n",
            "Requirement already satisfied: scikit-image in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 1)) (0.19.3)\n",
            "Requirement already satisfied: python_speech_features in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 2)) (0.6)\n",
            "Requirement already satisfied: pyworld in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 3)) (0.3.4)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 4)) (6.0.1)\n",
            "Requirement already satisfied: imageio in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 5)) (2.31.5)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 6)) (1.11.3)\n",
            "Requirement already satisfied: opencv-python in /usr/local/lib/python3.10/dist-packages (from -r /content/one-shot-talking-face/requirements.txt (line 8)) (4.8.0.76)\n",
            "Requirement already satisfied: numpy>=1.17.0 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (1.23.0)\n",
            "Requirement already satisfied: networkx>=2.2 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (3.1)\n",
            "Requirement already satisfied: pillow!=7.1.0,!=7.1.1,!=8.3.0,>=6.1.0 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (9.4.0)\n",
            "Requirement already satisfied: tifffile>=2019.7.26 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (2023.9.26)\n",
            "Requirement already satisfied: PyWavelets>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (1.4.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from scikit-image->-r /content/one-shot-talking-face/requirements.txt (line 1)) (23.2)\n",
            "Requirement already satisfied: cython>=0.24 in /usr/local/lib/python3.10/dist-packages (from pyworld->-r /content/one-shot-talking-face/requirements.txt (line 3)) (3.0.2)\n",
            "chmod: cannot access '/content/one-shot-talking-face/OpenFace/FeatureExtraction': No such file or directory\n",
            "mkdir: cannot create directory ‘/content/train’: File exists\n",
            "jq is already the newest version (1.6-2.1ubuntu3).\n",
            "libgtk2.0-0 is already the newest version (2.24.33-2ubuntu2).\n",
            "0 upgraded, 0 newly installed, 0 to remove and 18 not upgraded.\n"
          ]
        }
      ],
      "source": [
        "!pip install -q torch>=2.0.1+cu116 torchvision>=0.15.2+cu116 torchaudio==2.0.2 torchtext>=0.14.1 torchdata>=0.5.1 --extra-index-url https://download.pytorch.org/whl/cu116 -U\n",
        "!git lfs install\n",
        "%cd /content\n",
        "!git clone https://huggingface.co/camenduru/pocketsphinx-20.04-t4 pocketsphinx\n",
        "%cd  /content/pocketsphinx\n",
        "!sudo cmake --build build --target install\n",
        "%cd /content\n",
        "!git clone https://huggingface.co/camenduru/one-shot-talking-face-20.04-t4 one-shot-talking-face\n",
        "%cd /content/one-shot-talking-face\n",
        "!pip install -r /content/one-shot-talking-face/requirements.txt\n",
        "!chmod 755 /content/one-shot-talking-face/OpenFace/FeatureExtraction\n",
        "!mkdir /content/train\n",
        "!apt install -qq libgtk2.0-0 jq -y\n",
        "!pip install -q imageio-ffmpeg gradio numpy==1.23.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KqY9EWVSRcG2"
      },
      "outputs": [],
      "source": [
        "!wget https://github.com/camenduru/one-shot-talking-face-colab/raw/main/test/audio.wav -O /content/audio.wav\n",
        "!wget https://github.com/camenduru/one-shot-talking-face-colab/blob/main/test/image.png?raw=true -O /content/image.png"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e2UP5UrV2Z9W"
      },
      "outputs": [],
      "source": [
        "import gradio as gr\n",
        "import os, subprocess, torchaudio\n",
        "from PIL import Image\n",
        "\n",
        "block = gr.Blocks()\n",
        "\n",
        "def calculate(image_in, audio_in):\n",
        "    waveform, sample_rate = torchaudio.load(audio_in)\n",
        "    torchaudio.save(\"/content/audio.wav\", waveform, sample_rate, encoding=\"PCM_S\", bits_per_sample=16)\n",
        "    image = Image.open(image_in)\n",
        "    image.save(\"/content/image.png\")\n",
        "\n",
        "    pocketsphinx_run = subprocess.run(['pocketsphinx', '-phone_align', 'yes', 'single', '/content/audio.wav'], check=True, capture_output=True)\n",
        "    jq_run = subprocess.run(['jq', '[.w[]|{word: (.t | ascii_upcase | sub(\"<S>\"; \"sil\") | sub(\"<SIL>\"; \"sil\") | sub(\"\\\\\\(2\\\\\\)\"; \"\") | sub(\"\\\\\\(3\\\\\\)\"; \"\") | sub(\"\\\\\\(4\\\\\\)\"; \"\") | sub(\"\\\\\\[SPEECH\\\\\\]\"; \"SIL\") | sub(\"\\\\\\[NOISE\\\\\\]\"; \"SIL\")), phones: [.w[]|{ph: .t | sub(\"\\\\\\+SPN\\\\\\+\"; \"SIL\") | sub(\"\\\\\\+NSN\\\\\\+\"; \"SIL\"), bg: (.b*100)|floor, ed: (.b*100+.d*100)|floor}]}]'], input=pocketsphinx_run.stdout, capture_output=True)\n",
        "    with open(\"/content/test.json\", \"w\") as f:\n",
        "        f.write(jq_run.stdout.decode('utf-8').strip())\n",
        "\n",
        "    os.system(f\"cd /content/one-shot-talking-face && python3 -B test_script.py --img_path /content/image.png --audio_path /content/audio.wav --phoneme_path /content/test.json --save_dir /content/train\")\n",
        "    return \"/content/train/image_audio.mp4\"\n",
        "\n",
        "with block:\n",
        "  gr.Markdown(\n",
        "  \"\"\"\n",
        "  <style> body { text-align: right} </style>\n",
        "  map: 📄 [arxiv](https://arxiv.org/abs/2112.02749) &nbsp; ⇨ 👩‍💻 [github](https://github.com/FuxiVirtualHuman/AAAI22-one-shot-talking-face) &nbsp; ⇨ 🦒 [colab](https://github.com/camenduru/one-shot-talking-face-colab) &nbsp; ⇨ 🤗 [huggingface](https://huggingface.co/spaces/camenduru/one-shot-talking-face) &nbsp; | &nbsp; tools: 🌀 [duplicate this space](https://huggingface.co/spaces/camenduru/sandbox?duplicate=true) &nbsp; | 🐢 [tortoise tts](https://huggingface.co/spaces/mdnestor/tortoise) &nbsp; | 🎨 [text-to-image](https://huggingface.co/models?pipeline_tag=text-to-image&sort=downloads) &nbsp; | 🐣 [twitter](https://twitter.com/camenduru) &nbsp;\n",
        "  \"\"\")\n",
        "  with gr.Group():\n",
        "    with gr.Box():\n",
        "      with gr.Row().style(equal_height=True):\n",
        "        image_in = gr.Image(show_label=False, type=\"filepath\")\n",
        "        audio_in = gr.Audio(show_label=False, type='filepath')\n",
        "        video_out = gr.Video(show_label=False)\n",
        "      with gr.Row().style(equal_height=True):\n",
        "        btn = gr.Button(\"Generate\")\n",
        "  btn.click(calculate, inputs=[image_in, audio_in], outputs=[video_out])\n",
        "  block.launch(debug=True)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}