experiments/colab_demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "VQ-VAE timbre transfer demo",
      "provenance": [],
      "collapsed_sections": [
        "c-_FsWhVNMeH",
        "BQtNAd1hNG6R",
        "E6TaJ-A8NA52"
      ],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/cifkao/ss-vq-vae/blob/main/experiments/colab_demo.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nL589kr4K9i8"
      },
      "source": [
        "# Timbre transfer demo\n",
        "\n",
        "Copyright 2020 InterDigital R&D and Télécom Paris.  \n",
        "Author: Ondřej Cífka"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c-_FsWhVNMeH"
      },
      "source": [
        "## Install packages"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zaY5aWDDC7al"
      },
      "source": [
        "!git clone https://github.com/cifkao/ss-vq-vae.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZbnP2LiQSS2j"
      },
      "source": [
        "!pip install ./ss-vq-vae/src ddsp==0.14"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BQtNAd1hNG6R"
      },
      "source": [
        "## Download the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMig9ftDDb0o"
      },
      "source": [
        "logdir = 'ss-vq-vae/experiments/model'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l9FSW1ty4SlA"
      },
      "source": [
        "!wget https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/ssvqvae_model_state.pt -O $logdir/model_state.pt"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E6TaJ-A8NA52"
      },
      "source": [
        "## Load the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fll-AsHCNY9d"
      },
      "source": [
        "import os\n",
        "\n",
        "import confugue\n",
        "from ddsp.colab import colab_utils\n",
        "import librosa\n",
        "import torch\n",
        "\n",
        "from ss_vq_vae.models.vqvae_oneshot import Experiment"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pZ34SIFY6QrN"
      },
      "source": [
        "cfg = confugue.Configuration.from_yaml_file(os.path.join(logdir, 'config.yaml'))\n",
        "exp = cfg.configure(Experiment, logdir=logdir, device='cpu')\n",
        "exp.model.load_state_dict(torch.load(os.path.join(logdir, 'model_state.pt'), map_location=exp.device))\n",
        "exp.model.train(False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Lt4SMAfOXBI"
      },
      "source": [
        "INPUT_ROOT = 'https://adasp.telecom-paris.fr/rc-ext/demos_companion-pages/vqvae_examples/'\n",
        "INPUT_URLS = {\n",
        "    'Electric Guitar': INPUT_ROOT + 'real/content/UnicornRodeo_Maybe_UnicornRodeo_Maybe_Full_25_ElecGtr2CloseMic3.0148.mp3',\n",
        "    'Electric Organ': INPUT_ROOT + 'real/style/AllenStone_Naturally_Allen%20Stone_Naturally_Keys-Organ-Active%20DI.0253.mp3',\n",
        "    'Jazz Piano': INPUT_ROOT + 'real/style/MaurizioPagnuttiSextet_AllTheGinIsGone_MaurizioPagnuttiSextet_AllTheGinIsGone_Full_12_PianoMics1.08.mp3',\n",
        "    'Synth': INPUT_ROOT + 'real/content/Skelpolu_TogetherAlone_Skelpolu_TogetherAlone_Full_13_Synth.0190.mp3'\n",
        "}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1Cg9ITFZN122"
      },
      "source": [
        "## Choose or record inputs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ff7q62XD_DPB",
        "cellView": "form"
      },
      "source": [
        "#@title Content input\n",
        "content_input = 'Electric Guitar'  #@param [\"Record\", \"Electric Guitar\", \"Electric Organ\", \"Jazz Piano\", \"Synth\"]\n",
        "record_seconds = 8 #@param {type:\"number\"}\n",
        "\n",
        "if content_input == 'Record':\n",
        "    a_content = colab_utils.record(seconds=record_seconds, sample_rate=exp.sr, normalize_db=0.1)\n",
        "else:\n",
        "    !wget {INPUT_URLS[content_input]} -O content_input.mp3\n",
        "    a_content, _ = librosa.load('content_input.mp3', sr=exp.sr)\n",
        "colab_utils.play(a_content, sample_rate=exp.sr)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mnlaBLSI_X8G",
        "cellView": "form"
      },
      "source": [
        "#@title Style input\n",
        "style_input = 'Jazz Piano'  #@param [\"Record\", \"Electric Guitar\", \"Electric Organ\", \"Jazz Piano\", \"Synth\"]\n",
        "record_seconds = 8 #@param {type:\"number\"}\n",
        "\n",
        "if style_input == 'Record':\n",
        "    a_style = colab_utils.record(seconds=record_seconds, sample_rate=exp.sr, normalize_db=0.1)\n",
        "else:\n",
        "    !wget {INPUT_URLS[style_input]} -O style_input.mp3\n",
        "    a_style, _ = librosa.load('style_input.mp3', sr=exp.sr)\n",
        "colab_utils.play(a_style, sample_rate=16000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "neqYSLlmOvh-"
      },
      "source": [
        "## Run the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IRzrF8zADDTi"
      },
      "source": [
        "s_content = torch.as_tensor(exp.preprocess(a_content), device=exp.device)[None, :]\n",
        "s_style = torch.as_tensor(exp.preprocess(a_style), device=exp.device)[None, :]\n",
        "l_content, l_style = (torch.as_tensor([x.shape[2]], device=exp.device) for x in [s_content, s_style])\n",
        "with torch.no_grad():\n",
        "    s_output = exp.model(input_c=s_content, input_s=s_style,\n",
        "                         length_c=l_content, length_s=l_style)\n",
        "a_output = exp.postprocess(s_output.cpu().numpy()[0])\n",
        "colab_utils.play(a_output, sample_rate=16000)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}