prostate_mr_run_on_idc_data.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/bamf-health/aimi-prostate-mr/blob/idc-colab/prostate_mr_run_on_idc_data.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "D6dwah8L4o9g"
      },
      "source": [
        "Run AI segmentation on the ProstateX collection.\n",
        "\n",
        "Be sure to run this in a runtime with an attached GPU\n",
        "\n",
        "Querying and Download the MR scans is based on the [IDC  tutorial cookbook](https://github.com/ImagingDataCommons/IDC-Tutorials/blob/master/notebooks/cookbook.ipynb).\n",
        "\n",
        "## Prerequisites\n",
        "Please complete the prerequisites as described in this [documentation page](https://learn.canceridc.dev/introduction/getting-started-with-gcp).\n",
        "\n",
        "Insert that project ID in the cell below."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bNWiVkeH4euR"
      },
      "outputs": [],
      "source": [
        "#@title Enter your Project ID and authenticate with GCP\n",
        "# initialize this variable with your Google Cloud Project ID!\n",
        "my_ProjectID = ''\n",
        "\n",
        "import os\n",
        "os.environ[\"GCP_PROJECT_ID\"] = my_ProjectID\n",
        "\n",
        "from google.colab import auth\n",
        "auth.authenticate_user()\n",
        "\n",
        "import pandas as pd\n",
        "import subprocess\n",
        "from pathlib import Path\n",
        "from tempfile import TemporaryDirectory\n",
        "import shutil\n",
        "from tqdm.auto import tqdm"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OePqrhA94uSl"
      },
      "source": [
        "Install the `s5cmd` tool for efficient manifest downloads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu4iilTf4rN-",
        "outputId": "31c0f84a-7c68-484d-e6e3-7710075694fc"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "VERSION=\"s5cmd_2.2.2_Linux-64bit\"\n",
        "wget -N https://github.com/peak/s5cmd/releases/download/v2.2.2/${VERSION}.tar.gz\n",
        "tar zxf ${VERSION}.tar.gz\n",
        "mv s5cmd /usr/bin"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l4Qabt8i4zCm"
      },
      "source": [
        "Install dcm2niix for dicom conversion. Use the prebuild version because it was jpeg support."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pmzaajs04xTf",
        "outputId": "cfde5672-dcc6-42af-ce81-06d826648846"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "curl -fLO https://github.com/rordenlab/dcm2niix/releases/latest/download/dcm2niix_lnx.zip\n",
        "unzip -o dcm2niix_lnx.zip -d /usr/bin"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Kv8Cqns042g6"
      },
      "source": [
        "We can build a download manifest. To reproduce our results, get a list of the SeriesInstanceUIDs from the `qa-results.csv` file. This csv is in the prostate-mr.zip file at https://zenodo.org/record/8352041. Alternativly, we can get a copy directly from the github repo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cPqySInl40R0",
        "outputId": "7cdcf670-2e65-4a47-fba2-596234edba09"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "wget -N https://github.com/bamf-health/aimi-prostate-mr/raw/main/qa-results/qa-results.csv"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gSjT10xR5b3t"
      },
      "outputs": [],
      "source": [
        "qa_df = pd.read_csv('qa-results.csv')\n",
        "series_uids = qa_df.SeriesInstanceUID.tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 495
        },
        "id": "hvCC13zK56Jv",
        "outputId": "d237b8d3-0cf1-4316-d2ce-a50fc9626415"
      },
      "outputs": [],
      "source": [
        "# python API is the most flexible way to query IDC BigQuery metadata tables\n",
        "from google.cloud import bigquery\n",
        "bq_client = bigquery.Client(my_ProjectID)\n",
        "\n",
        "# enclose series_uids in quotes for use in sql query\n",
        "series_uids = [f\"'{x}'\" for x in series_uids]\n",
        "\n",
        "selection_query =f\"\"\"\n",
        "SELECT\n",
        "  # Organize the files in-place right after downloading\n",
        "  ANY_VALUE(CONCAT(\"cp s3\",REGEXP_SUBSTR(gcs_url, \"(://.*)/\"),\"/* \",collection_id,\"/\",PatientID,\"/\",StudyInstanceUID,\"/\",SeriesInstanceUID)) AS s5cmd_command\n",
        "FROM\n",
        "  `bigquery-public-data.idc_current.dicom_all`\n",
        "WHERE\n",
        "  SeriesInstanceUID IN ({','.join(series_uids)})\n",
        "GROUP BY\n",
        "  SeriesInstanceUID\n",
        "\"\"\"\n",
        "\n",
        "selection_result = bq_client.query(selection_query)\n",
        "selection_df = selection_result.result().to_dataframe()\n",
        "\n",
        "selection_df.to_csv(\"/content/s5cmd_gcp_manifest.txt\", header=False, index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KSNnSe2X6CS6"
      },
      "source": [
        "Download manifest with the `s5cmd` tool."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Bt1T1786Apj",
        "outputId": "45032d04-2133-4e53-e3bd-585e08f1ccda"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "# check if dicoms have already been downloaded\n",
        "if test -n \"$(find dcms -name '*.dcm' -print -quit)\"\n",
        "then\n",
        "    echo \"dicoms already downloaded\"\n",
        "else\n",
        "  mkdir -p dcms\n",
        "  cd dcms && s5cmd --no-sign-request --endpoint-url https://storage.googleapis.com run ../s5cmd_gcp_manifest.txt\n",
        "  cd -\n",
        "fi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P5IP_A3m6JrL"
      },
      "source": [
        "# Run model on ProstateX\n",
        "You can run the model on the downloaded scans with the below code.\n",
        "\n",
        ">If you want to run the model locally, this code is containerized in the project. Refer to the [readme](https://github.com/bamf-health/aimi-prostate-mr/tree/main#running-inference) for instructions on running the container locally."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HkfVjBsr6a8X"
      },
      "source": [
        "Install `nnunet` python package"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nNl4u13d6EkW"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install nnunet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Oa5l_1Bx6cRL"
      },
      "outputs": [],
      "source": [
        "# setup nnunet paths\n",
        "os.environ[\"nnUNet_raw_data_base\"] =\"/nnunet_data/nnUNet_raw_data_base/\"\n",
        "os.environ[\"nnUNet_preprocessed\"] =\"/nnunet_data/nnUNet_preprocessed/\"\n",
        "os.environ[\"RESULTS_FOLDER\"] =\"/nnunet_data/nnUNet_trained_models/\"\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e9rTRE516frE"
      },
      "source": [
        "download model weights from zenodo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5OtSk6tM6eHW",
        "outputId": "e0ec9215-cbdf-464a-afd4-81b086595cbd"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "# check if weights exist before starting a large downloaded\n",
        "if [ ! -f ${RESULTS_FOLDER}nnUNet/3d_fullres/Task788_ProstateX/nnUNetTrainerV2__nnUNetPlansv2.1/fold_0/model_final_checkpoint.model ]\n",
        "then\n",
        "    mkdir -p ${nnUNet_raw_data_base}\n",
        "    mkdir -p ${nnUNet_preprocessed}\n",
        "    mkdir -p ${RESULTS_FOLDER}nnUNet/\n",
        "    echo \"Downloading model weights\"\n",
        "    wget -N https://zenodo.org/record/8290093/files/Task788_Prostate.zip\n",
        "    unzip Task788_Prostate.zip -d ${RESULTS_FOLDER}nnUNet/\n",
        "else\n",
        "    echo \"model weights already downloaded\"\n",
        "fi"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "41wRu2jG6y5f"
      },
      "source": [
        "Install binaries for `itkimage2segimage` executable"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jXdjd87-6u_Y",
        "outputId": "88fad6cb-fcb5-4493-cee8-807f0efa11af"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "# Install binaries for itkimage2segimage package\n",
        "mkdir -p /app\n",
        "PACKAGE_TAR=\"dcmqi-1.2.5-linux.tar.gz\"\n",
        "ITKIMAGE2SEGIMAGE_URL=https://github.com/QIICR/dcmqi/releases/download/v1.2.5/${PACKAGE_TAR}\n",
        "wget -N ${ITKIMAGE2SEGIMAGE_URL} --no-check-certificate\n",
        "tar -zxvf ${PACKAGE_TAR} -C /app\n",
        "rm ${PACKAGE_TAR}"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "muBplqk_62Hu"
      },
      "source": [
        "Run inference on scans.\n",
        "\n",
        "First download  `run.py` and `ai-dicom-seg-meta.json` from the git repo"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcKoiSCq60Ui",
        "outputId": "5f33a8aa-c329-4357-b7a9-33b7be50f6d5"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "wget -N https://github.com/bamf-health/aimi-prostate-mr/raw/main/container/app/src/run.py\n",
        "wget -N https://github.com/bamf-health/aimi-prostate-mr/raw/main/container/app/dcm-meta/ai-dicom-seg-meta.json"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QugoJ8Hz68Ag"
      },
      "source": [
        "Then run model on all downloaded dicom series"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5HGYqNej66H0"
      },
      "outputs": [],
      "source": [
        "from run import main_dicom"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9R23ycg46-Is"
      },
      "outputs": [],
      "source": [
        "dcm_dir = Path('dcms')\n",
        "seg_dir = Path('preds')\n",
        "seg_meta = Path('ai-dicom-seg-meta.json')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BZdIX19z8GcC"
      },
      "outputs": [],
      "source": [
        "# for testing, just select a single series\n",
        "testing = True\n",
        "if testing:\n",
        "  test_dcm_dir = Path('test_dcms')\n",
        "  test_dcm_dir.mkdir(exist_ok=True)\n",
        "  test_series_dir = test_dcm_dir/\"1.3.6.1.4.1.14519.5.2.1.7311.5101.160028252338004527274326500702\"\n",
        "  if not test_series_dir.exists():\n",
        "    shutil.copytree(\"dcms/prostatex/ProstateX-0000/1.3.6.1.4.1.14519.5.2.1.7311.5101.158323547117540061132729905711/1.3.6.1.4.1.14519.5.2.1.7311.5101.160028252338004527274326500702\", test_dcm_dir/\"1.3.6.1.4.1.14519.5.2.1.7311.5101.160028252338004527274326500702\")\n",
        "  dcm_dir = test_dcm_dir"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 339
        },
        "id": "_AgBiOSmAuu3",
        "outputId": "6d451609-7dc2-4d5d-958e-3ab883748d74"
      },
      "outputs": [],
      "source": [
        "main_dicom(dcm_dir, seg_dir, seg_meta)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bH-IBVLq7BCb"
      },
      "source": [
        "Download segmentations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "aR-4U_sB7CCj"
      },
      "outputs": [],
      "source": [
        "%%shell\n",
        "zip -r preds.zip preds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "H-4cixe37Cwb"
      },
      "outputs": [],
      "source": [
        "from google.colab import files\n",
        "files.download('preds.zip')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "authorship_tag": "ABX9TyMU5q7iwJULhZpnrL86lsad",
      "gpuType": "T4",
      "include_colab_link": true,
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}