repo initialized + readme updated

.ipynb_checkpoints/website-reporting-checkpoint.ipynb

@@ -0,0 +1,202 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import numpy as np\n",
+    "import imageio\n",
+    "import os\n",
+    "import tqdm\n",
+    "from glob import glob\n",
+    "import shutil\n",
+    "import matplotlib.pyplot as plt\n",
+    "plt.rcParams['figure.figsize']=[12, 12]\n",
+    "from matplotlib.patches import Polygon\n",
+    "from PIL import Image, ImageDraw, ImageOps\n",
+    "import cv2 as cv2\n",
+    "import json\n",
+    "import requests\n",
+    "from datetime import datetime"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 132,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def find_poly_in_instance(img_filename, label_dict):\n",
+    "    shape_attributes = label_dict.get(img_filename)\n",
+    "    polygons = []\n",
+    "    for sh_attr in shape_attributes:\n",
+    "        xy = list(zip(sh_attr.get('shape_attributes')['all_points_x'], sh_attr.get('shape_attributes')['all_points_y']))\n",
+    "        polygons.append(xy)\n",
+    "\n",
+    "    return polygons"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 130,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = \" \"\n",
+    "response = requests.get(url)\n",
+    "data = response.json()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "27"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 111,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "datetime.now()\n",
+    "now = datetime.now()\n",
+    "dt_string = now.strftime(\"%Y%m%d-%H%M%S\")\n",
+    "\n",
+    "folder_name = dt_string\n",
+    "export_path = os.path.join('/media/scanvan/web/public/reporting', dt_string)\n",
+    "os.makedirs(export_path, exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 126,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "<ipython-input-126-9d5b9b80684a>:4: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0\n",
+      "Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`\n",
+      "  for i in tqdm.tqdm_notebook(range(len(data))):\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "ae407ffa21ca49b092f834ca4d784ace",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=27.0), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "root_path = '/media/scanvan/web/public/dev/'\n",
+    "SRCS = []\n",
+    "MASKS = []\n",
+    "for i in tqdm.tqdm_notebook(range(len(data))):\n",
+    "    \n",
+    "    im_path = root_path + data[i]['filepath']\n",
+    "    SRCS.append(im_path)\n",
+    "    im = Image.open(im_path)\n",
+    "    width, height  = im.size\n",
+    "    \n",
+    "    poly = []\n",
+    "    for order in data[i]['pitchYawOrders']:\n",
+    "        y = height * order['pitch'] / np.pi\n",
+    "        x = width * order['yaw'] / ( 2 * np.pi )\n",
+    "        poly.append( ( x , y ) )\n",
+    "        \n",
+    "    mask = Image.new('L', (width, height), 0)\n",
+    "    ImageDraw.Draw(mask).polygon(poly, outline=1, fill=1)\n",
+    "    mask_inv = cv2.bitwise_not(np.array(mask) * 255)\n",
+    "    \n",
+    "    mask_path = os.path.join('/media/scanvan/web/public/reporting', dt_string, data[i]['viewpointId'] + \"-mask.jpg\")\n",
+    "    MASKS.append(mask_path)\n",
+    "    imageio.imwrite(mask_path, mask_inv)\n",
+    "    \n",
+    "    # images are re-written at the same path\n",
+    "    !/home/descombe/Sources/image-suite/bin/image-missing-pixel -s $im_path -m $mask_path -e $im_path -k 16\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# useful to check annotations\n",
+    "extent = 0, width, 0, height\n",
+    "fig = plt.figure(frameon=False, figsize = (23,23))\n",
+    "\n",
+    "im1 = plt.imshow(im,\n",
+    "                 extent=extent)\n",
+    "\n",
+    "im2 = plt.imshow(mask,\n",
+    "                 extent=extent, alpha=0.5)\n",
+    "\n",
+    "plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

README.md

@@ -1,2 +1,79 @@
 # scanvan-image-processing
-Image anonymization tool for the ScanVan project
+
+Image anonymization tool for the ScanVan project. People and cars are blurred on panoramic images of the city of Sion, in order to be published on [website of the project](https://scanvan.dhlab.epfl.ch/) in compliance with the legal requirements for protection of personnal data.
+
+This image processing pipeline is compatible with 3D reconstruction made with [openMVG](https://github.com/openMVG/openMVG).
+
+### Architecture
+
+```
+main-folder
+│   
+└───images (jpg or png format required)
+│   
+└───anonymization (created at step 1.)
+│       │
+│       └──segmentation (step 1.)
+│       │
+│       └──masks (step 2.)
+│       │
+│       └──blur (step 3.)
+│       │
+│       └──gaussian_blur (step 4.)
+│
+└───omvg outputs
+    │
+    └───sfm_data.bin
+    │
+    └───sfm_data_bin.json (cf. note below)
+    
+```
+
+Note : *sfm_data.bin* must be converted to *sfm_data_bin.json* using *openMVG_main_ConvertSfMDataFormat -i sfm_data.bin -o sfm_data_bin.json* from [openMVG](https://github.com/openMVG/openMVG).
+
+### 1. image segmentation
+
+A segmentation model has been trained to detect people and vehicles. It has been trained using [dhSegment](https://dhsegment.readthedocs.io/en/latest/), the weights and training set are available [here](INSERT LINK).
+
+```
+python segmentation_2d.py --model_dir <path-to-dhsegment-model-weights>
+                                      --export_dir <path-to-main-folder>
+```
+![2d-segmentation](./img/probmaps.png)
+*probability maps of detection of people (left) and vehicles (right)*
+
+### 2. create masks from the segmentation results
+
+Masks are made from the probability maps, for each image.
+
+```
+python custom_masks.py --folder <path-to-main-folder>
+                       
+```
+![mask](./img/20200224-134957-886957-mask.png)
+
+### 3. Blur images
+
+Images are blurred with their corresponding mask.
+The blurring is done using a [missing pixel extrapolation](https://github.com/nils-hamel/image-suite/tree/master/src/image-missing-pixel).
+The *DST* folder has to be created before starting the blurring process.
+```
+./scanvan_mask $SRC $MASKS $DST
+
+```
+![blur](./img/20200224-134957-886957-blur.png)
+
+### 4. Blur camera (optional)
+
+For aesthetical reasons, a slight blur is applied on all the images to cover the camera and car roof.
+
+![gaussian_blur](./img/gaussian_blur_car_mask.png)
+
+*Left : mask covering the motionless car roof and camera, right : image with gaussian blur.*
+
+### 5. Website reporting tool
+
+A reporting tool is available to the public on the website to notify personnal data which must be blurred and were not detected at step 1. The annotations are checked and the images' masks are updated accordingly on a weekly basis. The process is done with [website-reporting.ipynb](https://github.com/adescombes/scanvan-image-processing/website-reporting.ipynb).
+
+
+                         

custom_masks.py

@@ -0,0 +1,65 @@
+### les images publiées sur le site scanvan doivent être floutées à certains endroits : 
+    # le masque cachant la caméra et le toit de la voiture 
+    # /media/scanvan/mask/20200219-151940_dhlab-car/mask.png
+    # les masques de segmentation comprenant des people ou des vehicles
+    # /media/scanvan/model/camera_40008603-40009302/%date/%folder/point-cloud-segmentation/scanvan-sbvp-single/2d-segmentation
+
+from glob import glob
+import imageio
+import os
+import sys
+sys.path.append('/home/descombe/point-cloud-segmentation/spherical-images/src_scanvan/')
+import matplotlib.pyplot as plt
+import numpy as np
+import cv2 as cv
+import tqdm
+import click
+
+@click.command()
+@click.option('--folder', help="Directory of the project, containing 2d segmentation and jpg images")
+def make_masks(folder: str):
+
+    masks_folder = folder + "/anonymization/segmentation/"
+
+    images = glob(folder + "/images/*.jpg")
+    export_path = folder + "/anonymization/masks/"
+
+    if not os.path.exists(export_path):
+        os.makedirs(export_path, exist_ok=True)
+
+    dhlab_mask_path = '/media/scanvan/mask/20200219-151940_dhlab-car/mask.png'
+    dhlab_mask = imageio.imread(dhlab_mask_path)
+
+    for image in tqdm.tqdm(images):
+
+        filename = os.path.basename(image)
+        probmap_people = cv.imread(masks_folder + filename.replace(".jpg","-probmap-people.png"), 0)
+        ret,mask_people = cv.threshold(probmap_people,0,255,cv.THRESH_BINARY+cv.THRESH_OTSU)
+
+        kernel = np.array([[0, 0, -1, 0, 0],[0, 0, -1, 0, 0],[0, 0, 1, 0, 0],[0, 0, 1, 0, 0],[0, 0, 1, 0, 0]], dtype=np.uint8)
+        mask_p_filter_it1 = cv.filter2D(mask_people, -1, kernel)
+        mask_p_filter = cv.filter2D(mask_p_filter_it1, -1, kernel)
+
+        probmap_cars = cv.imread(masks_folder + filename.replace(".jpg","-probmap-vehicle.png"), 0)
+        ret,mask_cars = cv.threshold(probmap_cars,0,255,cv.THRESH_BINARY+cv.THRESH_OTSU)
+
+        mask_thresh = cv.bitwise_or(mask_p_filter,mask_cars)
+        mask_inv = cv.bitwise_not(mask_thresh)
+
+        # resize at the same dimensions as the original image (and dhlab-car mask), add dhlab-car mask
+        mask_resized = np.zeros_like(dhlab_mask)
+        height_scaled = dhlab_mask.shape[0] / mask_inv.shape[0]
+        width_scaled = dhlab_mask.shape[1] / mask_inv.shape[1]
+
+        for i in range(mask_inv.shape[0]):
+            for j in range(mask_inv.shape[1]):
+                mask_resized[ int( np.floor( i * width_scaled ) ) : int( np.floor( ( i + 1 ) * width_scaled ) ) , int( np.floor( j * height_scaled ) ): int( np.floor( ( j + 1 ) * height_scaled ) ) ] += mask_inv[i,j]
+
+        #mask_total = cv.bitwise_and(dhlab_mask,mask_resized)
+        image_mask = export_path + filename.replace(".jpg", "-mask.png")
+        cv.imwrite(image_mask, mask_resized)
+
+
+
+if __name__ == '__main__':
+    make_masks()

gaussian_blur.py

@@ -0,0 +1,34 @@
+from glob import glob
+import os
+import numpy as np
+import cv2 as cv
+import click
+import tqdm
+
+@click.command()
+@click.option('--folder', help='folder of images to be blurred')
+def gaussian_blur(folder: str):
+
+#    model_name = max(folder.split('/'), key=len).replace('-normalized','')
+#    folder_scanvan = [x for x in glob('/media/scanvan/record/camera_40008603-40009302/*%s' % model_name.split('loop')[-1] ) if x.split('/')[-1][:8] == model_name[16:24]][0]
+    folder_out = os.path.join(folder, "gaussian_blur")
+    os.mkdir(folder_out)
+    mask_inv = cv.imread('/media/scanvan/mask/20200219-151940_dhlab-car/mask.png',0)
+    mask = cv.bitwise_not(mask_inv)
+
+#    images = glob(folder + "/images/*.jpg")
+    images = glob(folder + "/blur/*.png")
+    for img in tqdm.tqdm(images):
+
+        filename = os.path.basename(img).split('.')[0]
+#        bmp_img = folder_scanvan + "/" + filename + ".png"
+        bmp_img = img
+        bmp = cv.imread(bmp_img)
+        blur = cv.blur(bmp,(20,20),0)
+        out = bmp.copy()
+        out[mask!=0] = blur[mask!=0]
+
+        cv.imwrite(os.path.join(folder_out, filename + ".png"), out)
+
+if __name__ == '__main__':
+    gaussian_blur()