From 061973b574d19650206bc16fa7393e794976513e Mon Sep 17 00:00:00 2001 From: paul Date: Thu, 20 Oct 2022 14:07:49 +0200 Subject: [PATCH] add: table results recompute --- .../human36m/minmpjpe_and_calibration.ipynb | 320 +++++++++++++++--- 1 file changed, 272 insertions(+), 48 deletions(-) diff --git a/notebooks/human36m/minmpjpe_and_calibration.ipynb b/notebooks/human36m/minmpjpe_and_calibration.ipynb index 7cdc3e0..4074ce5 100644 --- a/notebooks/human36m/minmpjpe_and_calibration.ipynb +++ b/notebooks/human36m/minmpjpe_and_calibration.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 120, + "execution_count": 284, "metadata": { "collapsed": true }, @@ -16,7 +16,7 @@ }, { "cell_type": "code", - "execution_count": 166, + "execution_count": 285, "outputs": [], "source": [ "def get_h36m_mpjpe_results(\n", @@ -59,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 178, + "execution_count": 286, "outputs": [], "source": [ "def get_h36m_calibration_results(\n", @@ -89,7 +89,7 @@ }, { "cell_type": "code", - "execution_count": 144, + "execution_count": 287, "outputs": [], "source": [ "mpjpes = {}\n", @@ -113,7 +113,7 @@ }, { "cell_type": "code", - "execution_count": 145, + "execution_count": 288, "outputs": [ { "name": "stdout", @@ -127,7 +127,7 @@ "text/plain": " test hard occl\nmean 57.50 87.30 47.00\nstd 0.06 0.13 0.18", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testhardoccl
mean57.5087.3047.00
std0.060.130.18
\n
" }, - "execution_count": 145, + "execution_count": 288, "metadata": {}, "output_type": "execute_result" } @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 146, + "execution_count": 289, "outputs": [], "source": [ "mpjpes = {}\n", @@ -166,7 +166,7 @@ }, { "cell_type": "code", - "execution_count": 147, + "execution_count": 290, "outputs": [ { "name": "stdout", @@ -180,7 +180,7 @@ "text/plain": " test hard occl\nmean 53.00 79.30 41.80\nstd 0.06 0.05 0.04", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testhardoccl
mean53.0079.3041.80
std0.060.050.04
\n
" }, - "execution_count": 147, + "execution_count": 290, "metadata": {}, "output_type": "execute_result" } @@ -197,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 150, + "execution_count": 291, "outputs": [], "source": [ "mpjpes = {}\n", @@ -221,7 +221,7 @@ }, { "cell_type": "code", - "execution_count": 151, + "execution_count": 292, "outputs": [ { "name": "stdout", @@ -235,7 +235,7 @@ "text/plain": " test hard occl\nmean 48.50 72.60 39.90\nstd 0.02 0.09 0.05", "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testhardoccl
mean48.5072.6039.90
std0.020.090.05
\n
" }, - "execution_count": 151, + "execution_count": 292, "metadata": {}, "output_type": "execute_result" } @@ -252,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": 179, + "execution_count": 293, "outputs": [], "source": [ "cal = {}\n", @@ -278,7 +278,7 @@ }, { "cell_type": "code", - "execution_count": 182, + "execution_count": 294, "outputs": [ { "name": "stdout", @@ -289,16 +289,16 @@ }, { "data": { - "text/plain": " test occl\nmean 0.23 0.070\nstd 0.00 0.001", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.230.070
std0.000.001
\n
" + "text/plain": " test occl\nmean 0.230000 0.070000\nstd 0.000305 0.000539", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.2300000.070000
std0.0003050.000539
\n
" }, - "execution_count": 182, + "execution_count": 294, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"cGNF xlarge w Lsample\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"cGNF xlarge w Lsample\", sigfig=dict(mean=2, std=10))" ], "metadata": { "collapsed": false, @@ -309,7 +309,7 @@ }, { "cell_type": "code", - "execution_count": 187, + "execution_count": 295, "outputs": [], "source": [ "cal = {}\n", @@ -335,7 +335,7 @@ }, { "cell_type": "code", - "execution_count": 188, + "execution_count": 296, "outputs": [ { "name": "stdout", @@ -346,16 +346,16 @@ }, { "data": { - "text/plain": " test occl\nmean 0.08 0.03\nstd 0.00 0.00", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.080.03
std0.000.00
\n
" + "text/plain": " test occl\nmean 0.080000 0.030000\nstd 0.000046 0.000224", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.0800000.030000
std0.0000460.000224
\n
" }, - "execution_count": 188, + "execution_count": 296, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"cGNF w Lsample\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"cGNF w Lsample\", sigfig=dict(mean=2, std=10))" ], "metadata": { "collapsed": false, @@ -366,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 192, + "execution_count": 298, "outputs": [], "source": [ "cal = {}\n", @@ -392,7 +392,7 @@ }, { "cell_type": "code", - "execution_count": 193, + "execution_count": 299, "outputs": [ { "name": "stdout", @@ -403,16 +403,16 @@ }, { "data": { - "text/plain": " test occl\nmean 0.08 0.070\nstd 0.00 0.001", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.080.070
std0.000.001
\n
" + "text/plain": " test occl\nmean 0.080000 0.070000\nstd 0.000244 0.000506", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.0800000.070000
std0.0002440.000506
\n
" }, - "execution_count": 193, + "execution_count": 299, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"cGNF\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"cGNF\", sigfig=dict(mean=2, std=10))" ], "metadata": { "collapsed": false, @@ -423,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 227, + "execution_count": 300, "outputs": [], "source": [ "api = wandb.Api()\n", @@ -457,7 +457,7 @@ }, { "cell_type": "code", - "execution_count": 228, + "execution_count": 301, "outputs": [ { "name": "stdout", @@ -468,16 +468,16 @@ }, { "data": { - "text/plain": " test occl\nmean 0.180 0.26\nstd 0.001 0.00", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.1800.26
std0.0010.00
\n
" + "text/plain": " test occl\nmean 0.180000 0.260000\nstd 0.000552 0.000441", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
testoccl
mean0.1800000.260000
std0.0005520.000441
\n
" }, - "execution_count": 228, + "execution_count": 301, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"Wehrbein\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"Wehrbein\", sigfig=dict(mean=2, std=10))" ], "metadata": { "collapsed": false, @@ -488,7 +488,7 @@ }, { "cell_type": "code", - "execution_count": 241, + "execution_count": 302, "outputs": [], "source": [ "api = wandb.Api()\n", @@ -516,7 +516,7 @@ }, { "cell_type": "code", - "execution_count": 242, + "execution_count": 303, "outputs": [ { "name": "stdout", @@ -527,16 +527,16 @@ }, { "data": { - "text/plain": " NLL\nmean 0.07\nstd 0.00", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
NLL
mean0.07
std0.00
\n
" + "text/plain": " NLL\nmean 0.07000\nstd 0.00001", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
NLL
mean0.07000
std0.00001
\n
" }, - "execution_count": 242, + "execution_count": 303, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"Gaussian\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"Gaussian\", sigfig=dict(mean=2, std=10))" ], "metadata": { "collapsed": false, @@ -547,7 +547,7 @@ }, { "cell_type": "code", - "execution_count": 245, + "execution_count": 304, "outputs": [], "source": [ "api = wandb.Api()\n", @@ -575,7 +575,7 @@ }, { "cell_type": "code", - "execution_count": 249, + "execution_count": 305, "outputs": [ { "name": "stdout", @@ -586,16 +586,240 @@ }, { "data": { - "text/plain": " NLL\nmean 0.36\nstd NaN", - "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
NLL
mean0.36
stdNaN
\n
" + "text/plain": " NLL\nmean 0.360000\nstd 0.000286", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
NLL
mean0.360000
std0.000286
\n
" }, - "execution_count": 249, + "execution_count": 305, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "describe(cal, title=\"Sharma\", sigfig=dict(mean=2, std=3))" + "describe(cal, title=\"Sharma\", sigfig=dict(mean=2, std=10))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 306, + "outputs": [], + "source": [ + "api = wandb.Api()\n", + "\n", + "runs = list(\n", + " api.runs(\n", + " path=\"ppierzc/propose_human36m\",\n", + " filters={\n", + " \"display_name\": {\"$regex\": \"^wehrbein_occl_.*\"},\n", + " \"state\": \"finished\",\n", + " },\n", + " )\n", + ")\n", + "mpjpe = np.array([run.summary[\"occlusion\"] for run in runs])\n", + "\n", + "mpjpes = {}\n", + "mpjpes[\"Occl\"] = mpjpe" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 307, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Wehrbein MPJPE occl\n" + ] + }, + { + "data": { + "text/plain": " Occl\nmean 51.10\nstd 0.13", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
Occl
mean51.10
std0.13
\n
" + }, + "execution_count": 307, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "describe(mpjpes, title=\"Wehrbein MPJPE occl\", sigfig=dict(mean=1, std=2))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 308, + "outputs": [], + "source": [ + "api = wandb.Api()\n", + "\n", + "runs = list(\n", + " api.runs(\n", + " path=\"ppierzc/propose_human36m\",\n", + " filters={\n", + " \"display_name\": {\"$regex\": \"^gaussian_mpjpe_calibration_.*\"},\n", + " \"state\": \"finished\",\n", + " },\n", + " )\n", + ")\n", + "ece = np.array([run.summary[\"ECE\"] for run in runs])\n", + "\n", + "eces = {}\n", + "eces[\"mpjpe\"] = ece" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 263, + "outputs": [], + "source": [], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 309, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Gaussian MPJPE\n" + ] + }, + { + "data": { + "text/plain": " mpjpe\nmean 0.420000\nstd 0.000004", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
mpjpe
mean0.420000
std0.000004
\n
" + }, + "execution_count": 309, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "describe(eces, title=\"Gaussian MPJPE\", sigfig=dict(mean=2, std=10))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 310, + "outputs": [], + "source": [ + "api = wandb.Api()\n", + "\n", + "runs = list(\n", + " api.runs(\n", + " path=\"ppierzc/propose_human36m\",\n", + " filters={\n", + " \"display_name\": {\"$regex\": \"^gaussian_mpjpe_nll.*\"},\n", + " \"state\": \"finished\",\n", + " },\n", + " )\n", + ")\n", + "mpjpe = np.array([run.summary[\"minMPJPE\"] for run in runs])\n", + "\n", + "mpjpes = {}\n", + "mpjpes[\"NLL\"] = mpjpe\n", + "\n", + "runs = list(\n", + " api.runs(\n", + " path=\"ppierzc/propose_human36m\",\n", + " filters={\n", + " \"display_name\": {\"$regex\": \"^gaussian_mpjpe_mpjpe.*\"},\n", + " \"state\": \"finished\",\n", + " },\n", + " )\n", + ")\n", + "mpjpe = np.array([run.summary[\"minMPJPE\"] for run in runs])\n", + "mpjpes[\"minMPJPE\"] = mpjpe" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 314, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Gaussian\n" + ] + }, + { + "data": { + "text/plain": " NLL minMPJPE\nmean 60.139289 54.775911\nstd 0.001726 0.001843", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
NLLminMPJPE
mean60.13928954.775911
std0.0017260.001843
\n
" + }, + "execution_count": 314, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "describe(mpjpes, title='Gaussian', sigfig=dict(NLL=10, minMPJPE=10))" + ], + "metadata": { + "collapsed": false, + "pycharm": { + "name": "#%%\n" + } + } + }, + { + "cell_type": "code", + "execution_count": 312, + "outputs": [ + { + "data": { + "text/plain": "{'NLL': array([60.13749226, 60.14093531, 60.13943881]),\n 'minMPJPE': array([54.77691421, 54.77378377, 54.77703458])}" + }, + "execution_count": 312, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "mpjpes" ], "metadata": { "collapsed": false,