UN_all_cifar.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c469ec75",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''\n",
    "all using pitfalls data\n",
    "\n",
    "cifar100 and cifar10\n",
    "'''\n",
    "\n",
    "import os\n",
    "import sys\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import copy\n",
    "import time\n",
    "import platform\n",
    "\n",
    "from analysis.UN_utils         import w_l, \\\n",
    "                                      v_project2\n",
    "from analysis.util_calibration import ts_calibrate, \\\n",
    "                                      ets_calibrate, \\\n",
    "                                      mir_calibrate, \\\n",
    "                                      irova_calibrate\n",
    "from analysis.util_evaluation  import ece_eval_binary, \\\n",
    "                                      ece_hist_binary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7f794b81",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100, 10000, 10)\n",
      "(100, 10000, 10)\n",
      "[1.0000001  0.99999917 0.99999976 0.99999946 0.9999999  1.0000006\n",
      " 1.0000004  1.         0.99999917 0.99999994 0.99999976 1.0000002\n",
      " 0.9999992  0.9999999  1.0000005  0.9999997  0.99999946 1.0000005\n",
      " 0.9999999  1.0000004 ]\n"
     ]
    }
   ],
   "source": [
    "#%% ========== ========== ========== ========== ========== ========== ========== ==========\n",
    "# using pitfalls data\n",
    "from scipy.special import logsumexp, log_softmax, softmax\n",
    "\n",
    "data_dir    = './pitfalldee/megacache/'\n",
    "\n",
    "''' \n",
    "file list\n",
    "\n",
    "----- CIFAR100 -----\n",
    "'logits_CIFAR100-PreResNet110-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR100-VGG16BN-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR100-PreResNet164-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR100-WideResNet28x10-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR100-myResNet18-csgld-100-1.pkl.npy'\n",
    "\n",
    "----- CIFAR10 -----\n",
    "'logits_CIFAR10-PreResNet110-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR10-VGG16BN-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR10-PreResNet164-deepens-100-1.pkl.npy'\n",
    "'logits_CIFAR10-WideResNet28x10-deepens-100-1.pkl.npy'\n",
    "\n",
    "'''\n",
    "\n",
    "logits_file = 'logits_CIFAR10-PreResNet110-deepens-100-1.pkl.npy'\n",
    "\n",
    "# note: this is the raw output\n",
    "log_probs = np.load(data_dir+logits_file)\n",
    "print(log_probs.shape)\n",
    "\n",
    "# do softmax (since this data is small)\n",
    "log_probs_p = softmax(log_probs, axis = -1)\n",
    "print(log_probs_p.shape)\n",
    "print(np.sum(log_probs_p[0,:20,:], axis=-1)) # check prob sum\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "bff1962e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> reading label file from: ./labels/cifar10-labels.pk\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "#%% load label\n",
    "if 'CIFAR100-' in logits_file:\n",
    "    dataset     = 'cifar100'\n",
    "    num_feature = 100\n",
    "    n_class     = 100\n",
    "    \n",
    "if 'CIFAR10-' in logits_file:\n",
    "    dataset     = 'cifar10'\n",
    "    num_feature = 10\n",
    "    n_class     = 10\n",
    "\n",
    "fname = './labels/' + dataset + '-labels.pk'\n",
    "print('> reading label file from:', fname)\n",
    "with open(fname, 'rb') as f:\n",
    "    train_label = pickle.load(f)\n",
    "    test_label = pickle.load(f)\n",
    "\n",
    "test_label = test_label.astype(int)\n",
    "train_label = train_label.astype(int)\n",
    "\n",
    "label_total = np.eye(n_class)[test_label]\n",
    "\n",
    "print(label_total.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b8aae6d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> cut at: 10\n",
      "(10, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0109697 0.1131105 0.0540911 0.9628000 0.0050049\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0137652 0.1187795 0.0556528 0.9632000 0.0134574\n",
      "> acc changed: 22\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0142573 0.1188241 0.0556964 0.9628000 0.0137872 22\n",
      "> cut at: 20\n",
      "(20, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0124757 0.1098123 0.0529854 0.9642000 0.0044891\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0122656 0.1123410 0.0536506 0.9639000 0.0099984\n",
      "> acc changed: 9\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0121165 0.1123311 0.0536444 0.9642000 0.0096817 9\n",
      "> cut at: 30\n",
      "(30, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0126912 0.1074398 0.0523041 0.9642000 0.0042738\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0114973 0.1094113 0.0526672 0.9642000 0.0083415\n",
      "> acc changed: 13\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0119043 0.1094062 0.0526627 0.9642000 0.0083316 13\n",
      "> cut at: 40\n",
      "(40, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0132586 0.1070942 0.0521342 0.9641000 0.0041239\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0108600 0.1085312 0.0524417 0.9639000 0.0078179\n",
      "> acc changed: 9\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0108953 0.1085259 0.0524374 0.9641000 0.0076104 9\n",
      "> cut at: 50\n",
      "(50, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0131082 0.1060188 0.0519437 0.9638000 0.0048223\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0105708 0.1074706 0.0522784 0.9638000 0.0075961\n",
      "> acc changed: 5\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0107360 0.1074718 0.0522798 0.9638000 0.0075940 5\n",
      "> cut at: 60\n",
      "(60, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0131180 0.1058330 0.0519794 0.9645000 0.0037859\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0105098 0.1070941 0.0522778 0.9640000 0.0070593\n",
      "> acc changed: 5\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0109282 0.1070901 0.0522739 0.9645000 0.0065575 5\n",
      "> cut at: 70\n",
      "(70, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0123611 0.1056936 0.0518050 0.9642000 0.0038555\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0103756 0.1068218 0.0520204 0.9638000 0.0069573\n",
      "> acc changed: 7\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0103715 0.1068185 0.0520176 0.9642000 0.0065551 7\n",
      "> cut at: 80\n",
      "(80, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0120852 0.1054810 0.0516714 0.9646000 0.0038717\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0102920 0.1064135 0.0518525 0.9645000 0.0058935\n",
      "> acc changed: 6\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0101442 0.1064105 0.0518497 0.9646000 0.0057920 6\n",
      "> cut at: 90\n",
      "(90, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0117650 0.1054750 0.0516488 0.9644000 0.0038826\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0099385 0.1061781 0.0517928 0.9646000 0.0057132\n",
      "> acc changed: 4\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0097695 0.1061786 0.0517931 0.9644000 0.0059127 4\n",
      "> cut at: 100\n",
      "(100, 10000, 10)\n",
      "\n",
      ">>without truth<< ece | nll | mse | accu | hist \n",
      "0.0115303 0.1055102 0.0517404 0.9641000 0.0043982\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0101690 0.1062657 0.0518821 0.9642000 0.0059875\n",
      "> acc changed: 3\n",
      "\n",
      ">>with truth<< ece | nll | mse | accu | hist \n",
      "0.0102550 0.1062658 0.0518822 0.9641000 0.0060865 3\n"
     ]
    }
   ],
   "source": [
    "#%% cut in pipeline\n",
    "lines = []\n",
    "\n",
    "for C in [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]: # 10, 20, 30, 40, 50, 60, 70, 80, 90, 100\n",
    "    print('> cut at:', C)\n",
    "    log_prob3 = np.mean(log_probs_p[:C,:,:], axis=0)\n",
    "    print(log_probs_p[:C,:,:].shape)\n",
    "    \n",
    "    #% % all together ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~    \n",
    "    log_tD = np.zeros_like(log_prob3)    \n",
    "    for s in range(log_prob3.shape[0]):\n",
    "        values = log_probs_p[:C,s,:] # log_probs_p | all_eval_logit\n",
    "        v_mean = np.mean(values, axis=0) # 1st estimation\n",
    "        w      = w_l(values, v_mean)\n",
    "        v_     = np.dot(values.T, w) # 2nd estimation\n",
    "        for i in range(5):\n",
    "            w  = w_l(values, v_)\n",
    "            v_ = np.dot(values.T, w)\n",
    "        log_tD[s,:] = v_\n",
    "    \n",
    "    #% % without truth ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
    "    # mean of probabilities:\n",
    "    # note: np.mean(log_probs_p, axis=0) <=> log_prob3\n",
    "    ece, nll, mse, accu = ece_eval_binary(log_prob3, label_total) \n",
    "    hist_ece = ece_hist_binary(log_prob3, label_total)\n",
    "    line = '>>without truth<< ece | nll | mse | accu | hist \\n{:.7f} {:.7f} {:.7f} {:.7f} {:.7f}'.format(ece, nll, mse, accu, hist_ece.numpy()[0])\n",
    "    print('\\n'+line)\n",
    "    lines.append(line)\n",
    "    \n",
    "    #% % after truth ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
    "    ece, nll, mse, accu = ece_eval_binary(log_tD, label_total) \n",
    "    hist_ece = ece_hist_binary(log_tD, label_total)\n",
    "    line = '>>with truth<< ece | nll | mse | accu | hist \\n{:.7f} {:.7f} {:.7f} {:.7f} {:.7f}'.format(ece, nll, mse, accu, hist_ece.numpy()[0])\n",
    "    print('\\n'+line)\n",
    "    lines.append(line)\n",
    "    \n",
    "    #% % acc perserving ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~    \n",
    "    log_tD      = np.zeros_like(log_prob3)    \n",
    "    acc_changed = 0\n",
    "    for s in range(log_prob3.shape[0]):\n",
    "        values   = log_probs_p[:C,s,:] # log_probs_p | all_eval_logit\n",
    "        v_mean   = np.mean(values, axis=0) # 1st estimation\n",
    "        ind_mean = np.argmax(v_mean)\n",
    "        w        = w_l(values, v_mean)\n",
    "        v_       = np.dot(values.T, w) # 2nd estimation\n",
    "        for i in range(5):\n",
    "            w  = w_l(values, v_)\n",
    "            v_ = np.dot(values.T, w)\n",
    "        # acc perserving step here <<<<<\n",
    "        ind_ = np.argmax(v_)\n",
    "        if ind_ != ind_mean:\n",
    "            # print('> one sample found.')\n",
    "            acc_changed += 1\n",
    "            v_ = v_project2(v_, ind_mean, C=n_class)\n",
    "        # acc perserving step here <<<<<\n",
    "        log_tD[s,:] = v_\n",
    "    print('> acc changed:', acc_changed)\n",
    "    \n",
    "    #% % after acc perserving ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~\n",
    "    ece, nll, mse, accu = ece_eval_binary(log_tD, label_total) \n",
    "    hist_ece = ece_hist_binary(log_tD, label_total)\n",
    "    line = '>>with truth<< ece | nll | mse | accu | hist \\n{:.7f} {:.7f} {:.7f} {:.7f} {:.7f} {}'.format(ece, nll, mse, accu, hist_ece.numpy()[0], acc_changed)\n",
    "    print('\\n'+line)\n",
    "    lines.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "84b708c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0109697 0.1131105 0.0540911 0.9628000 0.0050049\n",
      "0.0137652 0.1187795 0.0556528 0.9632000 0.0134574\n",
      "0.0142573 0.1188241 0.0556964 0.9628000 0.0137872 22\n",
      "0.0124757 0.1098123 0.0529854 0.9642000 0.0044891\n",
      "0.0122656 0.1123410 0.0536506 0.9639000 0.0099984\n",
      "0.0121165 0.1123311 0.0536444 0.9642000 0.0096817 9\n",
      "0.0126912 0.1074398 0.0523041 0.9642000 0.0042738\n",
      "0.0114973 0.1094113 0.0526672 0.9642000 0.0083415\n",
      "0.0119043 0.1094062 0.0526627 0.9642000 0.0083316 13\n",
      "0.0132586 0.1070942 0.0521342 0.9641000 0.0041239\n",
      "0.0108600 0.1085312 0.0524417 0.9639000 0.0078179\n",
      "0.0108953 0.1085259 0.0524374 0.9641000 0.0076104 9\n",
      "0.0131082 0.1060188 0.0519437 0.9638000 0.0048223\n",
      "0.0105708 0.1074706 0.0522784 0.9638000 0.0075961\n",
      "0.0107360 0.1074718 0.0522798 0.9638000 0.0075940 5\n",
      "0.0131180 0.1058330 0.0519794 0.9645000 0.0037859\n",
      "0.0105098 0.1070941 0.0522778 0.9640000 0.0070593\n",
      "0.0109282 0.1070901 0.0522739 0.9645000 0.0065575 5\n",
      "0.0123611 0.1056936 0.0518050 0.9642000 0.0038555\n",
      "0.0103756 0.1068218 0.0520204 0.9638000 0.0069573\n",
      "0.0103715 0.1068185 0.0520176 0.9642000 0.0065551 7\n",
      "0.0120852 0.1054810 0.0516714 0.9646000 0.0038717\n",
      "0.0102920 0.1064135 0.0518525 0.9645000 0.0058935\n",
      "0.0101442 0.1064105 0.0518497 0.9646000 0.0057920 6\n",
      "0.0117650 0.1054750 0.0516488 0.9644000 0.0038826\n",
      "0.0099385 0.1061781 0.0517928 0.9646000 0.0057132\n",
      "0.0097695 0.1061786 0.0517931 0.9644000 0.0059127 4\n",
      "0.0115303 0.1055102 0.0517404 0.9641000 0.0043982\n",
      "0.0101690 0.1062657 0.0518821 0.9642000 0.0059875\n",
      "0.0102550 0.1062658 0.0518822 0.9641000 0.0060865 3\n"
     ]
    }
   ],
   "source": [
    "#%%\n",
    "for line in lines:\n",
    "    l = line.split('\\n')\n",
    "    print(l[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a462e47e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}