vander_plus_3_Pandas_12.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "69266ade",
   "metadata": {},
   "source": [
    "# Увеличение производительности библиотеки Pandas: ``eval()`` и ``query()``\n",
    "\n",
    "Обстракции (транслирование, группировки) Numpy и Pandas весьма производительны, но они зачастую требуют создания временных вспомогательных объектов, что приводит к дополнительным затратам процессорного времени и памяти.\n",
    "По состоянию на 0,13 от 2014г. библиотека включает эксперементальные инструменты, позволяющие ускорить работу."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e4e09e68",
   "metadata": {},
   "source": [
    "## Основания для использования ф-ий ``eval()`` и ``query()``: составные выражения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cde5e3d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "553d3b7c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.37454012, 0.95071431, 0.73199394, ..., 0.41807198, 0.42867126,\n",
       "       0.92944855])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# векторизованные операции\n",
    "rng = np.random.RandomState(42)\n",
    "x = rng.rand(1000000)\n",
    "y = rng.rand(1000000)\n",
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2222c603",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000000"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f3076aad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.51 ms ± 53 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d6022068",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.55 ms ± 55.7 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "50648138",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "212 ms ± 1.2 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "np.fromiter((xi + yi for xi, yi in zip(x,y)),\n",
    "            dtype=x.dtype, count=len(x))\n",
    "# вектроизованные операции быстрее чем списковые включения или списки в ~100раз"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "115b49fd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.07 ms ± 52.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "mask = (x > 0.5) & (y < 0.5)\n",
    "# однако данная обстракция менее эффективна при вычислении составных выражений (так в книге)\n",
    "# но все течет, все меняется 2.07 ms\n",
    "# актуальна ли эта тема теперь?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4b258e9",
   "metadata": {},
   "source": [
    "###### ОКАЗЫВАЕТСЯ ПЕРЕМЕННЫЙ ПОД ОПЕРАТОРОМ ``%%timeit`` ОКАЗЫВАЮТСЯ В ``LOCAL SCOPE`` !!! НЕОЧЕВИДНЫЙ ФАКТ!!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "165fce19",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False,  True,  True, ..., False, False, False])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask = (x > 0.5) & (y < 0.5)\n",
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a093f8a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext memory_profiler\n",
    "# гразунули memory_profiler получили доступ к %%memit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "588bad69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 147.13 MiB, increment: 0.08 MiB\n"
     ]
    }
   ],
   "source": [
    "%%memit\n",
    "x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "2d055588",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 149.04 MiB, increment: 1.91 MiB\n"
     ]
    }
   ],
   "source": [
    "%%memit\n",
    "mask = (x > 0.5) & (y < 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "94251610",
   "metadata": {},
   "outputs": [],
   "source": [
    "# такс..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ec655299",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting mprun_x_y.py\n"
     ]
    }
   ],
   "source": [
    "%%file mprun_x_y.py\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "def sum_x_y():\n",
    "    rng = np.random.RandomState(42)\n",
    "    x = rng.rand(1000000)\n",
    "    y = rng.rand(1000000)\n",
    "    x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "a9b68c6a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting mprun_x_and_y.py\n"
     ]
    }
   ],
   "source": [
    "%%file mprun_x_and_y.py\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "def mask_x_y():\n",
    "    rng = np.random.RandomState(42)\n",
    "    x = rng.rand(1000000)\n",
    "    y = rng.rand(1000000)\n",
    "    mask = (x > 0.5) & (y < 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f0c0e9f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mprun_x_y import sum_x_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "05999fdf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%mprun -f sum_x_y \n",
    "sum_x_y()\n",
    "# Filename: D:\\Python_projects\\Jake_VanderPlas\\mprun_x_y.py\n",
    "\n",
    "# Line #    Mem usage    Increment  Occurrences   Line Contents\n",
    "# =============================================================\n",
    "#      3    149.4 MiB    149.4 MiB           1   def sum_x_y():\n",
    "#      4    149.4 MiB      0.0 MiB           1       rng = np.random.RandomState(42)\n",
    "#      5    157.0 MiB      7.6 MiB           1       x = rng.rand(1000000)\n",
    "#      6    164.7 MiB      7.6 MiB           1       y = rng.rand(1000000)\n",
    "#      7    164.7 MiB      0.0 MiB           1       x + y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "458832eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mprun_x_and_y import mask_x_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "70b21e4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%mprun -f mask_x_y\n",
    "mask_x_y()\n",
    "# Filename: D:\\Python_projects\\Jake_VanderPlas\\mprun_x_and_y.py\n",
    "\n",
    "# Line #    Mem usage    Increment  Occurrences   Line Contents\n",
    "# =============================================================\n",
    "#      3    149.4 MiB    149.4 MiB           1   def mask_x_y():\n",
    "#      4    149.4 MiB      0.0 MiB           1       rng = np.random.RandomState(42)\n",
    "#      5    157.0 MiB      7.6 MiB           1       x = rng.rand(1000000)\n",
    "#      6    164.7 MiB      7.6 MiB           1       y = rng.rand(1000000)\n",
    "#      7    163.8 MiB     -0.9 MiB           1       mask = (x > 0.5) & (y < 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "7a4d64b2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.3.4'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "75ce147d",
   "metadata": {},
   "source": [
    "# ВЫВОД: ``pd.__version__ == '1.3.4'`` не имеет указанных недостатков."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "ca8aea44",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numexpr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "1bf53337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.41 ms ± 31.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "mask_numexp = numexpr.evaluate('(x > 0.5) & (y < 0.5)')\n",
    "# против родных 2.07 ms ± 52.5 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "bc57547d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting mprun_x_and_y_msk.py\n"
     ]
    }
   ],
   "source": [
    "%%file mprun_x_and_y_msk.py\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import numexpr\n",
    "def mask_numexp_f():\n",
    "    rng = np.random.RandomState(42)\n",
    "    x = rng.rand(1000000)\n",
    "    y = rng.rand(1000000)\n",
    "    mask_numexp = numexpr.evaluate('(x > 0.5) & (y < 0.5)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "4edca21a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mprun_x_and_y_msk import mask_numexp_f"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "0cd48f98",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "%%mprun -f mask_numexp_f\n",
    "mask_numexp_f()\n",
    "# Filename: D:\\Python_projects\\Jake_VanderPlas\\mprun_x_and_y_msk.py\n",
    "\n",
    "# Line #    Mem usage    Increment  Occurrences   Line Contents\n",
    "# =============================================================\n",
    "#      4    147.8 MiB    147.8 MiB           1   def mask_numexp_f():\n",
    "#      5    147.8 MiB      0.0 MiB           1       rng = np.random.RandomState(42)\n",
    "#      6    155.4 MiB      7.6 MiB           1       x = rng.rand(1000000)\n",
    "#      7    163.0 MiB      7.6 MiB           1       y = rng.rand(1000000)\n",
    "#      8    164.0 MiB      1.0 MiB           1       mask_numexp = numexpr.evaluate('(x > 0.5) & (y < 0.5)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "ba55002b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 147.76 MiB, increment: 0.00 MiB\n"
     ]
    }
   ],
   "source": [
    "%%memit\n",
    "mask_numexp = numexpr.evaluate('(x > 0.5) & (y < 0.5)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95ba5417",
   "metadata": {},
   "outputs": [],
   "source": [
    "# короче не пухнет нихрена. вот."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05e0d7e4",
   "metadata": {},
   "source": [
    "### Использование ф-ии .eval() для эффективных операций"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "7843673a",
   "metadata": {},
   "outputs": [],
   "source": [
    "nrows, ncols = 100000, 100\n",
    "rng = np.random.RandomState(42)\n",
    "df1, df2, df3, df4 = (pd.DataFrame(rng.rand(nrows, ncols))\n",
    "                      for i in range(4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "f606a364",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54.8 ms ± 889 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit df1 + df2 + df3 + df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "00df3093",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26.1 ms ± 895 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%timeit pd.eval('df1 + df2 + df3 + df4')\n",
    "# а тут дает ускорение в 2 раза"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "22c0d106",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 435.32 MiB, increment: 0.28 MiB\n"
     ]
    }
   ],
   "source": [
    "%memit df1 + df2 + df3 + df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "65e31446",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "peak memory: 435.33 MiB, increment: 0.00 MiB\n"
     ]
    }
   ],
   "source": [
    "%memit pd.eval('df1 + df2 + df3 + df4')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf0a45b1",
   "metadata": {},
   "source": [
    "#### Поддерживаемые ф-ией ``pd.eval()`` операции:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "6a5fb624",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1, df2, df3, df4, df5 = (pd.DataFrame(rng.randint(0, 1000, (100, 3)))\n",
    "                           for i in range(5))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2b651d3",
   "metadata": {},
   "source": [
    "###### Арифметические операторы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "56e28712",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = -df1 * df2 / (df3 + df4) - df5\n",
    "result2 = pd.eval('-df1 * df2 / (df3 + df4) - df5')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3d9235c2",
   "metadata": {},
   "source": [
    "###### Операторы сравнения."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "a94c7489",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = (df1 < df2) & (df2 <= df3) & (df3 != df4)\n",
    "result2 = pd.eval('df1 < df2 <= df3 != df4')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ea5c406",
   "metadata": {},
   "source": [
    "###### Побитовые операторы ``&`` и ``|`` , ``and`` , ``or``"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "8f75a790",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = (df1 < 0.5) & (df2 < 0.5) | (df3 < df4)\n",
    "result2 = pd.eval('(df1 < 0.5) & (df2 < 0.5) | (df3 < df4)')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "e0454c57",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result3 = pd.eval('(df1 < 0.5) and (df2 < 0.5) or (df3 < df4)')\n",
    "np.allclose(result1, result3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "250f2a66",
   "metadata": {},
   "source": [
    "######  Атрибуты объектов и индексы."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "1f334196",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = df2.T[0] + df3.iloc[1]\n",
    "result2 = pd.eval('df2.T[0] + df3.iloc[1]')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7baf32f5",
   "metadata": {},
   "source": [
    "###### и другие..."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ae11cde",
   "metadata": {},
   "source": [
    "### Использование метода ``DataFrame.eval()`` для выполнения операций по столбцам\n",
    "Возможность ссылаться на столбцы ``по имени``."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3e9bc2b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.375506</td>\n",
       "      <td>0.406939</td>\n",
       "      <td>0.069938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.069087</td>\n",
       "      <td>0.235615</td>\n",
       "      <td>0.154374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.677945</td>\n",
       "      <td>0.433839</td>\n",
       "      <td>0.652324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264038</td>\n",
       "      <td>0.808055</td>\n",
       "      <td>0.347197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.589161</td>\n",
       "      <td>0.252418</td>\n",
       "      <td>0.557789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          A         B         C\n",
       "0  0.375506  0.406939  0.069938\n",
       "1  0.069087  0.235615  0.154374\n",
       "2  0.677945  0.433839  0.652324\n",
       "3  0.264038  0.808055  0.347197\n",
       "4  0.589161  0.252418  0.557789"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame(rng.rand(1000, 3), columns=['A', 'B', 'C'])\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "c56d3651",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = (df['A'] + df['B']) / (df['C'] - 1)\n",
    "result2 = pd.eval(\"(df.A + df.B) / (df.C - 1)\")\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "9d93e113",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# более локаничная запись\n",
    "result3 = df.eval('(A + B) / (C - 1)')\n",
    "np.allclose(result1, result3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1843157",
   "metadata": {},
   "source": [
    "###### Мы ``обращались с названиями столбцов`` в вычисляемом выражении как с ``переменными``"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96868121",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "d6546bb4",
   "metadata": {},
   "source": [
    "### Присваивание в методе ``df.eval()``\n",
    "``df.eval()`` позволяет выполнять присваивание любому из столбцов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "382c7aa6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.375506</td>\n",
       "      <td>0.406939</td>\n",
       "      <td>0.069938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.069087</td>\n",
       "      <td>0.235615</td>\n",
       "      <td>0.154374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.677945</td>\n",
       "      <td>0.433839</td>\n",
       "      <td>0.652324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264038</td>\n",
       "      <td>0.808055</td>\n",
       "      <td>0.347197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.589161</td>\n",
       "      <td>0.252418</td>\n",
       "      <td>0.557789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.082646</td>\n",
       "      <td>0.036840</td>\n",
       "      <td>0.439733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.008826</td>\n",
       "      <td>0.896578</td>\n",
       "      <td>0.723374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.916424</td>\n",
       "      <td>0.978655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.758995</td>\n",
       "      <td>0.535431</td>\n",
       "      <td>0.347766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.369260</td>\n",
       "      <td>0.753815</td>\n",
       "      <td>0.280455</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            A         B         C\n",
       "0    0.375506  0.406939  0.069938\n",
       "1    0.069087  0.235615  0.154374\n",
       "2    0.677945  0.433839  0.652324\n",
       "3    0.264038  0.808055  0.347197\n",
       "4    0.589161  0.252418  0.557789\n",
       "..        ...       ...       ...\n",
       "995  0.082646  0.036840  0.439733\n",
       "996  0.008826  0.896578  0.723374\n",
       "997  0.907270  0.916424  0.978655\n",
       "998  0.758995  0.535431  0.347766\n",
       "999  0.369260  0.753815  0.280455\n",
       "\n",
       "[1000 rows x 3 columns]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "76130a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.375506</td>\n",
       "      <td>0.406939</td>\n",
       "      <td>0.069938</td>\n",
       "      <td>11.187620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.069087</td>\n",
       "      <td>0.235615</td>\n",
       "      <td>0.154374</td>\n",
       "      <td>1.973796</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.677945</td>\n",
       "      <td>0.433839</td>\n",
       "      <td>0.652324</td>\n",
       "      <td>1.704344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264038</td>\n",
       "      <td>0.808055</td>\n",
       "      <td>0.347197</td>\n",
       "      <td>3.087857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.589161</td>\n",
       "      <td>0.252418</td>\n",
       "      <td>0.557789</td>\n",
       "      <td>1.508776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.082646</td>\n",
       "      <td>0.036840</td>\n",
       "      <td>0.439733</td>\n",
       "      <td>0.271723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.008826</td>\n",
       "      <td>0.896578</td>\n",
       "      <td>0.723374</td>\n",
       "      <td>1.251641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.916424</td>\n",
       "      <td>0.978655</td>\n",
       "      <td>1.863469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.758995</td>\n",
       "      <td>0.535431</td>\n",
       "      <td>0.347766</td>\n",
       "      <td>3.722122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.369260</td>\n",
       "      <td>0.753815</td>\n",
       "      <td>0.280455</td>\n",
       "      <td>4.004473</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            A         B         C          D\n",
       "0    0.375506  0.406939  0.069938  11.187620\n",
       "1    0.069087  0.235615  0.154374   1.973796\n",
       "2    0.677945  0.433839  0.652324   1.704344\n",
       "3    0.264038  0.808055  0.347197   3.087857\n",
       "4    0.589161  0.252418  0.557789   1.508776\n",
       "..        ...       ...       ...        ...\n",
       "995  0.082646  0.036840  0.439733   0.271723\n",
       "996  0.008826  0.896578  0.723374   1.251641\n",
       "997  0.907270  0.916424  0.978655   1.863469\n",
       "998  0.758995  0.535431  0.347766   3.722122\n",
       "999  0.369260  0.753815  0.280455   4.004473\n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# создадим вычисляемый столбец\n",
    "df.eval('D = (A + B) / C', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "8f932a80",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>A</th>\n",
       "      <th>B</th>\n",
       "      <th>C</th>\n",
       "      <th>D</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.375506</td>\n",
       "      <td>0.406939</td>\n",
       "      <td>0.069938</td>\n",
       "      <td>-0.002810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.069087</td>\n",
       "      <td>0.235615</td>\n",
       "      <td>0.154374</td>\n",
       "      <td>-0.084369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.677945</td>\n",
       "      <td>0.433839</td>\n",
       "      <td>0.652324</td>\n",
       "      <td>0.143226</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.264038</td>\n",
       "      <td>0.808055</td>\n",
       "      <td>0.347197</td>\n",
       "      <td>-0.176180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.589161</td>\n",
       "      <td>0.252418</td>\n",
       "      <td>0.557789</td>\n",
       "      <td>0.223189</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>995</th>\n",
       "      <td>0.082646</td>\n",
       "      <td>0.036840</td>\n",
       "      <td>0.439733</td>\n",
       "      <td>0.168577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>996</th>\n",
       "      <td>0.008826</td>\n",
       "      <td>0.896578</td>\n",
       "      <td>0.723374</td>\n",
       "      <td>-0.709271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>997</th>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.916424</td>\n",
       "      <td>0.978655</td>\n",
       "      <td>-0.004912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>998</th>\n",
       "      <td>0.758995</td>\n",
       "      <td>0.535431</td>\n",
       "      <td>0.347766</td>\n",
       "      <td>0.060064</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>999</th>\n",
       "      <td>0.369260</td>\n",
       "      <td>0.753815</td>\n",
       "      <td>0.280455</td>\n",
       "      <td>-0.096031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            A         B         C         D\n",
       "0    0.375506  0.406939  0.069938 -0.002810\n",
       "1    0.069087  0.235615  0.154374 -0.084369\n",
       "2    0.677945  0.433839  0.652324  0.143226\n",
       "3    0.264038  0.808055  0.347197 -0.176180\n",
       "4    0.589161  0.252418  0.557789  0.223189\n",
       "..        ...       ...       ...       ...\n",
       "995  0.082646  0.036840  0.439733  0.168577\n",
       "996  0.008826  0.896578  0.723374 -0.709271\n",
       "997  0.907270  0.916424  0.978655 -0.004912\n",
       "998  0.758995  0.535431  0.347766  0.060064\n",
       "999  0.369260  0.753815  0.280455 -0.096031\n",
       "\n",
       "[1000 rows x 4 columns]"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# модификация столбца\n",
    "df.eval('D = (A - B) / D', inplace=True)\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b15c5f2b",
   "metadata": {},
   "source": [
    "### Локальные переменные в методе ``df.eval()`` (только в методе, ``не`` в ф-ии ``pd.eval()``)\n",
    "Символ ``@``отмечает *имя переменной*, а не *имя столбца*, позволяя тем самым вычислять выражения с использованием 2х пространств имен:\n",
    "- Пространств имен столбцов\n",
    "- Пространств имен объектов Python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "9e357d29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "column_mean = df.mean(1)\n",
    "result1 = df['A'] + column_mean\n",
    "result2 = df.eval('A + @column_mean')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a2573a6",
   "metadata": {},
   "source": [
    "### Метод DataFrame.query()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "f2774121",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result1 = df[(df.A < 0.5) & (df.B < 0.5)]\n",
    "result2 = pd.eval('df[(df.A < 0.5) & (df.B < 0.5)]')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "4b3e25af",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result2 = df.query('A < 0.5 and B < 0.5')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "662d25d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# позволяет использовать 2 пространства имен\n",
    "Cmean = df['C'].mean()\n",
    "result1 = df[(df.A < Cmean) & (df.B < Cmean)]\n",
    "result2 = df.query('A < @Cmean and B < @Cmean')\n",
    "np.allclose(result1, result2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b4358e7f",
   "metadata": {},
   "source": [
    "# Методы ``.eval`` и ``.query`` следует использовать только для очень больших массивов, инече выигрыш не так очевиден. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f953a1d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}