Skip to content

Commit

Permalink
[HOTFIX] import LoggerStorage instead of Logger in a notebook
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlejandroCN7
AlejandroCN7 committed Aug 21, 2024
1 parent d5d30cc commit 918df94
Show file tree
Hide file tree
Showing 2 changed files with 147 additions and 22 deletions.
137 changes: 134 additions & 3 deletions examples/logging_unused_variables.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,83 @@
"Old observation space shape: 17\n",
"Old observation variables: ['month', 'day_of_month', 'hour', 'outdoor_temperature', 'outdoor_humidity', 'wind_speed', 'wind_direction', 'diffuse_solar_radiation', 'direct_solar_radiation', 'htg_setpoint', 'clg_setpoint', 'air_temperature', 'air_humidity', 'people_occupant', 'co2_emission', 'HVAC_electricity_demand_rate', 'total_electricity_HVAC']\n",
"###########################################################################\n",
"\u001b[38;20m[WRAPPER CSVLogger] (INFO) : Wrapper initialized.\u001b[0m\n",
"\u001b[38;20m[WRAPPER CSVLogger] (INFO) : Wrapper initialized.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Failed to detect the name of this notebook, you can set it manually with the WANDB_NOTEBOOK_NAME environment variable to enable code saving.\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.\n",
"\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33malex_ugr\u001b[0m (\u001b[33msail_ugr\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"
]
},
{
"data": {
"text/html": [
"Tracking run with wandb version 0.17.5"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Run data is saved locally in <code>/workspaces/sinergym/examples/wandb/run-20240821_113506-p2d9p07z</code>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Syncing run <strong><a href='https://wandb.ai/sail_ugr/test-porject/runs/p2d9p07z' target=\"_blank\">test-reduction</a></strong> to <a href='https://wandb.ai/sail_ugr/test-porject' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" View project at <a href='https://wandb.ai/sail_ugr/test-porject' target=\"_blank\">https://wandb.ai/sail_ugr/test-porject</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" View run at <a href='https://wandb.ai/sail_ugr/test-porject/runs/p2d9p07z' target=\"_blank\">https://wandb.ai/sail_ugr/test-porject/runs/p2d9p07z</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[38;20m[WRAPPER WandBLogger] (INFO) : Wrapper initialized.\u001b[0m\n",
"\u001b[38;20m[WRAPPER ReduceObservationWrapper] (INFO) : Wrapper initialized.\u001b[0m\n",
"###########################################################################\n",
"Wrapped observation space shape: 14\n",
Expand Down Expand Up @@ -131,11 +207,11 @@
"\u001b[38;20m[WRAPPER NormalizeObservation] (INFO) : Saving normalization calibration data... [5zone-hot-continuous-stochastic-v1]\u001b[0m\n",
"###########################################################################\n",
"Reset observation length: 14\n",
"Removed variables info: {'outdoor_temperature': np.float64(0.009788675668636879), 'outdoor_humidity': np.float64(0.009998816541465044), 'air_temperature': np.float64(0.009987517842103989)}\n",
"Removed variables info: {'outdoor_temperature': np.float64(0.009694040064036152), 'outdoor_humidity': np.float64(0.009998816541465044), 'air_temperature': np.float64(0.00998743390464178)}\n",
"###########################################################################\n",
"###########################################################################\n",
"step observation length: 14\n",
"Removed variables info: {'outdoor_temperature': np.float64(0.9690823475299509), 'outdoor_humidity': np.float64(-0.9745626607331893), 'air_temperature': np.float64(-0.43564383626130754)}\n",
"Removed variables info: {'outdoor_temperature': np.float64(0.994618545162255), 'outdoor_humidity': np.float64(-0.9745626607331893), 'air_temperature': np.float64(-0.4008530311983138)}\n",
"###########################################################################\n",
"Progress: |***************************************************************************************************| 99%\n"
]
Expand Down Expand Up @@ -173,6 +249,61 @@
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"\u001b[38;20m[WRAPPER WandBLogger] (INFO) : Environment closed, dumping summary metrics in WandB Platform.\u001b[0m\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[34m\u001b[1mwandb\u001b[0m: Adding directory to artifact (/workspaces/sinergym/examples/Eplus-env-5zone-hot-continuous-stochastic-v1-res1)... Done. 0.0s\n"
]
},
{
"data": {
"text/html": [
"<style>\n",
" table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n",
" .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n",
" .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n",
" </style>\n",
"<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run history:</h3><br/><table class=\"wandb\"><tr><td>Agent_actions/Cooling_Setpoint_RL</td><td>▂▄▁▇▅█▁▂▃▅▅▅█▄▇▅▇▆▇▁█▃▇▃▇▇▆█▄█▄▄█▇▃</td></tr><tr><td>Agent_actions/Heating_Setpoint_RL</td><td>▃▃▄▃█▂▄▃▃▆▃▂▇▅▁█▁█▅▅▂▇▁█▇▅▅▇▇▃▇▇▄▅▃</td></tr><tr><td>Info/abs_comfort_penalty</td><td>▅██▂█▃██▂█▁█▇█▆█▇██▇██▅█▇███████▂█▇</td></tr><tr><td>Info/abs_energy_penalty</td><td>█████▆██▅█▆▇▁▆█▆▇▇▇▆▄▇▇▇▆▇█▆▆█████▃</td></tr><tr><td>Info/comfort_term</td><td>▅██▂█▃██▂█▁█▇█▆█▇██▇██▅█▇███████▂█▇</td></tr><tr><td>Info/energy_term</td><td>█████▆██▅█▆▇▁▆█▆▇▇▇▆▄▇▇▇▆▇█▆▆█████▃</td></tr><tr><td>Info/total_power_demand</td><td>▁▁▁▁▁▃▁▁▄▁▃▂█▃▁▃▂▂▂▃▅▂▂▂▃▂▁▃▃▁▁▁▁▁▆</td></tr><tr><td>Info/total_temperature_violation</td><td>▄▁▁▇▁▆▁▁▇▁█▁▂▁▃▁▂▁▁▂▁▁▄▁▂▁▁▁▁▁▁▁▇▁▂</td></tr><tr><td>Normalized_observations/HVAC_electricity_demand_rate</td><td>▁▁▁▁▁▃▁▂▄▂▃▂█▃▂▃▂▂▂▃▅▂▂▂▃▂▁▃▃▂▁▁▁▁▆</td></tr><tr><td>Normalized_observations/air_humidity</td><td>▆▆▅█▆▅▅▃▄▁▄▁▃▁▃▁▄▂▇▂▇▂▆▂▄▂▅▃▅▃▄▃▄▄▃</td></tr><tr><td>Normalized_observations/air_temperature</td><td>▆▄▂█▂▇▃▂█▃█▄▅▄▃▅▃▅▄▃▆▃█▄▆▄▂▄▂▃▃▁▇▁▅</td></tr><tr><td>Normalized_observations/clg_setpoint</td><td>▆▁█▆▂▅▅▅▆▇▆▃▂▁▄▅▆▇▅▄▄██▆▄▇▃▃▄▇▂▆█▆▂</td></tr><tr><td>Normalized_observations/co2_emission</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>Normalized_observations/day_of_month</td><td>██▁▄▆▂▄▇▂▅▇▂▅▇▂▅▇▂▅▇▂▅▇▂▅▇▃▅█▃▅█▃▆█</td></tr><tr><td>Normalized_observations/diffuse_solar_radiation</td><td>▄▁▁▄▁▅▁▃▂▁█▁▃▁▂▃▁▆▁▂▂▁▄▁▃▁▁▃▁▂▁▁▁▁▃</td></tr><tr><td>Normalized_observations/direct_solar_radiation</td><td>▆▁▁▁▁▇▁▆▄▁▄▁▇▁▃▆▁▅▁▆▃▁▁▁▇▁▁▅▁█▁▅▁▁▇</td></tr><tr><td>Normalized_observations/hour</td><td>▄▇▂▅▁▄▇▃▆▂▅█▄▇▂▅▁▄▇▃▆▂▅█▄▇▂▅▁▄▇▃▆▂▅</td></tr><tr><td>Normalized_observations/htg_setpoint</td><td>▄▂▂▃▂▁▃▂▄▁▇▂▆▃▆▁▁▁█▇█▅▃▆▆█▂▄▆▄▇▅▃▂▇</td></tr><tr><td>Normalized_observations/month</td><td>▁▁█▂▂▃▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂▂</td></tr><tr><td>Normalized_observations/outdoor_humidity</td><td>▆▆██▆▄▅▂▁▃▂▂▂▂▄▂▃▂█▆▄▅▃▆▂▃▆▂▆▂▄▄▃▅▂</td></tr><tr><td>Normalized_observations/outdoor_temperature</td><td>▄▃▁▄▃▆▃▃█▆▇▅▆▆▄█▅▆▄▄▆▄▆▄▅▅▂▅▂▄▃▁▃▁▃</td></tr><tr><td>Normalized_observations/people_occupant</td><td>█▂▁█▁▇▁▁█▁▇▁█▁▁▁▁▁▁▁█▁▇▁█▁▁▁▁▁▁▁█▁▇</td></tr><tr><td>Normalized_observations/total_electricity_HVAC</td><td>▁▁▁▁▁▃▁▂▄▂▃▂█▄▂▃▂▂▂▃▆▂▂▂▃▂▁▃▂▁▁▁▁▁▆</td></tr><tr><td>Normalized_observations/wind_direction</td><td>▄▇▄▃▄▂▅▇▇▄▆▇▅▆▄▇▃█▃▃▁▃▇▃▃▇▃▅▄▄▃▄█▄▆</td></tr><tr><td>Normalized_observations/wind_speed</td><td>▂▂▄▃▄▅▃▂▅▂▃▄▂▅▂▅▃▃▄▄▂▃█▃█▃▁▅▂▁▄▄▁▁▂</td></tr><tr><td>Observations/HVAC_electricity_demand_rate</td><td>▁▁▁▁▁▃▁▁▄▁▃▂█▃▁▃▂▂▂▃▅▂▂▂▃▂▁▃▃▁▁▁▁▁▆</td></tr><tr><td>Observations/air_humidity</td><td>▇▆▅██▆▇▄▅▁▅▁▃▁▃▁▄▁▇▂█▂▇▂▄▃▅▃▅▃▅▃▄▄▃</td></tr><tr><td>Observations/air_temperature</td><td>▅▄▁▇▁▇▃▁▇▃█▃▅▄▂▅▃▄▃▃▆▃█▄▆▄▂▄▂▃▃▁▇▁▅</td></tr><tr><td>Observations/clg_setpoint</td><td>▆▁█▆▂▅▅▅▆▇▆▃▂▁▄▅▆▇▅▄▄██▆▄▇▃▃▄▇▂▆█▆▂</td></tr><tr><td>Observations/co2_emission</td><td>▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁</td></tr><tr><td>Observations/day_of_month</td><td>▃▆▁▃▆▂▄▇▂▄▇▂▅▇▂▅▇▂▅▇▂▅▇▂▅▇▃▅█▃▅█▃▆█</td></tr><tr><td>Observations/diffuse_solar_radiation</td><td>▃▁▁▃▁▄▁▂▂▁█▁▃▁▁▃▁▆▁▂▂▁▄▁▄▁▁▃▁▂▁▁▁▁▃</td></tr><tr><td>Observations/direct_solar_radiation</td><td>▆▁▁▁▁▆▁▅▄▁▄▁▇▁▃▆▁▆▁▆▃▁▁▁▇▁▁▅▁█▁▅▁▁▇</td></tr><tr><td>Observations/hour</td><td>▄▇▂▅▁▄▇▃▆▂▅█▄▇▂▅▁▄▇▃▆▂▅█▄▇▂▅▁▄▇▃▆▂▅</td></tr><tr><td>Observations/htg_setpoint</td><td>▄▂▂▃▂▁▃▂▄▁▇▂▆▃▆▁▁▁█▇█▅▃▆▆█▂▄▆▄▇▅▃▂▇</td></tr><tr><td>Observations/month</td><td>▁▁▂▂▂▂▂▂▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▇▇▇▇▇▇███</td></tr><tr><td>Observations/outdoor_humidity</td><td>▇▇███▅▇▃▁▄▂▂▁▂▃▁▂▁▇▅▃▄▃▅▂▂▅▂▅▂▃▄▂▅▂</td></tr><tr><td>Observations/outdoor_temperature</td><td>▃▂▁▃▂▄▂▂▆▅▆▅▆▆▄█▅▇▅▅▇▅▇▅▆▆▃▅▂▅▃▁▃▁▄</td></tr><tr><td>Observations/people_occupant</td><td>█▂▁█▁▇▁▁█▁▇▁█▁▁▁▁▁▁▁█▁▇▁█▁▁▁▁▁▁▁█▁▇</td></tr><tr><td>Observations/total_electricity_HVAC</td><td>▁▁▁▁▁▂▁▁▃▁▃▁█▄▁▃▂▂▂▃▆▁▂▂▃▂▁▃▂▁▁▁▁▁▇</td></tr><tr><td>Observations/wind_direction</td><td>▃▆▃▃▃▁▅▆▆▄▆▆▅▆▄▇▃█▃▃▁▃▇▄▃▇▄▅▄▄▃▄█▄▆</td></tr><tr><td>Observations/wind_speed</td><td>▂▂▃▃▃▄▂▂▄▂▃▄▂▅▂▅▃▃▃▃▂▃█▃█▃▁▅▂▁▄▃▁▁▂</td></tr><tr><td>Reward/reward</td><td>▅██▂█▃██▂█▁█▆█▆█▇██▇▇█▅█▇██▇████▂█▆</td></tr><tr><td>Simulation_actions/Cooling_Setpoint_RL</td><td>▂▄▁▇▅█▁▂▃▅▅▅█▄▇▅▇▆▇▁█▃▇▃▇▇▆█▄█▄▄█▇▃</td></tr><tr><td>Simulation_actions/Heating_Setpoint_RL</td><td>▃▃▄▃█▂▄▃▃▆▃▂▇▅▁█▁█▅▅▂▇▁█▇▅▅▇▇▃▇▇▄▅▃</td></tr><tr><td>episode_summaries/comfort_violation_time(%)</td><td>▁</td></tr><tr><td>episode_summaries/cumulative_power_demand</td><td>▁</td></tr><tr><td>episode_summaries/episode_num</td><td>▁</td></tr><tr><td>episode_summaries/length(timesteps)</td><td>▁</td></tr><tr><td>episode_summaries/mean_abs_comfort_penalty</td><td>▁</td></tr><tr><td>episode_summaries/mean_abs_energy_penalty</td><td>▁</td></tr><tr><td>episode_summaries/mean_power_demand</td><td>▁</td></tr><tr><td>episode_summaries/mean_reward</td><td>▁</td></tr><tr><td>episode_summaries/mean_reward_comfort_term</td><td>▁</td></tr><tr><td>episode_summaries/mean_reward_energy_term</td><td>▁</td></tr><tr><td>episode_summaries/mean_temperature_violation</td><td>▁</td></tr><tr><td>episode_summaries/std_abs_comfort_penalty</td><td>▁</td></tr><tr><td>episode_summaries/std_abs_energy_penalty</td><td>▁</td></tr><tr><td>episode_summaries/std_power_demand</td><td>▁</td></tr><tr><td>episode_summaries/std_reward</td><td>▁</td></tr><tr><td>episode_summaries/std_reward_comfort_term</td><td>▁</td></tr><tr><td>episode_summaries/std_reward_energy_term</td><td>▁</td></tr><tr><td>episode_summaries/std_temperature_violation</td><td>▁</td></tr><tr><td>episode_summaries/time_elapsed(hours)</td><td>▁</td></tr></table><br/></div><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>Agent_actions/Cooling_Setpoint_RL</td><td>-0.4778</td></tr><tr><td>Agent_actions/Heating_Setpoint_RL</td><td>-0.25449</td></tr><tr><td>Info/abs_comfort_penalty</td><td>-0.8399</td></tr><tr><td>Info/abs_energy_penalty</td><td>-5115.03133</td></tr><tr><td>Info/comfort_term</td><td>-0.41995</td></tr><tr><td>Info/energy_term</td><td>-0.25575</td></tr><tr><td>Info/total_power_demand</td><td>5115.03133</td></tr><tr><td>Info/total_temperature_violation</td><td>0.8399</td></tr><tr><td>Normalized_observations/HVAC_electricity_demand_rate</td><td>1.32263</td></tr><tr><td>Normalized_observations/air_humidity</td><td>-0.97818</td></tr><tr><td>Normalized_observations/air_temperature</td><td>0.40859</td></tr><tr><td>Normalized_observations/clg_setpoint</td><td>-1.16473</td></tr><tr><td>Normalized_observations/co2_emission</td><td>0</td></tr><tr><td>Normalized_observations/day_of_month</td><td>1.74104</td></tr><tr><td>Normalized_observations/diffuse_solar_radiation</td><td>0.29647</td></tr><tr><td>Normalized_observations/direct_solar_radiation</td><td>1.75984</td></tr><tr><td>Normalized_observations/hour</td><td>0.21787</td></tr><tr><td>Normalized_observations/htg_setpoint</td><td>0.98495</td></tr><tr><td>Normalized_observations/month</td><td>1.59085</td></tr><tr><td>Normalized_observations/outdoor_humidity</td><td>-0.96832</td></tr><tr><td>Normalized_observations/outdoor_temperature</td><td>-0.27763</td></tr><tr><td>Normalized_observations/people_occupant</td><td>1.21883</td></tr><tr><td>Normalized_observations/total_electricity_HVAC</td><td>1.4688</td></tr><tr><td>Normalized_observations/wind_direction</td><td>0.61943</td></tr><tr><td>Normalized_observations/wind_speed</td><td>-1.03508</td></tr><tr><td>Observations/HVAC_electricity_demand_rate</td><td>5115.03125</td></tr><tr><td>Observations/air_humidity</td><td>16.681</td></tr><tr><td>Observations/air_temperature</td><td>24.3399</td></tr><tr><td>Observations/clg_setpoint</td><td>24.3404</td></tr><tr><td>Observations/co2_emission</td><td>0</td></tr><tr><td>Observations/day_of_month</td><td>31</td></tr><tr><td>Observations/diffuse_solar_radiation</td><td>86.5</td></tr><tr><td>Observations/direct_solar_radiation</td><td>898</td></tr><tr><td>Observations/hour</td><td>13</td></tr><tr><td>Observations/htg_setpoint</td><td>20.82611</td></tr><tr><td>Observations/month</td><td>12</td></tr><tr><td>Observations/outdoor_humidity</td><td>12</td></tr><tr><td>Observations/outdoor_temperature</td><td>19.19061</td></tr><tr><td>Observations/people_occupant</td><td>16</td></tr><tr><td>Observations/total_electricity_HVAC</td><td>5357560.5</td></tr><tr><td>Observations/wind_direction</td><td>240</td></tr><tr><td>Observations/wind_speed</td><td>1.5</td></tr><tr><td>Reward/reward</td><td>-0.6757</td></tr><tr><td>Simulation_actions/Cooling_Setpoint_RL</td><td>25.01243</td></tr><tr><td>Simulation_actions/Heating_Setpoint_RL</td><td>16.19347</td></tr><tr><td>episode_summaries/comfort_violation_time(%)</td><td>47.14041</td></tr><tr><td>episode_summaries/cumulative_power_demand</td><td>64705432.84029</td></tr><tr><td>episode_summaries/episode_num</td><td>1</td></tr><tr><td>episode_summaries/length(timesteps)</td><td>35041</td></tr><tr><td>episode_summaries/mean_abs_comfort_penalty</td><td>-0.73257</td></tr><tr><td>episode_summaries/mean_abs_energy_penalty</td><td>-1846.61623</td></tr><tr><td>episode_summaries/mean_power_demand</td><td>1846.61623</td></tr><tr><td>episode_summaries/mean_reward</td><td>-0.45862</td></tr><tr><td>episode_summaries/mean_reward_comfort_term</td><td>-0.36629</td></tr><tr><td>episode_summaries/mean_reward_energy_term</td><td>-0.09233</td></tr><tr><td>episode_summaries/mean_temperature_violation</td><td>0.73257</td></tr><tr><td>episode_summaries/std_abs_comfort_penalty</td><td>1.20454</td></tr><tr><td>episode_summaries/std_abs_energy_penalty</td><td>2470.25417</td></tr><tr><td>episode_summaries/std_power_demand</td><td>2470.25417</td></tr><tr><td>episode_summaries/std_reward</td><td>0.60726</td></tr><tr><td>episode_summaries/std_reward_comfort_term</td><td>0.60227</td></tr><tr><td>episode_summaries/std_reward_energy_term</td><td>0.12351</td></tr><tr><td>episode_summaries/std_temperature_violation</td><td>1.20454</td></tr><tr><td>episode_summaries/time_elapsed(hours)</td><td>8760.125</td></tr></table><br/></div></div>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
" View run <strong style=\"color:#cdcd00\">test-reduction</strong> at: <a href='https://wandb.ai/sail_ugr/test-porject/runs/p2d9p07z' target=\"_blank\">https://wandb.ai/sail_ugr/test-porject/runs/p2d9p07z</a><br/> View project at: <a href='https://wandb.ai/sail_ugr/test-porject' target=\"_blank\">https://wandb.ai/sail_ugr/test-porject</a><br/>Synced 5 W&B file(s), 0 media file(s), 21 artifact file(s) and 0 other file(s)"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"Find logs at: <code>./wandb/run-20240821_113506-p2d9p07z/logs</code>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
Expand Down
Loading

0 comments on commit 918df94

Please sign in to comment.