extract_CMEMS_ioc_cleanup.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copernicusmarine as cm\n",
    "import pandas as pd\n",
    "import xarray as xr\n",
    "import hvplot.xarray\n",
    "import hvplot.pandas\n",
    "import numpy as np\n",
    "import holoviews as hv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cm.login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ioc_cleanup = pd.read_csv('ioc_cleanup_2023.csv', index_col=0)\n",
    "ioc_cleanup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# subset\n",
    "ioc_re = ioc_cleanup[ioc_cleanup.ioc_code == 'boma']\n",
    "ioc_re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.makedirs('data/nc/', exist_ok=True)\n",
    "os.makedirs('data/parquet/', exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for it, item in ioc_cleanup.iterrows():\n",
    "    lonmax = np.ceil(item.longitude) + 1\n",
    "    lonmin = np.floor(item.longitude)\n",
    "    latmax = np.ceil(item.latitude)\n",
    "    latmin = np.floor(item.latitude)\n",
    "    cm.subset(\n",
    "        dataset_id=\"cmems_mod_glo_phy_anfc_0.083deg_PT1H-m\",\n",
    "        variables=[\"zos\"],\n",
    "        minimum_longitude=lonmin,\n",
    "        maximum_longitude=lonmax,\n",
    "        minimum_latitude=latmin,\n",
    "        maximum_latitude=latmax,\n",
    "        start_datetime=\"2022-01-01T00:00:00\",\n",
    "        end_datetime=\"2023-12-31T23:00:00\",\n",
    "        output_filename = f\"data/nc/{item.ioc_code}.nc\", \n",
    "        force_download=True\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_closest_coordinates(x, y, ds):\n",
    "    xx, yy = np.meshgrid(ds.longitude.values, ds.latitude.values)\n",
    "    xall, yall = xx.ravel(), yy.ravel()\n",
    "    data = ds.zos[-1, 0, :, :].values.ravel()\n",
    "    non_nan_mask = ~np.isnan(data)\n",
    "    xall_non_nan = xall[non_nan_mask]\n",
    "    yall_non_nan = yall[non_nan_mask]\n",
    "    distances_squared = (xall_non_nan - x)**2 + (yall_non_nan - y)**2\n",
    "    # print(distances_squared)\n",
    "    closest_idx = np.argmin(distances_squared)\n",
    "    lo_, la_ = xall_non_nan[closest_idx], yall_non_nan[closest_idx]\n",
    "    ilon = np.argmin(abs(ds.longitude.values - lo_))\n",
    "    ilat = np.argmin(abs(ds.latitude.values - la_))\n",
    "    return ilon, ilat\n",
    "\n",
    "def extract_parquet(stations: pd.DataFrame):\n",
    "    for it, item in stations.iterrows():\n",
    "        print(item.ioc_code)\n",
    "        ds = xr.open_dataset(f'data/nc/{item.ioc_code}.nc')\n",
    "        ilon, ilat = get_closest_coordinates(item.longitude, item.latitude, ds)\n",
    "        df = pd.DataFrame({'zos':ds.zos[:,0,ilat, ilon].values}, ds.time.values)\n",
    "        df.to_parquet(f'data/parquet/{item.ioc_code}.parquet')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_parquet(ioc_cleanup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if we have data\n",
    "for it, item in ioc_cleanup.iterrows():\n",
    "    print(it, item.ioc_code)\n",
    "    df = pd.read_parquet(f'./data/parquet/{item.ioc_code}.parquet')\n",
    "    df.hvplot()\n",
    "    if it > 200: \n",
    "        break"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "cmems",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}