diff --git a/pipelines/precipitation_model/impa/flows.py b/pipelines/precipitation_model/impa/flows.py index e422c7b..42853b6 100644 --- a/pipelines/precipitation_model/impa/flows.py +++ b/pipelines/precipitation_model/impa/flows.py @@ -66,15 +66,17 @@ "start_datetime", default=None, required=False, - #description="Datetime in YYYY-MM-dd HH:mm:ss format, UTC timezone", + # description="Datetime in YYYY-MM-dd HH:mm:ss format, UTC timezone", ) num_workers = Parameter( "num_workers", default=8, required=False, - #description="Number of workers to use for parallel processing", + # description="Number of workers to use for parallel processing", ) - cuda = Parameter("cuda", default=False, required=False) #, description="Use CUDA for prediction" + cuda = Parameter( + "cuda", default=False, required=False + ) # , description="Use CUDA for prediction" # Parameters for saving data on GCP materialize_after_dump = Parameter("materialize_after_dump", default=False, required=False) diff --git a/pipelines/precipitation_model/impa/src/data/HDFDataset2.py b/pipelines/precipitation_model/impa/src/data/HDFDataset2.py index 2a3dcb0..7f7b38e 100644 --- a/pipelines/precipitation_model/impa/src/data/HDFDataset2.py +++ b/pipelines/precipitation_model/impa/src/data/HDFDataset2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from pathlib import Path import h5py @@ -5,8 +6,16 @@ import torch from torch.utils import data -from pipelines.precipitation_model.impa.src.utils.dataframe_utils import N_AFTER, N_BEFORE, fetch_future_datetimes, fetch_reversed_past_datetimes -from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok, print_warning +from pipelines.precipitation_model.impa.src.utils.dataframe_utils import ( + N_AFTER, + N_BEFORE, + fetch_future_datetimes, + fetch_reversed_past_datetimes, +) +from pipelines.precipitation_model.impa.src.utils.general_utils import ( + print_ok, + print_warning, +) from pipelines.precipitation_model.impa.src.utils.hdf_utils import get_dataset_keys MIN_WEIGHT = 100 @@ -56,16 +65,27 @@ def __init__( else: suffix = "" - if len(set(["latent_field", "motion_field", "intensities"]).intersection(set(get_item_output))) > 0: + if ( + len( + set(["latent_field", "motion_field", "intensities"]).intersection( + set(get_item_output) + ) + ) + > 0 + ): if autoencoder_hash is None: autoencoder_hash = "001178e117f50cf17817f336b86a809f" parent_path = Path(filepath).parents[0] if "train" in filepath.stem: - self.latent_field_filepath = Path(f"{parent_path}/train_latent{suffix}_{autoencoder_hash}.hdf") + self.latent_field_filepath = Path( + f"{parent_path}/train_latent{suffix}_{autoencoder_hash}.hdf" + ) self.motion_field_filepath = Path(f"{parent_path}/train_motion{suffix}.hdf") self.intensities_filepath = Path(f"{parent_path}/train_intensities{suffix}.hdf") elif "val" in filepath.stem: - self.latent_field_filepath = Path(f"{parent_path}/val_latent{suffix}_{autoencoder_hash}.hdf") + self.latent_field_filepath = Path( + f"{parent_path}/val_latent{suffix}_{autoencoder_hash}.hdf" + ) self.motion_field_filepath = Path(f"{parent_path}/val_motion{suffix}.hdf") self.intensities_filepath = Path(f"{parent_path}/val_intensities{suffix}.hdf") else: @@ -267,9 +287,12 @@ def __len__(self): return len(self.keys) def get_sample_weights(self, overwrite_if_exists=False, verbose=True): - assert self.filepath.stem == "train", "Sample weights can only be calculated for the train dataset." + assert ( + self.filepath.stem == "train" + ), "Sample weights can only be calculated for the train dataset." weights_filepath = ( - self.filepath.parents[0] / f"train_sample_weights2-n_before={self.n_before}-n_after={self.n_after}.npy" + self.filepath.parents[0] + / f"train_sample_weights2-n_before={self.n_before}-n_after={self.n_after}.npy" ) if not overwrite_if_exists: if weights_filepath.is_file(): diff --git a/pipelines/precipitation_model/impa/src/data/HDFDatasetLocations.py b/pipelines/precipitation_model/impa/src/data/HDFDatasetLocations.py index b106717..0f08a7b 100644 --- a/pipelines/precipitation_model/impa/src/data/HDFDatasetLocations.py +++ b/pipelines/precipitation_model/impa/src/data/HDFDatasetLocations.py @@ -1,7 +1,13 @@ +# -*- coding: utf-8 -*- import numpy as np -from pipelines.precipitation_model.impa.src.data.HDFDatasetMultiple import HDFDatasetMultiple -from pipelines.precipitation_model.impa.src.utils.dataframe_utils import N_AFTER, N_BEFORE +from pipelines.precipitation_model.impa.src.data.HDFDatasetMultiple import ( + HDFDatasetMultiple, +) +from pipelines.precipitation_model.impa.src.utils.dataframe_utils import ( + N_AFTER, + N_BEFORE, +) elevation_file_small = "data/processed/elevations_data/elevation_{location}-res=2km-256x256.npy" elevation_file_large = "data/processed/elevations_data/elevation_{location}-res=4km-256x256.npy" diff --git a/pipelines/precipitation_model/impa/src/data/HDFDatasetMerged.py b/pipelines/precipitation_model/impa/src/data/HDFDatasetMerged.py index b8ecf51..abe6d34 100644 --- a/pipelines/precipitation_model/impa/src/data/HDFDatasetMerged.py +++ b/pipelines/precipitation_model/impa/src/data/HDFDatasetMerged.py @@ -1,7 +1,13 @@ +# -*- coding: utf-8 -*- import numpy as np -from pipelines.precipitation_model.impa.src.data.HDFDatasetMultiple import HDFDatasetMultiple -from pipelines.precipitation_model.impa.src.utils.dataframe_utils import N_AFTER, N_BEFORE +from pipelines.precipitation_model.impa.src.data.HDFDatasetMultiple import ( + HDFDatasetMultiple, +) +from pipelines.precipitation_model.impa.src.utils.dataframe_utils import ( + N_AFTER, + N_BEFORE, +) # For Rio de Janeiro only diff --git a/pipelines/precipitation_model/impa/src/data/HDFDatasetMultiple.py b/pipelines/precipitation_model/impa/src/data/HDFDatasetMultiple.py index 8cca6f4..a2b422f 100644 --- a/pipelines/precipitation_model/impa/src/data/HDFDatasetMultiple.py +++ b/pipelines/precipitation_model/impa/src/data/HDFDatasetMultiple.py @@ -8,7 +8,10 @@ import torch from torch.utils import data -from pipelines.precipitation_model.impa.src.utils.dataframe_utils import fetch_future_datetimes, fetch_reversed_past_datetimes +from pipelines.precipitation_model.impa.src.utils.dataframe_utils import ( + fetch_future_datetimes, + fetch_reversed_past_datetimes, +) from pipelines.precipitation_model.impa.src.utils.hdf_utils import get_dataset_keys @@ -76,7 +79,9 @@ def __init__( # print_warning("File not found in /dev/shm, using original path.") new_dataframe_filepaths_array[i] = str(filepath) - self.dataframe_shm_filepaths_array = new_dataframe_filepaths_array.reshape(dataframe_filepaths_array.shape) + self.dataframe_shm_filepaths_array = new_dataframe_filepaths_array.reshape( + dataframe_filepaths_array.shape + ) self.dataframe_filepaths_array = dataframe_filepaths_array self.n_before_array = n_before_array self.n_after_array = n_after_array @@ -126,7 +131,9 @@ def _load_keys(self): if j == 0: new_keys = set(hdf["split_info"]["split_datetime_keys"]) else: - new_keys = set(hdf["split_info"]["split_datetime_keys"]).intersection(new_keys) + new_keys = set(hdf["split_info"]["split_datetime_keys"]).intersection( + new_keys + ) else: if j == 0: new_keys = set(get_dataset_keys(hdf)) @@ -140,17 +147,30 @@ def _load_keys(self): self.past_keys = np.vstack( [ self.past_keys, - np.array(fetch_reversed_past_datetimes(new_keys, self.n_before_array.max(), timestep)), + np.array( + fetch_reversed_past_datetimes( + new_keys, self.n_before_array.max(), timestep + ) + ), ] ) except ValueError: - self.past_keys = np.array(fetch_reversed_past_datetimes(new_keys, self.n_before_array.max(), timestep)) + self.past_keys = np.array( + fetch_reversed_past_datetimes(new_keys, self.n_before_array.max(), timestep) + ) try: self.future_keys = np.vstack( - [self.future_keys, np.array(fetch_future_datetimes(new_keys, self.n_after_array.max(), timestep))] + [ + self.future_keys, + np.array( + fetch_future_datetimes(new_keys, self.n_after_array.max(), timestep) + ), + ] ) except ValueError: - self.future_keys = np.array(fetch_future_datetimes(new_keys, self.n_after_array.max(), timestep)) + self.future_keys = np.array( + fetch_future_datetimes(new_keys, self.n_after_array.max(), timestep) + ) try: self.ds_indices.append(len(new_keys) + self.ds_indices[-1]) except IndexError: @@ -162,11 +182,21 @@ def _get_hdf_index(self, index): return i def __getitem__(self, index): - X = torch.ones((self.ni, self.nj, (self.n_before_array * self.n_before_resolution_array).sum())) * torch.inf + X = ( + torch.ones( + (self.ni, self.nj, (self.n_before_array * self.n_before_resolution_array).sum()) + ) + * torch.inf + ) if self.leadtime_conditioning: pass else: - Y = torch.ones((self.ni, self.nj, (self.n_after_array * self.n_after_resolution_array).sum())) * torch.inf + Y = ( + torch.ones( + (self.ni, self.nj, (self.n_after_array * self.n_after_resolution_array).sum()) + ) + * torch.inf + ) if self.leadtime_conditioning: leadtime_index = index % self.n_after_array[0] @@ -186,17 +216,24 @@ def __getitem__(self, index): ) try: X[:, :, tensor_ind : tensor_ind + n_resolution] = torch.as_tensor( - np.array(hdf[self.past_keys[index][i]]).reshape((self.ni, self.nj, n_resolution)) + np.array(hdf[self.past_keys[index][i]]).reshape( + (self.ni, self.nj, n_resolution) + ) ) except KeyError: - X[:, :, tensor_ind : tensor_ind + n_resolution] = torch.ones((self.ni, self.nj, 1)) * np.nan + X[:, :, tensor_ind : tensor_ind + n_resolution] = ( + torch.ones((self.ni, self.nj, 1)) * np.nan + ) if self.leadtime_conditioning and j == 0: try: - Y = torch.as_tensor(np.array(hdf[self.future_keys[index][leadtime_index]])).reshape( - (self.ni, self.nj, self.n_after_resolution_array[0]) - ) + Y = torch.as_tensor( + np.array(hdf[self.future_keys[index][leadtime_index]]) + ).reshape((self.ni, self.nj, self.n_after_resolution_array[0])) except KeyError: - Y = torch.ones((self.ni, self.nj, self.n_after_resolution_array[0])) * np.nan + Y = ( + torch.ones((self.ni, self.nj, self.n_after_resolution_array[0])) + * np.nan + ) else: # for i, key in enumerate(self.future_keys[index]): n_resolution = self.n_after_resolution_array[j] @@ -204,7 +241,9 @@ def __getitem__(self, index): if j == 0: cumsum = 0 else: - cumsum = np.cumsum(self.n_after_array * self.n_after_resolution_array)[j - 1] + cumsum = np.cumsum(self.n_after_array * self.n_after_resolution_array)[ + j - 1 + ] tensor_ind = cumsum + i * n_resolution try: Y[:, :, tensor_ind : tensor_ind + n_resolution] = torch.as_tensor( @@ -212,7 +251,9 @@ def __getitem__(self, index): ).reshape((self.ni, self.nj, n_resolution)) except KeyError: - Y[:, :, tensor_ind : tensor_ind + n_resolution] = torch.ones((self.ni, self.nj, 1)) * np.nan + Y[:, :, tensor_ind : tensor_ind + n_resolution] = ( + torch.ones((self.ni, self.nj, 1)) * np.nan + ) if self.x_transform: X = self.x_transform(X) if self.y_transform: @@ -225,7 +266,9 @@ def __getitem__(self, index): day = int(date[6:8]) hour = int(date[9:11]) minute = int(date[11:13]) - date = torch.tensor([month / 12, day / 31, hour / 24, minute / 60], dtype=torch.float32).reshape((1, 1, -1)) + date = torch.tensor( + [month / 12, day / 31, hour / 24, minute / 60], dtype=torch.float32 + ).reshape((1, 1, -1)) date_tensor = date.expand((self.ni, self.nj, -1)) metadata_tensor = torch.cat( @@ -259,7 +302,9 @@ def get_sample_weights(self, min_sum=0): pre_weights_size = len(pre_weights) pre_weights = np.append(pre_weights, np.array([pre_weights.min()])) inds = np.load(inds_filepath).reshape(-1, 1) - deltas = np.arange(-self.n_before_array[0] + 1, self.n_after_array[0] + 1, dtype=int).reshape(1, -1) + deltas = np.arange( + -self.n_before_array[0] + 1, self.n_after_array[0] + 1, dtype=int + ).reshape(1, -1) slices = inds + deltas slices[np.logical_or(slices < 0, slices >= pre_weights_size)] = pre_weights_size summed_weights = pre_weights[slices].sum(axis=1).reshape(-1) diff --git a/pipelines/precipitation_model/impa/src/data/PredHDFDataset2.py b/pipelines/precipitation_model/impa/src/data/PredHDFDataset2.py index 0833bf6..b72f730 100644 --- a/pipelines/precipitation_model/impa/src/data/PredHDFDataset2.py +++ b/pipelines/precipitation_model/impa/src/data/PredHDFDataset2.py @@ -1,3 +1,4 @@ +# -*- coding: utf-8 -*- from pathlib import Path import h5py @@ -12,7 +13,10 @@ fetch_pred_keys, fetch_reversed_past_datetimes, ) -from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok, print_warning +from pipelines.precipitation_model.impa.src.utils.general_utils import ( + print_ok, + print_warning, +) from pipelines.precipitation_model.impa.src.utils.hdf_utils import get_dataset_keys MIN_WEIGHT = 100 @@ -59,7 +63,9 @@ def __init__( f"models/{model}/predictions/{dataset}/predict_{split}-ckpt={ckpt_file.replace('.ckpt','')}.hdf" ) else: - self.predict_filepath = Path(f"models/{model}/predictions/{dataset}/predict_{split}.hdf") + self.predict_filepath = Path( + f"models/{model}/predictions/{dataset}/predict_{split}.hdf" + ) self.n_predictions = n_predictions self.n_before = n_before self.n_after = n_after @@ -69,16 +75,27 @@ def __init__( self._load_keys() - if len(set(["latent_field", "motion_field", "intensities"]).intersection(set(get_item_output))) > 0: + if ( + len( + set(["latent_field", "motion_field", "intensities"]).intersection( + set(get_item_output) + ) + ) + > 0 + ): if autoencoder_hash is None: autoencoder_hash = "001178e117f50cf17817f336b86a809f" parent_path = filepath.parents[0] if "train" in filepath.stem: - self.latent_field_filepath = Path(f"{parent_path}/train_latent_{autoencoder_hash}.hdf") + self.latent_field_filepath = Path( + f"{parent_path}/train_latent_{autoencoder_hash}.hdf" + ) self.motion_field_filepath = Path(f"{parent_path}/train_motion.hdf") self.intensities_filepath = Path(f"{parent_path}/train_intensities.hdf") elif "val" in filepath.stem: - self.latent_field_filepath = Path(f"{parent_path}/val_latent_{autoencoder_hash}.hdf") + self.latent_field_filepath = Path( + f"{parent_path}/val_latent_{autoencoder_hash}.hdf" + ) self.motion_field_filepath = Path(f"{parent_path}/val_motion.hdf") self.intensities_filepath = Path(f"{parent_path}/val_intensities.hdf") else: @@ -287,9 +304,12 @@ def __len__(self): return len(self.keys) def get_sample_weights(self, overwrite_if_exists=False, verbose=True): - assert self.filepath.stem == "train", "Sample weights can only be calculated for the train dataset." + assert ( + self.filepath.stem == "train" + ), "Sample weights can only be calculated for the train dataset." weights_filepath = ( - self.filepath.parents[0] / f"train_sample_weights2-n_before={self.n_before}-n_after={self.n_after}.npy" + self.filepath.parents[0] + / f"train_sample_weights2-n_before={self.n_before}-n_after={self.n_after}.npy" ) if not overwrite_if_exists: if weights_filepath.is_file(): diff --git a/pipelines/precipitation_model/impa/src/data/PredHDFDatasetLocations.py b/pipelines/precipitation_model/impa/src/data/PredHDFDatasetLocations.py index 034776c..c962ba3 100644 --- a/pipelines/precipitation_model/impa/src/data/PredHDFDatasetLocations.py +++ b/pipelines/precipitation_model/impa/src/data/PredHDFDatasetLocations.py @@ -121,22 +121,39 @@ def _load_keys(self): self.keys = np.append(self.keys, new_keys) try: self.past_keys = np.vstack( - [self.past_keys, np.array(fetch_reversed_past_datetimes(new_keys, self.n_before, timestep))] + [ + self.past_keys, + np.array( + fetch_reversed_past_datetimes(new_keys, self.n_before, timestep) + ), + ] ) except ValueError: - self.past_keys = np.array(fetch_reversed_past_datetimes(new_keys, self.n_before, timestep)) + self.past_keys = np.array( + fetch_reversed_past_datetimes(new_keys, self.n_before, timestep) + ) try: self.future_keys = np.vstack( - [self.future_keys, np.array(fetch_future_datetimes(new_keys, self.n_after, timestep))] + [ + self.future_keys, + np.array(fetch_future_datetimes(new_keys, self.n_after, timestep)), + ] ) except ValueError: - self.future_keys = np.array(fetch_future_datetimes(new_keys, self.n_after, timestep)) + self.future_keys = np.array( + fetch_future_datetimes(new_keys, self.n_after, timestep) + ) try: self.pred_keys = np.vstack( - [self.pred_keys, np.array(fetch_pred_keys(new_keys, self.n_predictions, timestep))] + [ + self.pred_keys, + np.array(fetch_pred_keys(new_keys, self.n_predictions, timestep)), + ] ) except ValueError: - self.pred_keys = np.array(fetch_pred_keys(new_keys, self.n_predictions, timestep)) + self.pred_keys = np.array( + fetch_pred_keys(new_keys, self.n_predictions, timestep) + ) try: self.ds_indices.append(len(new_keys) + self.ds_indices[-1]) except IndexError: @@ -163,11 +180,13 @@ def __getitem__(self, index): for i, key in enumerate(self.past_keys[index]): tensor_ind = 2 * self.n_before - 2 * i - 2 try: - X[:, :, tensor_ind : tensor_ind + 2] = torch.as_tensor(np.array(hdf[key])).reshape( - (self.ni, self.nj, 2) - ) + X[:, :, tensor_ind : tensor_ind + 2] = torch.as_tensor( + np.array(hdf[key]) + ).reshape((self.ni, self.nj, 2)) except KeyError: - X[:, :, tensor_ind : tensor_ind + 2] = torch.ones((self.ni, self.nj, 2)) * np.nan + X[:, :, tensor_ind : tensor_ind + 2] = ( + torch.ones((self.ni, self.nj, 2)) * np.nan + ) if self.leadtime_conditioning: try: Y = torch.as_tensor(np.array(hdf[self.future_keys[index][leadtime_index]])) @@ -176,7 +195,9 @@ def __getitem__(self, index): else: for i, key in enumerate(self.future_keys[index]): try: - Y[:, :, 2 * i : 2 * i + 2] = torch.as_tensor(np.array(hdf[key])).reshape((self.ni, self.nj, 2)) + Y[:, :, 2 * i : 2 * i + 2] = torch.as_tensor(np.array(hdf[key])).reshape( + (self.ni, self.nj, 2) + ) except KeyError: Y[:, :, 2 * i : 2 * i + 2] = torch.ones((self.ni, self.nj, 2)) * np.nan with h5py.File(self.predict_filepaths[hdf_index], "r") as pred_hdf: @@ -198,7 +219,9 @@ def __getitem__(self, index): day = int(date[6:8]) hour = int(date[9:11]) minute = int(date[11:13]) - date = torch.tensor([month / 12, day / 31, hour / 24, minute / 60], dtype=torch.float32).reshape((1, 1, -1)) + date = torch.tensor( + [month / 12, day / 31, hour / 24, minute / 60], dtype=torch.float32 + ).reshape((1, 1, -1)) date_tensor = date.expand((self.ni, self.nj, -1)) metadata_tensor = torch.cat( diff --git a/pipelines/precipitation_model/impa/src/data/process/SatelliteData.py b/pipelines/precipitation_model/impa/src/data/process/SatelliteData.py index e73a25b..ec0541e 100644 --- a/pipelines/precipitation_model/impa/src/data/process/SatelliteData.py +++ b/pipelines/precipitation_model/impa/src/data/process/SatelliteData.py @@ -48,7 +48,9 @@ def load_data(cls, input_filepath: Path | str, value: str | None = None): def _load_previous_day(self): self.data = pd.concat( [ - pd.read_feather(f"{self.folder}/{self.product}/{self.day - timedelta(days=1)}.feather"), + pd.read_feather( + f"{self.folder}/{self.product}/{self.day - timedelta(days=1)}.feather" + ), self.data, ] ) @@ -57,7 +59,9 @@ def _load_cloud_height(self) -> pd.DataFrame: return pd.concat( [ pd.read_feather(f"{self.folder}/ABI-L2-ACHAF/{self.day}.feather"), - pd.read_feather(f"{self.folder}/ABI-L2-ACHAF/{self.day - timedelta(days=1)}.feather"), + pd.read_feather( + f"{self.folder}/ABI-L2-ACHAF/{self.day - timedelta(days=1)}.feather" + ), ] ) @@ -84,7 +88,9 @@ def correct_parallax(self): points = np.stack((height_lons, height_lats)).T h_interp = interpolate.griddata(points, heights, (lons, lats), method="linear") - new_lons, new_lats = get_parallax_corrected_lonlats(SAT_LON, SAT_LAT, SAT_ALT, lons, lats, h_interp) + new_lons, new_lats = get_parallax_corrected_lonlats( + SAT_LON, SAT_LAT, SAT_ALT, lons, lats, h_interp + ) if updated_lats is None: updated_lats = new_lats @@ -98,7 +104,9 @@ def correct_parallax(self): def interp_at_grid(self, band: str, timestamp: datetime, target_grid: NDArray): self._load_previous_day() - assert (timestamp >= self.data["creation"]).any(), "Timestamp passed precedes all timestamps in the data" + assert ( + timestamp >= self.data["creation"] + ).any(), "Timestamp passed precedes all timestamps in the data" closest_timestamp = self.data.loc[self.data["creation"] <= timestamp, "creation"].max() df = self.data[self.data["creation"] == closest_timestamp] column = f"{self.value}_{band}" if self.product == "ABI-L2-MCMIPF" else self.value @@ -113,7 +121,10 @@ def interp_at_grid(self, band: str, timestamp: datetime, target_grid: NDArray): points = np.stack((x, y)).T shape = target_grid.shape[:2] interp_values = interpolate.griddata( - points, values, (target_grid[:, :, 1].flatten(), target_grid[:, :, 0].flatten()), method="linear" + points, + values, + (target_grid[:, :, 1].flatten(), target_grid[:, :, 0].flatten()), + method="linear", ).reshape(shape) return interp_values diff --git a/pipelines/precipitation_model/impa/src/data/process/build_dataframe.py b/pipelines/precipitation_model/impa/src/data/process/build_dataframe.py index 5beb243..16863be 100644 --- a/pipelines/precipitation_model/impa/src/data/process/build_dataframe.py +++ b/pipelines/precipitation_model/impa/src/data/process/build_dataframe.py @@ -12,7 +12,9 @@ import numpy as np from tqdm import tqdm -from pipelines.precipitation_model.impa.src.data.process.SatelliteData import SatelliteData +from pipelines.precipitation_model.impa.src.data.process.SatelliteData import ( + SatelliteData, +) from pipelines.precipitation_model.impa.src.utils.general_utils import print_warning HEAVY_RAIN_TRAIN_LOG_MEAN = 0.15839338 @@ -23,7 +25,14 @@ def task_dt( - dt, ni: int, nj: int, sat_folder: str, grid_small: np.ndarray, grid_large: np.ndarray, value: str, band: str + dt, + ni: int, + nj: int, + sat_folder: str, + grid_small: np.ndarray, + grid_large: np.ndarray, + value: str, + band: str, ): date_str = dt.strftime("%Y-%m-%d") datetime_key = (dt).strftime("%Y%m%d/%H%M") @@ -74,7 +83,9 @@ def build_dataframe( pass # Build dataframe for last 6 hours of data - dt_rounded = datetime.datetime(dt.year, dt.month, dt.day, dt.hour, dt.minute - dt.minute % timestep) + dt_rounded = datetime.datetime( + dt.year, dt.month, dt.day, dt.hour, dt.minute - dt.minute % timestep + ) for i in range((60 * 6) // timestep): datetimes.append(dt_rounded - i * datetime.timedelta(minutes=timestep)) full_datetimes = list(datetimes) @@ -93,7 +104,8 @@ def build_dataframe( output_filepath.unlink() else: print_warning( - f"Warning: {output_filepath} already exists. Call with -o option to overwrite.", verbose=verbose + f"Warning: {output_filepath} already exists. Call with -o option to overwrite.", + verbose=verbose, ) exit(0) @@ -116,7 +128,8 @@ def build_dataframe( # these are the datetimes which we want to predict from relevant_datetimes = [datetimes[18], datetimes[0]] what.create_dataset( - "datetime_keys", data=np.asarray([dt.strftime("%Y%m%d/%H%M") for dt in relevant_datetimes], dtype="S") + "datetime_keys", + data=np.asarray([dt.strftime("%Y%m%d/%H%M") for dt in relevant_datetimes], dtype="S"), ) step = num_workers @@ -148,7 +161,9 @@ def build_dataframe( if __name__ == "__main__": parser = argparse.ArgumentParser() - parser.add_argument("--verbose", "-v", action="store_true", help="If true, prints more information.") + parser.add_argument( + "--verbose", "-v", action="store_true", help="If true, prints more information." + ) parser.add_argument( "--overwrite", "-o", @@ -169,7 +184,13 @@ def build_dataframe( default=1, help="Number of processes for parallelization.", ) - parser.add_argument("--location", "-loc", default="rio_de_janeiro", type=str, help="Location to build dataframe.") + parser.add_argument( + "--location", + "-loc", + default="rio_de_janeiro", + type=str, + help="Location to build dataframe.", + ) parser.add_argument("--timestep", "-ts", default=10, type=int, help="Timestep in minutes.") parser.add_argument("--value", "-val", default="RRQPE", type=str, help="Satellite value.") parser.add_argument("--band", "-b", default="RRQPE", type=str, help="Satellite band.") diff --git a/pipelines/precipitation_model/impa/src/eval/metrics/calc-metrics.py b/pipelines/precipitation_model/impa/src/eval/metrics/calc-metrics.py index 4dba326..9961d1b 100644 --- a/pipelines/precipitation_model/impa/src/eval/metrics/calc-metrics.py +++ b/pipelines/precipitation_model/impa/src/eval/metrics/calc-metrics.py @@ -9,6 +9,7 @@ import numpy as np import pandas as pd import tqdm + from pipelines.precipitation_model.impa.src.eval.metrics.metrics import metrics_dict from pipelines.precipitation_model.impa.src.utils.general_utils import print_warning from pipelines.precipitation_model.impa.src.utils.hdf_utils import get_dataset_keys diff --git a/pipelines/precipitation_model/impa/src/eval/update-real_time.py b/pipelines/precipitation_model/impa/src/eval/update-real_time.py index 54d37d4..8f377c6 100644 --- a/pipelines/precipitation_model/impa/src/eval/update-real_time.py +++ b/pipelines/precipitation_model/impa/src/eval/update-real_time.py @@ -6,8 +6,13 @@ import boto3 from botocore import UNSIGNED from botocore.config import Config -from pipelines.precipitation_model.impa.src.data.process.build_dataframe import build_dataframe -from pipelines.precipitation_model.impa.src.data.process.process_satellite import process_satellite + +from pipelines.precipitation_model.impa.src.data.process.build_dataframe import ( + build_dataframe, +) +from pipelines.precipitation_model.impa.src.data.process.process_satellite import ( + process_satellite, +) from pipelines.precipitation_model.impa.src.eval.predict_real_time import predict BUCKET_NAME = "noaa-goes16" diff --git a/pipelines/precipitation_model/impa/src/eval/update-real_time_.py b/pipelines/precipitation_model/impa/src/eval/update-real_time_.py index d92b504..24a5f76 100644 --- a/pipelines/precipitation_model/impa/src/eval/update-real_time_.py +++ b/pipelines/precipitation_model/impa/src/eval/update-real_time_.py @@ -10,8 +10,13 @@ from botocore import UNSIGNED from botocore.config import Config from joblib import Parallel, delayed -from pipelines.precipitation_model.impa.src.data.process.build_dataframe import build_dataframe -from pipelines.precipitation_model.impa.src.data.process.process_satellite import process_satellite + +from pipelines.precipitation_model.impa.src.data.process.build_dataframe import ( + build_dataframe, +) +from pipelines.precipitation_model.impa.src.data.process.process_satellite import ( + process_satellite, +) from pipelines.precipitation_model.impa.src.eval.predict_real_time import predict # from itertools import product diff --git a/pipelines/precipitation_model/impa/src/eval/viz/plot-real_time.py b/pipelines/precipitation_model/impa/src/eval/viz/plot-real_time.py index 9079a18..b09c5e8 100644 --- a/pipelines/precipitation_model/impa/src/eval/viz/plot-real_time.py +++ b/pipelines/precipitation_model/impa/src/eval/viz/plot-real_time.py @@ -10,6 +10,7 @@ import numpy as np import pandas as pd import tqdm + from pipelines.precipitation_model.impa.src.eval.metrics.metrics import metrics_dict from pipelines.precipitation_model.impa.src.utils.eval_utils import get_img from pipelines.precipitation_model.impa.src.utils.general_utils import print_warning diff --git a/pipelines/precipitation_model/impa/src/models/Evolution_Network/evolution_network.py b/pipelines/precipitation_model/impa/src/models/Evolution_Network/evolution_network.py index e0b5f70..29911ea 100644 --- a/pipelines/precipitation_model/impa/src/models/Evolution_Network/evolution_network.py +++ b/pipelines/precipitation_model/impa/src/models/Evolution_Network/evolution_network.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- import torch import torch.nn as nn -from pipelines.precipitation_model.impa.src.models.Evolution_Network.model_parts import DoubleConv, Down, OutConv, Up + +from pipelines.precipitation_model.impa.src.models.Evolution_Network.model_parts import ( + DoubleConv, + Down, + OutConv, + Up, +) class Evolution_Encoder_Decoder(nn.Module): diff --git a/pipelines/precipitation_model/impa/src/models/Evolution_Network/lightning.py b/pipelines/precipitation_model/impa/src/models/Evolution_Network/lightning.py index 4956ebe..cf56d4a 100644 --- a/pipelines/precipitation_model/impa/src/models/Evolution_Network/lightning.py +++ b/pipelines/precipitation_model/impa/src/models/Evolution_Network/lightning.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- import torch -from pipelines.precipitation_model.impa.src.models.Evolution_Network.evolution_network import Evolution_Encoder_Decoder -from pipelines.precipitation_model.impa.src.models.Evolution_Network.utils import make_grid, warp + +from pipelines.precipitation_model.impa.src.models.Evolution_Network.evolution_network import ( + Evolution_Encoder_Decoder, +) +from pipelines.precipitation_model.impa.src.models.Evolution_Network.utils import ( + make_grid, + warp, +) from pipelines.precipitation_model.impa.src.models.lightning_module import LModule diff --git a/pipelines/precipitation_model/impa/src/models/Metnet3/lightning.py b/pipelines/precipitation_model/impa/src/models/Metnet3/lightning.py index 342ad12..07d736c 100644 --- a/pipelines/precipitation_model/impa/src/models/Metnet3/lightning.py +++ b/pipelines/precipitation_model/impa/src/models/Metnet3/lightning.py @@ -3,6 +3,8 @@ import torch import torch.nn as nn +from torch.nn import Sequential + from pipelines.precipitation_model.impa.src.models.lightning_module import LModule from pipelines.precipitation_model.impa.src.models.Metnet3.Max_Vit.Max_Vit import MaxViT from pipelines.precipitation_model.impa.src.models.Metnet3.metnet3_pytorch import ( @@ -12,7 +14,6 @@ ResnetBlocks, Upsample2x, ) -from torch.nn import Sequential class model(LModule): diff --git a/pipelines/precipitation_model/impa/src/models/NowcastNet/lightning.py b/pipelines/precipitation_model/impa/src/models/NowcastNet/lightning.py index 4145c4d..f377ad8 100644 --- a/pipelines/precipitation_model/impa/src/models/NowcastNet/lightning.py +++ b/pipelines/precipitation_model/impa/src/models/NowcastNet/lightning.py @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- import torch -from pipelines.precipitation_model.impa.src.models.gan.gan_unet.model import NowcasnetGenerator, TemporalDiscriminator + +from pipelines.precipitation_model.impa.src.models.gan.gan_unet.model import ( + NowcasnetGenerator, + TemporalDiscriminator, +) from pipelines.precipitation_model.impa.src.models.lightning_module import LModule diff --git a/pipelines/precipitation_model/impa/src/models/UNET/lightning.py b/pipelines/precipitation_model/impa/src/models/UNET/lightning.py index 8eeb335..de28661 100644 --- a/pipelines/precipitation_model/impa/src/models/UNET/lightning.py +++ b/pipelines/precipitation_model/impa/src/models/UNET/lightning.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- import torch + from pipelines.precipitation_model.impa.src.models.lightning_module import LModule -from pipelines.precipitation_model.impa.src.models.UNET.unet_parts import DoubleConv, Down, OutConv, Up +from pipelines.precipitation_model.impa.src.models.UNET.unet_parts import ( + DoubleConv, + Down, + OutConv, + Up, +) class model(LModule): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/attention.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/attention.py index aa854a6..b695b10 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/attention.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/attention.py @@ -4,11 +4,12 @@ import torch import torch.nn.functional as F -from torch import einsum, nn from einops import rearrange, repeat +from torch import einsum, nn -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.utils import checkpoint - +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.utils import ( + checkpoint, +) def exists(val): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoenc_v2.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoenc_v2.py index a3a958f..d52393a 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoenc_v2.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoenc_v2.py @@ -2,11 +2,17 @@ import numpy as np import pytorch_lightning as pl import torch + from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.distributions import ( DiagonalGaussianDistribution, ) -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.loss import LPIPSWithDiscriminator -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.modules import Decoder, Encoder +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.loss import ( + LPIPSWithDiscriminator, +) +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.modules import ( + Decoder, + Encoder, +) class AutoencoderKL(pl.LightningModule): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/loss.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/loss.py index f7ac26b..09d4a74 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/loss.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/loss.py @@ -3,11 +3,12 @@ import torch import torch.nn.functional as F +from torch import nn + from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoencoder.discriminator import ( NLayerDiscriminator, weights_init, ) -from torch import nn def hinge_d_loss(logits_real, logits_fake): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/modules.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/modules.py index 891c041..ad12e94 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/modules.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/autoencoder/modules.py @@ -8,7 +8,9 @@ import torch.nn as nn # from ldm.util import instantiate_from_config -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.attention import LinearAttention +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.attention import ( + LinearAttention, +) class Encoder(nn.Module): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/ddim.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/ddim.py index c73923c..a2b4b94 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/ddim.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/ddim.py @@ -5,12 +5,13 @@ import numpy as np import torch +from tqdm import tqdm + from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.utils import ( make_ddim_sampling_parameters, make_ddim_timesteps, noise_like, ) -from tqdm import tqdm class DDIMSampler(object): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/lightning_v2.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/lightning_v2.py index 33892ef..95c9e08 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/lightning_v2.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/lightning_v2.py @@ -6,17 +6,26 @@ import torch.nn.functional as F from einops import rearrange from pytorch_lightning import LightningModule -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoenc_v2 import AutoencoderKL +from tqdm import tqdm + +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.autoenc_v2 import ( + AutoencoderKL, +) # from pipelines.precipitation_model.impa.src.models.context_LDM_concat.autoencoder.autoenc_old import AutoencoderKL -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.ddim import DDIMSampler +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.ddim import ( + DDIMSampler, +) from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.model import ( get_named_beta_schedule, linear_beta_schedule, ) -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.openaimodel import UNetModel -from pipelines.precipitation_model.impa.src.utils.data_utils import data_modification_options -from tqdm import tqdm +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.openaimodel import ( + UNetModel, +) +from pipelines.precipitation_model.impa.src.utils.data_utils import ( + data_modification_options, +) def extract(a, t, x_shape): diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/openaimodel.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/openaimodel.py index ef9b3a4..577de2b 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/openaimodel.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/openaimodel.py @@ -8,7 +8,10 @@ import torch as th import torch.nn as nn import torch.nn.functional as F -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.attention import SpatialTransformer + +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.attention import ( + SpatialTransformer, +) from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.utils import ( avg_pool_nd, checkpoint, diff --git a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/predict.py b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/predict.py index 9fc14c9..ef6f900 100644 --- a/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/predict.py +++ b/pipelines/precipitation_model/impa/src/models/context_LDM_concat_new/predict.py @@ -5,8 +5,13 @@ import pytorch_lightning as pl import torch from torch.utils.data import DataLoader -from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import HDFDatasetLocations -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.lightning_v2 import Diffusion_Model + +from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import ( + HDFDatasetLocations, +) +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.lightning_v2 import ( + Diffusion_Model, +) from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok from pipelines.precipitation_model.impa.src.utils.hdf_utils import array_to_pred_hdf diff --git a/pipelines/precipitation_model/impa/src/models/gan/gan_unet/model.py b/pipelines/precipitation_model/impa/src/models/gan/gan_unet/model.py index eabde3b..07c5238 100644 --- a/pipelines/precipitation_model/impa/src/models/gan/gan_unet/model.py +++ b/pipelines/precipitation_model/impa/src/models/gan/gan_unet/model.py @@ -3,13 +3,26 @@ import torch import torch.nn as nn -from pipelines.precipitation_model.impa.src.models.gan.gan_unet.unet_parts import DoubleConv, Down, OutConv, S, Up, Up2 -from pipelines.precipitation_model.impa.src.models.gan.parts_gan import AvgPool, L2_Block, L3_Block, Noise_Projector +from torch.nn.utils import spectral_norm + +from pipelines.precipitation_model.impa.src.models.gan.gan_unet.unet_parts import ( + DoubleConv, + Down, + OutConv, + S, + Up, + Up2, +) +from pipelines.precipitation_model.impa.src.models.gan.parts_gan import ( + AvgPool, + L2_Block, + L3_Block, + Noise_Projector, +) from pipelines.precipitation_model.impa.src.models.nowcasting.layers.generation.generative_network import ( Generative_Decoder, Generative_Encoder, ) -from torch.nn.utils import spectral_norm ni = 192 # size of image ndf = 32 # size of discriminator feature map diff --git a/pipelines/precipitation_model/impa/src/models/lightning_module.py b/pipelines/precipitation_model/impa/src/models/lightning_module.py index 0b21c98..df54a8e 100644 --- a/pipelines/precipitation_model/impa/src/models/lightning_module.py +++ b/pipelines/precipitation_model/impa/src/models/lightning_module.py @@ -3,7 +3,10 @@ import torch from einops import rearrange from pytorch_lightning import LightningModule -from pipelines.precipitation_model.impa.src.utils.data_utils import data_modification_options + +from pipelines.precipitation_model.impa.src.utils.data_utils import ( + data_modification_options, +) # define the LightningModule diff --git a/pipelines/precipitation_model/impa/src/models/mamba/lightning.py b/pipelines/precipitation_model/impa/src/models/mamba/lightning.py index 6f2fdac..deabd9c 100644 --- a/pipelines/precipitation_model/impa/src/models/mamba/lightning.py +++ b/pipelines/precipitation_model/impa/src/models/mamba/lightning.py @@ -3,8 +3,11 @@ import torch from einops import rearrange from pytorch_lightning import LightningModule + from pipelines.precipitation_model.impa.src.models.mamba.vmamba import VSSM -from pipelines.precipitation_model.impa.src.utils.data_utils import data_modification_options +from pipelines.precipitation_model.impa.src.utils.data_utils import ( + data_modification_options, +) class Vmamba_lightning(LightningModule): diff --git a/pipelines/precipitation_model/impa/src/models/mamba/predict.py b/pipelines/precipitation_model/impa/src/models/mamba/predict.py index 5abb79d..c501735 100644 --- a/pipelines/precipitation_model/impa/src/models/mamba/predict.py +++ b/pipelines/precipitation_model/impa/src/models/mamba/predict.py @@ -4,11 +4,16 @@ import pytorch_lightning as pl import torch -from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import HDFDatasetLocations -from pipelines.precipitation_model.impa.src.models.mamba.lightning import Vmamba_lightning +from torch.utils.data import DataLoader + +from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import ( + HDFDatasetLocations, +) +from pipelines.precipitation_model.impa.src.models.mamba.lightning import ( + Vmamba_lightning, +) from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok from pipelines.precipitation_model.impa.src.utils.hdf_utils import array_to_pred_hdf -from torch.utils.data import DataLoader MEAN = 0.08 STD = 0.39 diff --git a/pipelines/precipitation_model/impa/src/models/nowcasting/layers/generation/generative_network.py b/pipelines/precipitation_model/impa/src/models/nowcasting/layers/generation/generative_network.py index 6d9443c..7f527f9 100644 --- a/pipelines/precipitation_model/impa/src/models/nowcasting/layers/generation/generative_network.py +++ b/pipelines/precipitation_model/impa/src/models/nowcasting/layers/generation/generative_network.py @@ -3,8 +3,11 @@ import torch.nn as nn import torch.nn.functional as F + from pipelines.precipitation_model.impa.src.models.nowcasting.layers.evolution.module import * -from pipelines.precipitation_model.impa.src.models.nowcasting.layers.generation.module import GenBlock +from pipelines.precipitation_model.impa.src.models.nowcasting.layers.generation.module import ( + GenBlock, +) class Generative_Encoder(nn.Module): diff --git a/pipelines/precipitation_model/impa/src/models/predict.py b/pipelines/precipitation_model/impa/src/models/predict.py index 7e07a18..33ef557 100644 --- a/pipelines/precipitation_model/impa/src/models/predict.py +++ b/pipelines/precipitation_model/impa/src/models/predict.py @@ -7,13 +7,21 @@ import pytorch_lightning as pl import torch from einops import rearrange -from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import HDFDatasetLocations -from pipelines.precipitation_model.impa.src.data.PredHDFDatasetLocations import PredHDFDatasetLocations +from torch.utils.data import DataLoader + +from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import ( + HDFDatasetLocations, +) +from pipelines.precipitation_model.impa.src.data.PredHDFDatasetLocations import ( + PredHDFDatasetLocations, +) from pipelines.precipitation_model.impa.src.utils.data_utils import sat_dataframe from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok from pipelines.precipitation_model.impa.src.utils.hdf_utils import array_to_pred_hdf -from pipelines.precipitation_model.impa.src.utils.models_utils import get_ds, options_pretrained -from torch.utils.data import DataLoader +from pipelines.precipitation_model.impa.src.utils.models_utils import ( + get_ds, + options_pretrained, +) MEAN_LOG_SAT = 0.08 STD_LOG_SAT = 0.39 diff --git a/pipelines/precipitation_model/impa/src/models/pysteps_LK/predict.py b/pipelines/precipitation_model/impa/src/models/pysteps_LK/predict.py index 1b03482..4205a39 100644 --- a/pipelines/precipitation_model/impa/src/models/pysteps_LK/predict.py +++ b/pipelines/precipitation_model/impa/src/models/pysteps_LK/predict.py @@ -7,7 +7,10 @@ from joblib import Parallel, delayed from pysteps import motion, nowcasts from pysteps.utils import conversion, transformation -from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import HDFDatasetLocations + +from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import ( + HDFDatasetLocations, +) from pipelines.precipitation_model.impa.src.utils.general_utils import print_ok from pipelines.precipitation_model.impa.src.utils.hdf_utils import array_to_pred_hdf diff --git a/pipelines/precipitation_model/impa/src/utils/eval_utils.py b/pipelines/precipitation_model/impa/src/utils/eval_utils.py index 9be85a8..38ca89d 100644 --- a/pipelines/precipitation_model/impa/src/utils/eval_utils.py +++ b/pipelines/precipitation_model/impa/src/utils/eval_utils.py @@ -8,11 +8,20 @@ import pandas as pd import plotly.graph_objects as go from PIL import Image + from pipelines.precipitation_model.impa.src.eval.metrics.metrics import metrics_dict -from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.predict import main as latent_diffusion_predict -from pipelines.precipitation_model.impa.src.models.mamba.predict import main as mamba_predict -from pipelines.precipitation_model.impa.src.models.predict import main as general_predict -from pipelines.precipitation_model.impa.src.models.pysteps_LK.predict import main as pysteps_predict +from pipelines.precipitation_model.impa.src.models.context_LDM_concat_new.predict import ( + main as latent_diffusion_predict, +) +from pipelines.precipitation_model.impa.src.models.mamba.predict import ( + main as mamba_predict, +) +from pipelines.precipitation_model.impa.src.models.predict import ( + main as general_predict, +) +from pipelines.precipitation_model.impa.src.models.pysteps_LK.predict import ( + main as pysteps_predict, +) MAP_CENTER = {"lat": -22.914550816555533, "lon": -43.502443050594596} ZOOM = 8 diff --git a/pipelines/precipitation_model/impa/src/utils/models_utils.py b/pipelines/precipitation_model/impa/src/utils/models_utils.py index 2fa09ec..ac86fa7 100644 --- a/pipelines/precipitation_model/impa/src/utils/models_utils.py +++ b/pipelines/precipitation_model/impa/src/utils/models_utils.py @@ -1,9 +1,15 @@ # -*- coding: utf-8 -*- from pipelines.precipitation_model.impa.src.data.HDFDataset2 import HDFDataset2 -from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import HDFDatasetLocations -from pipelines.precipitation_model.impa.src.data.HDFDatasetMerged import HDFDatasetMerged +from pipelines.precipitation_model.impa.src.data.HDFDatasetLocations import ( + HDFDatasetLocations, +) +from pipelines.precipitation_model.impa.src.data.HDFDatasetMerged import ( + HDFDatasetMerged, +) from pipelines.precipitation_model.impa.src.data.PredHDFDataset2 import PredHDFDataset2 -from pipelines.precipitation_model.impa.src.data.PredHDFDatasetLocations import PredHDFDatasetLocations +from pipelines.precipitation_model.impa.src.data.PredHDFDatasetLocations import ( + PredHDFDatasetLocations, +) extension_dicts = { "context_LDM_VAE": ".pt", diff --git a/pipelines/precipitation_model/impa/tasks.py b/pipelines/precipitation_model/impa/tasks.py index 4f7c8fe..80971ba 100644 --- a/pipelines/precipitation_model/impa/tasks.py +++ b/pipelines/precipitation_model/impa/tasks.py @@ -11,10 +11,14 @@ from botocore.config import Config from prefect import task from prefeitura_rio.pipelines_utils.logging import log -from pipelines.precipitation_model.impa.src.data.process.build_dataframe import build_dataframe -from pipelines.precipitation_model.impa.src.data.process.process_satellite import process_satellite -from pipelines.precipitation_model.impa.src.eval.predict_real_time import predict +from pipelines.precipitation_model.impa.src.data.process.build_dataframe import ( + build_dataframe, +) +from pipelines.precipitation_model.impa.src.data.process.process_satellite import ( + process_satellite, +) +from pipelines.precipitation_model.impa.src.eval.predict_real_time import predict from pipelines.precipitation_model.impa.utils import download_file_from_s3 @@ -85,7 +89,7 @@ def process_data(year, day_of_year, num_workers, dt): num_workers (int): The number of workers to use for parallel processing. dt (datetime.datetime): The datetime object representing the date to process. - This function logs the processing activity, processes satellite data for specified + This function logs the processing activity, processes satellite data for specified products using `process_satellite`, and then builds a dataframe with `build_dataframe`. """ log("Processing satellite data...")