Skip to content

Commit

Permalink
Add grouping function for s3-path based search
Browse files Browse the repository at this point in the history
  • Loading branch information
stijnvanhoey committed Sep 18, 2023
1 parent 3c4234d commit a375d7a
Showing 1 changed file with 26 additions and 1 deletion.
27 changes: 26 additions & 1 deletion src/vptstools/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ def from_s3fs_enlisting(cls, h5_file_path):
return cls(
h5_file_path.split("/")[1],
*cls.parse_file_name(str(h5_file_path)),
h5_file_path.split("/")[1],
h5_file_path.split("/")[2],
)

@staticmethod
Expand Down Expand Up @@ -199,6 +199,31 @@ def extract_daily_group_from_inventory(file_path):
path_info.day,
)

def extract_daily_group_from_path(file_path):
"""Extract file name components to define a group
The coverage file counts the number of files available
per group (e.g. daily files per radar). This function is passed
to the Pandas ``groupby`` to translate the file path to a
countable set (e.g. source, radar-code, year month and day for
daily files per radar).
Parameters
----------
file_path : str
File path of the ODIM HDF5 file. Only the file name is taken
into account and a folder-path is ignored.
"""
path_info = OdimFilePath.from_s3fs_enlisting(file_path)
return (
path_info.source,
path_info.file_type,
path_info.radar_code,
path_info.year,
path_info.month,
path_info.day,
)


def _last_modified_from_inventory(df, modified_days_ago="2day"):
"""Filter manifest files on last modified
Expand Down

0 comments on commit a375d7a

Please sign in to comment.