From 48c09825969cef2b4ddd11324615a3ee599fd5c1 Mon Sep 17 00:00:00 2001 From: Jessy Barrette <30420025+JessyBarrette@users.noreply.github.com> Date: Wed, 29 May 2024 13:13:48 -0400 Subject: [PATCH 1/6] change logger handling --- harvester/cde_harvester/__main__.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/harvester/cde_harvester/__main__.py b/harvester/cde_harvester/__main__.py index 562370b8..0cfd92c9 100644 --- a/harvester/cde_harvester/__main__.py +++ b/harvester/cde_harvester/__main__.py @@ -24,6 +24,8 @@ load_dotenv() logging.getLogger("urllib3").setLevel(logging.WARNING) +logger = logging.getLogger() + sentry_sdk.init( dsn=os.environ.get("SENTRY_DSN"), integrations=[ @@ -38,15 +40,18 @@ def setup_logging(log_time, log_level): # setup logging - logging.basicConfig( - level=logging.getLevelName(log_level.upper()), - format=( - "%(asctime)s - %(name)s : %(message)s" - if log_time - else "%(name)s : %(message)s" - ), + logger.setLevel(logging.getLevelName(log_level.upper())) + + # Define the stream log format and level + c_handler = logging.StreamHandler() + c_handler.setLevel(logging.getLevelName(log_level.upper())) + c_format = logging.Formatter( + "%(asctime)s - %(name)s : %(message)s" + if log_time + else "%(name)s : %(message)s" ) - logger = logging.getLogger() + c_handler.setFormatter(c_format) + logger.addHandler(c_handler) return logger From 79195423e7ed85ba693233b80a6b8bee56edae6a Mon Sep 17 00:00:00 2001 From: Jessy Barrette <30420025+JessyBarrette@users.noreply.github.com> Date: Wed, 29 May 2024 14:52:46 -0400 Subject: [PATCH 2/6] clear handlers to remove duplicated handlers and add level for format --- harvester/cde_harvester/__main__.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/harvester/cde_harvester/__main__.py b/harvester/cde_harvester/__main__.py index 0cfd92c9..010a7f69 100644 --- a/harvester/cde_harvester/__main__.py +++ b/harvester/cde_harvester/__main__.py @@ -41,14 +41,13 @@ def setup_logging(log_time, log_level): # setup logging logger.setLevel(logging.getLevelName(log_level.upper())) - + logger.handlers.clear() # Define the stream log format and level c_handler = logging.StreamHandler() c_handler.setLevel(logging.getLevelName(log_level.upper())) c_format = logging.Formatter( - "%(asctime)s - %(name)s : %(message)s" - if log_time - else "%(name)s : %(message)s" + ("%(asctime)s - " if log_time else "") + + "%(levelname)-8s - %(name)s : %(message)s" ) c_handler.setFormatter(c_format) logger.addHandler(c_handler) From bca8dcd3daa7b2a0fc3d520ce893b3612ad4450d Mon Sep 17 00:00:00 2001 From: Jessy Barrette <30420025+JessyBarrette@users.noreply.github.com> Date: Mon, 3 Jun 2024 14:46:34 -0700 Subject: [PATCH 3/6] default platform to unknown and lower L06 platform names --- harvester/cde_harvester/dataset.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/harvester/cde_harvester/dataset.py b/harvester/cde_harvester/dataset.py index 2e3afa1a..ede3dedd 100644 --- a/harvester/cde_harvester/dataset.py +++ b/harvester/cde_harvester/dataset.py @@ -253,7 +253,7 @@ def get_eovs(self): return eovs def get_platform_code(self): - platform = self.globals["platform"] + platform = self.globals.get("platform") platform_vocabulary = self.globals.get("platform_vocabulary") if not (platform and platform_vocabulary): @@ -278,13 +278,16 @@ def get_platform_code(self): subset=["l06_label"] ) - if platform in list(platforms_nerc_ioos["l06_label"]): + if platform.lower() in list(platforms_nerc_ioos["l06_label"]): return platforms_nerc_ioos_no_duplicates.query( - f"l06_label=='{platform}'" + f"l06_label=='{platform.lower()}'" )["category"].item() else: self.logger.error("Found unsupported L06 platform: %s", platform) + self.logger.warning("Found platform='%s' without known vocabulary='%s', setting to 'unkown'", platform, platform_vocabulary) + return "unknown" + def get_metadata(self): "get all the global and variable metadata for a dataset" @@ -354,8 +357,7 @@ def get_metadata(self): filter(None, set([globals_dict.get(x) for x in organization_fields])) ) - if self.globals.get("platform"): - self.platform = self.get_platform_code() + self.platform = self.get_platform_code() def get_logger(self): logger = logging.getLogger(f"{self.erddap_server.domain} - {self.id}") From 1dcdba3ce89463a8d0ccab5ad113f68cc44ce033 Mon Sep 17 00:00:00 2001 From: Jessy Barrette <30420025+JessyBarrette@users.noreply.github.com> Date: Mon, 3 Jun 2024 15:22:55 -0700 Subject: [PATCH 4/6] fix typo and linter --- harvester/cde_harvester/dataset.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/harvester/cde_harvester/dataset.py b/harvester/cde_harvester/dataset.py index ede3dedd..af536399 100644 --- a/harvester/cde_harvester/dataset.py +++ b/harvester/cde_harvester/dataset.py @@ -285,9 +285,13 @@ def get_platform_code(self): else: self.logger.error("Found unsupported L06 platform: %s", platform) - self.logger.warning("Found platform='%s' without known vocabulary='%s', setting to 'unkown'", platform, platform_vocabulary) + self.logger.warning( + "Found platform='%s' without known vocabulary='%s', setting to 'unknown'", + platform, + platform_vocabulary, + ) return "unknown" - + def get_metadata(self): "get all the global and variable metadata for a dataset" From b19a73be96c4b09ffcb3a1c658044bf20160f415 Mon Sep 17 00:00:00 2001 From: Jessy Barrette <30420025+JessyBarrette@users.noreply.github.com> Date: Tue, 13 Aug 2024 16:41:15 -0400 Subject: [PATCH 5/6] Create CODEOWNER --- CODEOWNER | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 CODEOWNER diff --git a/CODEOWNER b/CODEOWNER new file mode 100644 index 00000000..1dcaef98 --- /dev/null +++ b/CODEOWNER @@ -0,0 +1,3 @@ +* @JessyBarrette +* @n-a-t-e +* @JorinHakai From 05bca106a9e77b5dd9f1b1b83cd1889c1a80ff5f Mon Sep 17 00:00:00 2001 From: Matthew Foster Date: Thu, 15 Aug 2024 12:38:28 -0700 Subject: [PATCH 6/6] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 44cd5037..1427ceba 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ If you just want to see how a dataset is harvested by CDE: - Rename .env.sample from the root directory to .env and change any settings if needed. If you are running on your local machine these settings don't need to change - Start a local database using `docker`: - `docker-compose up db -d` + `docker-compose up -d db` - Setup Python virtual env and install Python modules: ```sh