From 37bdaa08b9abf11b7adcff2da34198aef452d724 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:35:08 +0200 Subject: [PATCH 1/7] Common: Remove Python 2 compatibility --- common/check_new_dids | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/common/check_new_dids b/common/check_new_dids index 6f290cba..7acaa349 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright European Organization for Nuclear Research (CERN) 2013 # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,7 +13,6 @@ """ Probe to check the backlog of new dids. """ -from __future__ import print_function import sys import traceback From db8fd50d3c45c8a7c4159c9318fc3e1cc8ce7950 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:36:32 +0200 Subject: [PATCH 2/7] Common: Use standard file header --- common/check_new_dids | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/common/check_new_dids b/common/check_new_dids index 7acaa349..f1c2c699 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -1,14 +1,17 @@ #!/usr/bin/env python3 -# Copyright European Organization for Nuclear Research (CERN) 2013 +# Copyright European Organization for Nuclear Research (CERN) since 2012 # # Licensed under the Apache License, Version 2.0 (the "License"); -# You may not use this file except in compliance with the License. -# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Authors: -# - Vincent Garonne, , 2013 -# - Thomas Beermann, , 2019 -# - Eric Vaandering , 2020 +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """ Probe to check the backlog of new dids. From 95965db5ec0a4433941a6e18ba4f7d79e9258b35 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:39:08 +0200 Subject: [PATCH 3/7] Common: Port query to SQLAlchemy 2.0 syntax --- common/check_new_dids | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/common/check_new_dids b/common/check_new_dids index f1c2c699..978b05ee 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -21,10 +21,11 @@ import sys import traceback from prometheus_client import CollectorRegistry, Gauge, push_to_gateway +from sqlalchemy import func, null, select + from rucio.common.config import config_get from rucio.db.sqla import models from rucio.db.sqla.session import get_session -from rucio.db.sqla.util import get_count from utils.common import probe_metrics @@ -39,10 +40,18 @@ if __name__ == "__main__": try: registry = CollectorRegistry() session = get_session() - query = (session.query(models.DataIdentifier.scope) - .with_hint(models.DataIdentifier, "INDEX_FFS(DIDS DIDS_IS_NEW_IDX)", 'oracle') - .filter(models.DataIdentifier.is_new.isnot(None))) - result = get_count(query) + + query = select( + func.count() + ).select_from( + models.DataIdentifier + ).with_hint( + models.DataIdentifier, 'INDEX_FFS(DIDS DIDS_IS_NEW_IDX)', 'oracle' + ).where( + models.DataIdentifier.is_new != null() + ) + result = session.execute(query).scalar_one() + probe_metrics.gauge(name='transmogrifier.new_dids').set(result) Gauge('transmogrifier_new_dids', '', registry=registry).set(result) From 1027516cb7b74189b675b182126a974010d4391c Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:45:31 +0200 Subject: [PATCH 4/7] Common: Simplify configuration handling --- common/check_new_dids | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/common/check_new_dids b/common/check_new_dids index 978b05ee..0f1ed46d 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -23,7 +23,7 @@ import traceback from prometheus_client import CollectorRegistry, Gauge, push_to_gateway from sqlalchemy import func, null, select -from rucio.common.config import config_get +from rucio.common.config import config_get_list from rucio.db.sqla import models from rucio.db.sqla.session import get_session @@ -32,9 +32,7 @@ from utils.common import probe_metrics # Exit statuses OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3 -PROM_SERVERS = config_get('monitor', 'prometheus_servers', raise_exception=False, default='') -if PROM_SERVERS != '': - PROM_SERVERS = PROM_SERVERS.split(',') +PROM_SERVERS = config_get_list('monitor', 'prometheus_servers', raise_exception=False, default=[]) if __name__ == "__main__": try: @@ -55,12 +53,11 @@ if __name__ == "__main__": probe_metrics.gauge(name='transmogrifier.new_dids').set(result) Gauge('transmogrifier_new_dids', '', registry=registry).set(result) - if len(PROM_SERVERS): - for server in PROM_SERVERS: - try: - push_to_gateway(server.strip(), job='check_new_dids', registry=registry) - except: - continue + for server in PROM_SERVERS: + try: + push_to_gateway(server.strip(), job='check_new_dids', registry=registry) + except: + continue print(result) except: From b46f8fa8fb55ef9e90220c75d6e9d9848eebea11 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:48:36 +0200 Subject: [PATCH 5/7] Common: Avoid bare except keyword --- common/check_new_dids | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/check_new_dids b/common/check_new_dids index 0f1ed46d..a9fa33db 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -56,11 +56,11 @@ if __name__ == "__main__": for server in PROM_SERVERS: try: push_to_gateway(server.strip(), job='check_new_dids', registry=registry) - except: + except Exception: continue print(result) - except: + except Exception: print(traceback.format_exc()) sys.exit(UNKNOWN) sys.exit(OK) From 97a0e4048085b7d64074266373621f0ef07e7c1e Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 26 Jul 2024 15:49:33 +0200 Subject: [PATCH 6/7] Common: Use quotes consistently --- common/check_new_dids | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/check_new_dids b/common/check_new_dids index a9fa33db..895b7969 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -34,7 +34,7 @@ OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3 PROM_SERVERS = config_get_list('monitor', 'prometheus_servers', raise_exception=False, default=[]) -if __name__ == "__main__": +if __name__ == '__main__': try: registry = CollectorRegistry() session = get_session() From 439b165072539009a83f795602f115ade6c801e5 Mon Sep 17 00:00:00 2001 From: Dimitrios Christidis Date: Fri, 2 Aug 2024 10:07:12 +0200 Subject: [PATCH 7/7] Common: Remove unnecessary use of str.strip() --- common/check_new_dids | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/check_new_dids b/common/check_new_dids index 895b7969..b9238c15 100755 --- a/common/check_new_dids +++ b/common/check_new_dids @@ -55,7 +55,7 @@ if __name__ == '__main__': for server in PROM_SERVERS: try: - push_to_gateway(server.strip(), job='check_new_dids', registry=registry) + push_to_gateway(server, job='check_new_dids', registry=registry) except Exception: continue