Common: Check_expired_locked_rules modernization. #127

common/check_expired_locked_rules

@@ -1,56 +1,118 @@
-#!/usr/bin/env python
-# Copyright European Organization for Nuclear Research (CERN) 2013
+#!/usr/bin/env python3
+# Copyright European Organization for Nuclear Research (CERN) since 2012
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
 #
-# Authors:
-# - Cedric Serfon, <[email protected]>, 2015
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
-'''
+"""
 Probe to check the locked expired rules or datasets with locked rules
-'''
+"""
 
 import sys
+import traceback
+from collections import defaultdict
+from datetime import datetime
+
+from sqlalchemy import and_, select
+from sqlalchemy.sql import null, true
+
+from rucio.db.sqla import models
 from rucio.db.sqla.session import get_session
 
+from utils.common import PrometheusPusher
+
 # Exit statuses
 OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3
 
+if __name__ == "__main__":
 
-def main():
-    '''
-    Probe to check the locked expired rules or datasets with locked rules
-    '''
     status = OK
     session = get_session()
-    try:
-        query = "select rawtohex(id), scope, name, rse_expression from atlas_rucio.rules where locked=1 and expires_at<sys_extract_utc(localtimestamp)"
-        print 'Locked expired rules'
-        for row in session.execute(query):
-            status = CRITICAL
-            print row[0], row[1], row[2]
-    except Exception as error:
-        print error
-        status = UNKNOWN
-        sys.exit(status)
-    try:
-        query = """select rawtohex(c.id), c.scope, c.name, c.rse_expression from atlas_rucio.rules c,
-                   (select a.scope, a.name from atlas_rucio.dids a
-                   where a.expired_at is not null and a.expired_at < sys_extract_utc(localtimestamp)
-                   and exists (select 1 from atlas_rucio.rules b where a.scope=b.scope and a.name=b.name and locked=1)) d
-                   where c.scope=d.scope and c.name=d.name and locked=1"""
-        print 'Datasets expired with locked rules'
-        for row in session.execute(query):
-            status = CRITICAL
-            print row[0], row[1], row[2], row[3]
-    except Exception as error:
-        print error
-        status = UNKNOWN
-        sys.exit(status)
-    sys.exit(status)
 
+    # Select statement used for both metrics
+    base_statement = select(
+        models.ReplicationRule.id,
+        models.ReplicationRule.name,
+        models.ReplicationRule.scope,
+        models.ReplicationRule.rse_expression,
+    )
 
-if __name__ == "__main__":
-    main()
+    # Use prometheus pusher to send results to a remote service
+    with PrometheusPusher() as manager:
+        try:
+            statement = base_statement.where(
+                and_(
+                    models.ReplicationRule.locked == true(),
+                    models.ReplicationRule.expires_at < datetime.utcnow()  # see https://github.com/rucio/rucio/issues/6476
+                )
+            )
+            rule_counts = defaultdict(int)
+
+            # Print rules for nagios monitoring
+            print("Locked expired rules")
+            for row in session.execute(statement):
+                print(row.id, row.scope, row.name, row.rse_expression)
+                status = CRITICAL
+                # Keep track of the counts
+                rule_counts[row.rse_expression] += 1
+
+            # Add a summary entry so when there are no result from the query there is metric continuity
+            rule_counts["All"] = sum(rule_counts.values())
+
+            # Send to Prometheus pusher
+            for rse_expression, count in rule_counts.items():
+                (manager.gauge(
+                        "locked_expired_rules.{rse_expression}",
+                        documentation="Number of rules that are locked and expired, by RSE expression.")
+                    .labels(rse_expression=rse_expression)
+                    .set(count))
+
+        except Exception as error:
+            print(traceback.format_exc())
+            sys.exit(UNKNOWN)
+
+        try:
+            statement = base_statement.join(
+                models.DataIdentifier,
+                and_(
+                    models.ReplicationRule.scope == models.DataIdentifier.scope,
+                    models.ReplicationRule.name == models.DataIdentifier.name,
+                ),
+            ).where(
+                and_(
+                    models.ReplicationRule.locked == true(),
+                    models.DataIdentifier.expired_at != null(),
+                    models.DataIdentifier.expired_at < datetime.utcnow()  # see https://github.com/rucio/rucio/issues/6476
+                )
+            )
+            datasets_count = defaultdict(int)
+
+            print("Datasets expired with locked rules")
+            for row in session.execute(statement):
+                print(row.id, row.scope, row.name, row.rse_expression)
+                status = CRITICAL
+                datasets_count[row.rse_expression] += 1
+
+            datasets_count["All"] = sum(datasets_count.values())
+            for rse_expression, dids in datasets_count.items():
+                (manager.gauge(
+                        "locked_expired_rules.dids.{rse_expression}",
+                        documentation="Number of expired DIDs with locked rules, by RSE expression")
+                    .labels(rse_expression=rse_expression)
+                    .set(dids))
+
+        except Exception:
+            print(traceback.format_exc())
+            sys.exit(UNKNOWN)
+
+    sys.exit(status)