Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cms usage probes #136

Closed
wants to merge 2 commits into from
Closed

Cms usage probes #136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f93fed1
Fix probes
dynamic-entropy Jun 19, 2024
86987f8
Move to python3 explicitly, minor code updates
ericvaandering Jun 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 19 additions & 19 deletions common/check_deletable_replicas → cms/check_deletable_replicas
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright European Organization for Nuclear Research (CERN) 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand All @@ -9,18 +9,22 @@
# - Mario Lassnig, <[email protected]>, 2013-2014
# - Cedric Serfon, <[email protected]>, 2018

'''
Probe to check the queues of messages to submit by Hermes to the broker
'''
"""
Probe to update the RSE usage for expired replicas
"""

from __future__ import print_function
import sys

from rucio.db.sqla.session import get_session
from rucio.db.sqla.session import BASE, get_session
from sqlalchemy.sql import text

# Exit statuses
OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3

if BASE.metadata.schema:
schema = BASE.metadata.schema + '.'
else:
schema = ''

if __name__ == "__main__":
try:
Expand All @@ -36,35 +40,31 @@ if __name__ == "__main__":
SELECT
id AS rse_id
FROM
atlas_rucio.rses
{schema}rses
WHERE
deleted=0) a
LEFT OUTER JOIN
(
SELECT /*+ INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX) */
SELECT
rse_id,
COUNT(1) AS files,
SUM(bytes) AS bytes
FROM
ATLAS_RUCIO.REPLICAS
{schema}REPLICAS
WHERE
(
CASE
WHEN tombstone IS NOT NULL
THEN rse_id
END) IS NOT NULL
tombstone IS NOT NULL
AND tombstone < sys_extract_utc(localtimestamp) GROUP BY rse_id) b
ON
a.rse_id=b.rse_id)
LOOP
MERGE INTO atlas_rucio.RSE_USAGE
MERGE INTO {schema}RSE_USAGE
USING DUAL
ON (atlas_rucio.RSE_USAGE.rse_id = u.rse_id and source = 'expired')
ON ({schema}RSE_USAGE.rse_id = u.rse_id and source = 'expired')
WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at)
VALUES (u.rse_id, 'expired', u.bytes, u.files, u.updated_at, u.updated_at)
WHEN MATCHED THEN UPDATE SET used=u.bytes, files=u.files, updated_at=u.updated_at;

MERGE INTO ATLAS_RUCIO.RSE_USAGE_HISTORY H
MERGE INTO {schema}RSE_USAGE_HISTORY H
USING DUAL
ON (h.rse_id = u.rse_id and h.source = 'expired' and h.updated_at = u.updated_at)
WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at)
Expand All @@ -73,8 +73,8 @@ if __name__ == "__main__":
COMMIT;
END LOOP;
END;
'''
SESSION.execute(QUERY)
'''.format(schema=schema)
SESSION.execute(text(QUERY))
except Exception as error:
print(error)
sys.exit(UNKNOWN)
Expand Down
29 changes: 17 additions & 12 deletions common/check_obsolete_replicas → cms/check_obsolete_replicas
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# Copyright European Organization for Nuclear Research (CERN) 2013
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand All @@ -9,17 +9,22 @@
# - Vincent Garonne, <[email protected]>, 2015
# - Cedric Serfon, <[email protected]>, 2018

'''
"""
Probe to check the backlog of obsolete replicas.
'''
"""

import sys

from rucio.db.sqla.session import get_session
from rucio.db.sqla.session import BASE, get_session
from sqlalchemy.sql import text

# Exit statuses
OK, WARNING, CRITICAL, UNKNOWN = 0, 1, 2, 3

if BASE.metadata.schema:
schema = BASE.metadata.schema + '.'
else:
schema = ''

if __name__ == "__main__":
try:
Expand All @@ -35,7 +40,7 @@ if __name__ == "__main__":
SELECT
id AS rse_id
FROM
atlas_rucio.rses
{schema}rses
WHERE
deleted=0) a
LEFT OUTER JOIN
Expand All @@ -46,7 +51,7 @@ if __name__ == "__main__":
COUNT(1) AS files,
SUM(bytes) AS bytes
FROM
atlas_rucio.replicas
{schema}replicas
WHERE
(
CASE
Expand All @@ -60,14 +65,14 @@ if __name__ == "__main__":
a.rse_id=b.rse_id)

LOOP
MERGE INTO atlas_rucio.RSE_USAGE
MERGE INTO {schema}RSE_USAGE
USING DUAL
ON (atlas_rucio.RSE_USAGE.rse_id = u.rse_id and source = 'obsolete')
ON ({schema}RSE_USAGE.rse_id = u.rse_id and source = 'obsolete')
WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at)
VALUES (u.rse_id, 'obsolete', u.bytes, u.files, u.updated_at, u.updated_at)
WHEN MATCHED THEN UPDATE SET used=u.bytes, files=u.files, updated_at=u.updated_at;

MERGE INTO ATLAS_RUCIO.RSE_USAGE_HISTORY H
MERGE INTO {schema}RSE_USAGE_HISTORY H
USING DUAL
ON (h.rse_id = u.rse_id and h.source = 'obsolete' and h.updated_at = u.updated_at)
WHEN NOT MATCHED THEN INSERT(rse_id, source, used, files, updated_at, created_at)
Expand All @@ -76,9 +81,9 @@ if __name__ == "__main__":
COMMIT;
END LOOP;
END;
'''
SESSION.execute(QUERY)
'''.format(schema=schema)
SESSION.execute(text(QUERY))
except Exception as error:
print error
print(error)
sys.exit(UNKNOWN)
sys.exit(OK)