Skip to content

Commit

Permalink
Merge pull request #344 from PanDAWMS/tk
Browse files Browse the repository at this point in the history 
supporting plug out extra modules for other experiments via env var
  • Loading branch information
@tkorchug
tkorchug authored May 31, 2024
2 parents 649054d + 97cf386 commit 8653b82
Show file tree
Hide file tree
Showing 21 changed files with 118 additions and 98 deletions.
1 change: 1 addition & 0 deletions core/common/urls.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
urlpatterns = []
4 changes: 2 additions & 2 deletions core/dashboards/templates/EventService.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -861,14 +861,14 @@
newWinGrafana = window.open('', title + ', built at' + tmstp, strFeatures);
if (!newWinGrafana.opener) {newWinGrafana.opener = window; }
if (montype === 'jaccounting') {
newWinGrafana.document.write('<img src="{% url 'grafana_image' %}?url=https://monit-grafana.cern.ch/render/d-solo/000000696/job-accounting-historical-data?orgId=17&from=now-24h&to=now&var-bin=1h&var-groupby=gshare&var-country=All&var-federation=All&var-resources=GRID&var-resources=cloud&var-resources=hpc&var-tier=All&var-cloud=All&var-site=All&var-computingsite='+site+'&var-nucleus=All&var-cores=All&var-eventservice=All&var-groups=All&var-inputdatatypes=All&var-inputprojects=All&var-outputproject=All&var-gshare=All&var-resourceserporting=All&var-processingtype=All&var-jobtype=All&var-jobstatus=All&var-error_category=All&var-measurement_suffix=1h&var-measurement_suffix_CQ=1h&var-retention_policy=long&var-division_factor=1&var-hourly_multiplier=1&panelId=138&width=1000&height=500&tz=Europe%2FBerlin" id="show_image"><p>');
newWinGrafana.document.write('<img src="/grafana/img/?url=https://monit-grafana.cern.ch/render/d-solo/000000696/job-accounting-historical-data?orgId=17&from=now-24h&to=now&var-bin=1h&var-groupby=gshare&var-country=All&var-federation=All&var-resources=GRID&var-resources=cloud&var-resources=hpc&var-tier=All&var-cloud=All&var-site=All&var-computingsite='+site+'&var-nucleus=All&var-cores=All&var-eventservice=All&var-groups=All&var-inputdatatypes=All&var-inputprojects=All&var-outputproject=All&var-gshare=All&var-resourceserporting=All&var-processingtype=All&var-jobtype=All&var-jobstatus=All&var-error_category=All&var-measurement_suffix=1h&var-measurement_suffix_CQ=1h&var-retention_policy=long&var-division_factor=1&var-hourly_multiplier=1&panelId=138&width=1000&height=500&tz=Europe%2FBerlin" id="show_image"><p>');
}
else {
newWinGrafana.document.write("<html><head>");
newWinGrafana.document.write("<meta http-equiv='Content-Type' content='text/html; charset=windows-1251'>");
newWinGrafana.document.write("<title>"+ title +"</title></head>");
newWinGrafana.document.write("<body>");
newWinGrafana.document.write('<img alt="Loading image..." id="show_image" src="{% url 'grafana_image' %}?url=https://monit-grafana.cern.ch/render/d-solo/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h/h&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue=' + site + '&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All&panelId=8&width=1400&height=1400&tz=Europe%2FZurich&timeout=3600&timestamp=' + tmstp.toISOString() + '" id="show_image" style="width:900px;height:600px;"><p>');
newWinGrafana.document.write('<img alt="Loading image..." id="show_image" src="/grafana/img/?url=https://monit-grafana.cern.ch/render/d-solo/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h/h&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue=' + site + '&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All&panelId=8&width=1400&height=1400&tz=Europe%2FZurich&timeout=3600&timestamp=' + tmstp.toISOString() + '" id="show_image" style="width:900px;height:600px;"><p>');
newWinGrafana.document.write('<a target="_blank" href="https://monit-grafana.cern.ch/d/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h%2Fh&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue='+ site +'&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All">Jobs monitoring dashboard</a>');
newWinGrafana.document.write('<p><input type="button" onclick=reloadImage("show_image"); value="Reload">');
newWinGrafana.document.write('<input type=button value="Close" onclick="window.close();return false;">');
Expand Down
8 changes: 4 additions & 4 deletions core/dashboards/templates/JobSummaryRegion.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1116,7 +1116,7 @@
sDefaultContent: "---",
className: "num",
render: function(data, type, row, meta) {
return '<a href = "{% url 'harvesterWorkerList' %}?hours=12&status=submitted&computingsite=' + row[0] + '">' + row[8] + '</a>';
return '<a href = "/harvester/workers/?hours=12&status=submitted&computingsite=' + row[0] + '">' + row[8] + '</a>';
}
},
{
Expand All @@ -1128,7 +1128,7 @@
(row[21] !== 0) ? diffpct = (row[9] - row[21])*100./row[21] : diffpct = 0;
if (diffpct > 20) {cellstr += 'alert';}
else if (diffpct > 10) {cellstr += 'warning';}
cellstr += '" href = "{% url 'harvesterWorkerList' %}?hours=12&status=running&computingsite=' + row[0] + '">' + row[9] + '</a>';
cellstr += '" href = "/harvester/workers/?hours=12&status=running&computingsite=' + row[0] + '">' + row[9] + '</a>';
return cellstr
}
},
Expand Down Expand Up @@ -1387,14 +1387,14 @@
newWinGrafana = window.open('', title + ', built at' + tmstp, strFeatures);
if (!newWinGrafana.opener) {newWinGrafana.opener = window; }
if (montype === 'jaccounting') {
newWinGrafana.document.write('<img src="{% url 'grafana_image' %}?url=https://monit-grafana.cern.ch/render/d-solo/000000696/job-accounting-historical-data?orgId=17&from=now-24h&to=now&var-bin=1h&var-groupby=gshare&var-country=All&var-federation=All&var-resources=GRID&var-resources=cloud&var-resources=hpc&var-tier=All&var-cloud=All&var-site=All&var-computingsite='+site+'&var-nucleus=All&var-cores=All&var-eventservice=All&var-groups=All&var-inputdatatypes=All&var-inputprojects=All&var-outputproject=All&var-gshare=All&var-resourceserporting=All&var-processingtype=All&var-jobtype=All&var-jobstatus=All&var-error_category=All&var-measurement_suffix=1h&var-measurement_suffix_CQ=1h&var-retention_policy=long&var-division_factor=1&var-hourly_multiplier=1&panelId=138&width=1000&height=500&tz=Europe%2FBerlin" id="show_image"><p>');
newWinGrafana.document.write('<img src="/grafana/img/?url=https://monit-grafana.cern.ch/render/d-solo/000000696/job-accounting-historical-data?orgId=17&from=now-24h&to=now&var-bin=1h&var-groupby=gshare&var-country=All&var-federation=All&var-resources=GRID&var-resources=cloud&var-resources=hpc&var-tier=All&var-cloud=All&var-site=All&var-computingsite='+site+'&var-nucleus=All&var-cores=All&var-eventservice=All&var-groups=All&var-inputdatatypes=All&var-inputprojects=All&var-outputproject=All&var-gshare=All&var-resourceserporting=All&var-processingtype=All&var-jobtype=All&var-jobstatus=All&var-error_category=All&var-measurement_suffix=1h&var-measurement_suffix_CQ=1h&var-retention_policy=long&var-division_factor=1&var-hourly_multiplier=1&panelId=138&width=1000&height=500&tz=Europe%2FBerlin" id="show_image"><p>');
}
else {
newWinGrafana.document.write("<html><head>");
newWinGrafana.document.write("<meta http-equiv='Content-Type' content='text/html; charset=windows-1251'>");
newWinGrafana.document.write("<title>"+ title +"</title></head>");
newWinGrafana.document.write("<body>");
newWinGrafana.document.write('<img alt="Loading image..." id="show_image" src="{% url 'grafana_image' %}?url=https://monit-grafana.cern.ch/render/d-solo/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h/h&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue=' + site + '&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All&panelId=8&width=1400&height=1400&tz=Europe%2FZurich&timeout=3600&timestamp=' + tmstp.toISOString() + '" id="show_image" style="width:900px;height:600px;"><p>');
newWinGrafana.document.write('<img alt="Loading image..." id="show_image" src="/grafana/img/?url=https://monit-grafana.cern.ch/render/d-solo/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h/h&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue=' + site + '&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All&panelId=8&width=1400&height=1400&tz=Europe%2FZurich&timeout=3600&timestamp=' + tmstp.toISOString() + '" id="show_image" style="width:900px;height:600px;"><p>');
newWinGrafana.document.write('<a target="_blank" href="https://monit-grafana.cern.ch/d/VbKvjL2Zk/jobs-monitoring?orgId=17&from=now-24h%2Fh&to=now&var-retention=10m&var-cloud=All&var-tier=All&var-federation=All&var-nucleus=All&var-site=All&var-queue='+ site +'&var-queue_state=All&var-prod_source=' + prod_source + '&var-resource=' + res_cap + '&var-resource_type=All&var-type=All&var-states=All&var-pilot_manager=All&var-pilot_version=All&var-harvester=All&var-workflow=All&var-frontier=All&var-fts_server=All&var-container_type=All&var-pledge=All">Jobs monitoring dashboard</a>');
newWinGrafana.document.write('<p><input type="button" onclick=reloadImage("show_image"); value="Reload">');
newWinGrafana.document.write('<input type=button value="Close" onclick="window.close();return false;">');
Expand Down
1 change: 1 addition & 0 deletions core/dashboards/urls.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
urlpatterns = []
4 changes: 2 additions & 2 deletions core/datacarousel/templates/DataTapeCarouselle.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@
<th class="nowrap"> <a class="nowrap" href="{% url 'taskList' %}?tape=true&stagesource={$ x.source_rse $}&{$ getTimeForQuery() $}">{$ x.source_rse $}</a>
</th>
<td class="mid">
<a target="_blank" href="{% url 'staginprogressplot' %}?stagesource={$ x.source_rse $}&{$ getTimeForQuery() $}"><i class="fi-graph-trend"></i></a>
<a target="_blank" href="/staginprogressplot/?stagesource={$ x.source_rse $}&{$ getTimeForQuery() $}"><i class="fi-graph-trend"></i></a>
</td>
<td class="mid">
<a target="_blank" href="https://monit-grafana.cern.ch/d/FtSFfwdmk/ddm-transfers?orgId=17&var-binning=$__auto_interval_binning&var-groupby=dst_cloud&var-activity=Staging&var-src_tier=All&var-src_country=All&var-src_cloud=All&var-src_site=All{$ x.source_rse_breakdown $}&var-src_token=All&var-columns=src_cloud&var-dst_tier=All&var-dst_country=All&var-dst_cloud=All&var-dst_site=All&var-dst_endpoint=All&var-dst_token=All&var-rows=dst_cloud&var-measurement=ddm_transfer&var-retention_policy=raw&var-include=&var-exclude=none&var-exclude_es=All&var-include_es_dst=All&var-include_es_src=All&{$ getTimeForQuery(true) $}"><img src="{% static 'images/grafana.png' %}" width=14 height=14 border=0></a>
Expand Down Expand Up @@ -641,7 +641,7 @@
className: 'text icons1',
"render": function(data, type, full, meta) {
let entry = '<a href="{% url 'taskInfo' %}'+full['taskid']+'">'+full['taskid']+'</a> ';
entry += '<a target="_blank" href="{% url 'staginprogressplot' %}'+'?jeditaskid='+full['taskid']+'"><i class="fi-graph-trend"></i></a>';
entry += '<a target="_blank" href="/staginprogressplot/?jeditaskid='+full['taskid']+'"><i class="fi-graph-trend"></i></a>';
return entry;
},
},
Expand Down
5 changes: 3 additions & 2 deletions core/libs/task.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from core.common.models import JediDatasetContents, JediDatasets, JediTaskparams, JediDatasetLocality, JediTasks, \
JediJobRetryHistory
from core.pandajob.models import Jobsactive4, Jobsarchived, Jobswaiting4, Jobsdefined4, Jobsarchived4
from core.iDDS.utils import add_idds_info_to_tasks


from core.libs.exlib import insert_to_temp_table, get_tmp_table_name, round_to_n_digits, convert_sec
from core.libs.datetimestrings import parse_datetime
Expand Down Expand Up @@ -58,7 +58,7 @@ def cleanTaskList(tasks, **kwargs):
add_datasets_info = True
if 'sortby' in kwargs:
sortby = kwargs['sortby']
if 'add_idds_info' in kwargs:
if 'add_idds_info' in kwargs and 'core.iDDS' in settings.INSTALLED_APPS:
add_idds_info = kwargs['add_idds_info']
else:
add_idds_info = False
Expand Down Expand Up @@ -142,6 +142,7 @@ def cleanTaskList(tasks, **kwargs):

# check if tasks are part of idds and get workflow id
if add_idds_info:
from core.iDDS.utils import add_idds_info_to_tasks
tasks = add_idds_info_to_tasks(tasks)

if sortby is not None:
Expand Down
1 change: 1 addition & 0 deletions core/pandajob/urls.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
urlpatterns = []
1 change: 1 addition & 0 deletions core/schedresource/urls.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
urlpatterns = []
51 changes: 27 additions & 24 deletions core/settings/base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,32 +125,35 @@

INSTALLED_APPS_BIGPANDAMON_CORE = (
# BigPanDAmon core
'core.oauth',
'core.common',
'core.pandajob',
'core.schedresource',
'core.dashboards',
'core.status_summary',
"core.oauth",
"core.common",
"core.pandajob",
"core.schedresource",
"core.dashboards",
"core.status_summary",
"core.harvester",
"core.filebrowser",
)

INSTALLED_APPS_EXTRA = [
# "core.admin",
"core.art",
"core.buildmonitor",
"core.compare",
"core.datacarousel",
"core.errorsscattering",
"core.filebrowser",
"core.globalshares",
"core.grafana",
"core.harvester",
"core.iDDS",
"core.mlflowdynamic",
"core.reports",
"core.runningprod",
"core.panda_client",
"core.kafka"
]
if 'BIGMON_INSTALLED_APPS_LIST' in os.environ and os.environ['BIGMON_INSTALLED_APPS_LIST']:
INSTALLED_APPS_EXTRA = os.environ['BIGMON_INSTALLED_APPS_LIST'].split(',')
else:
INSTALLED_APPS_EXTRA = [
# "core.admin",
"core.art",
"core.buildmonitor",
"core.compare",
"core.datacarousel",
"core.errorsscattering",
"core.globalshares",
"core.grafana",
"core.iDDS",
"core.mlflowdynamic",
"core.reports",
"core.runningprod",
"core.panda_client",
"core.kafka"
]

CHANNEL_LAYERS = {
"default": {
Expand Down
Loading

0 comments on commit 8653b82

Please sign in to comment.