From c98ef05af6041a566ee36f57adc3fa2af47a547b Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Tue, 7 May 2024 19:50:38 -0500 Subject: [PATCH 01/10] feat: nopayloadclient service --- cmake/jana_plugin.cmake | 18 ++++++++++++++++++ src/services/CMakeLists.txt | 1 + src/services/payload/CMakeLists.txt | 18 ++++++++++++++++++ src/services/payload/Payload_service.cc | 13 +++++++++++++ src/services/payload/Payload_service.h | 25 +++++++++++++++++++++++++ src/services/payload/payload.cc | 17 +++++++++++++++++ 6 files changed, 92 insertions(+) create mode 100644 src/services/payload/CMakeLists.txt create mode 100644 src/services/payload/Payload_service.cc create mode 100644 src/services/payload/Payload_service.h create mode 100644 src/services/payload/payload.cc diff --git a/cmake/jana_plugin.cmake b/cmake/jana_plugin.cmake index ea5c0b2ffa..e50b0e1fe0 100644 --- a/cmake/jana_plugin.cmake +++ b/cmake/jana_plugin.cmake @@ -329,6 +329,24 @@ macro(plugin_add_irt _name) endmacro() +# Adds nopayloadclient for a plugin +macro(plugin_add_nopayloadclient _name) + + if(NOT CURL_FOUND) + find_package(CURL REQUIRED) + endif() + + find_library(npc_lib REQUIRED NAMES nopayloadclient) + find_path(npc_include REQUIRED NAMES nopayloadclient) + + # Add include directories + plugin_include_directories(${PLUGIN_NAME} PUBLIC curl ${npc_include}) + + # Add libraries + plugin_link_libraries(${PLUGIN_NAME} curl ${npc_lib}) + +endmacro() + # Adds podio, edm4hep, edm4eic for a plugin macro(plugin_add_event_model _name) diff --git a/src/services/CMakeLists.txt b/src/services/CMakeLists.txt index 4587344f41..57b65fd208 100644 --- a/src/services/CMakeLists.txt +++ b/src/services/CMakeLists.txt @@ -5,5 +5,6 @@ add_subdirectory(geometry/acts) add_subdirectory(geometry/richgeo) add_subdirectory(io/podio) add_subdirectory(log) +add_subdirectory(payload) add_subdirectory(rootfile) add_subdirectory(pid_lut) diff --git a/src/services/payload/CMakeLists.txt b/src/services/payload/CMakeLists.txt new file mode 100644 index 0000000000..bed3bd6e79 --- /dev/null +++ b/src/services/payload/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 3.16) + +# Automatically set plugin name the same as the directory name Don't forget +# string(REPLACE " " "_" PLUGIN_NAME ${PLUGIN_NAME}) if this dir has spaces in +# its name +get_filename_component(PLUGIN_NAME ${CMAKE_CURRENT_LIST_DIR} NAME) + +# Function creates ${PLUGIN_NAME}_plugin and ${PLUGIN_NAME}_library targets +# Setting default includes, libraries and installation paths +plugin_add(${PLUGIN_NAME} WITH_STATIC_LIBRARY) + +# The macro grabs sources as *.cc *.cpp *.c and headers as *.h *.hh *.hpp Then +# correctly sets sources for ${_name}_plugin and ${_name}_library targets Adds +# headers to the correct installation directory +plugin_glob_all(${PLUGIN_NAME}) + +# Find dependencies +plugin_add_nopayloadclient(${PLUGIN_NAME}) diff --git a/src/services/payload/Payload_service.cc b/src/services/payload/Payload_service.cc new file mode 100644 index 0000000000..0e2ab6fd88 --- /dev/null +++ b/src/services/payload/Payload_service.cc @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (C) 2024 Wouter Deconinck + +#include "Payload_service.h" + +#include +#include +#include + +Payload_service::Payload_service(JApplication *app) +: m_client("EICrecon") { + m_application = app; +} diff --git a/src/services/payload/Payload_service.h b/src/services/payload/Payload_service.h new file mode 100644 index 0000000000..4395c4960a --- /dev/null +++ b/src/services/payload/Payload_service.h @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: LGPL-3.0-or-later +// Copyright (C) 2024 Wouter Deconinck + +#pragma once + +#include +#include +#include +#include +#include + +class Payload_service : public JService +{ +public: + explicit Payload_service(JApplication *app); + ~Payload_service() { }; + +private: + + Payload_service() = default; + + nopayloadclient::NoPayloadClient m_client; + + JApplication* m_application; +}; diff --git a/src/services/payload/payload.cc b/src/services/payload/payload.cc new file mode 100644 index 0000000000..b1f6db3d37 --- /dev/null +++ b/src/services/payload/payload.cc @@ -0,0 +1,17 @@ +// Copyright 2022, David Lawrence +// Subject to the terms in the LICENSE file found in the top-level directory. +// +// + +#include +#include + +#include "Log_service.h" + + +extern "C" { +void InitPlugin(JApplication *app) { + InitJANAPlugin(app); + app->ProvideService(std::make_shared(app) ); +} +} From 75431a59579298f9fecb6a2501cc95c301d6cfc2 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Wed, 8 May 2024 07:19:52 -0500 Subject: [PATCH 02/10] fix: payload WITH_SHARED_LIBRARY Co-authored-by: Dmitry Kalinkin --- src/services/payload/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/payload/CMakeLists.txt b/src/services/payload/CMakeLists.txt index bed3bd6e79..e02351b0aa 100644 --- a/src/services/payload/CMakeLists.txt +++ b/src/services/payload/CMakeLists.txt @@ -7,7 +7,7 @@ get_filename_component(PLUGIN_NAME ${CMAKE_CURRENT_LIST_DIR} NAME) # Function creates ${PLUGIN_NAME}_plugin and ${PLUGIN_NAME}_library targets # Setting default includes, libraries and installation paths -plugin_add(${PLUGIN_NAME} WITH_STATIC_LIBRARY) +plugin_add(${PLUGIN_NAME} WITH_SHARED_LIBRARY) # The macro grabs sources as *.cc *.cpp *.c and headers as *.h *.hh *.hpp Then # correctly sets sources for ${_name}_plugin and ${_name}_library targets Adds From 5c21c4d1d54816925366079e20a86c7debdb08e8 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 9 May 2024 22:23:34 -0500 Subject: [PATCH 03/10] fix: FindCURL exposes CURL::libcurl --- cmake/jana_plugin.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/jana_plugin.cmake b/cmake/jana_plugin.cmake index e50b0e1fe0..4cb6848f65 100644 --- a/cmake/jana_plugin.cmake +++ b/cmake/jana_plugin.cmake @@ -340,10 +340,10 @@ macro(plugin_add_nopayloadclient _name) find_path(npc_include REQUIRED NAMES nopayloadclient) # Add include directories - plugin_include_directories(${PLUGIN_NAME} PUBLIC curl ${npc_include}) + plugin_include_directories(${PLUGIN_NAME} PUBLIC CURL::libcurl ${npc_include}) # Add libraries - plugin_link_libraries(${PLUGIN_NAME} curl ${npc_lib}) + plugin_link_libraries(${PLUGIN_NAME} CURL::libcurl ${npc_lib}) endmacro() From 418b92a24438cbba4d11596382fefbf828a9abca Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 9 May 2024 22:28:29 -0500 Subject: [PATCH 04/10] fix: CURL::libcurl in lib only --- cmake/jana_plugin.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/jana_plugin.cmake b/cmake/jana_plugin.cmake index 4cb6848f65..2ebbe3317a 100644 --- a/cmake/jana_plugin.cmake +++ b/cmake/jana_plugin.cmake @@ -340,7 +340,7 @@ macro(plugin_add_nopayloadclient _name) find_path(npc_include REQUIRED NAMES nopayloadclient) # Add include directories - plugin_include_directories(${PLUGIN_NAME} PUBLIC CURL::libcurl ${npc_include}) + plugin_include_directories(${PLUGIN_NAME} PUBLIC ${npc_include}) # Add libraries plugin_link_libraries(${PLUGIN_NAME} CURL::libcurl ${npc_lib}) From fd6ecf1068e89ac44ebc1c5c6a580fb5c51c1cdd Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Wed, 22 May 2024 19:28:12 -0500 Subject: [PATCH 05/10] feat(ci): setup postgres and nopayloaddb as services in CI --- .github/workflows/linux-eic-shell.yml | 63 +++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/.github/workflows/linux-eic-shell.yml b/.github/workflows/linux-eic-shell.yml index 8abe1f910f..5bb4564066 100644 --- a/.github/workflows/linux-eic-shell.yml +++ b/.github/workflows/linux-eic-shell.yml @@ -647,6 +647,69 @@ jobs: path: rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}_${{ matrix.benchmark_plugins }}.hists.root if-no-files-found: error + eicrecon-nopayloadclient: + runs-on: ubuntu-latest + needs: + - build + - npsim-gun + strategy: + matrix: + CC: [clang] + particle: [e] + detector_config: [craterlake] + services: + db: + image: postgres + env: + POSTGRES_DB: dbname + POSTGRES_USER: login + POSTGRES_PASSWORD: password + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + nopayloaddb-webapp: + image: ghcr.io/eic/nopayloaddb-webapp:latest + env: + POSTGRES_DB: dbname + POSTGRES_USER: login + POSTGRES_PASSWORD: password + POSTGRES_HOST: localhost + POSTGRES_PORT: 5432 + ports: + - 8000:8000 + steps: + - name: Checkout .github + uses: actions/checkout@v4 + with: + sparse-checkout: .github + - name: Download install directory + uses: actions/download-artifact@v4 + with: + name: install-${{ matrix.CC }}-eic-shell-Release-${{ env.platform }}-${{ env.release }} + - name: Uncompress install directory + run: tar -xaf install.tar.zst + - uses: actions/download-artifact@v4 + with: + name: sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root + - uses: cvmfs-contrib/github-action-cvmfs@v4 + - name: Run EICrecon + uses: eic/run-cvmfs-osg-eic-shell@main + with: + platform-release: "${{ env.platform }}:${{ env.release }}" + setup: "/opt/detector/epic-${{ env.detector-version }}/setup.sh" + run: | + cli_npc + export DETECTOR_CONFIG=${DETECTOR}_${{ matrix.detector_config }} + export LD_LIBRARY_PATH=$PWD/install/lib:$LD_LIBRARY_PATH + export JANA_PLUGIN_PATH=$PWD/install/lib/EICrecon/plugins:/usr/local/plugins + $PWD/install/bin/eicrecon -Ppodio:output_file=rec_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4eic.root sim_${{ matrix.particle }}_1GeV_20GeV_${{ matrix.detector_config }}.edm4hep.root -Pplugins=dump_flags,janadot,janatop -Pdump_flags:json=${{ matrix.particle }}_${{ matrix.detector_config }}_flags.json -Pjana:warmup_timeout=0 -Pjana:timeout=0 + eicrecon-gun: runs-on: ubuntu-latest needs: From 479e5d7856f63089c4ef6537c1b221e50449c99d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 May 2024 00:28:20 +0000 Subject: [PATCH 06/10] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .github/workflows/linux-eic-shell.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux-eic-shell.yml b/.github/workflows/linux-eic-shell.yml index 5bb4564066..a98bd5f6ed 100644 --- a/.github/workflows/linux-eic-shell.yml +++ b/.github/workflows/linux-eic-shell.yml @@ -675,7 +675,7 @@ jobs: --health-retries 5 nopayloaddb-webapp: image: ghcr.io/eic/nopayloaddb-webapp:latest - env: + env: POSTGRES_DB: dbname POSTGRES_USER: login POSTGRES_PASSWORD: password @@ -704,7 +704,7 @@ jobs: platform-release: "${{ env.platform }}:${{ env.release }}" setup: "/opt/detector/epic-${{ env.detector-version }}/setup.sh" run: | - cli_npc + cli_npc export DETECTOR_CONFIG=${DETECTOR}_${{ matrix.detector_config }} export LD_LIBRARY_PATH=$PWD/install/lib:$LD_LIBRARY_PATH export JANA_PLUGIN_PATH=$PWD/install/lib/EICrecon/plugins:/usr/local/plugins From a1e95707f9a131641c030865055e3e45a2f2781e Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Wed, 22 May 2024 19:30:33 -0500 Subject: [PATCH 07/10] fix: include full path to Log_service.h --- src/services/payload/payload.cc | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/services/payload/payload.cc b/src/services/payload/payload.cc index b1f6db3d37..542bfee67f 100644 --- a/src/services/payload/payload.cc +++ b/src/services/payload/payload.cc @@ -6,12 +6,11 @@ #include #include -#include "Log_service.h" - +#include "services/log/Log_service.h" extern "C" { -void InitPlugin(JApplication *app) { - InitJANAPlugin(app); - app->ProvideService(std::make_shared(app) ); +void InitPlugin(JApplication* app) { + InitJANAPlugin(app); + app->ProvideService(std::make_shared(app)); } } From 8496447477a0a8edd6adbffbf4614e350103b553 Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Wed, 22 May 2024 20:25:26 -0500 Subject: [PATCH 08/10] fix(ci): jobs: build: strategy: fail-fast: false --- .github/workflows/linux-eic-shell.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/linux-eic-shell.yml b/.github/workflows/linux-eic-shell.yml index a98bd5f6ed..79ad92a0da 100644 --- a/.github/workflows/linux-eic-shell.yml +++ b/.github/workflows/linux-eic-shell.yml @@ -62,6 +62,7 @@ jobs: runs-on: ubuntu-latest needs: env strategy: + fail-fast: false # include multiple compilers for one release version, # and one compiler for a range of release versions matrix: From 29d994c4b72bff1586916c1ad713c6d1c184d57e Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Wed, 22 May 2024 20:30:02 -0500 Subject: [PATCH 09/10] fix(ci): avoid fast-fail: false, but use 24.05.0 as 'old' env --- .github/workflows/linux-eic-shell.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/linux-eic-shell.yml b/.github/workflows/linux-eic-shell.yml index 79ad92a0da..1e463b1244 100644 --- a/.github/workflows/linux-eic-shell.yml +++ b/.github/workflows/linux-eic-shell.yml @@ -62,7 +62,6 @@ jobs: runs-on: ubuntu-latest needs: env strategy: - fail-fast: false # include multiple compilers for one release version, # and one compiler for a range of release versions matrix: @@ -86,7 +85,7 @@ jobs: release: nightly - CC: clang CMAKE_BUILD_TYPE: Release - release: 24.04.0-stable + release: 24.05.0-stable steps: - name: mmap rnd_bits workaround # https://github.com/actions/runner/issues/3207#issuecomment-2000066889 From 0cec95052111b6b52ba03eba1f9ea9f49313046d Mon Sep 17 00:00:00 2001 From: Wouter Deconinck Date: Thu, 23 May 2024 12:46:15 -0500 Subject: [PATCH 10/10] fix(ci): cli_npc checkConnection with default.json --- .github/workflows/linux-eic-shell.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux-eic-shell.yml b/.github/workflows/linux-eic-shell.yml index 1e463b1244..724c44f813 100644 --- a/.github/workflows/linux-eic-shell.yml +++ b/.github/workflows/linux-eic-shell.yml @@ -704,7 +704,24 @@ jobs: platform-release: "${{ env.platform }}:${{ env.release }}" setup: "/opt/detector/epic-${{ env.detector-version }}/setup.sh" run: | - cli_npc + export NOPAYLOADCLIENT_CONF=$PWD/default.json + cat <<- EOF > $NOPAYLOADCLIENT_CONF + { + "base_url": "localhost:8000", + "api_res": "/api/cdb_rest/", + "write_dir": "/tmp/remote_pl_store/", + "read_dir_list": ["/tmp/remote_pl_store/", "/cvmfs/place/holder/"], + "n_retries": 5, + "retry_sleep_mean": 30, + "cache_life_time": 10, + "cache_max_mb": 1, + "print_time_stamps": false, + "use_fake_backend": false, + "logger": "terminal", + "log_level": "INFO" + } + EOF + cli_npc checkConnection export DETECTOR_CONFIG=${DETECTOR}_${{ matrix.detector_config }} export LD_LIBRARY_PATH=$PWD/install/lib:$LD_LIBRARY_PATH export JANA_PLUGIN_PATH=$PWD/install/lib/EICrecon/plugins:/usr/local/plugins