diff --git a/Modules/Common/CMakeLists.txt b/Modules/Common/CMakeLists.txt index c0ddc6308e..3cc21015a7 100644 --- a/Modules/Common/CMakeLists.txt +++ b/Modules/Common/CMakeLists.txt @@ -30,6 +30,8 @@ target_sources(O2QcCommon src/ReferenceComparatorTask.cxx src/ReferenceComparatorTaskConfig.cxx src/ReferenceComparatorCheck.cxx + src/CheckerThresholdsConfig.cxx + src/TrendCheck.cxx src/NonEmpty.cxx src/MeanIsAbove.cxx src/TH1Reductor.cxx @@ -68,6 +70,7 @@ add_root_dictionary(O2QcCommon include/Common/ObjectComparatorKolmogorov.h include/Common/ReferenceComparatorTask.h include/Common/ReferenceComparatorCheck.h + include/Common/TrendCheck.h include/Common/MeanIsAbove.h include/Common/TH1Ratio.h include/Common/TH2Ratio.h diff --git a/Modules/Common/include/Common/LinkDef.h b/Modules/Common/include/Common/LinkDef.h index 3295ba72e4..267f0d7932 100644 --- a/Modules/Common/include/Common/LinkDef.h +++ b/Modules/Common/include/Common/LinkDef.h @@ -24,6 +24,7 @@ #pragma link C++ class o2::quality_control_modules::common::ObjectComparatorKolmogorov + ; #pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorTask + ; #pragma link C++ class o2::quality_control_modules::common::ReferenceComparatorCheck + ; +#pragma link C++ class o2::quality_control_modules::common::TrendCheck + ; #pragma link C++ class o2::quality_control_modules::common::WorstOfAllAggregator + ; #pragma link C++ class o2::quality_control_modules::common::IncreasingEntries + ; #pragma link C++ class o2::quality_control_modules::common::TH1SliceReductor + ; diff --git a/Modules/Common/include/Common/TrendCheck.h b/Modules/Common/include/Common/TrendCheck.h new file mode 100644 index 0000000000..f52c1bad17 --- /dev/null +++ b/Modules/Common/include/Common/TrendCheck.h @@ -0,0 +1,83 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendCheck.h +/// \author Andrea Ferrero +/// \brief Generic checker for trending graphs +/// + +#ifndef QC_MODULE_COMMON_TRENDCHECK_H +#define QC_MODULE_COMMON_TRENDCHECK_H + +#include "QualityControl/CheckInterface.h" + +#include +#include +#include + +class TGraph; +class TObject; + +using namespace o2::quality_control::core; + +namespace o2::quality_control_modules::common +{ + +class CheckerThresholdsConfig; + +/// \brief Generic checker for trending graphs +/// +/// \author Andrea Ferrero +class TrendCheck : public o2::quality_control::checker::CheckInterface +{ + public: + /// Default constructor + TrendCheck() = default; + /// Destructor + ~TrendCheck() override = default; + + void configure() override; + Quality check(std::map>* moMap) override; + void beautify(std::shared_ptr mo, Quality checkResult = Quality::Null) override; + std::string getAcceptedType() override; + + void startOfActivity(const Activity& activity) override; + void endOfActivity(const Activity& activity) override; + + ClassDefOverride(TrendCheck, 1); + + private: + enum ThresholdsMode { + ExpectedRange, + DeviationFromMean, + StdDeviation + }; + + std::array>, 2> getThresholds(std::string key, TGraph* graph); + void getGraphsFromObject(TObject* object, std::vector& graphs); + double getInteractionRate(); + + Activity mActivity; + ThresholdsMode mTrendCheckMode{ ExpectedRange }; + int mNPointsForAverage{ 0 }; + std::pair mQualityLabelPosition{ 0.12, 0.8 }; + std::pair mQualityLabelSize{ 0.5, 0.07 }; + std::shared_ptr mThresholds; + std::unordered_map>>> mAverageTrend; + std::unordered_map>>> mThresholdsTrendBad; + std::unordered_map>>> mThresholdsTrendMedium; + std::unordered_map mQualities; +}; + +} // namespace o2::quality_control_modules::common + +#endif // QC_MODULE_COMMON_TRENDCHECK_H diff --git a/Modules/Common/src/TrendCheck.cxx b/Modules/Common/src/TrendCheck.cxx new file mode 100644 index 0000000000..043b9c66a9 --- /dev/null +++ b/Modules/Common/src/TrendCheck.cxx @@ -0,0 +1,466 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file TrendCheck.cxx +/// \author Andrea Ferrero +/// + +#include "Common/TrendCheck.h" +#include "Common/CheckerThresholdsConfig.h" +#include "QualityControl/MonitorObject.h" +#include "QualityControl/Quality.h" +#include "QualityControl/QcInfoLogger.h" +#include +#include +#include +#include +#include +#include +#include + +namespace o2::quality_control_modules::common +{ + +void TrendCheck::configure() +{ +} + +void TrendCheck::startOfActivity(const Activity& activity) +{ + mActivity = activity; + + // initialize the thresholds configuration + mThresholds = std::make_shared(mCustomParameters, mActivity); + + // comparison method between actual values and thresholds + std::string parKey = "trendCheckMode"; + auto parOpt = mCustomParameters.atOptional(parKey, mActivity); + if (!parOpt) { + parOpt = mCustomParameters.atOptional(parKey); + } + mTrendCheckMode = ExpectedRange; + if (parOpt.has_value()) { + if (parOpt.value() == "DeviationFromMean") { + mTrendCheckMode = DeviationFromMean; + } else if (parOpt.value() == "StdDeviation") { + mTrendCheckMode = StdDeviation; + } else if (parOpt.value() != "ExpectedRange") { + ILOG(Warning, Devel) << "unrecognized threshold mode \"" << parOpt.value() << "\", using default \"ExpectedRange\" mode" << ENDM; + } + } + switch (mTrendCheckMode) { + case ExpectedRange: + ILOG(Info, Support) << "thresholds mode set to \"ExpectedRange\"" << ENDM; + break; + case DeviationFromMean: + ILOG(Info, Support) << "thresholds mode set to \"DeviationFromMean\"" << ENDM; + break; + case StdDeviation: + ILOG(Info, Support) << "thresholds mode set to \"StdDeviation\"" << ENDM; + break; + } + + // number of points, excluding the last one, used to compute the average and standard deviation of the trend + // all points are considered if the value is smaller or equal to zero + parKey = "nPointsForAverage"; + parOpt = mCustomParameters.atOptional(parKey, mActivity); + if (!parOpt) { + parOpt = mCustomParameters.atOptional(parKey); + } + if (parOpt.has_value()) { + if (parOpt.value() == "all") { + mNPointsForAverage = 0; + } else { + mNPointsForAverage = std::stoi(parOpt.value()); + } + } + + if (mNPointsForAverage == 0) { + ILOG(Info, Support) << "using all points for statistics calculation" << ENDM; + } else { + ILOG(Info, Support) << "using at most " << mNPointsForAverage << " points for statistics calculation" << ENDM; + } + + // position and size of the label showing the result of the check + parKey = "qualityLabelPosition"; + parOpt = mCustomParameters.atOptional(parKey, mActivity); + if (!parOpt) { + parOpt = mCustomParameters.atOptional(parKey); + } + if (parOpt.has_value()) { + auto position = o2::utils::Str::tokenize(parOpt.value(), ',', false, true); + if (position.size() == 2) { + mQualityLabelPosition = std::make_pair(std::stof(position[0]), std::stof(position[1])); + } + } + + parKey = "qualityLabelSize"; + parOpt = mCustomParameters.atOptional(parKey, mActivity); + if (!parOpt) { + parOpt = mCustomParameters.atOptional(parKey); + } + if (parOpt.has_value()) { + auto position = o2::utils::Str::tokenize(parOpt.value(), ',', false, true); + if (position.size() == 2) { + mQualityLabelSize = std::make_pair(std::stof(position[0]), std::stof(position[1])); + } + } +} + +void TrendCheck::endOfActivity(const Activity& activity) +{ + // reset all members + mActivity = Activity{}; + mAverageTrend.clear(); + mThresholdsTrendBad.clear(); + mThresholdsTrendMedium.clear(); + mQualities.clear(); +} + +static std::string getBaseName(std::string name) +{ + auto pos = name.rfind("/"); + return ((pos < std::string::npos) ? name.substr(pos + 1) : name); +} + +// compute the mean and standard deviation from a given number of points bewfore the last one +// all points are used if nPointsForAverage <= 0 +static std::optional> getGraphStatistics(TGraph* graph, int nPointsForAverage) +{ + std::optional> result; + + int nPoints = graph->GetN(); + // we need at least two points before the last one + if (nPoints < 3) { + return result; + } + + // index of the last-but-one point + int pointIndexMax = nPoints - 2; + // index of the starting point for the mean and stadrad deviation computation + int pointIndexMin = (nPointsForAverage > 0 && pointIndexMax >= nPointsForAverage) ? pointIndexMax - nPointsForAverage + 1 : 0; + // number of points used for the computation, must be greater than one + int N = pointIndexMax - pointIndexMin + 1; + if (N < 2) { + return result; + } + + // compute the mean of the points + double mean = 0; + for (int pointIndex = pointIndexMin; pointIndex <= pointIndexMax; pointIndex++) { + mean += graph->GetPointY(pointIndex); + } + mean /= N; + + // compute the standard deviation of the points + double stdDev = 0; + for (int pointIndex = pointIndexMin; pointIndex <= pointIndexMax; pointIndex++) { + double delta = graph->GetPointY(pointIndex) - mean; + stdDev += delta * delta; + } + stdDev /= (N - 1) * N; + stdDev = std::sqrt(stdDev); + + result = std::make_pair(mean, stdDev); + return result; +} + +double TrendCheck::getInteractionRate() +{ + return 750000; +} + +/// \brief compute the thresholds for a given graph, taking into account the current interaction rate +/// +/// \param key string identifying some plot-specific threshold values +/// \param graph the graph object to be checked +/// \return an array of optional (min,max) threshold values for Bad and Medium qualities +std::array>, 2> TrendCheck::getThresholds(std::string plotName, TGraph* graph) +{ + double rate = getInteractionRate(); + std::array>, 2> result = mThresholds->getThresholdsForPlot(plotName, rate); + if (!result[0]) { + ILOG(Warning, Support) << "Cannot retrieve thresholds for \"" << plotName << "\"" << ENDM; + return result; + } + + if (mTrendCheckMode != ExpectedRange) { + // the thresholds retrieved from the configuration need to be converted into absolute values + auto graphStatistics = getGraphStatistics(graph, mNPointsForAverage); + if (!graphStatistics) { + result[0].reset(); + result[1].reset(); + return result; + } + + if (mTrendCheckMode == DeviationFromMean) { + double mean = graphStatistics.value().first; + // the thresholds retrieved from the configuration are relative to the mean value of the last N points, we need to convert them into absolute values + if (result[0].has_value()) { + result[0].value().first = mean + result[0].value().first * std::fabs(mean); + result[0].value().second = mean + result[0].value().second * std::fabs(mean); + } + + if (result[1].has_value()) { + result[1].value().first = mean + result[1].value().first * std::fabs(mean); + result[1].value().second = mean + result[1].value().second * std::fabs(mean); + } + } else if (mTrendCheckMode == StdDeviation) { + // the thresholds retrieved from the configuration are expressed as number of sigmas from the mean value of the last N points, we need to convert them into absolute values + double mean = graphStatistics.value().first; + double stdDevOfMean = graphStatistics.value().second; + double lastPointValue = graph->GetPointY(graph->GetN() - 1); + double lastPointError = graph->GetErrorY(graph->GetN() - 1); + if (lastPointError < 0) { + lastPointError = 0; + } + const double totalError = sqrt(stdDevOfMean * stdDevOfMean + lastPointError * lastPointError); + + result[0].value().first = mean + result[0].value().first * totalError; + result[0].value().second = mean + result[0].value().second * totalError; + + if (result[1].has_value()) { + result[1].value().first = mean + result[1].value().first * totalError; + result[1].value().second = mean + result[1].value().second * totalError; + } + } + } + + return result; +} + +// helper function to retrieve a TGraph drawn inside a TPad +TGraph* getGraphFromPad(TPad* pad) +{ + if (!pad) { + return nullptr; + } + + TGraph* graph{ nullptr }; + int jList = pad->GetListOfPrimitives()->LastIndex(); + for (; jList >= 0; jList--) { + graph = dynamic_cast(pad->GetListOfPrimitives()->At(jList)); + if (graph) { + break; + } + } + + return graph; +} + +// retrieve a vector containing all the TGraph objects embedded in a given TObject +// the TObject can also be the TGraph itself +void TrendCheck::getGraphsFromObject(TObject* object, std::vector& graphs) +{ + TGraph* graph = dynamic_cast(object); + if (graph) { + graphs.push_back(graph); + } else { + // extract the TGraph from the canvas + TCanvas* canvas = dynamic_cast(object); + auto graph = getGraphFromPad(canvas); + if (graph) { + graphs.push_back(graph); + } else { + ILOG(Info, Devel) << "No TGraph found in the canvas, checking sub-pads" << ENDM; + // loop over the pads in the canvas and extract the TGraph from each pad + const int numberPads = canvas->GetListOfPrimitives()->GetEntries(); + for (int iPad = 0; iPad < numberPads; iPad++) { + auto pad = dynamic_cast(canvas->GetListOfPrimitives()->At(iPad)); + graph = getGraphFromPad(pad); + if (graph) { + graphs.push_back(graph); + } + } + } + } +} + +Quality TrendCheck::check(std::map>* moMap) +{ + for (auto& [moKey, mo] : *moMap) { + + std::vector graphs; + getGraphsFromObject(mo->getObject(), graphs); + if (graphs.empty()) { + continue; + } + + auto moName = mo->getName(); + auto key = getBaseName(moName); + + for (size_t graphIndex = 0; graphIndex < graphs.size(); graphIndex++) { + + TGraph* graph = graphs[graphIndex]; + if (!graph) { + continue; + } + + auto graphName = moName + "_" + std::to_string(graphIndex); + + // check that the graph is not empty + int nPoints = graph->GetN(); + if (nPoints < 1) { + continue; + } + + // get the value for the last point + double value = graph->GetPointY(nPoints - 1); + + // get acceptable range for the current plot + auto thresholds = getThresholds(key, graph); + // check that at least the thresholds for Bad quality are available + if (!thresholds[0]) { + continue; + } + + // Quality is good by default, unless the last point is outside the acceptable range + mQualities[graphName] = Quality::Good; + + mThresholdsTrendBad[graphName].emplace_back(graph->GetPointX(nPoints - 1), thresholds[0].value()); + if (thresholds[1].has_value()) { + mThresholdsTrendMedium[graphName].emplace_back(graph->GetPointX(nPoints - 1), thresholds[1].value()); + } + + if (value < thresholds[0]->first || value > thresholds[0]->second) { + mQualities[graphName] = Quality::Bad; + } else if (thresholds[1].has_value()) { + if (value < thresholds[1]->first || value > thresholds[1]->second) { + mQualities[graphName] = Quality::Medium; + } + } + } + } + + // we return the worse quality of all the objects we checked, but we preserve all FlagTypes + Quality result = mQualities.empty() ? Quality::Null : Quality::Good; + for (auto& [key, quality] : mQualities) { + (void)key; + for (const auto& flag : quality.getFlags()) { + result.addFlag(flag.first, flag.second); + } + if (quality.isWorseThan(result)) { + result.set(quality); + } + } + + return result; +} + +std::string TrendCheck::getAcceptedType() { return "TObject"; } + +static void drawThresholds(TGraph* graph, const std::vector>>& thresholds, int lineColor, int lineStyle) +{ + if (thresholds.empty()) { + return; + } + + double rangeMin = graph->GetMinimum(); + double rangeMax = graph->GetMaximum(); + + rangeMin = TMath::Min(rangeMin, TMath::MinElement(graph->GetN(), graph->GetY())); + rangeMax = TMath::Max(rangeMax, TMath::MaxElement(graph->GetN(), graph->GetY())); + + double* xValues = new double[thresholds.size()]; + double* yValuesMin = new double[thresholds.size()]; + double* yValuesMax = new double[thresholds.size()]; + + for (size_t index = 0; index < thresholds.size(); index++) { + const auto& thresholdsPoint = thresholds[index]; + xValues[index] = thresholdsPoint.first; + yValuesMin[index] = thresholdsPoint.second.first; + yValuesMax[index] = thresholdsPoint.second.second; + + // add some margin above an below the threshold values, if needed + auto delta = yValuesMax[index] - yValuesMin[index]; + rangeMin = TMath::Min(rangeMin, yValuesMin[index] - 0.1 * delta); + rangeMax = TMath::Max(rangeMax, yValuesMax[index] + 0.1 * delta); + } + + TPolyLine* lineMin = new TPolyLine(thresholds.size(), xValues, yValuesMin); + lineMin->SetLineColor(lineColor); + lineMin->SetLineStyle(lineStyle); + lineMin->SetLineWidth(2); + graph->GetListOfFunctions()->Add(lineMin); + + TPolyLine* lineMax = new TPolyLine(thresholds.size(), xValues, yValuesMax); + lineMax->SetLineColor(lineColor); + lineMax->SetLineStyle(lineStyle); + lineMax->SetLineWidth(2); + graph->GetListOfFunctions()->Add(lineMax); + + graph->SetMinimum(rangeMin); + graph->SetMaximum(rangeMax); +} + +// return the ROOT color index associated to a give quality level +static int getQualityColor(const Quality& q) +{ + if (q == Quality::Null) + return kViolet - 6; + if (q == Quality::Bad) + return kRed; + if (q == Quality::Medium) + return kOrange - 3; + if (q == Quality::Good) + return kGreen + 2; + + return 0; +} + +void TrendCheck::beautify(std::shared_ptr mo, Quality checkResult) +{ + std::vector graphs; + getGraphsFromObject(mo->getObject(), graphs); + if (graphs.empty()) { + return; + } + + auto moName = mo->getName(); + + for (size_t graphIndex = 0; graphIndex < graphs.size(); graphIndex++) { + + TGraph* graph = graphs[graphIndex]; + if (!graph || graph->GetN() < 1) { + return; + } + + auto graphName = moName + "_" + std::to_string(graphIndex); + + Quality quality = mQualities[graphName]; + + if (mThresholdsTrendMedium.count(graphName) > 0) { + const auto& thresholds = mThresholdsTrendMedium[graphName]; + drawThresholds(graph, thresholds, kOrange, 9); + } + + if (mThresholdsTrendBad.count(graphName) > 0) { + const auto& thresholds = mThresholdsTrendBad[graphName]; + drawThresholds(graph, thresholds, kRed, 9); + } + + // draw the label with the check result + TPaveText* label = new TPaveText(mQualityLabelPosition.first, + mQualityLabelPosition.second, + mQualityLabelPosition.first + mQualityLabelSize.first, + mQualityLabelPosition.second + mQualityLabelSize.second, + "brNDC"); + label->SetTextColor(getQualityColor(quality)); + label->AddText(quality.getName().c_str()); + label->SetFillStyle(0); + label->SetBorderSize(0); + label->SetTextAlign(12); + graph->GetListOfFunctions()->Add(label); + } +} + +} // namespace o2::quality_control_modules::common