From 03b2e9d64eefeef9be706d89d688dae373eb01d1 Mon Sep 17 00:00:00 2001 From: Adam Wegrzynek Date: Fri, 25 Jan 2019 09:15:45 +0100 Subject: [PATCH] Introduce tag numbering (#112) * Introduce tag numbering * Adapt docs * Adapt metric tests * Add freely global tags --- README.md | 6 +-- examples/2-TaggedMetrics.cxx | 7 ++-- examples/4-RateDerivedMetric.cxx | 2 +- examples/7-InternalBenchamrk.cxx | 6 +-- include/Monitoring/Metric.h | 26 +++++++++++-- include/Monitoring/Monitoring.h | 7 ++-- include/Monitoring/Tags.h | 66 +++++++++++++++++--------------- src/Backends/Flume.cxx | 8 ++-- src/Backends/InfluxDB.cxx | 4 +- src/Backends/StdOut.cxx | 10 ++--- src/DerivedMetrics.cxx | 16 +++++--- src/Metric.cxx | 16 +++++++- src/Monitoring.cxx | 13 ++++--- test/testDerived.cxx | 2 +- test/testMetric.cxx | 12 +++--- test/testMonitoring.cxx | 3 +- 16 files changed, 126 insertions(+), 78 deletions(-) diff --git a/README.md b/README.md index fcfc9c2bc..f15060500 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,7 @@ A metric consist of 5 parameters: name, value, timestamp, verbosity and tags. | verbosity | DEBUG / INFO / PROD | no | INFO | | tags | vector | no | - | -A metric can be constructed by providing required parameters: +A metric can be constructed by providing required parameters (value and name): ```cpp Metric{10, "name"} ``` @@ -108,7 +108,7 @@ Metrics need to match backends verbosity in order to be sent, eg. backend with ` #### Tags Each metric can be tagged with any number of [predefined tags](include/Monitoring/Tags.h). -In order to do so use `addTags(std::initializer_list&& tags)` method. +In order to do so use `addTag(tags::Key, tags::Value)` or `addTag(tags::Key, unsigned short)` methods. The latter method allows assigning numeric value to a tag. See the example: [examples/2-TaggedMetrics.cxx](examples/2-TaggedMetrics.cxx). @@ -165,7 +165,7 @@ Glabal tags are tags that are added to each metric. The following tags are set t - `hostname` - `name` - process name -You can add your own global tag by calling `addGlobalTag(std::string name, std::string value)`. +You can add your own global tag by calling `addGlobalTag(std::string_view key, std::string_view value)` or `addGlobalTag(tags::Key, tags::Value)`. ### Process monitoring ```cpp diff --git a/examples/2-TaggedMetrics.cxx b/examples/2-TaggedMetrics.cxx index 2e298c041..252f626d1 100644 --- a/examples/2-TaggedMetrics.cxx +++ b/examples/2-TaggedMetrics.cxx @@ -14,12 +14,13 @@ int main() { auto monitoring = MonitoringFactory::Get("stdout://"); /// Add global tags - monitoring->addGlobalTag("example", "yes"); - monitoring->addGlobalTag(tags::Subsystem::DPL); + monitoring->addGlobalTag("name", "test"); + monitoring->addGlobalTag(tags::Key::Subsystem, tags::Value::DPL); // now send an application specific metric with additional tags // 10 is the value // myMetric is the name of the metric // then add predefined tag - monitoring->send(Metric{10, "myMetric"}.addTags({tags::Detector::TPC})); + monitoring->send(Metric{10, "myMetric"}.addTag(tags::Key::Detector, tags::Value::TPC)); + monitoring->send(Metric{10, "myMetric"}.addTag(tags::Key::CRU, 123)); } diff --git a/examples/4-RateDerivedMetric.cxx b/examples/4-RateDerivedMetric.cxx index afefaf52d..ed3157eba 100644 --- a/examples/4-RateDerivedMetric.cxx +++ b/examples/4-RateDerivedMetric.cxx @@ -14,7 +14,7 @@ int main() { // now send at least two metrics to see the result for (int i = 0; i < 101; i += 10) { - monitoring->send(Metric{i, "myMetric"}.addTags({tags::Subsystem::Readout}), DerivedMetricMode::RATE); + monitoring->send(Metric{i, "myMetric"}.addTag(tags::Key::Subsystem, tags::Value::Readout), DerivedMetricMode::RATE); std::this_thread::sleep_for(std::chrono::milliseconds(100)); } } diff --git a/examples/7-InternalBenchamrk.cxx b/examples/7-InternalBenchamrk.cxx index 3489018e9..dfa71443c 100644 --- a/examples/7-InternalBenchamrk.cxx +++ b/examples/7-InternalBenchamrk.cxx @@ -17,10 +17,10 @@ void test(std::unique_ptr& monitoring) { } void testWithTags(std::unique_ptr& monitoring) { - monitoring->addGlobalTag("benchmark", "yes"); + monitoring->addGlobalTag("name", "benchmark"); for (int i = 0; i < 100000; i++) { - monitoring->send(Metric{10, "myMetricInt"}.addTags({tags::Detector::TPC})); - monitoring->send(Metric{10.10, "myMetricFloat"}.addTags({tags::Subsystem::QC})); + monitoring->send(Metric{10, "myMetricInt"}.addTag(tags::Key::Detector, tags::Value::TPC)); + monitoring->send(Metric{10.10, "myMetricFloat"}.addTag(tags::Key::Subsystem, tags::Value::QC)); } } diff --git a/include/Monitoring/Metric.h b/include/Monitoring/Metric.h index 94b5dd54b..27c6198e4 100644 --- a/include/Monitoring/Metric.h +++ b/include/Monitoring/Metric.h @@ -18,10 +18,15 @@ namespace o2 namespace monitoring { +/// Metric and Backedn verbosity enum class Verbosity : short { PROD, INFO, DEBUG }; + +/// Metric types enum MetricType { INT = 0, STRING = 1, DOUBLE = 2, UINT64_T = 3 }; +class DerivedMetrics; + /// \brief Represents a metric including value, type of the value, name, timestamp and tags class Metric { @@ -72,12 +77,19 @@ class Metric /// Tag list getter /// \return tags - const std::vector& getTags() const; + const std::vector>& getTags() const; + + /// Add user defined tags + /// \param key enum tag key + /// \param value emum tag value + /// \return r-value to "this" - to be able to chain methods + Metric&& addTag(tags::Key key, tags::Value value); /// Add user defined tags - /// \param tags r-value to vector of tags + /// \param key enum tag key + /// \param value numeric value /// \return r-value to "this" - to be able to chain methods - Metric&& addTags(std::vector&& tags); + Metric&& addTag(tags::Key key, unsigned short int number); /// Verbosity getter Verbosity getVerbosity(); @@ -92,6 +104,12 @@ class Metric /// Default metric verbosity static Verbosity DEFAULT_VERBOSITY; protected: + /// Allow DerivedMetrics access to setTags + friend class o2::monitoring::DerivedMetrics; + + /// Set full vector of tags + Metric&& setTags(std::vector>&& tags); + /// Metric value boost::variant< int, std::string, double, uint64_t > mValue; @@ -102,7 +120,7 @@ class Metric std::chrono::time_point mTimestamp; /// Metric tags - std::vector mTags; + std::vector> mTags; /// Metric verbosity Verbosity mVerbosity; diff --git a/include/Monitoring/Monitoring.h b/include/Monitoring/Monitoring.h index 63622f3a2..6c05134fb 100644 --- a/include/Monitoring/Monitoring.h +++ b/include/Monitoring/Monitoring.h @@ -81,9 +81,10 @@ class Monitoring /// \param value tag value void addGlobalTag(std::string_view name, std::string_view value); - /// Adds predefined global tag - /// \param tag tag index (use predefined enums form tag:: namespace) - void addGlobalTag(const unsigned int tag); + /// Adds global tag + /// \param name tag name + /// \param value tag value + void addGlobalTag(tags::Key key, tags::Value value); /// Returns a metric which will be periodically sent to backends /// \param name metric name diff --git a/include/Monitoring/Tags.h b/include/Monitoring/Tags.h index b887d63c1..e6d28e7d9 100644 --- a/include/Monitoring/Tags.h +++ b/include/Monitoring/Tags.h @@ -18,37 +18,43 @@ namespace tags { using namespace std::string_view_literals; - // Detector tag indexes - static constexpr std::string_view detectorTag = "detector"; - enum Detector { ACO = 0 , AD, CPV, EMC, FMD, HMP, MCH, MTR, PHS, PMD, ITS, T0, TOF, TPC, TRD, V0}; - - // Subsystem tag indexes - static constexpr std::string_view subsystemTag = "subsystem"; - enum Subsystem { QC = 16, Readout, DPL, CRU }; - - // Single tag array - static constexpr std::array, 20> TAG_ARRAY = {{ - {detectorTag, "ACO"sv}, // 0 - {detectorTag, "AD"sv}, // 1 - {detectorTag, "CPV"sv}, // 2 - {detectorTag, "EMC"sv}, // 3 - {detectorTag, "FMD"sv}, // 4 - {detectorTag, "HMP"sv}, // 5 - {detectorTag, "MCH"sv}, // 6 - {detectorTag, "MTR"sv}, // 7 - {detectorTag, "PHS"sv}, // 8 - {detectorTag, "PHS"sv}, // 9 - {detectorTag, "PMD"sv}, // 10 - {detectorTag, "ITS"sv}, // 11 - {detectorTag, "TOF"sv}, // 12 - {detectorTag, "TPC"sv}, // 13 - {detectorTag, "TRD"sv}, // 14 - {detectorTag, "V0"sv}, // 15 - {subsystemTag, "QC"sv}, - {subsystemTag, "Readout"sv}, - {subsystemTag, "DLP"sv}, - {subsystemTag, "CRU"sv} + // Tag keys + enum class Key : unsigned short int { Hostname, Rolename, Name, Detector, Subsystem, CRU, FLP, EPN }; + + /// Tag keys array + static constexpr std::array TAG_KEY = { + "hostname"sv, "rolenane"sv, "name"sv, "detector"sv, "subsystem"sv, "CRU"sv, "FLP"sv, "EPN"sv + }; + + // Tag values + enum class Value : unsigned short int { ACO, AD, CPV, EMC, FMD, HMP, MCH, MTR, PHS, PMD, ITS, T0, TOF, TPC, TRD, V0, QC, Readout, DPL, CRU}; + + // Tag value array + static constexpr std::array TAG_VALUE = {{ + "ACO"sv, + "AD"sv, // 1 + "CPV"sv, // 2 + "EMC"sv, // 3 + "FMD"sv, // 4 + "HMP"sv, // 5 + "MCH"sv, // 6 + "MTR"sv, // 7 + "PHS"sv, // 8 + "PHS"sv, // 9 + "PMD"sv, // 10 + "ITS"sv, // 11 + "TOF"sv, // 12 + "TPC"sv, // 13 + "TRD"sv, // 14 + "V0"sv, // 15 + "QC"sv, + "Readout"sv, + "DPL"sv, + "CRU"sv }}; + static constexpr std::string_view GetValue(const int value) { + return value >= 0 ? TAG_VALUE[value] : std::to_string(0 - value); + } } } // namespace monitoring diff --git a/src/Backends/Flume.cxx b/src/Backends/Flume.cxx index ba066362b..a9c7975fb 100644 --- a/src/Backends/Flume.cxx +++ b/src/Backends/Flume.cxx @@ -33,8 +33,8 @@ std::string Flume::metricToJson(const Metric& metric) header.put("name", metric.getName()); header.put("value_value", boost::lexical_cast(metric.getValue())); - for (const auto& tagIndex : metric.getTags()) { - header.put("tag_" + std::string(tags::TAG_ARRAY[tagIndex].first.data()), tags::TAG_ARRAY[tagIndex].second.data()); + for (const auto& [key, value] : metric.getTags()) { + header.put("tag_" + std::string(tags::TAG_KEY[key].data()), tags::GetValue(value).data()); } event.push_back(std::make_pair("headers", header)); event.put("body", ""); @@ -67,8 +67,8 @@ std::string Flume::metricsToJson(std::string measurement, std::vector&& boost::property_tree::ptree header = globalHeader; header.put("timestamp", std::to_string(convertTimestamp(metrics.front().getTimestamp()))); header.put("name", measurement); - for (const auto& tagIndex : metrics.front().getTags()) { - header.put("tag_" + std::string(tags::TAG_ARRAY[tagIndex].first.data()), tags::TAG_ARRAY[tagIndex].second.data()); + for (const auto& [key, value] : metrics.front().getTags()) { + header.put("tag_" + std::string(tags::TAG_KEY[key].data()), tags::GetValue(value).data()); } for (auto& metric : metrics) { header.put("value_" + metric.getName(), boost::lexical_cast(metric.getValue())); diff --git a/src/Backends/InfluxDB.cxx b/src/Backends/InfluxDB.cxx index 1db4a4198..fc7fb81a5 100644 --- a/src/Backends/InfluxDB.cxx +++ b/src/Backends/InfluxDB.cxx @@ -97,8 +97,8 @@ std::string InfluxDB::toInfluxLineProtocol(const Metric& metric) { escape(name); convert << name << "," << tagSet; - for (const auto& tagIndex : metric.getTags()) { - convert << "," << tags::TAG_ARRAY[tagIndex].first << "=" << tags::TAG_ARRAY[tagIndex].second; + for (const auto& [key, value] : metric.getTags()) { + convert << "," << tags::TAG_KEY[key] << "=" << tags::GetValue(value); } std::string value = boost::lexical_cast(metric.getValue()); diff --git a/src/Backends/StdOut.cxx b/src/Backends/StdOut.cxx index 339d11772..ebcc27edd 100644 --- a/src/Backends/StdOut.cxx +++ b/src/Backends/StdOut.cxx @@ -47,11 +47,11 @@ void StdOut::send(std::vector&& metrics) { void StdOut::sendMultiple(std::string measurement, std::vector&& metrics) { std::string metricTags{}; - for (const auto& tagIndex : metrics.front().getTags()) { + for (const auto& [key, value] : metrics.front().getTags()) { metricTags += ','; - metricTags += tags::TAG_ARRAY[tagIndex].first; + metricTags += tags::TAG_KEY[key]; metricTags += "="; - metricTags += tags::TAG_ARRAY[tagIndex].second; + metricTags += tags::GetValue(value); } for (auto& metric : metrics) { mStream << "[METRIC] " << measurement << '/' << metric.getName() << ',' << metric.getType() << ' ' @@ -65,8 +65,8 @@ void StdOut::send(const Metric& metric) mStream << "[METRIC] " << metric.getName() << ',' << metric.getType() << " " << metric.getValue() << ' ' << convertTimestamp(metric.getTimestamp()) << ' ' << tagString; - for (const auto& tagIndex : metric.getTags()) { - mStream << ',' << tags::TAG_ARRAY[tagIndex].first << "=" << tags::TAG_ARRAY[tagIndex].second; + for (const auto& [key, value] : metric.getTags()) { + mStream << ',' << tags::TAG_KEY[key] << "=" << tags::GetValue(value); } mStream << '\n'; } diff --git a/src/DerivedMetrics.cxx b/src/DerivedMetrics.cxx index f51fbb2c7..ab008fb5b 100644 --- a/src/DerivedMetrics.cxx +++ b/src/DerivedMetrics.cxx @@ -27,14 +27,17 @@ Metric DerivedMetrics::process(Metric& metric, DerivedMetricMode mode) { { DerivedMetricMode::INCREMENT, [this](Metric& metric) { auto tags = metric.getTags(); - std::string key = metric.getName() + std::string(tags.begin(), tags.end()); + std::string key = metric.getName(); + std::for_each(tags.begin(), tags.end(), [&key](auto const & pair) { + key += pair.second; + }); auto search = mStorage.find(key); if (search != mStorage.end()) { auto currentValue = metric.getValue(); auto storedValue = search->second.getValue(); auto value = boost::apply_visitor(VariantVisitorAdd(), currentValue, storedValue); mStorage.erase(search); - Metric result = Metric{value, metric.getName() + "Increment", metric.getVerbosity()}.addTags(std::move(tags)); + Metric result = Metric{value, metric.getName() + "Increment", metric.getVerbosity()}.setTags(std::move(tags)); mStorage.insert(std::make_pair(key, result)); return result; } @@ -45,7 +48,10 @@ Metric DerivedMetrics::process(Metric& metric, DerivedMetricMode mode) { DerivedMetricMode::RATE, [this](Metric& metric) { // disallow string auto tags = metric.getTags(); - std::string key = metric.getName() + std::string(tags.begin(), tags.end()); + std::string key = metric.getName(); + std::for_each(tags.begin(), tags.end(), [&key](auto const & pair) { + key += pair.second; + }); if (metric.getType() == MetricType::STRING) { throw MonitoringException("DerivedMetrics", "Not able to process string values"); } @@ -54,7 +60,7 @@ Metric DerivedMetrics::process(Metric& metric, DerivedMetricMode mode) { auto search = mStorage.find(key); if (search == mStorage.end()) { mStorage.insert(std::make_pair(key, metric)); - return Metric{(double) 0.0, metric.getName() + "Rate", metric.getVerbosity()}.addTags(std::move(tags)); + return Metric{(double) 0.0, metric.getName() + "Rate", metric.getVerbosity()}.setTags(std::move(tags)); } auto timestampDifference = std::chrono::duration_cast( @@ -74,7 +80,7 @@ Metric DerivedMetrics::process(Metric& metric, DerivedMetricMode mode) { // swap metrics mStorage.erase(key); mStorage.insert(std::make_pair(key, metric)); - return Metric{rate, metric.getName() + "Rate", metric.getVerbosity()}.addTags(std::move(tags)); + return Metric{rate, metric.getName() + "Rate", metric.getVerbosity()}.setTags(std::move(tags)); } } }; diff --git a/src/Metric.cxx b/src/Metric.cxx index a6aa66e10..53184416a 100644 --- a/src/Metric.cxx +++ b/src/Metric.cxx @@ -61,13 +61,25 @@ Verbosity Metric::getVerbosity() return mVerbosity; } -Metric&& Metric::addTags(std::vector&& tags) +Metric&& Metric::addTag(tags::Key key, tags::Value value) +{ + mTags.push_back({static_cast::type>(key), static_cast::type>(value)}); + return std::move(*this); +} + +Metric&& Metric::addTag(tags::Key key, unsigned short number) +{ + mTags.push_back({static_cast::type>(key), 0 - number}); + return std::move(*this); +} + +Metric&& Metric::setTags(std::vector>&& tags) { mTags = std::move(tags); return std::move(*this); } -const std::vector& Metric::getTags() const +const std::vector>& Metric::getTags() const { return mTags; } diff --git a/src/Monitoring.cxx b/src/Monitoring.cxx index 0dad705fa..96657a3f9 100644 --- a/src/Monitoring.cxx +++ b/src/Monitoring.cxx @@ -37,7 +37,7 @@ void Monitoring::enableBuffering(const std::size_t size) { mBufferSize = size; mBuffering = true; - for (short i = 0; i < static_cast::type>(Verbosity::DEBUG); i++) { + for (std::underlying_type::type i = 0; i < static_cast::type>(Verbosity::DEBUG); i++) { mStorage[i].reserve(size); } MonLogger::Get() << "Buffering enabled (" << mStorage[0].capacity() << ")" << MonLogger::End(); @@ -77,17 +77,20 @@ void Monitoring::enableProcessMonitoring(const unsigned int interval) { #endif } -void Monitoring::addGlobalTag(std::string_view name, std::string_view value) +void Monitoring::addGlobalTag(std::string_view key, std::string_view value) { for (auto& backend: mBackends) { - backend->addGlobalTag(name, value); + backend->addGlobalTag(key, value); } } -void Monitoring::addGlobalTag(const unsigned int tag) +void Monitoring::addGlobalTag(tags::Key key, tags::Value value) { for (auto& backend: mBackends) { - backend->addGlobalTag(tags::TAG_ARRAY[tag].first, tags::TAG_ARRAY[tag].second); + backend->addGlobalTag( + tags::TAG_KEY[static_cast::type>(key)], + tags::TAG_VALUE[static_cast::type>(value)] + ); } } diff --git a/test/testDerived.cxx b/test/testDerived.cxx index 1a673dac2..35264fb3b 100644 --- a/test/testDerived.cxx +++ b/test/testDerived.cxx @@ -51,7 +51,7 @@ BOOST_AUTO_TEST_CASE(derivedRateDouble) { try { std::this_thread::sleep_for(std::chrono::milliseconds(100)); o2::monitoring::Metric metric(results[i].value, name); - o2::monitoring::Metric metricTagged = Metric{resultsTagged[i].value, name}.addTags({o2::monitoring::tags::Subsystem::Readout}); + o2::monitoring::Metric metricTagged = Metric{resultsTagged[i].value, name}.addTag(o2::monitoring::tags::Key::Subsystem, o2::monitoring::tags::Value::Readout); o2::monitoring::Metric derived = derivedHandler.process(metric, DerivedMetricMode::RATE); o2::monitoring::Metric derivedTagged = derivedHandler.process(metricTagged, DerivedMetricMode::RATE); BOOST_CHECK_EQUAL(derived.getName(), "metricDoubleRate"); diff --git a/test/testMetric.cxx b/test/testMetric.cxx index b0485b7a3..84cc29aa7 100644 --- a/test/testMetric.cxx +++ b/test/testMetric.cxx @@ -75,17 +75,17 @@ BOOST_AUTO_TEST_CASE(retrieveWrongType) { } BOOST_AUTO_TEST_CASE(tags) { - Metric metric = Metric{10, "myMetric"}.addTags({o2::monitoring::tags::Detector::TPC, o2::monitoring::tags::Detector::TRD}); + Metric metric = Metric{10, "myMetric"}.addTag(o2::monitoring::tags::Key::Detector, o2::monitoring::tags::Value::TRD); auto tags = metric.getTags(); int sum = 0; for (auto const& tag: tags) { - sum += tag; + sum += tag.second; } - BOOST_CHECK_EQUAL(sum, 27); + BOOST_CHECK_EQUAL(sum, 14); } BOOST_AUTO_TEST_CASE(customCopyConstructor) { - Metric metric = Metric{10, "myMetric"}.addTags({o2::monitoring::tags::Detector::TPC, o2::monitoring::tags::Detector::TRD}); + Metric metric = Metric{10, "myMetric"}.addTag(o2::monitoring::tags::Key::Detector, 123).addTag(o2::monitoring::tags::Key::Detector, o2::monitoring::tags::Value::TRD); Metric assigned{1, "assingedMetric"}; auto copied = metric; assigned = metric; @@ -98,9 +98,9 @@ BOOST_AUTO_TEST_CASE(customCopyConstructor) { auto tags = copied.getTags(); int sum = 0; for (auto const& tag: tags) { - sum += tag; + sum += tag.second; } - BOOST_CHECK_EQUAL(sum, 27); + BOOST_CHECK_EQUAL(sum, -109); } BOOST_AUTO_TEST_CASE(verbosity) { diff --git a/test/testMonitoring.cxx b/test/testMonitoring.cxx index 33ab51da1..37424a990 100644 --- a/test/testMonitoring.cxx +++ b/test/testMonitoring.cxx @@ -22,7 +22,8 @@ BOOST_AUTO_TEST_CASE(createMonitoring) std::string stringMetric("monitoringString"); double doubleMetric = static_cast (rand()) / static_cast (RAND_MAX); - monitoring->addGlobalTag("device", "sampleDevice"); + monitoring->addGlobalTag("name", "Readout"); + monitoring->addGlobalTag(tags::Key::Name, tags::Value::Readout); monitoring->send({intMetric, "myCrazyMetricI"}); monitoring->send({stringMetric, "myCrazyMetricS"});