From bf4a3a55f08f2df3de98fd531b50386f2e9ca63a Mon Sep 17 00:00:00 2001 From: Costa Tsaousis Date: Thu, 12 Dec 2024 18:20:43 +0200 Subject: [PATCH] Streaming Improvements No 5 (#19193) * rrdhost state id is now used to detect not available functions * acquire release for rrdhost state * initialize rddhost state for local hosts * track send misses * log for functions that return 503 * fix rrd_collector_finished() call from stream threads --- CMakeLists.txt | 2 + src/collectors/ebpf.plugin/ebpf_socket.c | 1 - src/database/rrd.h | 7 ++- src/database/rrdfunctions-internals.h | 1 + src/database/rrdfunctions.c | 33 ++++++++++- src/database/rrdhost-state-id.c | 73 ++++++++++++++++++++++++ src/database/rrdhost-state-id.h | 19 ++++++ src/database/rrdhost.c | 3 + src/libnetdata/common.h | 4 ++ src/plugins.d/pluginsd_parser.c | 27 +++++---- src/plugins.d/pluginsd_parser.h | 2 +- src/streaming/stream-receiver.c | 9 ++- src/streaming/stream-sender.c | 4 +- src/streaming/stream-thread.c | 14 ++++- src/streaming/stream-thread.h | 4 +- src/web/api/queries/backfill.c | 11 ++-- 16 files changed, 182 insertions(+), 32 deletions(-) create mode 100644 src/database/rrdhost-state-id.c create mode 100644 src/database/rrdhost-state-id.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 3f1d42ad4faa0f..a51827c8eeb694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1431,6 +1431,8 @@ set(RRD_PLUGIN_FILES src/database/rrdcollector-internals.h src/database/rrd-database-mode.h src/database/rrd-database-mode.c + src/database/rrdhost-state-id.c + src/database/rrdhost-state-id.h ) if(ENABLE_DBENGINE) diff --git a/src/collectors/ebpf.plugin/ebpf_socket.c b/src/collectors/ebpf.plugin/ebpf_socket.c index f0d376f43a92d3..a587e1b5dd059e 100644 --- a/src/collectors/ebpf.plugin/ebpf_socket.c +++ b/src/collectors/ebpf.plugin/ebpf_socket.c @@ -1759,7 +1759,6 @@ end_socket_loop: ; // the empty statement is here to allow code to be compiled b else { ebpf_release_pid_data(local_pid, fd, key.pid, EBPF_MODULE_SOCKET_IDX); ebpf_socket_release_publish(curr); - local_pid->socket = NULL; } memset(values, 0, length); memcpy(&key, &next_key, sizeof(key)); diff --git a/src/database/rrd.h b/src/database/rrd.h index 319a89dcf6ab3f..e41816e4e174c8 100644 --- a/src/database/rrd.h +++ b/src/database/rrd.h @@ -11,6 +11,7 @@ extern "C" { #include "rrd-database-mode.h" #include "streaming/stream-traffic-types.h" #include "streaming/stream-sender-commit.h" +#include "rrdhost-state-id.h" // non-existing structs instead of voids // to enable type checking at compile time @@ -1161,6 +1162,11 @@ struct rrdhost { STRING *program_name; // the program name that collects metrics for this host STRING *program_version; // the program version that collects metrics for this host + REFCOUNT state_refcount; + RRDHOST_STATE state_id; // every time data collection (stream receiver) (dis)connects, + // this gets incremented - it is used to detect stale functions, + // stale backfilling requests, etc. + int32_t utc_offset; // the offset in seconds from utc RRDHOST_OPTIONS options; // configuration option for this RRDHOST (no atomics on this) @@ -1238,7 +1244,6 @@ struct rrdhost { struct { pid_t tid; - uint32_t state_id; // every time the receiver connects/disconnects, this is incremented time_t last_connected; // the time the last sender was connected time_t last_disconnected; // the time the last sender was disconnected diff --git a/src/database/rrdfunctions-internals.h b/src/database/rrdfunctions-internals.h index 79eb52aa481266..f748e3d18df3de 100644 --- a/src/database/rrdfunctions-internals.h +++ b/src/database/rrdfunctions-internals.h @@ -29,6 +29,7 @@ struct rrd_host_function { rrd_function_execute_cb_t execute_cb; void *execute_cb_data; + RRDHOST_STATE rrdhost_state_id; struct rrd_collector *collector; }; diff --git a/src/database/rrdfunctions.c b/src/database/rrdfunctions.c index 508ec98f68a1c8..ed6e583bdd3f88 100644 --- a/src/database/rrdfunctions.c +++ b/src/database/rrdfunctions.c @@ -15,11 +15,12 @@ // ---------------------------------------------------------------------------- static void rrd_functions_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, void *func, void *rrdhost) { - RRDHOST *host = rrdhost; (void)host; + RRDHOST *host = rrdhost; struct rrd_host_function *rdcf = func; rrd_collector_started(); rdcf->collector = rrd_collector_acquire_current_thread(); + rdcf->rrdhost_state_id = rrdhost_state_id(host); if(!rdcf->priority) rdcf->priority = RRDFUNCTIONS_PRIORITY_DEFAULT; @@ -57,6 +58,17 @@ static bool rrd_functions_conflict_callback(const DICTIONARY_ITEM *item __maybe_ changed = true; } + if(rdcf->rrdhost_state_id != rrdhost_state_id(host)) { + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "FUNCTIONS: function '%s' of host '%s' changed state id from %u to %u", + dictionary_acquired_item_name(item), rrdhost_hostname(host), + rdcf->rrdhost_state_id, + rrdhost_state_id(host)); + + rdcf->rrdhost_state_id = rrdhost_state_id(host); + changed = true; + } + if(rdcf->execute_cb != new_rdcf->execute_cb) { nd_log(NDLS_DAEMON, NDLP_DEBUG, "FUNCTIONS: function '%s' of host '%s' changed execute callback", @@ -260,6 +272,8 @@ int rrd_functions_find_by_name(RRDHOST *host, BUFFER *wb, const char *name, size strncpyz(buffer, name, sizeof(buffer) - 1); char *s = NULL; + RRDHOST_STATE state_id = rrdhost_state_id(host); + bool found = false; *item = NULL; if(host->functions) { @@ -268,10 +282,23 @@ int rrd_functions_find_by_name(RRDHOST *host, BUFFER *wb, const char *name, size found = true; struct rrd_host_function *rdcf = dictionary_acquired_item_value(*item); - if(rrd_collector_running(rdcf->collector)) { + if(rrd_collector_running(rdcf->collector) && rdcf->rrdhost_state_id == state_id) { break; } else { + + nd_log(NDLS_DAEMON, NDLP_DEBUG, + "Function '%s' is not available. " + "host '%s', collector = { tid: %d, running: %s }, host tid { rcv: %d, snd: %d }, host state { id: %u, expected %u }, hops: %d", + name, + rrdhost_hostname(host), + rrd_collector_tid(rdcf->collector), + rrd_collector_running(rdcf->collector) ? "yes" : "no", + host->stream.rcv.status.tid, host->stream.snd.status.tid, + state_id, rdcf->rrdhost_state_id, + host->system_info->hops + ); + dictionary_acquired_item_release(host->functions, *item); *item = NULL; } @@ -314,7 +341,7 @@ bool rrd_function_available(RRDHOST *host, const char *function) { const DICTIONARY_ITEM *item = dictionary_get_and_acquire_item(host->functions, function); if(item) { struct rrd_host_function *rdcf = dictionary_acquired_item_value(item); - if(rrd_collector_running(rdcf->collector)) + if(rrd_collector_running(rdcf->collector) && rdcf->rrdhost_state_id == rrdhost_state_id(host)) ret = true; dictionary_acquired_item_release(host->functions, item); diff --git a/src/database/rrdhost-state-id.c b/src/database/rrdhost-state-id.c new file mode 100644 index 00000000000000..f3b4830a78979c --- /dev/null +++ b/src/database/rrdhost-state-id.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "rrdhost-state-id.h" +#include "rrd.h" + +RRDHOST_STATE rrdhost_state_id(struct rrdhost *host) { + return __atomic_load_n(&host->state_id, __ATOMIC_RELAXED); +} + +bool rrdhost_state_connected(RRDHOST *host) { + __atomic_add_fetch(&host->state_id, 1, __ATOMIC_RELAXED); + + int32_t expected = __atomic_load_n(&host->state_refcount, __ATOMIC_RELAXED); + int32_t desired; + + do { + if(expected >= 0) { + internal_fatal(true, "Cannot get the node connected"); + return false; + } + + desired = 0; + + } while(!__atomic_compare_exchange_n( + &host->state_refcount, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + return true; +} + +bool rrdhost_state_disconnected(RRDHOST *host) { + __atomic_add_fetch(&host->state_id, 1, __ATOMIC_RELAXED); + + int32_t expected = __atomic_load_n(&host->state_refcount, __ATOMIC_RELAXED); + int32_t desired; + + do { + if(expected < 0) { + internal_fatal(true, "Cannot get the node disconnected"); + return false; + } + + desired = -1; + + } while(!__atomic_compare_exchange_n( + &host->state_refcount, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + return true; +} + +bool rrdhost_state_acquire(RRDHOST *host, RRDHOST_STATE wanted_state_id) { + int32_t expected = __atomic_load_n(&host->state_refcount, __ATOMIC_RELAXED); + int32_t desired; + + do { + if(expected < 0) + return false; + + desired = expected + 1; + + } while(!__atomic_compare_exchange_n( + &host->state_refcount, &expected, desired, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); + + if(rrdhost_state_id(host) != wanted_state_id) { + rrdhost_state_release(host); + return false; + } + + return true; +} + +void rrdhost_state_release(RRDHOST *host) { + __atomic_sub_fetch(&host->state_refcount, 1, __ATOMIC_RELAXED); +} diff --git a/src/database/rrdhost-state-id.h b/src/database/rrdhost-state-id.h new file mode 100644 index 00000000000000..1f6d828c454e7d --- /dev/null +++ b/src/database/rrdhost-state-id.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +#ifndef NETDATA_RRDHOST_STATE_ID_H +#define NETDATA_RRDHOST_STATE_ID_H + +#include "libnetdata/libnetdata.h" + +typedef uint32_t RRDHOST_STATE; + +struct rrdhost; +RRDHOST_STATE rrdhost_state_id(struct rrdhost *host); + +bool rrdhost_state_connected(struct rrdhost *host); +bool rrdhost_state_disconnected(struct rrdhost *host); + +bool rrdhost_state_acquire(struct rrdhost *host, RRDHOST_STATE wanted_state_id); +void rrdhost_state_release(struct rrdhost *host); + +#endif //NETDATA_RRDHOST_STATE_ID_H diff --git a/src/database/rrdhost.c b/src/database/rrdhost.c index e7c196a4e9dc36..be26f3c31a976b 100644 --- a/src/database/rrdhost.c +++ b/src/database/rrdhost.c @@ -327,6 +327,8 @@ static RRDHOST *rrdhost_create( } RRDHOST *host = callocz(1, sizeof(RRDHOST)); + host->state_refcount = -1; + __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(RRDHOST), __ATOMIC_RELAXED); strncpyz(host->machine_guid, guid, GUID_LEN + 1); @@ -840,6 +842,7 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool return 1; rrdhost_flag_set(localhost, RRDHOST_FLAG_COLLECTOR_ONLINE); + rrdhost_state_connected(localhost); ml_host_start(localhost); dyncfg_host_init(localhost); diff --git a/src/libnetdata/common.h b/src/libnetdata/common.h index b46eb9b9f06b29..1dbe2b0e808c47 100644 --- a/src/libnetdata/common.h +++ b/src/libnetdata/common.h @@ -400,6 +400,10 @@ typedef uint32_t uid_t; // -------------------------------------------------------------------------------------------------------------------- +typedef int32_t REFCOUNT; + +// -------------------------------------------------------------------------------------------------------------------- + #if defined(OS_WINDOWS) #include #include diff --git a/src/plugins.d/pluginsd_parser.c b/src/plugins.d/pluginsd_parser.c index 78f97bb5ade0a1..cd82a3186703d7 100644 --- a/src/plugins.d/pluginsd_parser.c +++ b/src/plugins.d/pluginsd_parser.c @@ -205,6 +205,7 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si rrdhost_option_set(host, RRDHOST_OPTION_VIRTUAL_HOST); rrdhost_flag_set(host, RRDHOST_FLAG_COLLECTOR_ONLINE); + rrdhost_state_connected(host); ml_host_start(host); dyncfg_host_init(host); @@ -376,16 +377,19 @@ static inline PARSER_RC pluginsd_chart(char **words, size_t num_words, PARSER *p } static void backfill_callback(size_t successful_dims __maybe_unused, size_t failed_dims __maybe_unused, struct backfill_request_data *brd) { - if (brd->rrdhost_receiver_state_id == __atomic_load_n(&brd->host->stream.rcv.status.state_id, __ATOMIC_RELAXED)) { - if (!replicate_chart_request(send_to_plugin, brd->parser, brd->host, brd->st, - brd->first_entry_child, brd->last_entry_child, brd->child_wall_clock_time, - 0, 0)) { - netdata_log_error( - "PLUGINSD: 'host:%s' failed to initiate replication for 'chart:%s'", - rrdhost_hostname(brd->host), - rrdset_id(brd->st)); - } + if(!rrdhost_state_acquire(brd->host, brd->rrdhost_receiver_state_id)) + return; + + if (!replicate_chart_request(send_to_plugin, brd->parser, brd->host, brd->st, + brd->first_entry_child, brd->last_entry_child, brd->child_wall_clock_time, + 0, 0)) { + netdata_log_error( + "PLUGINSD: 'host:%s' failed to initiate replication for 'chart:%s'", + rrdhost_hostname(brd->host), + rrdset_id(brd->st)); } + + rrdhost_state_release(brd->host); } static inline PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_words, PARSER *parser) { @@ -417,7 +421,7 @@ static inline PARSER_RC pluginsd_chart_definition_end(char **words, size_t num_w rrdhost_receiver_replicating_charts_plus_one(st->rrdhost); struct backfill_request_data brd = { - .rrdhost_receiver_state_id =__atomic_load_n(&host->stream.rcv.status.state_id, __ATOMIC_RELAXED), + .rrdhost_receiver_state_id = rrdhost_state_id(host), .parser = parser, .host = host, .st = st, @@ -1173,8 +1177,6 @@ void pluginsd_process_cleanup(PARSER *parser) { pluginsd_cleanup_v2(parser); pluginsd_host_define_cleanup(parser); - rrd_collector_finished(); - #ifdef NETDATA_LOG_STREAM_RECEIVE if(parser->user.stream_log_fp) { fclose(parser->user.stream_log_fp); @@ -1188,6 +1190,7 @@ void pluginsd_process_cleanup(PARSER *parser) { void pluginsd_process_thread_cleanup(void *pptr) { PARSER *parser = CLEANUP_FUNCTION_GET_PTR(pptr); pluginsd_process_cleanup(parser); + rrd_collector_finished(); } bool parser_reconstruct_node(BUFFER *wb, void *ptr) { diff --git a/src/plugins.d/pluginsd_parser.h b/src/plugins.d/pluginsd_parser.h index eb219494a67bc6..9198428113229b 100644 --- a/src/plugins.d/pluginsd_parser.h +++ b/src/plugins.d/pluginsd_parser.h @@ -5,7 +5,7 @@ #include "daemon/common.h" -#define WORKER_PARSER_FIRST_JOB 35 +#define WORKER_PARSER_FIRST_JOB 36 // this has to be in-sync with the same at stream-thread.c #define WORKER_RECEIVER_JOB_REPLICATION_COMPLETION 25 diff --git a/src/streaming/stream-receiver.c b/src/streaming/stream-receiver.c index d4e20213eaeccb..d6fb223ce20798 100644 --- a/src/streaming/stream-receiver.c +++ b/src/streaming/stream-receiver.c @@ -358,8 +358,6 @@ static void streaming_parser_init(struct receiver_state *rpt) { pluginsd_keywords_init(parser, PARSER_INIT_STREAMING); - rrd_collector_started(); - rpt->thread.compressed.start = 0; rpt->thread.compressed.used = 0; rpt->thread.compressed.enabled = stream_decompression_initialize(rpt); @@ -446,6 +444,7 @@ void stream_receiver_move_to_running_unsafe(struct stream_thread *sth, struct re sth->id, rrdhost_hostname(rpt->host), rpt->client_ip, rpt->client_port); stream_receive_log_database_gap(rpt); + rrdhost_state_connected(rpt->host); // keep this last, since it sends commands back to the child streaming_parser_init(rpt); @@ -477,6 +476,8 @@ static void stream_receiver_remove(struct stream_thread *sth, struct receiver_st , rpt->client_port ? rpt->client_port : "-" , why ? why : ""); + rrdhost_state_disconnected(rpt->host); + internal_fatal(META_GET(&sth->run.meta, (Word_t)&rpt->thread.meta) == NULL, "Receiver to be removed is not found in the list of receivers"); META_DEL(&sth->run.meta, (Word_t)&rpt->thread.meta); @@ -806,8 +807,6 @@ bool rrdhost_set_receiver(RRDHOST *host, struct receiver_state *rpt) { rrdhost_receiver_lock(host); if (!host->receiver) { - __atomic_add_fetch(&host->stream.rcv.status.state_id, 1, __ATOMIC_RELAXED); - rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN); host->stream.rcv.status.connections++; @@ -870,8 +869,8 @@ void rrdhost_clear_receiver(struct receiver_state *rpt) { // Make sure that we detach this thread and don't kill a freshly arriving receiver if (host->receiver == rpt) { - __atomic_add_fetch(&host->stream.rcv.status.state_id, 1, __ATOMIC_RELAXED); rrdhost_flag_clear(host, RRDHOST_FLAG_COLLECTOR_ONLINE); + rrdhost_receiver_unlock(host); { // run all these without having the receiver lock diff --git a/src/streaming/stream-sender.c b/src/streaming/stream-sender.c index 1c6060349393aa..a5984cdfed6b06 100644 --- a/src/streaming/stream-sender.c +++ b/src/streaming/stream-sender.c @@ -540,8 +540,10 @@ bool stream_sender_process_poll_events(struct stream_thread *sth, struct sender_ return false; } } - else + else { + sth->snd.send_misses++; break; + } } } diff --git a/src/streaming/stream-thread.c b/src/streaming/stream-thread.c index a633d51a9e3929..b08488a5783aef 100644 --- a/src/streaming/stream-thread.c +++ b/src/streaming/stream-thread.c @@ -450,10 +450,15 @@ void *stream_thread(void *ptr) { "ops processed", "messages", WORKER_METRIC_INCREMENTAL_TOTAL); - worker_register_job_custom_metric(WORKER_SENDER_JOB_RECEIVERS_WAITING_LIST_SIZE, + worker_register_job_custom_metric(WORKER_STREAM_JOB_RECEIVERS_WAITING_LIST_SIZE, "receivers waiting to be added", "nodes", WORKER_METRIC_ABSOLUTE); + worker_register_job_custom_metric(WORKER_STREAM_JOB_SEND_MISSES, + "send misses", "misses", + WORKER_METRIC_INCREMENTAL_TOTAL); + + if(pipe(sth->pipe.fds) != 0) { nd_log(NDLS_DAEMON, NDLP_ERR, "STREAM THREAD[%zu]: cannot create required pipe.", sth->id); sth->pipe.fds[PIPE_READ] = -1; @@ -487,6 +492,8 @@ void *stream_thread(void *ptr) { sth->snd.bytes_received = 0; sth->snd.bytes_sent = 0; + rrd_collector_started(); + while(!exit_thread && !nd_thread_signaled_to_cancel() && service_running(SERVICE_STREAMING)) { usec_t now_ut = now_monotonic_usec(); @@ -521,7 +528,8 @@ void *stream_thread(void *ptr) { worker_set_metric(WORKER_SENDER_JOB_BYTES_SENT, (NETDATA_DOUBLE)sth->snd.bytes_sent); worker_set_metric(WORKER_SENDER_JOB_REPLAY_DICT_SIZE, (NETDATA_DOUBLE)replay_entries); - worker_set_metric(WORKER_SENDER_JOB_RECEIVERS_WAITING_LIST_SIZE, (NETDATA_DOUBLE)receivers_waiting); + worker_set_metric(WORKER_STREAM_JOB_RECEIVERS_WAITING_LIST_SIZE, (NETDATA_DOUBLE)receivers_waiting); + worker_set_metric(WORKER_STREAM_JOB_SEND_MISSES, (NETDATA_DOUBLE)sth->snd.send_misses); replay_entries = 0; sth->snd.bytes_received = 0; sth->snd.bytes_sent = 0; @@ -597,6 +605,8 @@ void *stream_thread(void *ptr) { worker_unregister(); + rrd_collector_finished(); + return NULL; } diff --git a/src/streaming/stream-thread.h b/src/streaming/stream-thread.h index 14f04e05cc0661..8a0e41c4431481 100644 --- a/src/streaming/stream-thread.h +++ b/src/streaming/stream-thread.h @@ -83,7 +83,8 @@ struct stream_opcode { #define WORKER_SENDER_JOB_BYTES_COMPRESSION_RATIO 32 #define WORKER_SENDER_JOB_REPLAY_DICT_SIZE 33 #define WORKER_SENDER_JOB_MESSAGES 34 -#define WORKER_SENDER_JOB_RECEIVERS_WAITING_LIST_SIZE 35 +#define WORKER_STREAM_JOB_RECEIVERS_WAITING_LIST_SIZE 35 +#define WORKER_STREAM_JOB_SEND_MISSES 36 // IMPORTANT: to add workers, you have to edit WORKER_PARSER_FIRST_JOB accordingly @@ -125,6 +126,7 @@ struct stream_thread { struct { size_t bytes_received; size_t bytes_sent; + size_t send_misses; } snd; struct { diff --git a/src/web/api/queries/backfill.c b/src/web/api/queries/backfill.c index f24ec47d313058..8b5ee43a485b1e 100644 --- a/src/web/api/queries/backfill.c +++ b/src/web/api/queries/backfill.c @@ -48,7 +48,7 @@ bool backfill_request_add(RRDSET *st, backfill_callback_t cb, struct backfill_re if(backfill_globals.running) { struct backfill_request *br = aral_mallocz(backfill_globals.ar_br); br->data = *data; - br->rrdhost_receiver_state_id =__atomic_load_n(&st->rrdhost->stream.rcv.status.state_id, __ATOMIC_RELAXED); + br->rrdhost_receiver_state_id = rrdhost_state_id(st->rrdhost); br->rsa = rrdset_find_and_acquire(st->rrdhost, string2str(st->id)); if(br->rsa) { br->cb = cb; @@ -95,19 +95,20 @@ bool backfill_request_add(RRDSET *st, backfill_callback_t cb, struct backfill_re bool backfill_execute(struct backfill_dim_work *bdm) { RRDSET *st = rrdset_acquired_to_rrdset(bdm->br->rsa); - if(bdm->br->rrdhost_receiver_state_id !=__atomic_load_n(&st->rrdhost->stream.rcv.status.state_id, __ATOMIC_RELAXED)) + if(!rrdhost_state_acquire(st->rrdhost, bdm->br->rrdhost_receiver_state_id)) return false; + size_t success = 0; RRDDIM *rd = rrddim_acquired_to_rrddim(bdm->rda); - size_t success = 0; for (size_t tier = 1; tier < storage_tiers; tier++) - if(backfill_tier_from_smaller_tiers(rd, tier, now_realtime_sec())) + if (backfill_tier_from_smaller_tiers(rd, tier, now_realtime_sec())) success++; - if(success > 0) + if (success > 0) rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS); + rrdhost_state_release(st->rrdhost); return success > 0; }