fix(dashmate)!: drive logs configuration

Dockerfile

@@ -203,23 +203,22 @@ FROM alpine:${ALPINE_VERSION} AS drive-abci
 LABEL maintainer="Dash Developers <[email protected]>"
 LABEL description="Drive ABCI Rust"
 
-WORKDIR /var/lib/dash
-
 RUN apk add --no-cache libgcc libstdc++
 
+ENV DB_PATH=/var/lib/dash/rs-drive-abci/db
+
+RUN mkdir -p /var/log/dash \
+    /var/lib/dash/rs-drive-abci
+
 COPY --from=build-drive-abci /artifacts/drive-abci /usr/bin/drive-abci
 COPY --from=build-drive-abci /platform/packages/rs-drive-abci/.env.example /var/lib/dash/rs-drive-abci/.env
 
-# Double-check that we don't have missing deps
-RUN ldd /usr/bin/drive-abci
-
 # Create a volume
-VOLUME /var/lib/dash
+VOLUME /var/lib/dash/rs-drive-abci/db
 VOLUME /var/log/dash
 
-RUN mkdir -p /var/log/dash
-
-ENV DB_PATH=/var/lib/dash/rs-drive-abci/db
+# Double-check that we don't have missing deps
+RUN ldd /usr/bin/drive-abci
 
 #
 # Create new non-root user

docs/DRIVE-ABCI.md

@@ -10,7 +10,7 @@ Configuration is implemented using environment variables. Environment variables
 
 ### Logging
 
-The logging configuration allows you to configure multiple logging destinations. Each destination is defined using a series of environment variables prefixed with ABCI_LOG_*key*, where *key* is an arbitrary name for the destination.
+The logging configuration allows you to configure multiple logging destinations. Each destination is defined using a series of environment variables prefixed with ABCI_LOG_*key*_*option*, where *key* is an arbitrary name for the destination.
 
 The log destination configuration consists of the following parameters:
 
@@ -22,22 +22,31 @@ Specifies the destination of the logs. It can have one of the following values:
 * **stderr**: Logs will be sent to the standard error
 * An absolute path to an existing directory where logs will be stored, for example `/var/log/dash/`
 
-#### VERBOSITY (optional)
+#### LEVEL (optional)
 
-Specifies the verbosity level of the logs. It is an integer value ranging from 0 to 5. Defaults to 0. Higher values indicate more detailed and verbose logs. For more information on verbosity levels, refer to the `-v` option in the `drive-abci --help` command.
+Specifies the logs verbosity level preset:
+
+* **silent**: No logs.
+* **custom**: Uses RUST_LOG env or info level if not set.
+* **error**: Only errors.
+* **warn**: Warnings and errors. Errors for 3rd party dependencies.
+* **info**: Info level and lower. Warnings for 3rd party dependencies. Default.
+* **debug**: Debug level and lower. Info level for 3rd party dependencies.
+* **trace**: Trace level and lower. Debug level for 3rd party dependencies.
+* **paranoid**: Trace level for everything.
 
 #### COLOR (optional)
 
-Specifies whether or not to use colorful output for the logs. It is an optional boolean parameter. If not provided, the output colorization will be autodetected.
+Specifies whether to use colorful output for the logs. It is an optional boolean parameter. If not provided, the output colorization will be autodetected.
 
 #### FORMAT (optional)
 
 Specifies the output format to use for the logs. It can have one of the following values:
 
-* **Full**: Logs will be formatted with full details.
-* **Compact**: Logs will be formatted in a compact manner.
-* **Pretty**: Logs will be formatted in a human-readable and visually appealing manner.
-* **Json**: Logs will be formatted in JSON format.
+* **full**: Logs will be formatted with full details.
+* **compact**: Logs will be formatted in a compact manner.
+* **pretty**: Logs will be formatted in a human-readable and visually appealing manner.
+* **json**: Logs will be formatted in JSON format.
 
 For more detailed description for different formats, refer to the [tracing-subscriber](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/fmt/format/index.html#formatters) documentation.
 
@@ -54,7 +63,7 @@ To configure multiple logging destinations, each destination should have its own
 For example, to configure a logging destination with the key "example", you would set the following environment variables:
 
 * ABCI_LOG_EXAMPLE_DESTINATION: Specifies the destination of logs for the "example" destination.
-* ABCI_LOG_EXAMPLE_VERBOSITY: Specifies the verbosity level for the "example" destination.
+* ABCI_LOG_EXAMPLE_LEVEL: Specifies the verbosity level for the "example" destination.
 * ABCI_LOG_EXAMPLE_COLOR: Specifies whether to use colorful output for the "example" destination (optional).
 * ABCI_LOG_EXAMPLE_FORMAT: Specifies the output format for the "example" destination (optional).
 * ABCI_LOG_EXAMPLE_MAX_FILES: Specifies the maximum number of daily log files to store for the "example" destination (optional).
@@ -67,9 +76,9 @@ Here's an example configuration for a logging destination named "EXAMPLE":
 
 ```bash
 export ABCI_LOG_EXAMPLE_DESTINATION="/var/log/example"
-export ABCI_LOG_EXAMPLE_VERBOSITY=3
+export ABCI_LOG_EXAMPLE_LEVEL=debug
 export ABCI_LOG_EXAMPLE_COLOR=false
-export ABCI_LOG_EXAMPLE_FORMAT="Pretty"
+export ABCI_LOG_EXAMPLE_FORMAT="pretty"
 export ABCI_LOG_EXAMPLE_MAX_FILES=10
 ```
 

packages/dashmate/configs/getConfigFileMigrationsFactory.js

@@ -137,6 +137,20 @@ function getConfigFileMigrationsFactory(homeDir, defaultConfigs) {
           });
         return configFile;
       },
+      '0.25.0-dev.23': (configFile) => {
+        Object.entries(configFile.configs)
+          .forEach(([, options]) => {
+            if (options.platform.drive.abci.log.jsonFile.level === 'fatal') {
+              options.platform.drive.abci.log.jsonFile.level = 'error';
+            }
+
+            if (options.platform.drive.abci.log.prettyFile.level === 'fatal') {
+              options.platform.drive.abci.log.prettyFile.level = 'error';
+            }
+          });
+
+        return configFile;
+      },
     };
   }
 

packages/dashmate/docker-compose.yml

@@ -45,8 +45,9 @@ services:
       org.dashmate.service.title: "Drive ABCI"
     restart: unless-stopped
     volumes:
-      - drive_abci_data:/var/lib/dash/rs-drive-abci
-      - drive_abci_logs:/var/log/dash
+      - drive_abci_data:/var/lib/dash/rs-drive-abci/db
+      - ${PLATFORM_DRIVE_ABCI_LOG_PRETTY_DIRECTORY_PATH:?err}:/var/log/dash/drive/pretty
+      - ${PLATFORM_DRIVE_ABCI_LOG_JSON_DIRECTORY_PATH:?err}:/var/log/dash/drive/json
     environment:
       - BLOCK_SPACING_MS=3000 # TODO: sync with tenderdash
       - CHAIN_ID=devnet # TODO: sync with tenderdash chain id
@@ -68,8 +69,15 @@ services:
       - WITHDRAWALS_SECOND_PUBLIC_KEY=${PLATFORM_WITHDRAWALS_SECOND_PUBLIC_KEY}
       - QUORUM_SIZE=5 # TODO: sync with Tenderdash
       - QUORUM_TYPE=${PLATFORM_DRIVE_ABCI_VALIDATOR_SET_LLMQ_TYPE:?err}
-      - NETWORK=${NETWORK}
-      - TENDERDASH_P2P_PORT=${PLATFORM_DRIVE_TENDERDASH_P2P_PORT}
+      - NETWORK=${NETWORK:?err}
+      - TENDERDASH_P2P_PORT=${PLATFORM_DRIVE_TENDERDASH_P2P_PORT:?err}
+      - ABCI_LOG_STDOUT_LEVEL=${PLATFORM_DRIVE_ABCI_LOG_STDOUT_LEVEL:?err}
+      - ABCI_LOG_PRETTY_DESTINATION=/var/log/dash/drive/pretty/${PLATFORM_DRIVE_ABCI_LOG_PRETTY_FILE_NAME:?err}
+      - ABCI_LOG_PRETTY_LEVEL=${PLATFORM_DRIVE_ABCI_LOG_PRETTY_FILE_LEVEL:?err}
+      - ABCI_LOG_PRETTY_FORMAT=pretty
+      - ABCI_LOG_JSON_DESTINATION=/var/log/dash/drive/json/${PLATFORM_DRIVE_ABCI_LOG_JSON_FILE_NAME:?err}
+      - ABCI_LOG_JSON_LEVEL=${PLATFORM_DRIVE_ABCI_LOG_JSON_FILE_LEVEL:?err}
+      - ABCI_LOG_JSON_FORMAT=json
     stop_grace_period: 30s
     profiles:
       - platform

packages/dashmate/src/commands/stop.js

@@ -65,7 +65,6 @@ StopCommand.flags = {
     description: 'stop only platform',
     default: false,
   }),
-
 };
 
 module.exports = StopCommand;

packages/dashmate/src/config/configJsonSchema.js

@@ -77,7 +77,7 @@ module.exports = {
       properties: {
         level: {
           type: 'string',
-          enum: ['fatal', 'error', 'warn', 'info', 'debug', 'trace', 'silent'],
+          enum: ['error', 'warn', 'info', 'debug', 'trace', 'silent'],
         },
         path: {
           type: 'string',

packages/dashmate/src/listr/tasks/setup/setupLocalPresetTaskFactory.js

@@ -228,23 +228,19 @@ function setupLocalPresetTaskFactory(
                   config.set('platform.drive.tenderdash.mode', 'validator');
 
                   // Setup logs
+                  const drivePrettyFilePath = homeDir.joinPath('logs', config.getName(), 'drive-pretty.log');
+                  config.set('platform.drive.abci.log.prettyFile.path', drivePrettyFilePath);
+
+                  const driveJsonFilePath = homeDir.joinPath('logs', config.getName(), 'drive-json.log');
+                  config.set('platform.drive.abci.log.jsonFile.path', driveJsonFilePath);
+
                   if (ctx.debugLogs) {
                     config.set('platform.drive.abci.log.stdout.level', 'trace');
                     config.set('platform.drive.abci.log.prettyFile.level', 'trace');
 
                     config.set('platform.drive.tenderdash.log.level', 'debug');
                   }
 
-                  if (!config.get('platform.drive.abci.log.prettyFile.path')) {
-                    const drivePrettyLogFile = homeDir.joinPath('logs', config.getName(), 'drive_pretty.log');
-                    config.set('platform.drive.abci.log.prettyFile.path', drivePrettyLogFile);
-                  }
-
-                  if (!config.get('platform.drive.abci.log.jsonFile.path')) {
-                    const driveJsonLogFile = homeDir.joinPath('logs', config.getName(), 'drive_json.log');
-                    config.set('platform.drive.abci.log.jsonFile.path', driveJsonLogFile);
-                  }
-
                   config.set('platform.dpns.masterPublicKey', dpnsDerivedMasterPrivateKey.privateKey.toPublicKey().toString());
                   config.set('platform.dpns.secondPublicKey', dpnsDerivedSecondPrivateKey.privateKey.toPublicKey().toString());
 

packages/rs-drive-abci/.env.example

@@ -3,20 +3,10 @@ ABCI_BIND_ADDRESS="tcp://0.0.0.0:26658"
 ABCI_PROMETHEUS_BIND_ADDRESS="http://0.0.0.0:29090"
 
 # stderr logging for humans
-ABCI_LOG_STDERR_DESTINATION=stderr
-ABCI_LOG_STDERR_VERBOSITY=1
-ABCI_LOG_STDERR_FORMAT=Pretty
-ABCI_LOG_STDERR_COLOR=true
-
-# stdout logging for machines (elasticsearch)
 ABCI_LOG_STDOUT_DESTINATION=stdout
-ABCI_LOG_STDOUT_VERBOSITY=4
-ABCI_LOG_STDOUT_FORMAT=Json
-
-# file logging for humans, but very verbose; no log rotation
-ABCI_LOG_FILE_DESTINATION=/var/log/dash
-ABCI_LOG_FILE_VERBOSITY=5
-ABCI_LOG_FILE_MAX_FILES=0
+ABCI_LOG_STDOUT_LEVEL=info
+ABCI_LOG_STDOUT_FORMAT=pretty
+ABCI_LOG_STDOUT_COLOR=true
 
 DB_PATH=/tmp/db
 

packages/rs-drive-abci/Cargo.toml

@@ -39,7 +39,7 @@ prost = { version = "0.11.6", default-features = false }
 tracing = { version = "0.1.37", default-features = false, features = [] }
 clap = { version = "4.1.8", optional = true, features = ["derive"] }
 envy = { version = "0.4.2" }
-dotenvy = { version = "0.15.6", optional = true }
+dotenvy = { version = "0.15.7", optional = true }
 dapi-grpc = { path = "../dapi-grpc" }
 platform-serialization = { path = "../rs-platform-serialization" }
 platform-serialization-derive = { path = "../rs-platform-serialization-derive" }
@@ -59,6 +59,7 @@ anyhow = { version = "1.0.70" }
 lazy_static = "1.4.0"
 itertools = { version = "0.10.5" }
 file-rotate = { version = "0.7.3" }
+reopen = { version = "1.0.3" }
 delegate = { version = "0.9.0" }
 regex = { version = "1.8.1" }
 metrics = { version = "0.21" }

packages/rs-drive-abci/src/config.rs

@@ -280,9 +280,16 @@ mod tests {
     fn test_config_from_env() {
         // ABCI log configs are parsed manually, so they deserve separate handling
         // Notat that STDOUT is also defined in .env.example, but env var should overwrite it.
-        let log_ids = &["STDOUT", "UPPERCASE", "lowercase", "miXedC4s3", "123"];
-        for id in log_ids {
-            env::set_var(format!("ABCI_LOG_{}_DESTINATION", id), "bytes");
+        let vectors = &[
+            ("STDOUT", "pretty"),
+            ("UPPERCASE", "json"),
+            ("lowercase", "pretty"),
+            ("miXedC4s3", "full"),
+            ("123", "compact"),
+        ];
+        for vector in vectors {
+            env::set_var(format!("ABCI_LOG_{}_DESTINATION", vector.0), "bytes");
+            env::set_var(format!("ABCI_LOG_{}_FORMAT", vector.0), vector.1);
         }
 
         let envfile = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(".env.example");
@@ -293,8 +300,8 @@ mod tests {
         let config = super::PlatformConfig::from_env().unwrap();
         assert!(config.verify_sum_trees);
         assert_ne!(config.quorum_type(), QuorumType::UNKNOWN);
-        for id in log_ids {
-            assert_eq!(config.abci.log[*id].destination.as_str(), "bytes");
+        for id in vectors {
+            assert_eq!(config.abci.log[id.0].destination.as_str(), "bytes");
         }
     }
 }