Merge pull request #15 from PearsonEducation/1.6

1.6
PearsonEducation · Jun 7, 2024 · ae91762 · ae91762
2 parents 45b337b + cbbdb75
commit ae91762
Show file tree

Hide file tree

Showing 569 changed files with 166,769 additions and 23,934 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,13 @@
-/target/
-derby.log
-StatsAgg.iml
 .idea
-nb-configuration.xml
+StatsAgg.iml
+nbactions.xml
+*.log
+/logs/*.log
+/conf/application-dev.ini
+/conf/application-dev.properties
+/conf/database-dev.ini
+/conf/database-dev.properties
+/conf/flyway-dev.properties
+/conf/logback-config-dev.xml
+/nbproject/
+/target/
diff --git a/ChangeLog.txt b/ChangeLog.txt
@@ -1,3 +1,16 @@
+StatsAgg 1.6  - released xxxx-yy-zz
+ * New feature: Alert/MetricGroup/NotificationGroup templating. This feature makes it easy to create & manage hundreds-to-thousands of Alerts/MetricGroups/NotificationGroups using variable substitutions into templates.
+ * New feature: Integration with PagerDuty. StatsAgg can now send alerts directly to the PagerDuty "events" api. StatsAgg can also acknowledge & auto-resolve the alerts through the PagerDuty "events" api.
+ * New feature: Improved feedback to users when creating alerts/suspensions/etc with invalid criteria.
+ * New feature: Notification group alert associations UI -- so you can see what alerts are tied to a notification group 
+ * StatsAgg now requires Java 17 (or newer) for it's runtime
+ * Converted the StatsAgg project from using a middleware app (Tomcat) to being a standalone project that uses an embedded HTTP server (Jetty). As a result, the installation/upgrade/setup steps will be significantly different.
+ * Switched StatsAgg to use HikariCP for it's JDBC connection pool. Note that some connection pool variables in the database.properties file have changed.
+ * Significant refactoring on how StatsAgg interacts with it's database. This should improve StatsAgg's performance & make it easier to improve/update in the future. 
+ * Database schema changes are now made automatically via Flyway. If updating from a previous StatsAgg installation (1.5 or older), see the manual for additional steps.
+ * Apache Derby databases will be upgraded to version 10.16.1.1 (this happens automatically)
+ * Libraries were updated on both the frontend (javascript/css/etc) and the backend (java). 
+
 StatsAgg 1.5  - released 2020-09-25
  * New feature : A json-based API for programatically interacting with StatsAgg
  * New feature -- 'metric suspensions'. With a 'metric suspension', you can write a regular-expression that will match metric-keys & prevent them from being considered for alerting.

diff --git a/...app/WEB-INF/config/application.properties → conf/application.ini b/...app/WEB-INF/config/application.properties → conf/application.ini
@@ -6,6 +6,11 @@
 flush_time_agg = 10000
 debug_mode_enabled = false
 
+### UI & API HTTP Interface configuration options
+ui_and_api_http_enabled = true
+ui_and_api_http_port = 8080
+ui_and_api_http_context = StatsAgg
+
 ### 'Sending data to Graphite' configuration options
 graphite_output_module_1 = false,graphite.someDomain.com,2003,2,1000,true,true
 
@@ -16,7 +21,7 @@ opentsdb_http_output_module_1 = false,http://opentsdb.some-domain.com:4242/api/p
 ### 'Sending data to InfluxDB' configuration options
 influxdb_v1_output_module_1 = false,http://influxdb.some-domain.com:8086/,2,10
 
-### 'Listener' configuration options
+### 'Metric collector listener' configuration options
 statsd_tcp_listener_enabled = true
 statsd_tcp_listener_port = 8125
 statsd_udp_listener_enabled = true
@@ -97,6 +102,9 @@ alert_smtp_use_starttls = false
 alert_smtp_from_address = [email protected]
 alert_smtp_from_name = StatsAgg
 
+### PagerDuty integration options
+pagerduty_integration_enabled = false
+
 ### 'Website GUI' configuration options
 custom_action_url_1 = http://google.com,Goto Google
 

diff --git a/...webapp/WEB-INF/config/database.properties → conf/database.properties b/...webapp/WEB-INF/config/database.properties → conf/database.properties
@@ -2,17 +2,7 @@
 ### Details for what each option does can be found in the "example_database.properties" file.
 
 
-#### Connection Variables
-cp_max_connections = 50
-cp_acquire_retry_attempts = 3
-cp_acquire_retry_delay = 250
-cp_connection_timeout = 5000
-cp_enable_statistics = false
-cp_default_auto_commit = false
-connection_validity_check_timeout = 5
-
-
-#### JDBC Configuration 
+### JDBC Configuration 
 db_type = derby_embedded
 #db_hostname =
 #db_port =
@@ -23,7 +13,24 @@ db_password =
 db_attributes = create=true;upgrade=true;
 
 
-#### Apache Derby Configuration
+### HikariCP Variables
+### These variables shouldn't be changed unless you REALLY know what you're doing.
+poolName = StatsAgg
+maximumPoolSize = 50
+maxLifetime = 30000
+connectionTimeout = 5000
+registerMbeans = true
+
+
+### Flyway Variables
+### These variables shouldn't be changed unless you REALLY know what you're doing.
+flyway_migrate_enabled = true
+flyway_repair_enabled = true
+
+
+### Apache Derby Configuration
+### Ignore these variables if you're using MySQL.
+### These variables shouldn't be changed unless you REALLY know what you're doing.
 derby.system.home = ${db_localpath}
 derby.infolog.append = true
 derby.stream.error.file = ${db_localpath}${file.separator}${db_name}${file.separator}log${file.separator}db_log.txt      

diff --git a/...INF/config/example_application.properties → conf/example-application.ini b/...INF/config/example_application.properties → conf/example-application.ini
@@ -8,9 +8,33 @@
 #
 # debug_mode_enabled : When enabled, debugging info is output to the log files & console. Included are: raw metrics on arrival, aggregated metrics to output.
 #                      Default : false
+#
+# log_metric_format_errors : When enabled, errors related to invalid incoming metric data will be written to logs. 
+#                            In the case of bad incoming data, this can cause excessive logging, and can be turned disabled.
+#                            Default : true
 flush_time_agg = 10000
 debug_mode_enabled = false
+log_metric_format_errors = true
+
 
+### UI & API HTTP Interface configuration options
+#
+# ui_and_api_http_enabled : Controls whether or not the UI & API HTTP interface is enabled or not.
+#                           Disabling it will mean that any existing alerts/suspensions/etc will continue to function -- you just won't be able to enable/disable/edit/etc them.
+#                           The UI/API has no impact on metric ingestion (Graphite, OpenTSDB, etc) or metric output -- those will function regardless of whether this is enabled or not.
+#                           Default : true
+#
+# ui_and_api_http_port : The port that the HTTP user-interface & apis will listen on.
+#                        Default : 8080
+#
+# ui_and_api_http_context : The HTTP context path that StatsAgg will run it's UI & API on. 
+#                           Example: http://mydomain.com/THIS-VALUE (where "THIS-VALUE" is the http context) 
+#                           Historically, this would be "StatsAgg", but it can be any simple string that conforms to uri conventions.
+#                           If you want to run the UI & API off the root context (http://mydomain.com/), leave this field commented out (or blank)
+#                           Default: StatsAgg
+ui_and_api_http_enabled = true
+ui_and_api_http_port = 8080
+ui_and_api_http_context = StatsAgg
 
 
 ### 'Sending data to Graphite' configuration options
@@ -83,7 +107,7 @@ influxdb_v1_output_module_1 = false,http://influxdb.some-domain.com:8086/,2,10
 
 
 
-### 'Listener' configuration options
+### 'Metric collector listener' configuration options
 #
 # statsd_tcp_listener_enabled : Defines whether this application will listen for & collect StatsD metrics via TCP.
 #                               This listener accepts TCP traffic in the StatsD format.          
@@ -451,6 +475,15 @@ alert_smtp_from_name = StatsAgg
 
 
 
+### PagerDuty integration options
+#
+# pagerduty_integration_enabled : Enables or disables PagerDuty integration. 
+#                                 Controls if PagerDuty stuff is shown in the user-interface & if StatsAgg will try to interact with PagerDuty as alerts trigger.
+#                                 Default : false
+pagerduty_integration_enabled = false
+
+
+
 ### 'Website GUI' configuration options
 #
 # The StatsAgg website user interface can be customized to include custom url links in the 'actions menu'.

diff --git a/...EB-INF/config/example_database.properties → conf/example-database.properties b/...EB-INF/config/example_database.properties → conf/example-database.properties
@@ -1,53 +1,16 @@
-### Connection Variables
-#
-# cp_max_connections : The maximum number of connections to the database that the connection-pool is allowed to make.
-#                      Default : 25
-#
-# cp_acquire_retry_attempts : After attempting to acquire a connection and failing, try to connect this many times before giving up.
-#                             Default : 3
-#
-# cp_acquire_retry_delay : Sets the number of milliseconds to wait before attempting to obtain a connection after a failure.
-#                          Default : 250
-#
-# cp_connection_timeout : Sets the maximum time (in milliseconds) to wait before a call 'get a connection' is timed out. 
-#                         Setting this to '0' means 'wait forever' (disables the time-out)
-#                         Default : 5000
-#
-# cp_enable_statistics : Specifies whether metrics about connection pool performance are gathered. Enabling this gives greater insight into
-#                        the performance of the connection pool, but at the cost of performance.
-#                        Default : false
-#
-# cp_default_auto_commit : Specifies whether the connection pool defaults to having auto-commit enabled or disabled. True=enabled, false=disabled.
-#                          Default : false
-#
-# connection_validity_check_timeout : Database connections are often tested to see if they are 'healthy' (aka 'valid'). When a validity test is issued against a connection,  
-#                                     the validity test can fail if too much time has passed. This variable controls how long the application will wait (in seconds) 
-#                                     before declaring that a connection is invalid. Setting this variable to '0' means that the validity test will wait forever (no time-out).
-#                                     Default : 5
-cp_max_connections = 50
-cp_acquire_retry_attempts = 3
-cp_acquire_retry_delay = 250
-cp_connection_timeout = 5000
-cp_enable_statistics = false
-cp_default_auto_commit = false
-connection_validity_check_timeout = 5
-
-
-
 ### JDBC Configuration 
 #
-# db_type : The type of database being used. Choices include Apache Derby Embedded, Apache Derby Network, MySQL, or PostgreSQL.
-#           Valid options: derby_embedded, derby_network, mysql, postgres
+# db_type : The type of database being used. Choices include Apache Derby Embedded, Apache Derby Network, and MySQL.
+#           Valid options: derby_embedded, derby_network, mysql
 #           Example: derby_embedded	 
 #
-# db_hostname : Use if using Apache Derby Network, MySQL, or PostgreSQL. Should be a valid hostname or IP-address.
+# db_hostname : Use if using Apache Derby Network or MySQL. Should be a valid hostname or IP-address.
 #               Do not set if using Apache Derby Embedded.
 #               Example: 192.168.0.100
 #
 # db_port : Use if using Apache Derby Network, MySQL, or PostgreSQL. Do not set if using Apache Derby Embedded. Should be a valid port #.
 #           Tips-- Apache Derby Network usually runs on port 1527
 #                  MySQL usually runs on port 3306
-#                  PostgreSQL usually runs on port 5432
 #           Example: 3306
 #
 # db_localpath : Use if using Apache Derby Embedded. This should be a path to a folder on your computer where Derby will store its database data.
@@ -84,11 +47,44 @@ db_attributes = create=true;upgrade=true;
 
 
 
+### Flyway Variables
+#
+# These are variables that control a few aspects of Flyway (7.x). Flyway is used to update the StatsAgg database when new builds are deployed. 
+# Advanced Flyway options are controlled in a separate configuration file -- flyway.properties.
+# These variables shouldn't be changed unless you REALLY know what you're doing.
+#
+# flyway_migrate_enabled : Enables running Flyway migrate -- which is used to update the StatsAgg database tables when new builds are deployed. This should typically be left enabled.
+#                          More details: https://flywaydb.org/documentation/command/migrate
+#                          Default: true
+#
+# flyway_repair_enabled : Enables running Flyway repair -- which is used to fix previous database table updates that had issues (ie - a failed 'migrate'). This should typically be left enabled.
+#                         More details: https://flywaydb.org/documentation/command/repair
+#                         Default: true
+flyway_migrate_enabled = true
+flyway_repair_enabled = true
+
+
+
+### HikariCP Variables
+#
+# As of StatsAgg 1.6, StatsAgg uses HikariCP (3.x) as it's connection pool. 
+# HikariCP-specific configurations can be set here. The availabile options are listed here: https://github.com/brettwooldridge/HikariCP
+# These variables shouldn't be changed unless you REALLY know what you're doing.
+poolName = StatsAgg
+maximumPoolSize = 50
+maxLifetime = 30000
+connectionTimeout = 5000
+registerMbeans = true
+
+
+
 ### Apache Derby Configuration
-# These are derby configuration values that override the default derby system-wide configuration (the default derby.properties values).
-# If using another database (MySQL, PostgreSQL, etc), setting these variables will have no effect.
+#
+# These are Derby configuration values that override the default Derby system-wide configuration (the default derby.properties values).
+# If using another database engine (ie- MySQL), setting these variables will have no effect.
 # View the 'Derby properties' page of Apache Derby Reference Manual for more details 
-# http://db.apache.org/derby/docs/10.11/ref/crefproper22250.html
+# http://db.apache.org/derby/docs/10.15/ref/crefproper22250.html
+# These variables shouldn't be changed unless you REALLY know what you're doing.
 #
 # Note : Limited variable substitution is allowed. 
 #        Use ${db_localpath} to get the value of db_localpath (specified in the JDBC connection section)

diff --git a/conf/example-flyway.properties b/conf/example-flyway.properties
@@ -0,0 +1,7 @@
+### Advanced Flyway configuration options
+#
+# These configuration options control aspects of Flyway that don't normally need adjusting. 
+# This file is only exposed for troubleshooting/debugging, and possibly to make life easier for some advanced StatsAgg users. Regular users should not modify this file. 
+# That said, most of the modifyable Flyway options are listed here: https://flywaydb.org/documentation/configuration/configfile.html
+flyway.outOfOrder = true
+flyway.validateOnMigrate = true
diff --git a/conf/flyway.properties b/conf/flyway.properties
@@ -0,0 +1,7 @@
+### Advanced Flyway configuration options
+#
+# Theses configuration options control aspects of Flyway that don't normally need adjusting. 
+# This file is only exposed for troubleshooting/debugging, and possibly to make life easier for some advanced StatsAgg users. Regular users should not modify this file. 
+# That said, most of the modifyable Flyway options are listed here: https://flywaydb.org/documentation/configuration/configfile.html
+flyway.outOfOrder = true
+flyway.validateOnMigrate = true
diff --git a/.../webapp/WEB-INF/config/logback_config.xml → conf/logback-config.xml b/.../webapp/WEB-INF/config/logback_config.xml → conf/logback-config.xml
@@ -1,42 +1,40 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-
-<configuration>
-
-    <contextName>StatsAgg</contextName>
-
-    <jmxConfigurator />
-
-    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
-
-        <file>${catalina.base}${file.separator}logs${file.separator}statsagg.log</file>
-
-        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
-            <fileNamePattern>${catalina.base}${file.separator}logs${file.separator}statsagg.%i.log</fileNamePattern>
-            <minIndex>1</minIndex>
-            <maxIndex>10</maxIndex>
-        </rollingPolicy>
-
-        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
-            <maxFileSize>100MB</maxFileSize>
-        </triggeringPolicy>
-
-        <encoder>
-            <pattern>%date{"yyyy-MM-dd' 'HH:mm:ss.SSSZ"} : %level : %logger{100}:%line : %message%n</pattern>
-        </encoder>
-
-    </appender> 
-
-    <!--
-    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
-        <encoder>
-            <pattern>%date{"yyyy-MM-dd' 'HH:mm:ss.SSSZ"} : %level : %logger{100}:%line : %message%n</pattern>
-        </encoder>
-    </appender>
-    -->
-
-    <root level="INFO">
-        <appender-ref ref="FILE" />
-        <!-- <appender-ref ref="STDOUT" /> -->
-    </root>
-
-</configuration>
+<?xml version="1.0" encoding="iso-8859-1"?>
+
+<configuration>
+
+    <contextName>StatAagg</contextName>
+
+    <jmxConfigurator />
+
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+
+        <file>logs${file.separator}statsagg.log</file>
+
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>logs${file.separator}statsagg.%i.log</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>10</maxIndex>
+        </rollingPolicy>
+
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>10MB</maxFileSize>
+        </triggeringPolicy>
+
+        <encoder>
+            <pattern>%date{ISO8601} : %level : %logger{100}:%line : %message%n</pattern>
+        </encoder>
+
+    </appender> 
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%date{ISO8601} : %level : %logger{100}:%line : %message%n</pattern>
+        </encoder>
+    </appender>
+
+    <root level="INFO">
+        <appender-ref ref="FILE" />
+        <appender-ref ref="STDOUT" />
+    </root>
+
+</configuration>