From 68f533fbe573be206b26eb4a5b48e18d010e8930 Mon Sep 17 00:00:00 2001 From: Ellis Date: Thu, 6 Feb 2020 16:36:14 -0500 Subject: [PATCH 1/2] Setup Retry logic for both loggly appenders --- .gitignore | 1 + build.gradle | 4 +- .../ext/loggly/AbstractLogglyAppender.java | 38 ++-- .../logback/ext/loggly/LogglyAppender.java | 57 +++--- .../ext/loggly/LogglyBatchAppender.java | 165 +++++++++++------- 5 files changed, 166 insertions(+), 99 deletions(-) diff --git a/.gitignore b/.gitignore index a120bfd..94a2dfb 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ target out build local.properties +/.nb-gradle/ \ No newline at end of file diff --git a/build.gradle b/build.gradle index 009b629..af3fd2c 100644 --- a/build.gradle +++ b/build.gradle @@ -29,8 +29,8 @@ subprojects { apply plugin: 'java' apply from: "${rootProject.rootDir}/gradle/deploy.gradle" - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + sourceCompatibility = JavaVersion.VERSION_1_6 + targetCompatibility = JavaVersion.VERSION_1_6 tasks.withType(JavaCompile) { options.encoding = 'UTF-8' // Warn about deprecations diff --git a/loggly/src/main/java/ch/qos/logback/ext/loggly/AbstractLogglyAppender.java b/loggly/src/main/java/ch/qos/logback/ext/loggly/AbstractLogglyAppender.java index 2bd2c30..55d1458 100644 --- a/loggly/src/main/java/ch/qos/logback/ext/loggly/AbstractLogglyAppender.java +++ b/loggly/src/main/java/ch/qos/logback/ext/loggly/AbstractLogglyAppender.java @@ -35,6 +35,7 @@ * @author Cyrille Le Clerc */ public abstract class AbstractLogglyAppender extends UnsynchronizedAppenderBase { + public static final String DEFAULT_ENDPOINT_PREFIX = "https://logs-01.loggly.com/"; public static final String DEFAULT_LAYOUT_PATTERN = "%d{\"yyyy-MM-dd'T'HH:mm:ss.SSS'Z'\",UTC} %-5level [%thread] %logger: %m%n"; protected static final Charset UTF_8 = Charset.forName("UTF-8"); @@ -48,6 +49,7 @@ public abstract class AbstractLogglyAppender extends UnsynchronizedAppenderBa private String proxyHost; protected Proxy proxy; private int httpReadTimeoutInMillis = 1000; + private int httpMaxNumberOfRetries = 3; @Override public void start() { @@ -90,7 +92,7 @@ protected byte[] toBytes(final InputStream is) throws IOException { int count; byte[] buf = new byte[512]; - while((count = is.read(buf, 0, buf.length)) != -1) { + while ((count = is.read(buf, 0, buf.length)) != -1) { baos.write(buf, 0, count); } baos.flush(); @@ -135,14 +137,13 @@ protected String buildEndpointUrl(String inputKey) { return new StringBuilder(DEFAULT_ENDPOINT_PREFIX).append(getEndpointPrefix()) .append(inputKey).toString(); } - + /** - * Returns the URL path prefix for the Loggly endpoint to which the - * implementing class will send log events. This path prefix varies - * for the different Loggly services. The final endpoint URL is built - * by concatenating the {@link #DEFAULT_ENDPOINT_PREFIX} with the - * endpoint prefix from {@link #getEndpointPrefix()} and the - * {@link #inputKey}. + * Returns the URL path prefix for the Loggly endpoint to which the + * implementing class will send log events. This path prefix varies for the + * different Loggly services. The final endpoint URL is built by + * concatenating the {@link #DEFAULT_ENDPOINT_PREFIX} with the endpoint + * prefix from {@link #getEndpointPrefix()} and the {@link #inputKey}. * * @return the URL path prefix for the Loggly endpoint */ @@ -194,8 +195,9 @@ public int getProxyPort() { public void setProxyPort(int proxyPort) { this.proxyPort = proxyPort; } + public void setProxyPort(String proxyPort) { - if(proxyPort == null || proxyPort.trim().isEmpty()) { + if (proxyPort == null || proxyPort.trim().isEmpty()) { // handle logback configuration default value like "${logback.loggly.proxy.port:-}" proxyPort = "0"; } @@ -207,7 +209,7 @@ public String getProxyHost() { } public void setProxyHost(String proxyHost) { - if(proxyHost == null || proxyHost.trim().isEmpty()) { + if (proxyHost == null || proxyHost.trim().isEmpty()) { // handle logback configuration default value like "${logback.loggly.proxy.host:-}" proxyHost = null; } @@ -221,4 +223,20 @@ public int getHttpReadTimeoutInMillis() { public void setHttpReadTimeoutInMillis(int httpReadTimeoutInMillis) { this.httpReadTimeoutInMillis = httpReadTimeoutInMillis; } + + public int getHttpMaxNumberOfRetries() { + return httpMaxNumberOfRetries; + } + + public void setHttpMaxNumberOfRetries(int httpMaxNumberOfRetries) { + if (httpMaxNumberOfRetries <= 0) { + this.httpMaxNumberOfRetries = 0; + } else { + this.httpMaxNumberOfRetries = httpMaxNumberOfRetries; + } + } + + protected boolean canRetry(int currentRetryCount) { + return currentRetryCount <= getHttpMaxNumberOfRetries(); + } } diff --git a/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyAppender.java b/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyAppender.java index 6a71733..c1a76d8 100644 --- a/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyAppender.java +++ b/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyAppender.java @@ -21,7 +21,8 @@ import java.net.URL; /** - * An Appender that posts logging messages to Loggly, a cloud logging service. + * An Appender that posts logging messages to + * Loggly, a cloud logging service. * * @author Mårten Gustafson * @author Les Hazlewood @@ -30,7 +31,7 @@ public class LogglyAppender extends AbstractLogglyAppender { public static final String ENDPOINT_URL_PATH = "inputs/"; - + public LogglyAppender() { } @@ -41,28 +42,37 @@ protected void append(E eventObject) { } private void postToLoggly(final String event) { - try { - assert endpointUrl != null; - URL endpoint = new URL(endpointUrl); - final HttpURLConnection connection; - if (proxy == null) { - connection = (HttpURLConnection) endpoint.openConnection(); - } else { - connection = (HttpURLConnection) endpoint.openConnection(proxy); + int currentRetryCount = 0; + boolean success = false; + while (!success && canRetry(currentRetryCount)) { + try { + assert endpointUrl != null; + URL endpoint = new URL(endpointUrl); + final HttpURLConnection connection; + if (proxy == null) { + connection = (HttpURLConnection) endpoint.openConnection(); + } else { + connection = (HttpURLConnection) endpoint.openConnection(proxy); + } + connection.setRequestMethod("POST"); + connection.setDoOutput(true); + connection.addRequestProperty("Content-Type", this.layout.getContentType()); + connection.connect(); + sendAndClose(event, connection.getOutputStream()); + connection.disconnect(); + final int responseCode = connection.getResponseCode(); + if (responseCode == 200) { + success = true; + } else if (!canRetry(currentRetryCount)) { + final String message = readResponseBody(connection.getInputStream()); + addError("Loggly post failed (HTTP " + responseCode + "). Response body:\n" + message); + } + } catch (final IOException e) { + if (!canRetry(currentRetryCount)) { + addError("IOException while attempting to communicate with Loggly", e); + } } - connection.setRequestMethod("POST"); - connection.setDoOutput(true); - connection.addRequestProperty("Content-Type", this.layout.getContentType()); - connection.connect(); - sendAndClose(event, connection.getOutputStream()); - connection.disconnect(); - final int responseCode = connection.getResponseCode(); - if (responseCode != 200) { - final String message = readResponseBody(connection.getInputStream()); - addError("Loggly post failed (HTTP " + responseCode + "). Response body:\n" + message); - } - } catch (final IOException e) { - addError("IOException while attempting to communicate with Loggly", e); + currentRetryCount++; } } @@ -79,4 +89,3 @@ protected String getEndpointPrefix() { return ENDPOINT_URL_PATH; } } - diff --git a/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyBatchAppender.java b/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyBatchAppender.java index 318a583..0b5c51c 100644 --- a/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyBatchAppender.java +++ b/loggly/src/main/java/ch/qos/logback/ext/loggly/LogglyBatchAppender.java @@ -41,12 +41,17 @@ /** *

- * Logback batch appender for Loggly HTTP API. + * Logback batch appender for + * Loggly HTTP API. *

- *

Note:Loggly's Syslog API is much more scalable than the HTTP API which should mostly be used in - * low-volume or non-production systems. The HTTP API can be very convenient to workaround firewalls.

- *

If the {@link LogglyBatchAppender} saturates and discards log messages, the following warning message is - * appended to both Loggly and {@link System#err}:
+ *

+ * Note:Loggly's Syslog API is much more scalable than the HTTP + * API which should mostly be used in low-volume or non-production systems. The + * HTTP API can be very convenient to workaround firewalls.

+ *

+ * If the {@link LogglyBatchAppender} saturates and discards log messages, the + * following warning message is appended to both Loggly and {@link System#err}: + *
* "$date - OutputStream is full, discard previous logs"

*

Configuration settings

* @@ -58,13 +63,14 @@ * * * - * + * * * * * - * * * @@ -76,42 +82,51 @@ * * * - * + * * * * * - * + * * * * * * + * "ch.qos.logback:type=LogglyBatchAppender,name=LogglyBatchAppender@#hashcode#". + * Default: true. * * * * - * + * * * * * - * + * * * * * - * + * * * * * - * + * * *
inputKeyStringLoggly input key. "inputKey" or endpointUrl is required. Sample - * "12345678-90ab-cdef-1234-567890abcdef"Loggly input key. "inputKey" or endpointUrl is + * required. Sample "12345678-90ab-cdef-1234-567890abcdef"
endpointUrlStringLoggly HTTP API endpoint URL. "inputKey" or endpointUrl is required. Sample: + * Loggly HTTP API endpoint URL. "inputKey" or + * endpointUrl is required. Sample: * "https://logs.loggly.com/inputs/12345678-90ab-cdef-1234-567890abcdef"
proxyHostStringhostname of a proxy server. If blank, no proxy is used (See {@link URL#openConnection(java.net.Proxy)}.hostname of a proxy server. If blank, no proxy is used (See + * {@link URL#openConnection(java.net.Proxy)}.
proxyPortintport of a proxy server. Must be a valid int but is ignored if proxyHost is blank or null.port of a proxy server. Must be a valid int but is ignored if + * proxyHost is blank or null.
jmxMonitoringbooleanEnable registration of a monitoring MBean named - * "ch.qos.logback:type=LogglyBatchAppender,name=LogglyBatchAppender@#hashcode#". Default: true.
maxNumberOfBucketsintMax number of buckets of in the byte buffer. Default value: 8.Max number of buckets of in the byte buffer. Default value: + * 8.
maxBucketSizeInKilobytesintMax size of each bucket. Default value: 1024 Kilobytes (1MB).Max size of each bucket. Default value: 1024 Kilobytes + * (1MB).
flushIntervalInSecondsintInterval of the buffer flush to Loggly API. Default value: 3.Interval of the buffer flush to Loggly API. Default value: + * 3.
connReadTimeoutSecondsintHow Long the HTTP Connection will wait on reads. Default value: 1 second.How Long the HTTP Connection will wait on reads. Default value: + * 1 second.
- * Default configuration consumes up to 8 buffers of 1024 Kilobytes (1MB) each, which seems very reasonable even for small JVMs. - * If logs are discarded, try first to shorten the flushIntervalInSeconds parameter to "2s" or event "1s". + * Default configuration consumes up to 8 buffers of 1024 Kilobytes (1MB) each, + * which seems very reasonable even for small JVMs. If logs are discarded, try + * first to shorten the flushIntervalInSeconds parameter to "2s" or + * event "1s". *

*

Configuration Sample

* 

@@ -137,11 +152,13 @@
  * 
Implementation decisions

  * 
    
  * 
  • Why buffer the generated log messages as bytes instead of using the
- * {@code ch.qos.logback.core.read.CyclicBufferAppender} and buffering the {@code ch.qos.logback.classic.spi.ILoggingEvent} ?
- * Because it is much easier to control the size in memory
    • 
+ * {@code ch.qos.logback.core.read.CyclicBufferAppender} and buffering the
+ * {@code ch.qos.logback.classic.spi.ILoggingEvent} ? Because it is much easier
+ * to control the size in memory
  * 
  • 
- * Why buffer in a byte array instead of directly writing in a {@link BufferedOutputStream} on the {@link HttpURLConnection} ?
- * Because the Loggly API may not like such kind of streaming approach.
+ * Why buffer in a byte array instead of directly writing in a
+ * {@link BufferedOutputStream} on the {@link HttpURLConnection} ? Because the
+ * Loggly API may not like such kind of streaming approach.
  * 
    • 
  * 

  *
@@ -150,7 +167,7 @@
 public class LogglyBatchAppender extends AbstractLogglyAppender implements LogglyBatchAppenderMBean {
 
     public static final String ENDPOINT_URL_PATH = "bulk/";
-    
+
     private boolean debug = false;
 
     private int flushIntervalInSeconds = 3;
@@ -192,7 +209,7 @@ protected void append(E eventObject) {
         // Issue #21: Make sure messages end with new-line to delimit
         // individual log events within the batch sent to loggly.
         if (!msg.endsWith("\n")) {
-          msg += "\n";
+            msg += "\n";
         }
 
         try {
@@ -336,44 +353,65 @@ protected HttpURLConnection getHttpConnection(URL url) throws IOException {
 
     /**
      * Send log entries to Loggly
+     *
      * @param in log input stream
      */
     protected void processLogEntries(InputStream in) {
         long nanosBefore = System.nanoTime();
+        int currentRetryCount = 0;
+        boolean success = false;
         try {
-
-            HttpURLConnection conn = getHttpConnection(new URL(endpointUrl));
-            /* Set connection Read Timeout */
-            conn.setReadTimeout(connReadTimeoutSeconds*1000);
-            BufferedOutputStream out = new BufferedOutputStream(conn.getOutputStream());
-
-            long len = IoUtils.copy(in, out);
-            sentBytes.addAndGet(len);
-
-            out.flush();
-            out.close();
-
-            int responseCode = conn.getResponseCode();
-            String response = super.readResponseBody(conn.getInputStream());
-            switch (responseCode) {
-                case HttpURLConnection.HTTP_OK:
-                case HttpURLConnection.HTTP_ACCEPTED:
-                    sendSuccessCount.incrementAndGet();
-                    break;
-                default:
-                    sendExceptionCount.incrementAndGet();
-                    addError("LogglyAppender server-side exception: " + responseCode + ": " + response);
-            }
-            // force url connection recycling
-            try {
-                conn.getInputStream().close();
-                conn.disconnect();
-            } catch (Exception e) {
-                // swallow exception
+            while (!success && canRetry(currentRetryCount)) {
+                try {
+                    HttpURLConnection conn = getHttpConnection(new URL(endpointUrl));
+                    /* Set connection Read Timeout */
+                    conn.setReadTimeout(connReadTimeoutSeconds * 1000);
+                    BufferedOutputStream out = new BufferedOutputStream(conn.getOutputStream());
+
+                    //If we are retrying the request, the input stream needs to be reset in order to be read again
+                    if (currentRetryCount > 0) {
+                        in.reset();
+                    } 
+                    
+                    long len = IoUtils.copy(in, out);
+                    sentBytes.addAndGet(len);
+                    
+                    out.flush();
+                    out.close();
+
+                    int responseCode = conn.getResponseCode();
+                    String response = super.readResponseBody(conn.getInputStream());
+                    switch (responseCode) {
+                        case HttpURLConnection.HTTP_OK:
+                        case HttpURLConnection.HTTP_ACCEPTED:
+                            success = true;
+                            sendSuccessCount.incrementAndGet();
+                            break;
+                        default:
+                            if (!canRetry(currentRetryCount)) {
+                                sendExceptionCount.incrementAndGet();
+                                addError("LogglyAppender server-side exception: " + responseCode + ": " + response);
+                            } else if (isDebug()) {
+                                addWarn("LogglyAppender server-side exception - Retrying...: " + responseCode + ": " + response);
+                            }
+                    }
+                    // force url connection recycling
+                    try {
+                        conn.getInputStream().close();
+                        conn.disconnect();
+                    } catch (Exception e) {
+                        // swallow exception
+                    }
+                } catch (Exception e) {
+                    if (!canRetry(currentRetryCount)) {
+                        sendExceptionCount.incrementAndGet();
+                        addError("LogglyAppender client-side exception", e);
+                    } else if (isDebug()) {
+                        addWarn("LogglyAppender client-side exception - Retrying...", e);
+                    }
+                }
+                currentRetryCount++;
             }
-        } catch (Exception e) {
-            sendExceptionCount.incrementAndGet();
-            addError("LogglyAppender client-side exception", e);
         } finally {
             sendDurationInNanos.addAndGet(System.nanoTime() - nanosBefore);
         }
@@ -449,17 +487,18 @@ public void setConnReadTimeoutSeconds(int connReadTimeoutSeconds) {
     }
 
     private String getDebugInfo() {
-        return "{" +
-                "sendDurationInMillis=" + TimeUnit.MILLISECONDS.convert(sendDurationInNanos.get(), TimeUnit.NANOSECONDS) +
-                ", sendSuccessCount=" + sendSuccessCount +
-                ", sendExceptionCount=" + sendExceptionCount +
-                ", sentBytes=" + sentBytes +
-                ", discardedBucketsCount=" + getDiscardedBucketsCount() +
-                ", currentLogEntriesBufferSizeInBytes=" + getCurrentLogEntriesBufferSizeInBytes() +
-                '}';
+        return "{"
+                + "sendDurationInMillis=" + TimeUnit.MILLISECONDS.convert(sendDurationInNanos.get(), TimeUnit.NANOSECONDS)
+                + ", sendSuccessCount=" + sendSuccessCount
+                + ", sendExceptionCount=" + sendExceptionCount
+                + ", sentBytes=" + sentBytes
+                + ", discardedBucketsCount=" + getDiscardedBucketsCount()
+                + ", currentLogEntriesBufferSizeInBytes=" + getCurrentLogEntriesBufferSizeInBytes()
+                + '}';
     }
 
     public class LogglyExporter implements Runnable {
+
         @Override
         public void run() {
             try {

From 089db6e0559a5884087c24c4210c867baf169ea8 Mon Sep 17 00:00:00 2001
From: Ellis 
Date: Thu, 6 Feb 2020 16:38:46 -0500
Subject: [PATCH 2/2] Change source compatability to java 1.8

---
 build.gradle | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/build.gradle b/build.gradle
index af3fd2c..009b629 100644
--- a/build.gradle
+++ b/build.gradle
@@ -29,8 +29,8 @@ subprojects {
   apply plugin: 'java'
   apply from: "${rootProject.rootDir}/gradle/deploy.gradle"
 
-  sourceCompatibility = JavaVersion.VERSION_1_6
-  targetCompatibility = JavaVersion.VERSION_1_6
+  sourceCompatibility = JavaVersion.VERSION_1_8
+  targetCompatibility = JavaVersion.VERSION_1_8
   tasks.withType(JavaCompile) {
     options.encoding = 'UTF-8'
     // Warn about deprecations