diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 5d46bfec..00000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: 2.1 -tagged_build_filters: &tagged_build_filters - tags: - only: /[0-9]+\.[0-9]+\.[0-9]+/ - branches: - ignore: /.*/ -test_build_filters: &test_build_filters - branches: - only: /.*/ - tags: - ignore: /[0-9]+\.[0-9]+\.[0-9]+/ -orbs: - slack: circleci/slack@3.4.2 -jobs: - test: - docker: - - image: circleci/node:12.13.0 - steps: - - checkout - - run: npm install - - run: npm run test -workflows: - test: - jobs: - - test: - filters: *test_build_filters diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..c32c435d --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,77 @@ +def TRIGGER_PATTERN = ".*@logdnabot.*" +def PROJECT_NAME = "logdna-agent" + +pipeline { + agent {label 'ec2-fleet'} + + options { + timestamps() + ansiColor 'xterm' + } + + triggers { + issueCommentTrigger(TRIGGER_PATTERN) + } + + environment { + NPM_CONFIG_CACHE = '.npm' + NPM_CONFIG_USERCONFIG = '.npmrc' + SPAWN_WRAP_SHIM_ROOT = '.npm' + } + + stages { + stage('Test Suite') { + matrix { + axes { + axis { + name 'NODE_VERSION' + values '14', '16' + } + } + + when { + not { + changelog '\\[skip ci\\]' + } + } + + + agent { + docker { + image "us.gcr.io/logdna-k8s/node:${NODE_VERSION}-ci" + customWorkspace "${PROJECT_NAME}-${BUILD_NUMBER}" + } + } + + stages { + stage('Install') { + steps { + sh "mkdir -p ${NPM_CONFIG_CACHE} coverage" + sh 'npm install' + } + } + + stage('Test') { + steps { + sh 'npm test' + } + + post { + always { + publishHTML target: [ + allowMissing: false, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: 'coverage/lcov-report', + reportFiles: 'index.html', + reportName: "coverage-node-v${NODE_VERSION}" + ] + } + } + } + } + } + } + + } +} diff --git a/README.md b/README.md index e8b9203a..24d8e546 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # LogDNA Agent LogDNA Agent streams from log files to your LogDNA account. Works with Linux, Windows, and macOS Servers. - +## :warning: Deprecation Warning +logdna-agent will soon be deprecated and will cease to have support. Please refer to [LogDNA Agent V2](https://github.com/logdna/logdna-agent-v2) for all future implementations. ## Getting Started ### From an Official Release diff --git a/index.js b/index.js index 62cd8d9f..737a3505 100644 --- a/index.js +++ b/index.js @@ -1,11 +1,12 @@ +/* eslint-disable no-multi-str */ // Adding this to be able to better format the console logs 'use strict' // External Modules +const fs = require('fs') +const os = require('os') const async = require('async') const debug = require('debug')('logdna:index') -const fs = require('fs') const getos = require('getos') -const os = require('os') const {Command} = require('commander') const properties = require('properties') const request = require('request') @@ -13,7 +14,7 @@ const request = require('request') // Internal Modules const start = require('./lib/start.js') const pkg = require('./package.json') -const utils = require('./lib/utils') +const utils = require('./lib/utils.js') // Constants const HOSTNAME_IP_REGEX = /[^0-9a-zA-Z\-.]/g @@ -29,34 +30,76 @@ process.title = 'logdna-agent' commander._name = 'logdna-agent' commander .version(pkg.version, '-v, --version') - .description('This agent collect and ship logs for processing. Defaults to /var/log if run without parameters.') - .option('-c, --config ', `uses alternate config file (default: ${config.DEFAULT_CONF_FILE})`) + .description( + 'This agent collects and ships logs for processing.' + + ' Defaults to /var/log if run without parameters.' + ) + .option( + '-c, --config ' + , `Uses alternate config file (default: ${config.DEFAULT_CONF_FILE})` + ) .option('-k, --key ', 'sets your LogDNA Ingestion Key in the config') - .option('-d, --logdir ', 'adds log directories to config, supports glob patterns', utils.appender(), []) - .option('-f, --logfile ', 'adds log files to config', utils.appender(), []) - .option('-e, --exclude ', 'exclude files from logdir', utils.appender(), []) + .option( + '-d, --logdir ' + , 'Adds log directories to config, supports glob patterns' + , utils.appender() + , [] + ) + .option('-f, --logfile ', 'Adds log files to config', utils.appender(), []) + .option('-e, --exclude ', 'Exclude files from logdir', utils.appender(), []) .option('-r, --exclude-regex ', 'filter out lines matching pattern') - .option('-n, --hostname ', `uses alternate hostname (default: ${os.hostname().replace('.ec2.internal', '')})`) - .option('-t, --tags ', 'add tags for this host, separate multiple tags by comma', utils.appender(), []) - .option('-l, --list [params]', 'show the saved configuration (all unless params specified)', utils.appender(), false) - .option('-u, --unset ', 'clear some saved configurations (use "all" to unset all except key)', utils.appender(), []) - .option('-s, --set [key=value]', 'set config variables', utils.appender(), false) + .option( + '-n, --hostname ' + , `Uses alternate hostname (default: ${os.hostname().replace('.ec2.internal', '')})` + ) + .option( + '-t, --tags ', + 'Add tags for this host, separate multiple tags by comma' + , utils.appender(), + [] + ) + .option( + '-l, --list [params]' + , 'Show the saved configuration (all unless params specified)' + , utils.appender(), + + false + ) + .option( + '-u, --unset ' + , 'Clear some saved configurations (use "all" to unset all except key)' + , utils.appender(), + + [] + ) + .option('-s, --set [key=value]', 'Set config variables', utils.appender(), false) .on('--help', () => { console.log(' Examples:') console.log() console.log(' $ logdna-agent --key YOUR_INGESTION_KEY') console.log(' $ logdna-agent -d /home/ec2-user/logs') - console.log(' $ logdna-agent -d /home/ec2-user/logs -d /path/to/another/log_dir # multiple logdirs in 1 go') - console.log(' $ logdna-agent -d "/var/log/*.txt" # supports glob patterns') - console.log(' $ logdna-agent -d "/var/log/**/myapp.log" # myapp.log in any subfolder') + console.log('\ + $ logdna-agent -d /home/ec2-user/logs -d /path/to/another/log_dir\ + # multiple logdirs in 1 go\ + ') + console.log(' $ logdna-agent -d "/var/log/*.txt" ' + + '# supports glob patterns') + console.log(' $ logdna-agent -d "/var/log/**/myapp.log" ' + + '# myapp.log in any subfolder') console.log(' $ logdna-agent -f /usr/local/nginx/logs/access.log') - console.log(' $ logdna-agent -f /usr/local/nginx/logs/access.log -f /usr/local/nginx/logs/error.log') - console.log(' $ logdna-agent -t production # tags') + console.log(' $ logdna-agent -f /usr/local/nginx/logs/access.log ' + + '-f /usr/local/nginx/logs/error.log') + console.log(' $ logdna-agent -t production ' + + '# tags') console.log(' $ logdna-agent -t staging,2ndtag') - console.log(' $ logdna-agent -l tags,key,logdir # show specific config parameters') - console.log(' $ logdna-agent -l # show all') - console.log(' $ logdna-agent -u tags,logdir # unset specific entries from config') - console.log(' $ logdna-agent -u all # unset all except LogDNA API Key') + console.log(' $ logdna-agent -l tags,key,logdir ' + + '# show specific config parameters') + console.log(' $ logdna-agent -l ' + + '# show all') + console.log(' $ logdna-agent -u tags,logdir ' + + '# unset specific entries from config') + console.log(' $ logdna-agent -u all ' + + '# unset all except LogDNA API Key') console.log() }) @@ -89,7 +132,8 @@ function loadConfig(program) { } if (!program.key && !parsedConfig.key) { - console.error('LogDNA Ingestion Key not set! Use -k to set or use environment variable LOGDNA_AGENT_KEY.') + console.error('LogDNA Ingestion Key not set! ' + + 'Use -k to set or use environment variable LOGDNA_AGENT_KEY.') return } @@ -99,11 +143,15 @@ function loadConfig(program) { if (!program.hostname && !parsedConfig.hostname) { if (fs.existsSync(HOSTNAME_PATH) && fs.statSync(HOSTNAME_PATH).isFile()) { - parsedConfig.hostname = fs.readFileSync(HOSTNAME_PATH).toString().trim().replace(HOSTNAME_IP_REGEX, '') + parsedConfig.hostname = fs.readFileSync(HOSTNAME_PATH) + .toString() + .trim() + .replace(HOSTNAME_IP_REGEX, '') } else if (os.hostname()) { parsedConfig.hostname = os.hostname().replace('.ec2.internal', '') } else { - console.error('Hostname information cannot be found! Use -n to set or use environment variable LOGDNA_HOSTNAME.') + console.error('Hostname information cannot be found! ' + + 'Use -n to set or use environment variable LOGDNA_HOSTNAME.') return } } @@ -140,7 +188,8 @@ function loadConfig(program) { const kvPair = setOption.split('=') if (kvPair.length === 2) { parsedConfig[kvPair[0]] = kvPair[1] - saveMessages.push(`Config variable: ${kvPair[0]} = ${kvPair[1]} has been saved to config.`) + saveMessages.push(`Config variable: ${kvPair[0]} = ${kvPair[1]} ` + + 'has been saved to config.') } else { saveMessages.push(`Unknown setting: ${setOption}. Usage: -s [key=value]`) } @@ -191,7 +240,8 @@ function loadConfig(program) { // a slash. The user should be forced to provide open and closing slashes, // otherwise it's too hard to know what's part of the pattern text. parsedConfig.exclude_regex = re.replace(/^\//, '').replace(/\/$/, '') - saveMessages.push(`Exclude pattern: /${parsedConfig.exclude_regex}/ been saved to config.`) + saveMessages + .push(`Exclude pattern: /${parsedConfig.exclude_regex}/ been saved to config.`) } if (program.hostname) { @@ -260,15 +310,20 @@ function loadConfig(program) { config.awstype = responseBody.instanceType } - const networkInterface = Object.values(os.networkInterfaces()).reduce((networkData, interfaces) => { - interfaces.forEach(interfce => networkData.push(interfce)) - return networkData - }, []).filter((interfce) => { - return interfce.family && interfce.family === 'IPv4' && interfce.mac && interfce.address && ( - interfce.address.startsWith('10.') - || interfce.address.startsWith('172.') - || interfce.address.startsWith('192.168.')) - })[0] + const networkInterface = Object.values(os.networkInterfaces()) + .reduce((networkData, interfaces) => { + interfaces.forEach((interfce) => { return networkData.push(interfce) }) + return networkData + }, []).filter((interfce) => { + return interfce.family + && interfce.family === 'IPv4' + && interfce.mac && interfce.address + && ( + interfce.address.startsWith('10.') + || interfce.address.startsWith('172.') + || interfce.address.startsWith('192.168.') + ) + })[0] if (networkInterface) { if (networkInterface.mac) { config.mac = networkInterface.mac } @@ -287,14 +342,21 @@ function loadConfig(program) { }) } -process.on('uncaughtException', err => utils.log(`uncaught error: ${(err.stack || '').split('\r\n')}`, 'error')) +process.on('uncaughtException', (err) => { + return utils.log(`uncaught error: ${(err.stack || '').split('\r\n')}`, 'error') +}) if (require.main === module) { commander.parse(process.argv) - const isAdmin = os.platform() === 'win32' ? require('is-administrator')() : process.getuid() === 0 + const isAdmin = os.platform() === 'win32' + ? require('is-administrator')() + : process.getuid() === 0 if (!isAdmin) { - console.log('You must be an Administrator (root, sudo) to run this agent! See -h or --help for more info.') + console.log('You must be an Administrator (root, sudo) to run this agent! ' + + 'See -h or --help for more info.') + + process.exitCode = 1 return } diff --git a/lib/config.js b/lib/config.js index f722ba57..8cc08cc1 100644 --- a/lib/config.js +++ b/lib/config.js @@ -5,24 +5,35 @@ const os = require('os') const path = require('path') module.exports = { - AWS_INSTANCE_CHECK_URL: 'http://169.254.169.254/latest/dynamic/instance-identity/document/' + AWS_INSTANCE_CHECK_URL: + 'http://169.254.169.254/latest/dynamic/instance-identity/document/' , COMPRESS: isNaN(process.env.COMPRESS) || process.env.COMPRESS === '1' -, DEFAULT_CONF_FILE: os.platform() !== 'win32' ? '/etc/logdna.conf' : path.join(process.env.ALLUSERSPROFILE, '/logdna/logdna.conf') -, DEFAULT_LOG_PATH: os.platform() !== 'win32' ? '/var/log' : path.join(process.env.ALLUSERSPROFILE, '/logs') -, DEFAULT_WINTAIL_FILE: os.platform() !== 'win32' ? '/etc/winTail.ps1' : path.join(process.env.ALLUSERSPROFILE, '/logdna/winTail.ps1') +, DEFAULT_CONF_FILE: os.platform() !== 'win32' + ? '/etc/logdna.conf' + : path.join(process.env.ALLUSERSPROFILE, '/logdna/logdna.conf') +, DEFAULT_LOG_PATH: os.platform() !== 'win32' + ? '/var/log' + : path.join(process.env.ALLUSERSPROFILE, '/logs') +, DEFAULT_WINTAIL_FILE: os.platform() !== 'win32' + ? '/etc/winTail.ps1' + : path.join(process.env.ALLUSERSPROFILE, '/logdna/winTail.ps1') , FILEQUEUE_PATH: os.platform() !== 'win32' ? (process.env.FILEQUEUE_PATH || '/tmp') : path.join(process.env.ALLUSERSPROFILE, (process.env.FILEQUEUE_PATH || '/tmp')) , FLUSH_INTERVAL: process.env.FLUSH_INTERVAL || 1000 , FLUSH_LIMIT: process.env.FLUSH_LIMIT , PROXY: process.env.HTTPS_PROXY || process.env.HTTP_PROXY -, LOGDNA_LOGENDPOINT: process.env.LDLOGPATH || process.env.INGESTION_ENDPOINT || '/logs/agent' +, LOGDNA_LOGENDPOINT: process.env.LDLOGPATH + || process.env.INGESTION_ENDPOINT + || '/logs/agent' , LOGDNA_LOGHOST: process.env.LDLOGHOST || process.env.INGESTION_HOST || 'logs.logdna.com' , LOGDNA_LOGPORT: process.env.LDLOGPORT || process.env.INGESTION_PORT || 443 , LOGDNA_LOGSSL: isNaN(process.env.LDLOGSSL) ? true : +process.env.LDLOGSSL , RESCAN_INTERVAL: process.env.RESCAN_INTERVAL || 60000 // 1 min , RESCAN_INTERVAL_K8S: process.RESCAN_INTERVAL_K8S || 10000 // 10 sec -, SOCKET_PATH: process.env.LOGDNA_DOCKER_SOCKET || process.env.DOCKER_SOCKET || '/var/run/docker.sock' +, SOCKET_PATH: process.env.LOGDNA_DOCKER_SOCKET + || process.env.DOCKER_SOCKET + || '/var/run/docker.sock' , TAIL_MODE: process.env.TAIL_MODE || 'trs' , TRS_READ_INTERVAL: process.env.TRS_READ_INTERVAL || 1000 // 1 sec , TRS_READ_TIMEOUT: process.env.TRS_READ_TIMEOUT || 300000 // 5 min diff --git a/lib/file-utilities.js b/lib/file-utilities.js index 99df02da..e4418eb4 100644 --- a/lib/file-utilities.js +++ b/lib/file-utilities.js @@ -1,15 +1,17 @@ 'use strict' +const fs = require('fs') +const os = require('os') const async = require('async') const debug = require('debug')('logdna:lib:file-utilities') -const fs = require('fs') const glob = require('glob') -const os = require('os') +const split2 = require('split2') +const TailFile = require('@logdna/tail-file') + const spawn = require('child_process').spawn const log = require('./utils').log -const TailReadStream = require('./tailreadstream/tailreadstream') -const Splitter = require('./tailreadstream/line-splitter') + const client = require('./logger-client.js') const GLOB_CHARS_REGEX = /[*?[\]()]/ @@ -102,32 +104,43 @@ function getFiles(config, dir, callback) { function streamFiles(config, logfiles, callback) { logfiles.forEach((file) => { - let tail - - if (os.platform() !== 'win32' && config.TAIL_MODE === 'unix') { - debug('tailing: ' + file) - tail = spawn('tail', ['-n0', '-F', file]) - tail.stdout.on('data', (data) => { - const lines = data.toString().trim().split('\n') - for (const line of lines) { - if (config.exclude_regex && config.exclude_regex.test(line)) continue - client.log.agentLog({ - line - , f: file - }) - } - }) + const startPos = 0 + const tail = new TailFile(file, { + encoding: 'utf8' + , startPos + }) + debug(`tailing: ${file}`) + tail.on('error', (error) => { + log(`tail error: ${file}: ${error}`, 'error') + }) + + tail.on('end', (error) => { + if (error) { + log( + `file does not exist, stopped tailing: ${file} after ${tail.timeout}ms` + , 'warn' + ) + files = files.filter((element) => { return element !== file }) + } + }) - tail.stderr.on('data', (error) => { - log(`tail error: ${file}: ${error.toString().trim()}`, 'error') - }) + tail.on('rename', () => { + log(`log rotated: ${file} by rename`) + }) - tails.push(tail) + tail.on('truncate', () => { + log(`log rotated: ${file} by truncation`) + }) - } else { - debug(`tailing: ${file}`) - tail = TailReadStream.tail(file, config) - tail.pipe(new Splitter()) + try { + tail + .start() + .then(callback) + .catch((err) => { + log.error(err, 'Tail could not start. Does the file exist?') + }) + + tail.pipe(split2()) .on('data', (line) => { if (config.exclude_regex && config.exclude_regex.test(line)) return client.log.agentLog({ @@ -135,31 +148,13 @@ function streamFiles(config, logfiles, callback) { , f: file }) }) - - tail.on('error', (error) => { - log(`tail error: ${file}: ${error}`, 'error') - }) - - tail.on('end', (error) => { - if (error) { - log(`file does not exist, stopped tailing: ${file} after ${tail.timeout}ms`, 'warn') - files = files.filter(element => element !== file) - } - }) - - tail.on('rename', () => { - log(`log rotated: ${file} by rename`) - }) - - tail.on('truncate', () => { - log(`log rotated: ${file} by truncation`) - }) - tailStreams.push(tail) + } catch (err) { + + log.error(err, 'Tail could not start. Does the file exist?') } + return callback && callback() }) - - return callback && callback() } function streamAllLogs(config, callback) { @@ -169,20 +164,25 @@ function streamAllLogs(config, callback) { async.each(config.logdir, (dir, done) => { getFiles(config, dir, (err, logfiles) => { if (!err && logfiles.length > 0) { + conf = config debug(`all ${dir} files`) debug(logfiles) // figure out new files that we're not already tailing - var diff = logfiles.filter(element => files.indexOf(element) < 0) + var diff = logfiles.filter((element) => { return files.indexOf(element) < 0 }) // unique filenames between logdir(s) newfiles = newfiles.concat(diff) - newfiles = newfiles.filter((element, index) => newfiles.indexOf(element) === index) + newfiles = newfiles.filter((element, index) => { + return newfiles.indexOf(element) === index + }) debug(`newfiles after processing ${dir}`) debug(newfiles) if (diff.length > 0) { - log(`streaming ${dir}: ${diff.length}${(!firstrun ? ` new file(s), ${logfiles.length} total` : '')} file(s)`) + log(`streaming ${dir}: ${diff.length}${( + !firstrun ? ` new file(s), ${logfiles.length} total` : '')} + file(s)`) } } done() @@ -205,35 +205,21 @@ function streamAllLogs(config, callback) { var journalctl, lastchunk, i if (config.usejournald === 'files') { - journalctl = spawn('journalctl', ['-n0', '-D', '/var/log/journal', '-o', 'json', '-f']) + journalctl = spawn( + 'journalctl', + ['-n0', '-D', '/var/log/journal', '-o', 'json', '-f'] + ) } else { journalctl = spawn('journalctl', ['-n0', '-o', 'json', '-f']) } - const processChunk = (data) => { - try { - data = JSON.parse(data) - } catch (e) { - return {t: Date.now(), l: data, f: 'systemd'} - } - if (data.__REALTIME_TIMESTAMP && parseInt(data.__REALTIME_TIMESTAMP) > 10000000000000) { - // convert to ms - data.__REALTIME_TIMESTAMP = parseInt(data.__REALTIME_TIMESTAMP / 1000) - } - return { - t: data.__REALTIME_TIMESTAMP - , line: data.MESSAGE - , f: data.CONTAINER_NAME || data._SYSTEMD_UNIT || data.SYSLOG_IDENTIFIER || 'UNKNOWN_SYSTEMD_APP' - , pid: data._PID && parseInt(data._PID) - , prival: data.PRIORITY && parseInt(data.PRIORITY) - , containerid: data.CONTAINER_ID_FULL - } - } - journalctl.stdout.on('data', (data) => { data = data.toString().trim().split('\n') for (i = 0; i < data.length; i++) { - if (data[i].substring(0, 1) === '{' && data[i].substring(data[i].length - 1) === '}') { + if ( + data[i].substring(0, 1) === '{' + && data[i].substring(data[i].length - 1) === '}' + ) { // full chunk client.log.agentLog(processChunk(data[i])) if (lastchunk) { lastchunk = null } // clear @@ -271,17 +257,41 @@ function streamAllLogs(config, callback) { }, config.RESCAN_INTERVAL) // rescan for files every once in awhile } -function gracefulShutdown(signal) { +function processChunk(input) { + let data + try { + data = JSON.parse(input) + } catch (e) { + return {t: Date.now(), l: input, f: 'systemd'} + } + if (data.__REALTIME_TIMESTAMP && parseInt(data.__REALTIME_TIMESTAMP) > 10000000000000) { + // convert to ms + data.__REALTIME_TIMESTAMP = parseInt(data.__REALTIME_TIMESTAMP / 1000) + } + return { + t: data.__REALTIME_TIMESTAMP + , line: data.MESSAGE + , f: data.CONTAINER_NAME + || data._SYSTEMD_UNIT + || data.SYSLOG_IDENTIFIER + || 'UNKNOWN_SYSTEMD_APP' + , pid: data._PID && parseInt(data._PID) + , prival: data.PRIORITY && parseInt(data.PRIORITY) + , containerid: data.CONTAINER_ID_FULL + } +} +async function gracefulShutdown(signal) { log(`got ${signal} signal, shutting down...`) clearTimeout(conf.rescanTimer) - tails.forEach((tail) => { - tail.kill('SIGTERM') + for (const tail of tails) { + await tail.quit('SIGTERM') debug(`tail pid ${tail.pid} killed`) - }) + } + tails.length = 0 for (const stream of tailStreams) { - debug(`Destroying ${stream._filepath}`) - stream.destroy() + await stream.quit() + debug(`tail pid ${stream.pid} killed`) } } diff --git a/lib/tailreadstream/line-splitter.js b/lib/tailreadstream/line-splitter.js deleted file mode 100644 index c6217736..00000000 --- a/lib/tailreadstream/line-splitter.js +++ /dev/null @@ -1,55 +0,0 @@ -'use strict' -// from https://github.com/msimerson/safe-log-reader/blob/master/lib/line-splitter.js - -// https://nodejs.org/api/stream.html#stream_object_mode - -var StringDecoder = require('string_decoder').StringDecoder -var Transform = require('stream').Transform -var util = require('util') - -function LineSplitter(options) { - if (!options) { options = {} } - if (!options.transform) { options.transform = {objectMode: true} } - - Transform.call(this, options.transform) - - this._encoding = options.encoding || 'utf8' - this._seperator = options.seperator || '\n' - this._buffer = '' - this._decoder = new StringDecoder(this._encoding) - - this.bytes = options.bytes || 0 -} - -util.inherits(LineSplitter, Transform) - -LineSplitter.prototype._transform = function(chunk, encoding, done) { - this.bytes += chunk.length - - if (encoding !== this._encoding) { - // this is likely 'buffer' when the source file is an archive - this._buffer += this._decoder.write(chunk) - } else { - // already decoded by fs.createReadStream - this._buffer += chunk - } - - var lines = this._buffer.split(this._seperator) - this._buffer = lines.pop() - - for (var i = 0; i < lines.length; i++) { - this.push(lines[i]) - } - done() -} - -LineSplitter.prototype._flush = function(done) { - // trailing text (after last seperator) - var rem = this._buffer.trim() - if (rem) { this.push(rem) } - done() -} - -module.exports = function(options) { - return new LineSplitter(options) -} diff --git a/lib/tailreadstream/tailreadstream.js b/lib/tailreadstream/tailreadstream.js deleted file mode 100644 index a15c8829..00000000 --- a/lib/tailreadstream/tailreadstream.js +++ /dev/null @@ -1,251 +0,0 @@ -'use strict' -/* - * The base code for this is taken from the 'node-growing-file' module (https://github.com/felixge/node-growing-file) by felixge - * Due to the inactivity of the repo and our desire to switch to ES6 syntax, the code has been ported over with a few minor tweaks to the calling params - */ - -const fs = require('fs') -const debug = require('debug')('logdna:tailreadstream') -const Readable = require('stream').Readable -const config = require('../config') - -const DOES_NOT_EXIST_ERROR = 'ENOENT' -const FILE_LOCKED_ERROR = 'EBUSY' - -class TailReadStream extends Readable { - constructor(filepath, options) { - options = options || {} - - super() - - this.readable = true - - this._filepath = filepath - this._stream = null - this._offset = 0 - - this._interval = config.TRS_READ_INTERVAL || options.interval - this._timeout = config.TRS_READ_TIMEOUT || options.timeout - this._watchinterval = config.TRS_WATCH_INTERVAL || options.watchinterval - this._tailheadsize = config.TRS_TAILHEAD_SIZE || options.tailheadsize - this._tailheadage = config.TRS_TAILHEAD_AGE || options.tailheadage - this._idle = 0 - - this._reading = false - this._paused = false - this._ended = false - this._watching = false - this._exiting = false - this._retryTimer = null - this._watchTimer = null - this._readTimer = null - } - - static tail(filepath, fromstart, options) { - if (typeof fromstart === 'object') { // shift args - options = fromstart - fromstart = false - } - var file = new this(filepath, options) - if (fromstart) { - if (typeof fromstart === 'boolean') { - // read from start - debug(`${filepath}: reading from beginning of file`) - file._readFromOffsetUntilEof() - } else { - // read from offset - debug(`${filepath}: reading from offset ${fromstart}`) - file._readFromOffsetUntilEof(+fromstart) - } - } else { - // tail from end - file._getFileSizeAndReadUntilEof() - } - return file - } - - get offset() { return this._offset } - get timeout() { return this._timeout } - set timeout(timeout) { this._timeout = timeout } - - _destroy() { - this._exiting = true - clearTimeout(this._retryTimer) - clearTimeout(this._watchTimer) - clearTimeout(this._readTimer) - this.readable = false - this._stream = null - this.timeout = 0 - } - - pause() { - this._paused = true - this._stream.pause() - } - - resume() { - if (!this._stream) return - this._paused = false - this._stream.resume() - this._readFromOffsetUntilEof() - } - - _readFromOffsetUntilEof(offset) { - if (!isNaN(offset)) { - this._offset = offset - } - - if (this._paused || this._reading) { - return - } - - this._reading = true - - this._stream = fs.createReadStream(this._filepath, { - start: this._offset - }) - - this._stream.on('error', this._handleError.bind(this)) - this._stream.on('data', this._handleData.bind(this)) - this._stream.on('end', this._handleEnd.bind(this)) - } - - _getFileSizeAndReadUntilEof() { - var that = this - - if (this._exiting) return - - fs.stat(this._filepath, (err, stats) => { - if (err) { - that.readable = false - - if (that._hasTimedOut()) { - debug(`${that._filepath}: file does not exist, timed out after ${that._timeout}ms`) - that.destroy() - that.emit('end', err) - return - } - - if (err.code === DOES_NOT_EXIST_ERROR) { - debug(`${that._filepath}: file does not exist, waiting for it to appear...`) - this._readTimer = setTimeout(that._getFileSizeAndReadUntilEof.bind(that), that._interval) - that._idle += that._interval - return - } - - that.emit('error', err) - return - } - - if (stats.size < that._tailheadsize && Date.now() - stats.birthtime.getTime() < that._tailheadage) { - debug(`${that._filepath}: file is smaller than ${that._tailheadsize} bytes and is ` - + `${(Date.now() - stats.birthtime.getTime())}ms old, reading from beginning` - ) - that._readFromOffsetUntilEof(0) // tail from beginning of file if small enough (e.g. newly created files) - } else { - that._readFromOffsetUntilEof(stats.size) - } - }) - } - - _retryInInterval() { - if (this._exiting) return - this._retryTimer = setTimeout(this._readFromOffsetUntilEof.bind(this), this._interval) - } - - _handleError(error) { - this._reading = false - - if (this._hasTimedOut()) { - debug(`${this._filepath}: file no longer exists, timed out after ${this._timeout}ms`) - this.destroy() - this.emit('end', error) - return - } - - if (error.code === DOES_NOT_EXIST_ERROR) { - debug(`${this._filepath}: file renamed, waiting for it to reappear...`) - if (this.readable) { - this.readable = false - this.emit('rename') - this._offset = 0 // reset on rename - } - this._idle += this._interval - this._retryInInterval() - return - } - - if (error.code === FILE_LOCKED_ERROR) { - debug(`${this._filepath}: file locked, waiting for it to be released...`) - this._idle += this._interval - this._retryInInterval() - return - } - - this.readable = false - - this.emit('error', error) - } - - _handleData(data) { - this.readable = true - - this._offset += data.length - this._idle = 0 - - debug(`${this._filepath}: reading ${data.length} bytes`) - this.emit('data', data) - } - - _handleEnd() { - this._reading = false - - if (!this._watching) { - this._watching = true - this._watchFile() - } - - if (!this._hasTimedOut()) { - this._retryInInterval() - return - } - - this.destroy() - this.emit('end') - } - - _hasTimedOut() { - return this._idle >= this._timeout - } - - _watchFile() { - var that = this - - if (this._exiting || !this.readable) { - this._watching = false - return - } - - fs.stat(this._filepath, (err, stats) => { - if (err) { - this._watchTimer = setTimeout(that._watchFile.bind(that), that._watchinterval) - return - } - - if (stats.size < that._offset) { - that.emit('truncate', stats.size) - if (stats.size < that._tailheadsize) { - debug(`${that._filepath}: file truncated but smaller than ${that._tailheadsize} bytes, reading from beginning`) - that._offset = 0 - } else { - debug(`${that._filepath}: file truncated but larger than ${that._tailheadsize} bytes, reading from ${stats.size}`) - that._offset = stats.size - } - } - - this._watchTimer = setTimeout(that._watchFile.bind(that), that._watchinterval) - }) - } -} - -module.exports = TailReadStream diff --git a/lib/utils.js b/lib/utils.js index 9d293fed..205e2f01 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -18,8 +18,8 @@ function merge(objects) { return objects.reduce((merged, obj) => { if (!Array.isArray(obj)) obj = [obj] const arr = merged.pop().concat(obj) - merged.push(arr.filter((element, index) => arr.indexOf(element) === index)) - return merged.filter((element, index) => merged.indexOf(element) === index) + merged.push(arr.filter((element, index) => { return arr.indexOf(element) === index })) + return merged.filter((element, index) => { return merged.indexOf(element) === index }) }, [[]])[0] } @@ -42,7 +42,7 @@ function split(str, delimiter, hasWhitespace) { element = element.replace(/\+/, ' ') } return element.trim() - }).filter(element => element) + }).filter((element) => { return element }) } // Custom Appender @@ -53,20 +53,25 @@ function appender(xs) { return xs } } - +function padStart(number, expectedLength) { + const strNumber = String(Math.abs(number)).padStart(expectedLength, '0') + return (number < 0) ? `-${strNumber}` : strNumber +} // Custom Logger function log(message, level) { const dateObject = new Date() - const padStart = (number, expectedLength) => { - const strNumber = String(Math.abs(number)).padStart(expectedLength, '0') - return (number < 0) ? `-${strNumber}` : strNumber - } - const date = `${dateObject.getFullYear()}-${padStart(dateObject.getMonth() + 1, 2)}-${padStart(dateObject.getDate(), 2)}` - const time = `${padStart(dateObject.getHours(), 2)}:${padStart(dateObject.getMinutes(), 2)}:${padStart(dateObject.getSeconds(), 2)}` + const date = `${dateObject.getFullYear()}-${ + padStart(dateObject.getMonth() + 1, 2) + }-${padStart(dateObject.getDate(), 2)}` + const time = `${padStart(dateObject.getHours(), 2)}:${ + padStart(dateObject.getMinutes(), 2) + }:${padStart(dateObject.getSeconds(), 2)}` const tzOffset = dateObject.getTimezoneOffset() * -100 / 60 const timezone = `${(tzOffset > 0 ? '+' : '')}${padStart(tzOffset, 4)}` // Properly handle Error objects. The client may have additional meta information to log - if (typeof message === 'object' && message.constructor && message.constructor.name === 'Error') { + if (typeof message === 'object' + && message.constructor + && message.constructor.name === 'Error') { console.error(`${date} ${time} ${timezone} [error]`, message) return } @@ -75,7 +80,7 @@ function log(message, level) { // Pick the Keys to List Values of function pick2list(options, config) { - const lowOptions = options.map(value => value.toLowerCase()) + const lowOptions = options.map((value) => { return value.toLowerCase() }) if (lowOptions.indexOf('all') > -1) { return { cfg: config @@ -102,12 +107,20 @@ function pick2list(options, config) { // Custom Processing - Combining all processes function processOption(options, config, hasWhitespace) { - const newValues = merge(options.map(option => split(option, ',', hasWhitespace))) - const oldValues = (config ? (typeof config === 'string' ? split(config, ',', false) : config) : []) - const diff = newValues.filter(value => oldValues.indexOf(value) < 0).filter(value => value) + const newValues = merge(options.map((option) => { + return split(option, ',', hasWhitespace) + })) + const oldValues = (config + ? (typeof config === 'string' + ? split(config, ',', false) + : config) + : []) + const diff = newValues.filter((value) => { + return oldValues.indexOf(value) < 0 + }).filter(Boolean) return { - values: merge([oldValues, newValues]).filter(element => element) + values: merge([oldValues, newValues]).filter((element) => { return element }) , diff: preparePostMessage(diff) } } @@ -146,8 +159,10 @@ function stringify(obj) { // Custom UnSetting Configuration function unsetConfig(options, config) { - options = merge(options.map(option => split(option, ',', false).filter(element => element !== 'key'))) - const lowOptions = options.map(value => value.toLowerCase()) + options = merge(options.map((option) => { + return split(option, ',', false).filter((element) => { return element !== 'key' }) + })) + const lowOptions = options.map((value) => { return value.toLowerCase() }) if (lowOptions.indexOf('all') > -1) { return { cfg: { @@ -162,7 +177,9 @@ function unsetConfig(options, config) { return obj }, {}) const newValues = (config ? Object.keys(config) : []) - const diff = oldValues.filter(value => newValues.indexOf(value) < 0).filter(value => value) + const diff = oldValues.filter((value) => { + return newValues.indexOf(value) < 0 + }).filter(Boolean) return { cfg: config diff --git a/package.json b/package.json index 7e74e3ab..3c529039 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "logdna-agent", - "version": "2.2.1", + "version": "2.2.2", "description": "LogDNA Agent streams from log files to your LogDNA account. Works with Linux, Windows, and macOS Servers", "main": "index.js", "scripts": { @@ -19,23 +19,21 @@ }, "homepage": "https://github.com/logdna/logdna-agent", "eslintConfig": { - "extends": [ - "logdna" - ], "root": true, "ignorePatterns": [ "node_modules/", "coverage/" ], + "extends": [ + "logdna" + ], "parserOptions": { "ecmaVersion": 2019 - }, - "rules": { - "node/no-deprecated-api": 0 } }, "dependencies": { "@logdna/logger": "^2.2.2", + "@logdna/tail-file": "^2.2.0", "async": "^3.2.0", "commander": "^4.1.1", "debug": "^4.1.1", @@ -43,11 +41,12 @@ "glob": "^7.1.6", "is-administrator": "^1.0.1", "properties": "^1.2.1", - "request": "^2.88.2" + "request": "^2.88.2", + "split2": "^4.1.0" }, "devDependencies": { - "eslint": "^7.6.0", - "eslint-config-logdna": "^1.0.0", + "eslint": "^7.32.0", + "eslint-config-logdna": "^6.1.0", "nock": "^13.0.4", "tap": "^14.10.8" } diff --git a/test/unit/file-utilities.js b/test/unit/file-utilities.js index 682ffb21..75d33a8a 100644 --- a/test/unit/file-utilities.js +++ b/test/unit/file-utilities.js @@ -1,8 +1,8 @@ 'use strict' +const fs = require('fs') const path = require('path') const {test} = require('tap') -const fs = require('fs') const fileUtilities = require('../../lib/file-utilities.js') test('getFiles()', async (t) => { @@ -24,7 +24,7 @@ test('getFiles()', async (t) => { tt.error(err, 'No error') tt.type(files, Array, 'files is an array') tt.equal(files.length, 5, 'File count is correct') - tt.deepEqual(files.sort(), [ + tt.same(files.sort(), [ path.join(tempDir, 'somelog1.log') , path.join(tempDir, 'somelog2') , path.join(tempDir, 'somelog3-file') @@ -62,7 +62,7 @@ test('getFiles()', async (t) => { tt.error(err, 'No error') tt.type(files, Array, 'files is an array') tt.equal(files.length, 1, 'File count is correct') - tt.deepEqual(files[0], path.join(tempDir, 'somelog1.txt'), 'Filename is correct') + tt.same(files[0], path.join(tempDir, 'somelog1.txt'), 'Filename is correct') tt.end() }) }) @@ -83,7 +83,7 @@ test('getFiles()', async (t) => { tt.error(err, 'No error') tt.type(files, Array, 'files is an array') tt.equal(files.length, 3, 'File count is correct') - tt.deepEqual(files.sort(), [ + tt.same(files.sort(), [ path.join(tempDir, 'somelog1.txt') , path.join(tempDir, 'somelink.txt') , path.join(tempDir, 'subdir', 'somelog2.txt') @@ -102,7 +102,7 @@ test('getFiles()', async (t) => { tt.error(err, 'No error') tt.type(files, Array, 'files is an array') tt.equal(files.length, 1, 'File count is correct') - tt.deepEqual(files, [file], 'Single filename is correct') + tt.same(files, [file], 'Single filename is correct') tt.end() }) }) diff --git a/test/unit/index.js b/test/unit/index.js index 23b4db6a..32978005 100644 --- a/test/unit/index.js +++ b/test/unit/index.js @@ -50,7 +50,7 @@ test('loadConfig() settings based on user input', async (t) => { , 'key = abc123' , 'hostname = myMachine' ].sort() - tt.deepEqual(contents, expected, 'Config file was written correctly') + tt.same(contents, expected, 'Config file was written correctly') tt.end() }, 200) }) @@ -71,7 +71,7 @@ test('loadConfig() settings based on user input', async (t) => { , 'key = abc123' , 'hostname = myMachine' ].sort() - tt.deepEqual(contents, expected, 'log directories added to the config file') + tt.same(contents, expected, 'log directories added to the config file') tt.end() }, 200) }) @@ -175,7 +175,7 @@ test('loadConfig() hostname decisions', async (t) => { , 'key = abc123' , `hostname = ${os.hostname()}` ].sort() - tt.deepEqual(contents, expected, 'Host name set to os.hostname()') + tt.same(contents, expected, 'Host name set to os.hostname()') tt.end() }, 200) }) @@ -223,7 +223,7 @@ test('Process options through commander', (t) => { , 'propkey = val' ].sort() - t.deepEqual(contents, expected, 'Commander set options') + t.same(contents, expected, 'Commander set options') t.end() }, 200) }) diff --git a/tools/files/darwin/logdna-agent.rb b/tools/files/darwin/logdna-agent.rb index d638213c..629638ca 100644 --- a/tools/files/darwin/logdna-agent.rb +++ b/tools/files/darwin/logdna-agent.rb @@ -2,7 +2,7 @@ # frozen_string_literal: true cask "logdna-agent" do - version "2.2.1" + version "2.2.2" sha256 "b5c44e27cd6f4a92ff2eb12be09ff0be7e8a56309db33c139db42f5110a037fb" # github.com/logdna/logdna-agent/ was verified as official when first introduced to the cask @@ -18,6 +18,7 @@ launchctl: "com.logdna.logdna-agentd" caveats <<~EOS + DEPRECATION WARNING!!! logdna-agent will soon be deprecated. please refer to LogDNA Agent V2 https://github.com/logdna/logdna-agent-v2 When you first start logdna-agent, you must set your LogDNA Ingestion Key with the command: sudo logdna-agent -k diff --git a/tools/files/win32/logdna-agent.nuspec b/tools/files/win32/logdna-agent.nuspec index 5ffd2668..4af88119 100644 --- a/tools/files/win32/logdna-agent.nuspec +++ b/tools/files/win32/logdna-agent.nuspec @@ -4,7 +4,7 @@ logdna-agent LogDNA Agent for Windows - 2.2.1 + 2.2.2 smusali,leeliu leeliu LogDNA Agent for Windows diff --git a/tools/scripts/darwin.sh b/tools/scripts/darwin.sh index 652c0574..a405a670 100644 --- a/tools/scripts/darwin.sh +++ b/tools/scripts/darwin.sh @@ -5,7 +5,7 @@ ARCH=x64 INPUT_TYPE=dir LICENSE=MIT -NODE_VERSION=12.16.2 +NODE_VERSION=14.15.3 OSXPKG_IDENTIFIER_PREFIX=com.logdna OUTPUT_TYPE=osxpkg PACKAGE_NAME=logdna-agent diff --git a/tools/scripts/debian.sh b/tools/scripts/debian.sh index dc216237..6a2adbf2 100644 --- a/tools/scripts/debian.sh +++ b/tools/scripts/debian.sh @@ -2,11 +2,11 @@ # THIS SHOULD RUN ON DEBIAN FROM THE PROJECT DIRECTORY # VARIABLES -ARCH=x64 +ARCH=x86 DEB_SIGNATURE_ID=EF506BE8 INPUT_TYPE=dir LICENSE=MIT -NODE_VERSION=12.16.2 +NODE_VERSION=14.15.3 OUTPUT_TYPE=deb PACKAGE_NAME=logdna-agent S3_BUCKET=repo.logdna.com diff --git a/tools/scripts/redhat.sh b/tools/scripts/redhat.sh index 0cf6f520..020af713 100644 --- a/tools/scripts/redhat.sh +++ b/tools/scripts/redhat.sh @@ -5,7 +5,7 @@ ARCH=x64 INPUT_TYPE=dir LICENSE=MIT -NODE_VERSION=8.3.0 +NODE_VERSION=14.15.3 OUTPUT_TYPE=rpm PACKAGE_NAME=logdna-agent S3_BUCKET=repo.logdna.com diff --git a/tools/scripts/win32.sh b/tools/scripts/win32.sh index 50b9dd48..760c7418 100644 --- a/tools/scripts/win32.sh +++ b/tools/scripts/win32.sh @@ -3,7 +3,7 @@ # VARIABLES ARCH=x64 -NODE_VERSION=12.16.2 +NODE_VERSION=14.15.3 PACKAGE_NAME=logdna-agent VERSION=$(cat tools/files/win32/logdna-agent.nuspec | grep "" | cut -d'>' -f2 | cut -d'<' -f1)