diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c1e5760c..0355c46b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -60,12 +60,12 @@ deploy:docker: script: - sh scripts/replace_templates.sh ${CI_JOB_TOKEN} - docker info - - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin artifacts.bitsensor.io:1443 + - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin docker.bitsensor.io - docker build -t elastalert . - - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:latest - - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:$(git describe --abbrev=0) - - docker push artifacts.bitsensor.io:1443/elastalert:latest - - docker push artifacts.bitsensor.io:1443/elastalert:$(git describe --abbrev=0) + - docker tag elastalert docker.bitsensor.io/elastalert:latest + - docker tag elastalert docker.bitsensor.io/elastalert:$(git describe --abbrev=0) + - docker push docker.bitsensor.io/elastalert:latest + - docker push docker.bitsensor.io/elastalert:$(git describe --abbrev=0) only: - tags tags: @@ -76,10 +76,10 @@ deploy:docker:snapshot: script: - sh scripts/replace_templates.sh ${CI_JOB_TOKEN} - docker info - - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin artifacts.bitsensor.io:1443 + - echo $BITSENSOR_PASSWORD | docker login -u $BITSENSOR_USERNAME --password-stdin docker.bitsensor.io - docker build -t elastalert . - - docker tag elastalert artifacts.bitsensor.io:1443/elastalert:snapshot - - docker push artifacts.bitsensor.io:1443/elastalert:snapshot + - docker tag elastalert docker.bitsensor.io/elastalert:snapshot + - docker push docker.bitsensor.io/elastalert:snapshot only: - develop tags: diff --git a/Dockerfile b/Dockerfile index 7e9b8a69..c83321d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:latest as py-ea -ARG ELASTALERT_VERSION=v0.1.33 +ARG ELASTALERT_VERSION=v0.1.36 ENV ELASTALERT_VERSION=${ELASTALERT_VERSION} # URL from which to download Elastalert. ARG ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/$ELASTALERT_VERSION.zip @@ -26,12 +26,10 @@ RUN sed -i 's/jira>=1.0.10/jira>=1.0.10,<1.0.15/g' setup.py && \ FROM node:alpine LABEL maintainer="BitSensor " -# Set this environment variable to True to set timezone on container start. -ENV SET_CONTAINER_TIMEZONE False -# Default container timezone as found under the directory /usr/share/zoneinfo/. -ENV CONTAINER_TIMEZONE Etc/UTC +# Set timezone for this container +ENV TZ Etc/UTC -RUN apk add --update --no-cache curl tzdata python2 make +RUN apk add --update --no-cache curl tzdata python2 make libmagic COPY --from=py-ea /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages COPY --from=py-ea /opt/elastalert /opt/elastalert diff --git a/README.md b/README.md index f9ce037f..38f32e80 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,10 @@ You can use the following config options: "dataPath": { // The path to a folder that the server can use to store data and temporary files. "relative": true, // Whether to use a path relative to the `elastalertPath` folder. "path": "/server_data" // The path to the data folder. - } + }, + "es_host": "elastalert", // For getting metadata and field mappings, connect to this ES server + "es_port": 9200, // Port for above + "writeback_index": "elastalert_status" // Writeback index to examine for /metadata endpoint } ``` @@ -200,11 +203,25 @@ This server exposes the following REST API's: "days": "1" // Whether to send real alerts - "alert": false + "alert": false, + + // Return results in structured JSON + "format": "json", + + // Limit returned results to this amount + "maxResults": 1000 } } ``` +- **GET `/metadata/:type`** + + Returns metadata from elasticsearch related to elasalert's state. `:type` should be one of: elastalert_status, elastalert, elastalert_error, or silence. See [docs about the elastalert metadata index](https://elastalert.readthedocs.io/en/latest/elastalert_status.html). + +- **GET `/mapping/:index`** + + Returns field mapping from elasticsearch for a given index. + - **[WIP] GET `/config`** Gets the ElastAlert configuration from `config.yaml` in `elastalertPath` (from the config). diff --git a/config/config.json b/config/config.json index 51a8c3da..65dfe601 100644 --- a/config/config.json +++ b/config/config.json @@ -12,5 +12,8 @@ "templatesPath": { "relative": true, "path": "/rule_templates" - } -} \ No newline at end of file + }, + "es_host": "elastalert", + "es_port": 9200, + "writeback_index": "elastalert_status" +} diff --git a/package.json b/package.json index 965973ac..5f7cb001 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@bitsensor/elastalert", - "version": "0.0.12", + "version": "0.0.13", "description": "A server that runs ElastAlert and exposes REST API's for manipulating rules and alerts.", "license": "MIT", "main": "index.js", @@ -11,7 +11,7 @@ }, "repository": { "type": "git", - "url": "git+https://git.bitsensor.io/back-end/elastalert.git" + "url": "git+https://github.com/bitsensor/elastalert.git" }, "directories": { "lib": "./lib", @@ -23,6 +23,8 @@ "babel-register": "^6.14.0", "body-parser": "^1.15.2", "bunyan": "^1.8.1", + "cors": "^2.8.4", + "elasticsearch": "^15.1.1", "express": "^4.14.0", "fs-extra": "^5.0.0", "joi": "^13.1.2", diff --git a/src/common/config/schema.js b/src/common/config/schema.js index 9734ec46..fe757a4d 100644 --- a/src/common/config/schema.js +++ b/src/common/config/schema.js @@ -3,6 +3,9 @@ import Joi from 'joi'; const schema = Joi.object().keys({ 'appName': Joi.string().default('elastalert-server'), + 'es_host': Joi.string().default('elastalert'), + 'es_port': Joi.number().default(9200), + 'writeback_index': Joi.string().default('elastalert_status'), 'port': Joi.number().default(3030), 'elastalertPath': Joi.string().default('/opt/elastalert'), 'rulesPath': Joi.object().keys({ diff --git a/src/common/elasticsearch_client.js b/src/common/elasticsearch_client.js new file mode 100644 index 00000000..7b7449e8 --- /dev/null +++ b/src/common/elasticsearch_client.js @@ -0,0 +1,9 @@ +import elasticsearch from 'elasticsearch'; +import config from './config'; + +export function getClient() { + var client = new elasticsearch.Client({ + hosts: [ `http://${config.get('es_host')}:${config.get('es_port')}`] + }); + return client; +} diff --git a/src/controllers/test/index.js b/src/controllers/test/index.js index 9e6c6f3d..053bb487 100644 --- a/src/controllers/test/index.js +++ b/src/controllers/test/index.js @@ -33,6 +33,15 @@ export default class TestController { processOptions.push('-m', 'elastalert.test_rule', '--config', 'config.yaml', tempFilePath, '--days', options.days); + if (options.format === 'json') { + processOptions.push('--formatted-output'); + } + + if (options.maxResults > 0) { + processOptions.push('--max-query-size'); + processOptions.push(options.maxResults); + } + if (options.alert) { processOptions.push('--alert'); } @@ -61,7 +70,12 @@ export default class TestController { testProcess.on('exit', function (statusCode) { if (statusCode === 0) { - resolve(stdoutLines.join('\n')); + if (options.format === 'json') { + resolve(stdoutLines.join('')); + } + else { + resolve(stdoutLines.join('\n')); + } } else { reject(stderrLines.join('\n')); logger.error(stderrLines.join('\n')); diff --git a/src/elastalert_server.js b/src/elastalert_server.js index d3747b45..8db07e9d 100644 --- a/src/elastalert_server.js +++ b/src/elastalert_server.js @@ -9,6 +9,7 @@ import ProcessController from './controllers/process'; import RulesController from './controllers/rules'; import TemplatesController from './controllers/templates'; import TestController from './controllers/test'; +import cors from 'cors'; let logger = new Logger('Server'); @@ -58,6 +59,7 @@ export default class ElastalertServer { // Start the server when the config is loaded config.ready(function () { try { + self._express.use(cors()); self._express.use(bodyParser.json()); self._express.use(bodyParser.urlencoded({ extended: true })); self._setupRouter(); diff --git a/src/handlers/mapping/get.js b/src/handlers/mapping/get.js new file mode 100644 index 00000000..7f7f741f --- /dev/null +++ b/src/handlers/mapping/get.js @@ -0,0 +1,20 @@ +import { getClient } from '../../common/elasticsearch_client'; + +export default function metadataHandler(request, response) { + /** + * @type {ElastalertServer} + */ + + var client = getClient(); + + client.indices.getMapping({ + index: request.params.index + }).then(function(resp) { + response.send(resp); + }, function(err) { + response.send({ + error: err + }); + }); + +} diff --git a/src/handlers/metadata/get.js b/src/handlers/metadata/get.js new file mode 100644 index 00000000..36bd8eb3 --- /dev/null +++ b/src/handlers/metadata/get.js @@ -0,0 +1,42 @@ +import config from '../../common/config'; +import { getClient } from '../../common/elasticsearch_client'; + + +function getQueryString(request) { + if (request.params.type === 'elastalert_error') { + return '*:*'; + } + else { + return `rule_name:${request.query.rule_name || '*'}`; + } +} + +export default function metadataHandler(request, response) { + /** + * @type {ElastalertServer} + */ + var client = getClient(); + + client.search({ + index: config.get('writeback_index'), + type: request.params.type, + body: { + from : request.query.from || 0, + size : request.query.size || 100, + query: { + query_string: { + query: getQueryString(request) + } + }, + sort: [{ '@timestamp': { order: 'desc' } }] + } + }).then(function(resp) { + resp.hits.hits = resp.hits.hits.map(h => h._source); + response.send(resp.hits); + }, function(err) { + response.send({ + error: err + }); + }); + +} diff --git a/src/handlers/templates/id/post.js b/src/handlers/templates/id/post.js index 6412d27d..e4459bb5 100644 --- a/src/handlers/templates/id/post.js +++ b/src/handlers/templates/id/post.js @@ -26,8 +26,8 @@ export default function templatePostHandler(request, response) { }); }) .catch(function (error) { - if (error.error === 'ruleNotFound') { - server.templatesController.createRule(request.params.id, body) + if (error.error === 'templateNotFound') { + server.templatesController.createTemplate(request.params.id, body) .then(function () { logger.sendSuccessful(); response.send({ diff --git a/src/handlers/test/post.js b/src/handlers/test/post.js index 1d8bf74a..1ac35115 100644 --- a/src/handlers/test/post.js +++ b/src/handlers/test/post.js @@ -8,7 +8,9 @@ let logger = new RouteLogger('/test', 'POST'); const optionsSchema = Joi.object().keys({ testType: Joi.string().valid('all', 'schemaOnly', 'countOnly').default('all'), days: Joi.number().min(1).default(1), - alert: Joi.boolean().default(false) + alert: Joi.boolean().default(false), + format: Joi.string().default(''), + maxResults: Joi.number().default(0) }).default(); function analyzeRequest(request) { diff --git a/src/routes/routes.js b/src/routes/routes.js index 3a435340..5161560a 100644 --- a/src/routes/routes.js +++ b/src/routes/routes.js @@ -14,6 +14,8 @@ import templateDeleteHandler from '../handlers/templates/id/delete'; import testPostHandler from '../handlers/test/post'; import configGetHandler from '../handlers/config/get'; import configPostHandler from '../handlers/config/post'; +import metadataHandler from '../handlers/metadata/get'; +import mappingHandler from '../handlers/mapping/get'; /** * A server route. @@ -74,6 +76,16 @@ let routes = [ path: 'download', method: ['POST'], handler: [downloadRulesHandler] + }, + { + path: 'metadata/:type', + method: ['GET'], + handler: [metadataHandler] + }, + { + path: 'mapping/:index', + method: ['GET'], + handler: [mappingHandler] } ];