[SIEM] Add scripts for on boarding prepackage timeline (#67496)

* add prepackaged timelines * generate ndjson * expose end api points * fix types * fix types * fix unit test * install prepackage timelines * plumbing for prepackaged timelines * read ndjson by line * fix unit test * update templates * fix types * fix types * fix integration test * update script * name it back * add timeline status into rule status api * fix update messages * fix unit tests * fix integration test * rename types * update prepackaged timelines * update prepackaged timelines script * update scripts * fix update for elastic template * move timeline utils * export timelines scripts * update module path * fix intefration test * add delete all timelines script * readme * add get_timeline_by_templatetimeline_id * add getTimelineByIdRoute * rename file * add unit test * fix types * fix types * update readme * fix error id * fix unit test * update path * update i18n * update readme * load prepacked timelines by default * add unit tests * Update x-pack/plugins/security_solution/server/lib/detection_engine/rules/prepackaged_timelines/README.md Co-authored-by: Xavier Mouligneau <[email protected]> * review * update prepacked timelines Co-authored-by: Elastic Machine <[email protected]> Co-authored-by: Xavier Mouligneau <[email protected]>
elastic · Jul 14, 2020 · 9c91fd9 · 9c91fd9
1 parent 2c9dac2
commit 9c91fd9
Show file tree

Hide file tree

Showing 83 changed files with 3,634 additions and 666 deletions.
diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts
@@ -117,6 +117,7 @@ export const TIMELINE_URL = '/api/timeline';
 export const TIMELINE_DRAFT_URL = `${TIMELINE_URL}/_draft`;
 export const TIMELINE_EXPORT_URL = `${TIMELINE_URL}/_export`;
 export const TIMELINE_IMPORT_URL = `${TIMELINE_URL}/_import`;
+export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged`;
 
 /**
  * Default signals index key for kibana.dev.yml

diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/common/schemas.ts
@@ -407,6 +407,11 @@ export const rules_custom_installed = PositiveInteger;
 export const rules_not_installed = PositiveInteger;
 export const rules_not_updated = PositiveInteger;
 
+export const timelines_installed = PositiveInteger;
+export const timelines_updated = PositiveInteger;
+export const timelines_not_installed = PositiveInteger;
+export const timelines_not_updated = PositiveInteger;
+
 export const note = t.string;
 export type Note = t.TypeOf<typeof note>;
 

diff --git a/...curity_solution/common/detection_engine/schemas/response/prepackaged_rules_schema.test.ts b/...curity_solution/common/detection_engine/schemas/response/prepackaged_rules_schema.test.ts
@@ -6,14 +6,22 @@
 
 import { pipe } from 'fp-ts/lib/pipeable';
 import { left } from 'fp-ts/lib/Either';
-import { PrePackagedRulesSchema, prePackagedRulesSchema } from './prepackaged_rules_schema';
+import {
+  PrePackagedRulesAndTimelinesSchema,
+  prePackagedRulesAndTimelinesSchema,
+} from './prepackaged_rules_schema';
 import { exactCheck } from '../../../exact_check';
 import { foldLeftRight, getPaths } from '../../../test_utils';
 
 describe('prepackaged_rules_schema', () => {
   test('it should validate an empty prepackaged response with defaults', () => {
-    const payload: PrePackagedRulesSchema = { rules_installed: 0, rules_updated: 0 };
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const payload: PrePackagedRulesAndTimelinesSchema = {
+      rules_installed: 0,
+      rules_updated: 0,
+      timelines_installed: 0,
+      timelines_updated: 0,
+    };
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -22,12 +30,14 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should not validate an extra invalid field added', () => {
-    const payload: PrePackagedRulesSchema & { invalid_field: string } = {
+    const payload: PrePackagedRulesAndTimelinesSchema & { invalid_field: string } = {
       rules_installed: 0,
       rules_updated: 0,
       invalid_field: 'invalid',
+      timelines_installed: 0,
+      timelines_updated: 0,
     };
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -36,8 +46,13 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_installed" number', () => {
-    const payload: PrePackagedRulesSchema = { rules_installed: -1, rules_updated: 0 };
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const payload: PrePackagedRulesAndTimelinesSchema = {
+      rules_installed: -1,
+      rules_updated: 0,
+      timelines_installed: 0,
+      timelines_updated: 0,
+    };
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -48,8 +63,13 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_updated"', () => {
-    const payload: PrePackagedRulesSchema = { rules_installed: 0, rules_updated: -1 };
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const payload: PrePackagedRulesAndTimelinesSchema = {
+      rules_installed: 0,
+      rules_updated: -1,
+      timelines_installed: 0,
+      timelines_updated: 0,
+    };
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -60,9 +80,14 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response if "rules_installed" is not there', () => {
-    const payload: PrePackagedRulesSchema = { rules_installed: 0, rules_updated: 0 };
+    const payload: PrePackagedRulesAndTimelinesSchema = {
+      rules_installed: 0,
+      rules_updated: 0,
+      timelines_installed: 0,
+      timelines_updated: 0,
+    };
     delete payload.rules_installed;
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -73,9 +98,14 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response if "rules_updated" is not there', () => {
-    const payload: PrePackagedRulesSchema = { rules_installed: 0, rules_updated: 0 };
+    const payload: PrePackagedRulesAndTimelinesSchema = {
+      rules_installed: 0,
+      rules_updated: 0,
+      timelines_installed: 0,
+      timelines_updated: 0,
+    };
     delete payload.rules_updated;
-    const decoded = prePackagedRulesSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 

diff --git a/...ns/security_solution/common/detection_engine/schemas/response/prepackaged_rules_schema.ts b/...ns/security_solution/common/detection_engine/schemas/response/prepackaged_rules_schema.ts
@@ -7,14 +7,28 @@
 import * as t from 'io-ts';
 
 /* eslint-disable @typescript-eslint/camelcase */
-import { rules_installed, rules_updated } from '../common/schemas';
+import {
+  rules_installed,
+  rules_updated,
+  timelines_installed,
+  timelines_updated,
+} from '../common/schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
-export const prePackagedRulesSchema = t.exact(
-  t.type({
-    rules_installed,
-    rules_updated,
-  })
+const prePackagedRulesSchema = t.type({
+  rules_installed,
+  rules_updated,
+});
+
+const prePackagedTimelinesSchema = t.type({
+  timelines_installed,
+  timelines_updated,
+});
+
+export const prePackagedRulesAndTimelinesSchema = t.exact(
+  t.intersection([prePackagedRulesSchema, prePackagedTimelinesSchema])
 );
 
-export type PrePackagedRulesSchema = t.TypeOf<typeof prePackagedRulesSchema>;
+export type PrePackagedRulesAndTimelinesSchema = t.TypeOf<
+  typeof prePackagedRulesAndTimelinesSchema
+>;
diff --git a/...solution/common/detection_engine/schemas/response/prepackaged_rules_status_schema.test.ts b/...solution/common/detection_engine/schemas/response/prepackaged_rules_status_schema.test.ts
@@ -7,21 +7,24 @@
 import { pipe } from 'fp-ts/lib/pipeable';
 import { left } from 'fp-ts/lib/Either';
 import {
-  PrePackagedRulesStatusSchema,
-  prePackagedRulesStatusSchema,
+  PrePackagedRulesAndTimelinesStatusSchema,
+  prePackagedRulesAndTimelinesStatusSchema,
 } from './prepackaged_rules_status_schema';
 import { exactCheck } from '../../../exact_check';
 import { foldLeftRight, getPaths } from '../../../test_utils';
 
 describe('prepackaged_rules_schema', () => {
   test('it should validate an empty prepackaged response with defaults', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: 0,
       rules_not_installed: 0,
       rules_not_updated: 0,
       rules_custom_installed: 0,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -30,14 +33,17 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should not validate an extra invalid field added', () => {
-    const payload: PrePackagedRulesStatusSchema & { invalid_field: string } = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema & { invalid_field: string } = {
       rules_installed: 0,
       rules_not_installed: 0,
       rules_not_updated: 0,
       rules_custom_installed: 0,
       invalid_field: 'invalid',
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -46,13 +52,16 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_installed" number', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: -1,
       rules_not_installed: 0,
       rules_not_updated: 0,
       rules_custom_installed: 0,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -63,13 +72,16 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_not_installed"', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: 0,
       rules_not_installed: -1,
       rules_not_updated: 0,
       rules_custom_installed: 0,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -80,13 +92,16 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_not_updated"', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: 0,
       rules_not_installed: 0,
       rules_not_updated: -1,
       rules_custom_installed: 0,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -97,13 +112,16 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response with a negative "rules_custom_installed"', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: 0,
       rules_not_installed: 0,
       rules_not_updated: 0,
       rules_custom_installed: -1,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 
@@ -114,14 +132,17 @@ describe('prepackaged_rules_schema', () => {
   });
 
   test('it should NOT validate an empty prepackaged response if "rules_installed" is not there', () => {
-    const payload: PrePackagedRulesStatusSchema = {
+    const payload: PrePackagedRulesAndTimelinesStatusSchema = {
       rules_installed: 0,
       rules_not_installed: 0,
       rules_not_updated: 0,
       rules_custom_installed: 0,
+      timelines_installed: 0,
+      timelines_not_installed: 0,
+      timelines_not_updated: 0,
     };
     delete payload.rules_installed;
-    const decoded = prePackagedRulesStatusSchema.decode(payload);
+    const decoded = prePackagedRulesAndTimelinesStatusSchema.decode(payload);
     const checked = exactCheck(payload, decoded);
     const message = pipe(checked, foldLeftRight);
 

diff --git a/...rity_solution/common/detection_engine/schemas/response/prepackaged_rules_status_schema.ts b/...rity_solution/common/detection_engine/schemas/response/prepackaged_rules_status_schema.ts
@@ -12,16 +12,29 @@ import {
   rules_custom_installed,
   rules_not_installed,
   rules_not_updated,
+  timelines_installed,
+  timelines_not_installed,
+  timelines_not_updated,
 } from '../common/schemas';
 /* eslint-enable @typescript-eslint/camelcase */
 
-export const prePackagedRulesStatusSchema = t.exact(
-  t.type({
-    rules_custom_installed,
-    rules_installed,
-    rules_not_installed,
-    rules_not_updated,
-  })
+export const prePackagedTimelinesStatusSchema = t.type({
+  timelines_installed,
+  timelines_not_installed,
+  timelines_not_updated,
+});
+
+const prePackagedRulesStatusSchema = t.type({
+  rules_custom_installed,
+  rules_installed,
+  rules_not_installed,
+  rules_not_updated,
+});
+
+export const prePackagedRulesAndTimelinesStatusSchema = t.exact(
+  t.intersection([prePackagedRulesStatusSchema, prePackagedTimelinesStatusSchema])
 );
 
-export type PrePackagedRulesStatusSchema = t.TypeOf<typeof prePackagedRulesStatusSchema>;
+export type PrePackagedRulesAndTimelinesStatusSchema = t.TypeOf<
+  typeof prePackagedRulesAndTimelinesStatusSchema
+>;